control shadow preservation display with a flag

5 files changed, 80 insertions, 19 deletions

diff --git a/python/fatcat_web/graphics.py b/python/fatcat_web/graphics.py
index 7d6e5702..a8e217e3 100644
--- a/python/fatcat_web/graphics.py
+++ b/python/fatcat_web/graphics.py
@@ -37,14 +37,17 @@ def ia_coverage_histogram(rows: List[Tuple]) -> pygal.Graph:
     chart.add('Missing', [y['missing'] for y in years])
     return chart
 
-def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_year_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
     """
     Note: this returns a raw pygal chart; it does not render it to SVG/PNG
     """
 
     years = sorted(rows, key=lambda x: x['year'])
 
-    CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+    if merge_shadows:
+        CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+    else:
+        CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
     label_count = len(years)
     if len(years) > 30:
         label_count = 10
@@ -55,20 +58,26 @@ def preservation_by_year_histogram(rows: List[Dict]) -> pygal.Graph:
     chart.x_title = "Year"
     #chart.y_title = "Count"
     chart.x_labels = [str(y['year']) for y in years]
-    chart.add('None', [y['none'] for y in years])
-    chart.add('Shadow', [y['shadows_only'] for y in years])
+    if merge_shadows:
+        chart.add('None', [y['none'] + y['shadows_only'] for y in years])
+    else:
+        chart.add('None', [y['none'] for y in years])
+        chart.add('Shadow', [y['shadows_only'] for y in years])
     chart.add('Dark', [y['dark'] for y in years])
     chart.add('Bright', [y['bright'] for y in years])
     return chart
 
-def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_date_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
     """
     Note: this returns a raw pygal chart; it does not render it to SVG/PNG
     """
 
     dates = sorted(rows, key=lambda x: x['date'])
 
-    CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+    if merge_shadows:
+        CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+    else:
+        CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
     label_count = len(dates)
     if len(dates) > 30:
         label_count = 10
@@ -79,20 +88,26 @@ def preservation_by_date_histogram(rows: List[Dict]) -> pygal.Graph:
     chart.x_title = "Date"
     #chart.y_title = "Count"
     chart.x_labels = [str(y['date']) for y in dates]
-    chart.add('None', [y['none'] for y in dates])
-    chart.add('Shadow', [y['shadows_only'] for y in dates])
+    if merge_shadows:
+        chart.add('None', [y['none'] + y['shadows_only'] for y in dates])
+    else:
+        chart.add('None', [y['none'] for y in dates])
+        chart.add('Shadow', [y['shadows_only'] for y in dates])
     chart.add('Dark', [y['dark'] for y in dates])
     chart.add('Bright', [y['bright'] for y in dates])
     return chart
 
-def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph:
+def preservation_by_volume_histogram(rows: List[Dict], merge_shadows: bool = False) -> pygal.Graph:
     """
     Note: this returns a raw pygal chart; it does not render it to SVG/PNG
     """
 
     volumes = sorted(rows, key=lambda x: x['volume'])
 
-    CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
+    if merge_shadows:
+        CleanStyle.colors = ("red", "darkolivegreen", "limegreen")
+    else:
+        CleanStyle.colors = ("red", "darkred", "darkolivegreen", "limegreen")
     label_count = len(volumes)
     if len(volumes) >= 30:
         label_count = 10
@@ -103,8 +118,11 @@ def preservation_by_volume_histogram(rows: List[Dict]) -> pygal.Graph:
     chart.x_title = "Volume"
     #chart.y_title = "Count"
     chart.x_labels = [str(y['volume']) for y in volumes]
-    chart.add('None', [y['none'] for y in volumes])
-    chart.add('Shadow', [y['shadows_only'] for y in volumes])
+    if merge_shadows:
+        chart.add('None', [y['none'] + y['shadows_only'] for y in volumes])
+    else:
+        chart.add('None', [y['none'] for y in volumes])
+        chart.add('Shadow', [y['shadows_only'] for y in volumes])
     chart.add('Dark', [y['dark'] for y in volumes])
     chart.add('Bright', [y['bright'] for y in volumes])
     return chart
diff --git a/python/fatcat_web/routes.py b/python/fatcat_web/routes.py
index 1281a617..bcfa133b 100644
--- a/python/fatcat_web/routes.py
+++ b/python/fatcat_web/routes.py
@@ -754,10 +754,16 @@ def coverage_search():
         coverage_type_preservation = get_elastic_preservation_by_type(query)
         if query.recent:
             date_histogram = get_elastic_preservation_by_date(query)
-            date_histogram_svg = preservation_by_date_histogram(date_histogram).render_data_uri()
+            date_histogram_svg = preservation_by_date_histogram(
+                date_histogram,
+                merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+            ).render_data_uri()
         else:
             year_histogram = get_elastic_preservation_by_year(query)
-            year_histogram_svg = preservation_by_year_histogram(year_histogram).render_data_uri()
+            year_histogram_svg = preservation_by_year_histogram(
+                year_histogram,
+                merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+            ).render_data_uri()
     return render_template(
         'coverage_search.html',
         query=query,
@@ -886,7 +892,10 @@ def container_ident_preservation_by_year_svg(ident):
     except Exception as ae:
         app.log.error(ae)
         abort(503)
-    return preservation_by_year_histogram(histogram).render_response()
+    return preservation_by_year_histogram(
+        histogram,
+        merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+    ).render_response()
 
 @app.route('/container/<ident>/preservation_by_volume.json', methods=['GET', 'OPTIONS'])
 @crossdomain(origin='*',headers=['access-control-allow-origin','Content-Type'])
@@ -914,7 +923,10 @@ def container_ident_preservation_by_volume_svg(ident):
     except Exception as ae:
         app.log.error(ae)
         abort(503)
-    return preservation_by_volume_histogram(histogram).render_response()
+    return preservation_by_volume_histogram(
+        histogram,
+        merge_shadows=Config.FATCAT_MERGE_SHADOW_PRESERVATION,
+    ).render_response()
 
 @app.route('/release/<ident>.bib', methods=['GET'])
 def release_bibtex(ident):
diff --git a/python/fatcat_web/search.py b/python/fatcat_web/search.py
index b0d27b2e..7d9ce69f 100644
--- a/python/fatcat_web/search.py
+++ b/python/fatcat_web/search.py
@@ -402,6 +402,9 @@ def get_elastic_search_coverage(query: ReleaseQuery) -> dict:
     for k in ('bright', 'dark', 'shadows_only', 'none'):
         if not k in preservation_bucket:
             preservation_bucket[k] = 0
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        preservation_bucket['none'] += preservation_bucket['shadows_only']
+        preservation_bucket['shadows_only'] = 0
     stats = {
         'total': resp.hits.total,
         'preservation': preservation_bucket,
@@ -464,6 +467,9 @@ def get_elastic_container_stats(ident, issnl=None):
     for k in ('bright', 'dark', 'shadows_only', 'none'):
         if not k in preservation_bucket:
             preservation_bucket[k] = 0
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        preservation_bucket['none'] += preservation_bucket['shadows_only']
+        preservation_bucket['shadows_only'] = 0
     release_type_bucket = agg_to_dict(resp.aggregations.release_type)
     stats = {
         'ident': ident,
@@ -602,6 +608,10 @@ def get_elastic_preservation_by_year(query) -> List[dict]:
         year_dicts[num] = dict(year=num, bright=0, dark=0, shadows_only=0, none=0)
     for row in buckets:
         year_dicts[int(row['key']['year'])][row['key']['preservation']] = int(row['doc_count'])
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        for k in year_dicts.keys():
+            year_dicts[k]['none'] += year_dicts[k]['shadows_only']
+            year_dicts[k]['shadows_only'] = 0
     return sorted(year_dicts.values(), key=lambda x: x['year'])
 
 
@@ -675,6 +685,10 @@ def get_elastic_preservation_by_date(query) -> List[dict]:
         this_date = this_date + datetime.timedelta(days=1)
     for row in buckets:
         date_dicts[row['key']['date'][0:10]][row['key']['preservation']] = int(row['doc_count'])
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        for k in date_dicts.keys():
+            date_dicts[k]['none'] += date_dicts[k]['shadows_only']
+            date_dicts[k]['shadows_only'] = 0
     return sorted(date_dicts.values(), key=lambda x: x['date'])
 
 def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict]:
@@ -728,6 +742,10 @@ def get_elastic_container_preservation_by_volume(container_id: str) -> List[dict
     for row in buckets:
         if row['key']['volume'].isdigit():
             volume_dicts[int(row['key']['volume'])][row['key']['preservation']] = int(row['doc_count'])
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        for k in volume_dicts.keys():
+            volume_dicts[k]['none'] += volume_dicts[k]['shadows_only']
+            volume_dicts[k]['shadows_only'] = 0
     return sorted(volume_dicts.values(), key=lambda x: x['volume'])
 
 def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:
@@ -797,4 +815,8 @@ def get_elastic_preservation_by_type(query: ReleaseQuery) -> List[dict]:
     for k in type_set:
         for p in ('bright', 'dark', 'shadows_only', 'none'):
             type_dicts[k]['total'] += type_dicts[k][p]
+    if app.config['FATCAT_MERGE_SHADOW_PRESERVATION']:
+        for k in type_set:
+            type_dicts[k]['none'] += type_dicts[k]['shadows_only']
+            type_dicts[k]['shadows_only'] = 0
     return sorted(type_dicts.values(), key=lambda x: x['total'], reverse=True)
diff --git a/python/fatcat_web/templates/entity_macros.html b/python/fatcat_web/templates/entity_macros.html
index c3ae099a..718c071c 100644
--- a/python/fatcat_web/templates/entity_macros.html
+++ b/python/fatcat_web/templates/entity_macros.html
@@ -265,7 +265,6 @@ yellow
 
   {% set frac_bright = stats.bright/stats.total %}
   {% set frac_dark = stats.dark/stats.total %}
-  {% set frac_shadows_only = stats.shadows_only/stats.total %}
   {% set frac_none = stats.none/stats.total %}
 
   <div class="ui {{ extra_class }} multiple progress" data-percent="0,0,0,0" style="margin-bottom: 0.1em;">
@@ -275,9 +274,12 @@ yellow
     <div class="green bar" style="border-radius: 0; min-width: 0; width: {{ (frac_dark*100)|round(method='ceil') }}%; background-color: darkgreen;" title="dark">
       <div class="progress">{# {{ (frac_dark*100)|int }}% #}</div>
     </div>
+    {% if stats.shadows_only %}
+  {% set frac_shadows_only = stats.shadows_only/stats.total %}
     <div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_shadows_only*100)|round(method='ceil') }}%; background-color: darkred;" title="shadows only">
       <div class="progress">{# {{ (frac_shadows_only*100)|int }}% #}</div>
     </div>
+    {% endif %}
     <div class="red bar" style="border-radius: 0; min-width: 0; width: {{ (frac_none*100)|round(method='ceil') }}%;" title="no preservation">
       <div class="progress">{# {{ (frac_none*100)|int }}% #}</div>
     </div>
@@ -290,7 +292,6 @@ yellow
 
   {% set frac_bright = stats.bright/stats.total %}
   {% set frac_dark = stats.dark/stats.total %}
-  {% set frac_shadows_only = stats.shadows_only/stats.total %}
   {% set frac_none = stats.none/stats.total %}
 
   <table class="ui very basic very compact collapsing table" style="font-weight: bold; margin-left: 1em;">
@@ -305,11 +306,14 @@ yellow
         <td class="right aligned" >{{ "{:,}".format(stats.dark) }}
         <td class="right aligned" >{{ (frac_dark*100)|round(2,method='ceil') }}%
         <td>preserved but not publicly accessible (dark)
+      {% if stats.shadows_only %}
+      {% set frac_shadows_only = stats.shadows_only/stats.total %}
       <tr>
         <td style="background-color: darkred;">
         <td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }}
         <td class="right aligned" >{{ (frac_shadows_only*100)|round(2,method='ceil') }}%
         <td>only independently preserved in "shadow" libraries
+      {% endif %}
       <tr>
         <td style="background-color: #db2828;">
         <td class="right aligned" >{{ "{:,}".format(stats.none) }}
@@ -324,7 +328,6 @@ yellow
 
   {% set frac_bright = stats.bright/stats.total %}
   {% set frac_dark = stats.dark/stats.total %}
-  {% set frac_shadows_only = stats.shadows_only/stats.total %}
   {% set frac_none = stats.none/stats.total %}
 
   <table class="ui very basic very compact collapsing table">
@@ -337,10 +340,13 @@ yellow
         <td style="background-color: darkgreen;">
         <td class="right aligned" >{{ "{:,}".format(stats.dark) }}
         <td>preserved, inaccessible (dark)
+      {% if stats.shadows_only %}
+      {% set frac_shadows_only = stats.shadows_only/stats.total %}
       <tr>
         <td style="background-color: darkred;">
         <td class="right aligned" >{{ "{:,}".format(stats.shadows_only) }}
         <td>shadow library only
+      {% endif %}
       <tr>
         <td style="background-color: #db2828;">
         <td class="right aligned" >{{ "{:,}".format(stats.none) }}
diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py
index 344f1c2a..22a704d9 100644
--- a/python/fatcat_web/web_config.py
+++ b/python/fatcat_web/web_config.py
@@ -52,6 +52,9 @@ class Config(object):
     IA_XAUTH_CLIENT_ID = os.environ.get("IA_XAUTH_CLIENT_ID", default=None)
     IA_XAUTH_CLIENT_SECRET = os.environ.get("IA_XAUTH_CLIENT_SECRET", default=None)
 
+    # controls granularity of "shadow_only" preservation category
+    FATCAT_MERGE_SHADOW_PRESERVATION = os.environ.get("FATCAT_MERGE_SHADOW_PRESERVATION", default=False)
+
     # CSRF on by default, but only for WTF forms (not, eg, search, lookups, GET
     # forms)
     WTF_CSRF_CHECK_DEFAULT = False