Merge branch 'COMP-385' into 'develop'

COMP-385 : Export data entry for all (sub)question regardless

See merge request !122

compendium_v2/routes/common.py

@@ -3,9 +3,11 @@ Utilities used by multiple route blueprints.
 """
 import functools
 import logging
+from itertools import product
 
 from compendium_v2 import db
-from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectionMethod, ConnectivityCoverage
+from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectionMethod, ConnectivityCoverage, \
+    UserCategory
 from compendium_v2.db.presentation_models import NREN, AlienWave, BudgetEntry, Capacity, CentralProcurement, \
     CertificateProviders, ChargingStructure, CommercialChargingLevel, CommercialConnectivity, ConnectedProportion, \
     ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, \
@@ -107,6 +109,39 @@ def fetch_data_from_table(table_model, extract_function):
     return []
 
 
+def fetch_data_from_table_with_user_category(table_model, extract_function):
+    distinct_years = (
+        db.session.query(distinct(table_model.year))
+        .order_by(table_model.year.asc())
+        .all()
+    )
+
+    result = []
+
+    for years, user_category in product(distinct_years, UserCategory):
+        year = years[0]
+        entries = (
+            db.session.query(NREN, year, table_model)
+            .outerjoin(table_model, and_(NREN.id == table_model.nren_id, table_model.year == year,
+                                         user_category == table_model.user_category))
+            .order_by(NREN.name.asc())
+            .all()
+        )
+        for nren, year, entry_model in entries:
+            if entry_model is None:  # Placeholder entry
+                data = {
+                    'nren': nren.name,
+                    'nren_country': nren.country,
+                    'year': year,
+                    'user_category': user_category.value,
+                }
+            else:
+                data = extract_function(nren, year, entry_model)
+
+            result.append(data)
+    return result
+
+
 def extract_model_data(nren, year, entry_model):
     if isinstance(entry_model, DarkFibreLease):
         return {

compendium_v2/routes/data_download.py

@@ -14,7 +14,8 @@ from compendium_v2.db.presentation_models import AlienWave, BudgetEntry, Capacit
     Standards, SubOrganization, TrafficRatio, TrafficStatistics, \
     TrafficVolume, Policy, WeatherMap
 from compendium_v2.routes import common
-from compendium_v2.routes.common import extract_model_data, extract_special_model_data, fetch_data_from_table
+from compendium_v2.routes.common import extract_model_data, extract_special_model_data, fetch_data_from_table, \
+    fetch_data_from_table_with_user_category
 from flask import Blueprint
 
 routes = Blueprint('data_download', __name__)
@@ -45,23 +46,23 @@ def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
     result_set.append({'name': 'Commercial Charging Level', 'data': entries})
 
     # Fetch and extract data from the ConnectedProportion table and add it to the result set
-    entries = fetch_data_from_table(ConnectedProportion, extract_model_data)
+    entries = fetch_data_from_table_with_user_category(ConnectedProportion, extract_model_data)
     result_set.append({'name': 'Market Share', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLevel table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLevel, extract_model_data)
-    result_set.append({'name': 'Level of IP Connectivity by Institution Type', 'data': entries})
+    entries = fetch_data_from_table_with_user_category(ConnectivityLevel, extract_model_data)
+    result_set.append({'name': 'Level of IP Connectivity ', 'data': entries})
 
     # Fetch and extract data from the ConnectionCarrier table and add it to the result set
-    entries = fetch_data_from_table(ConnectionCarrier, extract_model_data)
-    result_set.append({'name': 'Methods of Carrying IP Traffic to Users', 'data': entries})
+    entries = fetch_data_from_table_with_user_category(ConnectionCarrier, extract_model_data)
+    result_set.append({'name': 'Carrying IP Traffic to Users', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLoad table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLoad, extract_model_data)
+    entries = fetch_data_from_table_with_user_category(ConnectivityLoad, extract_model_data)
     result_set.append({'name': 'Traffic Load', 'data': entries})
 
     # Fetch and extract data from the ConnectivityGrowth table
-    entries = fetch_data_from_table(ConnectivityGrowth, extract_model_data)
+    entries = fetch_data_from_table_with_user_category(ConnectivityGrowth, extract_model_data)
     result_set.append({'name': 'Est Traffic Growth 3 years', 'data': entries})
 
     # Fetch and extract data from the CommercialConnectivity table and add it to the result set