fix for pulling all NREN response

compendium_v2/routes/common.py

@@ -5,16 +5,21 @@ import functools
 import logging
 
 from compendium_v2 import db
-from compendium_v2.db.presentation_model_enums import ConnectionMethod
-from compendium_v2.db.presentation_models import NREN, AlienWave, Capacity, CentralProcurement, CertificateProviders, \
+from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectionMethod, ConnectivityCoverage
+from compendium_v2.db.presentation_models import NREN, AlienWave, BudgetEntry, Capacity, CentralProcurement, \
+    CertificateProviders, ChargingStructure, CommercialChargingLevel, CommercialConnectivity, ConnectedProportion, \
+    ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, \
     CrisisExercises, \
-    DarkFibreInstalled, EOSCListings, ExternalConnections, FibreLight, MonitoringTools, NetworkAutomation, \
-    NetworkFunctionVirtualisation, NetworkMapUrls, NonREPeers, OpsAutomation, PassiveMonitoring, PertTeam, \
-    DarkFibreLease, RemoteCampuses, SecurityControls, ServiceManagement, ServiceUserTypes, SiemVendors, Standards, \
-    TrafficRatio, TrafficStatistics, WeatherMap, PreviewYear
+    DarkFibreInstalled, ECProject, EOSCListings, ExternalConnections, FibreLight, FundingSource, InstitutionURLs, \
+    MonitoringTools, NRENService, NetworkAutomation, \
+    NetworkFunctionVirtualisation, NetworkMapUrls, NonREPeers, NrenStaff, OpsAutomation, ParentOrganization, \
+    PassiveMonitoring, PertTeam, \
+    DarkFibreLease, Policy, RemoteCampuses, SecurityControls, ServiceManagement, ServiceUserTypes, SiemVendors, \
+    Standards, SubOrganization, \
+    TrafficRatio, TrafficStatistics, TrafficVolume, WeatherMap, PreviewYear
 from flask import Response, request
 from flask_login import current_user  # type: ignore
-from sqlalchemy import select
+from sqlalchemy import select, distinct, and_
 
 logger = logging.getLogger(__name__)
 
@@ -78,34 +83,36 @@ def get_data(table_class):
 
 
 def fetch_data_from_table(table_model, extract_function):
-    # Assuming 'table_model' has a relationship with NREN
-    entries = (
-        db.session.query(table_model)
-        .outerjoin(NREN, table_model.nren_id == NREN.id)
-        .order_by(NREN.name.asc(), table_model.year.desc())
+    distinct_years = (
+        db.session.query(distinct(table_model.year))
+        .order_by(table_model.year.asc())
+        .all()
     )
 
-    is_admin = (not current_user.is_anonymous) and current_user.is_admin
-    preview = is_admin and request.args.get('preview') is not None
-
-    if not preview:
-        preview_subquery = select(PreviewYear.year).correlate(None)
-        entries = entries.filter(~table_model.year.in_(preview_subquery))
-
-    entries = entries.all()
+    result = []
+    for distinct_year in distinct_years:
+        year = distinct_year[0]
+        entries = (
+            db.session.query(NREN, year, table_model)
+            .outerjoin(table_model, and_(NREN.id == table_model.nren_id, table_model.year == year))
+            .order_by(NREN.name.asc())
+            .all()
+        )
+        result.extend(entries)
 
-    if table_model == ExternalConnections or table_model == RemoteCampuses:
-        return [item for entry in entries for item in extract_function(entry)]
+    if result is not None:
+        if table_model == ExternalConnections or table_model == RemoteCampuses:
+            return [item for entry in result for item in extract_function(entry[0], entry[1], entry[2])]
+        return [extract_function(entry[0], entry[1], entry[2]) for entry in result]
+    return []
 
-    return [extract_function(entry) for entry in entries]
 
-
-def extract_model_data(entry_model):
+def extract_model_data(nren, year, entry_model):
     if isinstance(entry_model, DarkFibreLease):
         return {
             'nren': entry_model.nren.name,
             'nren_country': entry_model.nren.country,
-            'year': int(entry_model.year),
+            'year': int(year),
             'iru_or_lease': entry_model.iru_or_lease,
             'fibre_length_in_country': entry_model.fibre_length_in_country,
             'fibre_length_outside_country': entry_model.fibre_length_outside_country,
@@ -206,23 +213,6 @@ def extract_model_data(entry_model):
             'typical_backbone_capacity': entry_model.typical_backbone_capacity,
         }
 
-    if isinstance(entry_model, ExternalConnections):
-        flat_remote_campuses_data = []
-        for connection in entry_model.connections:
-            flat_remote_campuses_data.append({
-                'nren': entry_model.nren.name,
-                'nren_country': entry_model.nren.country,
-                'year': int(entry_model.year),
-                'link_name': connection['link_name'],
-                'capacity': connection['capacity'],
-                'from_organization': connection['from_organization'],
-                'to_organization': connection['to_organization'],
-                'interconnection_method': ConnectionMethod(connection['interconnection_method']).value
-                if connection['interconnection_method'] is not None else None
-            })
-
-        return flat_remote_campuses_data
-
     if isinstance(entry_model, NonREPeers):
         return {
             'nren': entry_model.nren.name,
@@ -263,19 +253,6 @@ def extract_model_data(entry_model):
             if entry_model.network_automation is not None else None,
             'network_automation_specifics': ', '.join(entry_model.network_automation_specifics),
         }
-    if isinstance(entry_model, RemoteCampuses):
-        flat_remote_campuses_data = []
-        for connection in entry_model.connections:
-            flat_remote_campuses_data.append({
-                'nren': entry_model.nren.name,
-                'nren_country': entry_model.nren.country,
-                'year': int(entry_model.year),
-                'remote_campus_connectivity': entry_model.remote_campus_connectivity,
-                'country': connection['country'],
-                'local_r_and_e_connection': connection['local_r_and_e_connection']
-            })
-
-        return flat_remote_campuses_data
     if isinstance(entry_model, CrisisExercises):
         return {
             'nren': entry_model.nren.name,
@@ -333,3 +310,238 @@ def extract_model_data(entry_model):
             'business_continuity_plans_specifics': entry_model.business_continuity_plans_specifics,
             'crisis_management_procedure': entry_model.crisis_management_procedure,
         }
+    if isinstance(entry_model, TrafficVolume):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'to_customers': float(entry_model.to_customers),
+            'from_customers': float(entry_model.from_customers),
+            'to_external': float(entry_model.to_external),
+            'from_external': float(entry_model.from_external)
+        }
+    if isinstance(entry_model, NrenStaff):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'permanent_fte': float(entry_model.permanent_fte),
+            'subcontracted_fte': float(entry_model.subcontracted_fte),
+            'technical_fte': float(entry_model.technical_fte),
+            'non_technical_fte': float(entry_model.non_technical_fte)
+        }
+    if isinstance(entry_model, Policy):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'strategic_plan': entry_model.strategic_plan,
+            'environmental': entry_model.environmental,
+            'equal_opportunity': entry_model.equal_opportunity,
+            'connectivity': entry_model.connectivity,
+            'acceptable_use': entry_model.acceptable_use,
+            'privacy_notice': entry_model.privacy_notice,
+            'data_protection': entry_model.data_protection,
+            'gender_equality': entry_model.gender_equality
+        }
+    if isinstance(entry_model, SubOrganization):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'name': entry_model.organization,
+            'role': entry_model.role
+        }
+    if isinstance(entry_model, ParentOrganization):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'name': entry_model.organization
+        }
+    if isinstance(entry_model, NRENService):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'product_name': entry_model.product_name,
+            'additional_information': entry_model.additional_information,
+            'official_description': entry_model.official_description,
+            'service_name': entry_model.service.name,
+            'service_category': entry_model.service.category.value,
+            'service_description': entry_model.service.description
+        }
+    if isinstance(entry_model, InstitutionURLs):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'urls': entry_model.urls
+        }
+    if isinstance(entry_model, ECProject):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'project': entry_model.project
+        }
+    if isinstance(entry_model, CommercialConnectivity):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'commercial_r_and_e': entry_model.commercial_r_and_e.value
+            if entry_model.commercial_r_and_e is not None else None,
+            'commercial_general': entry_model.commercial_general.value
+            if entry_model.commercial_general is not None else None,
+            'commercial_collaboration': entry_model.commercial_collaboration.value
+            if entry_model.commercial_collaboration is not None else None,
+            'commercial_service_provider': entry_model.commercial_service_provider.value
+            if entry_model.commercial_service_provider is not None else None,
+            'university_spin_off': entry_model.university_spin_off.value
+            if entry_model.university_spin_off is not None else None,
+        }
+    if isinstance(entry_model, ConnectivityGrowth):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'user_category': entry_model.user_category.value,
+            'growth': entry_model.growth,
+        }
+
+    if isinstance(entry_model, ConnectivityLoad):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'user_category': entry_model.user_category.value,
+            'average_load_from_institutions': entry_model.average_load_from_institutions,
+            'average_load_to_institutions': entry_model.average_load_to_institutions,
+            'peak_load_from_institutions': entry_model.peak_load_from_institutions,
+            'peak_load_to_institutions': entry_model.peak_load_to_institutions,
+        }
+    if isinstance(entry_model, ConnectionCarrier):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'user_category': entry_model.user_category.value,
+            'carry_mechanism': entry_model.carry_mechanism.value
+            if entry_model.carry_mechanism is not None else CarryMechanism.other.value,
+        }
+
+    if isinstance(entry_model, ConnectivityLevel):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'user_category': entry_model.user_category.value,
+            'typical_speed': entry_model.typical_speed,
+            'highest_speed': entry_model.highest_speed,
+            'highest_speed_proportion': entry_model.highest_speed_proportion,
+        }
+
+    if isinstance(entry_model, ConnectedProportion):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'user_category': entry_model.user_category.value
+            if entry_model.user_category is not None else None,
+            'coverage': entry_model.coverage.value
+            if entry_model.coverage is not None else ConnectivityCoverage.unsure.value,
+            'number_connected': entry_model.number_connected,
+            'market_share': entry_model.market_share,
+            'users_served': entry_model.users_served
+        }
+    if isinstance(entry_model, CommercialChargingLevel):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'collaboration': entry_model.collaboration.value
+            if entry_model.collaboration is not None else None,
+            'service_supplier': entry_model.service_supplier.value
+            if entry_model.service_supplier is not None else None,
+            'direct_peering': entry_model.direct_peering.value
+            if entry_model.direct_peering is not None else None,
+        }
+
+    if isinstance(entry_model, ChargingStructure):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': int(entry_model.year),
+            'fee_type': entry_model.fee_type.value
+            if entry_model.fee_type is not None else None,
+        }
+    if isinstance(entry_model, FundingSource):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'year': entry_model.year,
+            'client_institutions': float(entry_model.client_institutions),
+            'european_funding': float(entry_model.european_funding),
+            'gov_public_bodies': float(entry_model.gov_public_bodies),
+            'commercial': float(entry_model.commercial),
+            'other': float(entry_model.other)
+        }
+
+    if isinstance(entry_model, BudgetEntry):
+        return {
+            'nren': entry_model.nren.name,
+            'nren_country': entry_model.nren.country,
+            'budget': float(entry_model.budget),
+            'year': entry_model.year,
+        }
+    else:
+        return {
+            'nren': nren.name,
+            'nren_country': nren.country,
+            'year': int(year),
+        }
+
+
+def extract_special_model_data(nren, year, entry_model):
+    if isinstance(entry_model, ExternalConnections):
+        flat_remote_campuses_data = []
+        for connection in entry_model.connections:
+            flat_remote_campuses_data.append({
+                'nren': entry_model.nren.name,
+                'nren_country': entry_model.nren.country,
+                'year': int(entry_model.year),
+                'link_name': connection['link_name'],
+                'capacity': connection['capacity'],
+                'from_organization': connection['from_organization'],
+                'to_organization': connection['to_organization'],
+                'interconnection_method': ConnectionMethod(connection['interconnection_method']).value
+                if connection['interconnection_method'] is not None else None
+            })
+        return flat_remote_campuses_data
+    if isinstance(entry_model, RemoteCampuses):
+        flat_remote_campuses_data = []
+        if entry_model.connections:
+            for connection in entry_model.connections:
+                flat_remote_campuses_data.append({
+                    'nren': entry_model.nren.name,
+                    'nren_country': entry_model.nren.country,
+                    'year': int(entry_model.year),
+                    'remote_campus_connectivity': entry_model.remote_campus_connectivity,
+                    'country': connection['country'],
+                    'local_r_and_e_connection': connection['local_r_and_e_connection']
+                })
+            return flat_remote_campuses_data
+        else:
+            return [{
+                'nren': entry_model.nren.name,
+                'nren_country': entry_model.nren.country,
+                'year': int(entry_model.year),
+                'remote_campus_connectivity': entry_model.remote_campus_connectivity,
+            }]
+    else:
+        return [{
+            'nren': nren.name,
+            'nren_country': nren.country,
+            'year': int(year),
+        }]

compendium_v2/routes/data_download.py

@@ -13,24 +13,7 @@ from compendium_v2.db.presentation_models import AlienWave, BudgetEntry, Capacit
     Standards, SubOrganization, TrafficRatio, TrafficStatistics, \
     TrafficVolume, Policy, WeatherMap
 from compendium_v2.routes import common
-from compendium_v2.routes.budget import extract_data as budget_view
-from compendium_v2.routes.charging import extract_data as charging_view
-from compendium_v2.routes.common import extract_model_data, fetch_data_from_table
-from compendium_v2.routes.connectivity import commercial_charging_level_extract_data
-from compendium_v2.routes.connectivity import commercial_connectivity_extract_data
-from compendium_v2.routes.connectivity import connected_proportion_extract_data
-from compendium_v2.routes.connectivity import connection_carrier_extract_data
-from compendium_v2.routes.connectivity import connectivity_growth_extract_data
-from compendium_v2.routes.connectivity import connectivity_level_extract_data
-from compendium_v2.routes.connectivity import connectivity_load_extract_data
-from compendium_v2.routes.ec_projects import ec_project_extract_data
-from compendium_v2.routes.funding import extract_data as funding_view
-from compendium_v2.routes.institutions_urls import institution_extract_data
-from compendium_v2.routes.nren_services import nren_service_extract_data
-from compendium_v2.routes.organization import extract_parent_organization, extract_sub_organization
-from compendium_v2.routes.policy import policy_extract_data
-from compendium_v2.routes.staff import staff_extract_data
-from compendium_v2.routes.traffic import traffic_volume_extract_data
+from compendium_v2.routes.common import extract_model_data, extract_special_model_data, fetch_data_from_table
 from flask import Blueprint
 
 routes = Blueprint('data_download', __name__)
@@ -43,77 +26,85 @@ def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
     result_set: List[Optional[Dict[str, Any]]] = []
 
     # Fetch and extract data from the BudgetEntry table and add it to the result set
-    entries = fetch_data_from_table(BudgetEntry, budget_view)
+    entries = fetch_data_from_table(BudgetEntry, extract_model_data)
     result_set.append({'name': 'Budget', 'data': entries})
 
     # Fetch and extract data from the FundingSource table and add it to the result set
-    entries = fetch_data_from_table(FundingSource, funding_view)
+    entries = fetch_data_from_table(FundingSource, extract_model_data)
     result_set.append({'name': 'Funding Source', 'data': entries})
 
     # Fetch and extract data from the ChargingStructure table and add it to the result set
-    entries = fetch_data_from_table(ChargingStructure, charging_view)
+    entries = fetch_data_from_table(ChargingStructure, extract_model_data)
     result_set.append({'name': 'Charging Structure', 'data': entries})
 
     # Fetch and extract data from the CommercialChargingLevel table and add it to the result set
-    entries = fetch_data_from_table(CommercialChargingLevel, commercial_charging_level_extract_data)
+    entries = fetch_data_from_table(CommercialChargingLevel, extract_model_data)
     result_set.append({'name': 'Commercial Charging Level', 'data': entries})
 
     # Fetch and extract data from the ConnectedProportion table and add it to the result set
-    entries = fetch_data_from_table(ConnectedProportion, connected_proportion_extract_data)
+    entries = fetch_data_from_table(ConnectedProportion, extract_model_data)
     result_set.append({'name': 'Connected Proportion', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLevel table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLevel, connectivity_level_extract_data)
+    entries = fetch_data_from_table(ConnectivityLevel, extract_model_data)
     result_set.append({'name': 'Connectivity Level', 'data': entries})
 
     # Fetch and extract data from the ConnectionCarrier table and add it to the result set
-    entries = fetch_data_from_table(ConnectionCarrier, connection_carrier_extract_data)
+    entries = fetch_data_from_table(ConnectionCarrier, extract_model_data)
     result_set.append({'name': 'Connection Carrier', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLoad table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLoad, connectivity_load_extract_data)
+    entries = fetch_data_from_table(ConnectivityLoad, extract_model_data)
     result_set.append({'name': 'Connectivity Load', 'data': entries})
 
     # Fetch and extract data from the ConnectivityGrowth table
-    entries = fetch_data_from_table(ConnectivityGrowth, connectivity_growth_extract_data)
+    entries = fetch_data_from_table(ConnectivityGrowth, extract_model_data)
     result_set.append({'name': 'Connectivity Growth', 'data': entries})
 
     # Fetch and extract data from the CommercialConnectivity table and add it to the result set
-    entries = fetch_data_from_table(CommercialConnectivity, commercial_connectivity_extract_data)
+    entries = fetch_data_from_table(CommercialConnectivity, extract_model_data)
     result_set.append({'name': 'Commercial Connectivity', 'data': entries})
 
     # Fetch and extract data from the ECProject table and add it to the result set
-    entries = fetch_data_from_table(ECProject, ec_project_extract_data)
+    entries = fetch_data_from_table(ECProject, extract_model_data)
     result_set.append({'name': 'EC Project', 'data': entries})
 
     # Fetch and extract data from the InstitutionURLs table and add it to the result set
-    entries = fetch_data_from_table(InstitutionURLs, institution_extract_data)
+    entries = fetch_data_from_table(InstitutionURLs, extract_model_data)
     result_set.append({'name': 'Institutions', 'data': entries})
 
     # Fetch and extract data from the NRENService table and add it to the result set
-    entries = fetch_data_from_table(NRENService, nren_service_extract_data)
+    entries = fetch_data_from_table(NRENService, extract_model_data)
     result_set.append({'name': 'Service', 'data': entries})
 
     # Fetch and extract data from the ParentOrganization table and add it to the result set
-    entries = fetch_data_from_table(ParentOrganization, extract_parent_organization)
+    entries = fetch_data_from_table(ParentOrganization, extract_model_data)
     result_set.append({'name': 'Parent Organization', 'data': entries})
 
     # Fetch and extract data from the SubOrganization table and add it to the result set
-    entries = fetch_data_from_table(SubOrganization, extract_sub_organization)
+    entries = fetch_data_from_table(SubOrganization, extract_model_data)
     result_set.append({'name': 'Sub-Organization', 'data': entries})
 
     # Fetch and extract data from the Policy table and add it to the result set
-    entries = fetch_data_from_table(Policy, policy_extract_data)
+    entries = fetch_data_from_table(Policy, extract_model_data)
     result_set.append({'name': 'Policy', 'data': entries})
 
     # Fetch and extract data from the NrenStaff table and add it to the result set
-    entries = fetch_data_from_table(NrenStaff, staff_extract_data)
+    entries = fetch_data_from_table(NrenStaff, extract_model_data)
     result_set.append({'name': 'Staff', 'data': entries})
 
     # Fetch and extract data from the TrafficVolume table and add it to the result set
-    entries = fetch_data_from_table(TrafficVolume, traffic_volume_extract_data)
+    entries = fetch_data_from_table(TrafficVolume, extract_model_data)
     result_set.append({'name': 'Traffic Volume', 'data': entries})
 
+    # Fetch and extract data from the ExternalConnections table and add it to the result set
+    entries = fetch_data_from_table(ExternalConnections, extract_special_model_data)
+    result_set.append({'name': 'External Connections', 'data': entries})
+
+    # Fetch and extract data from the RemoteCampuses table and add it to the result set
+    entries = fetch_data_from_table(RemoteCampuses, extract_special_model_data)
+    result_set.append({'name': 'Remote Campuses', 'data': entries})
+
     # Fetch and extract data from the DarkFibreLease table and add it to the result set
     entries = fetch_data_from_table(DarkFibreLease, extract_model_data)
     result_set.append({'name': 'Dark Fibre Lease', 'data': entries})
@@ -162,10 +153,6 @@ def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
     entries = fetch_data_from_table(Capacity, extract_model_data)
     result_set.append({'name': 'Capacity', 'data': entries})
 
-    # Fetch and extract data from the ExternalConnections table and add it to the result set
-    entries = fetch_data_from_table(ExternalConnections, extract_model_data)
-    result_set.append({'name': 'External Connections', 'data': entries})
-
     # Fetch and extract data from the NonREPeers table and add it to the result set
     entries = fetch_data_from_table(NonREPeers, extract_model_data)
     result_set.append({'name': 'Non R&E Peers', 'data': entries})
@@ -186,10 +173,6 @@ def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
     entries = fetch_data_from_table(NetworkAutomation, extract_model_data)
     result_set.append({'name': 'Network Automation', 'data': entries})
 
-    # Fetch and extract data from the RemoteCampuses table and add it to the result set
-    entries = fetch_data_from_table(RemoteCampuses, extract_model_data)
-    result_set.append({'name': 'Remote Campuses', 'data': entries})
-
     # Fetch and extract data from the CrisisExercises table and add it to the result set
     entries = fetch_data_from_table(CrisisExercises, extract_model_data)
     result_set.append({'name': 'Crisis Exercises', 'data': entries})