diff --git a/Changelog.md b/Changelog.md
index e4a3140f4878795479fd040b6ffe82e2db70243c..241519c3d8f1acf4b388d5ce1733fc8593ac2870 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -2,6 +2,9 @@
All notable changes to this project will be documented in this file.
+## [0.47] - 2024-01-30
+- COMP-288: Mass data download network and other models
+
## [0.46] - 2024-01-26
- COMP-288: Mass data download
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index 53d3b6eac29d9ffa23ad4739a177f9cd13d714b7..3209017f360ee7d97148dc93e897edc6aa5e9fb1 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -3,12 +3,23 @@ Utilities used by multiple route blueprints.
"""
import functools
import logging
-from compendium_v2 import db
-from compendium_v2.db.presentation_models import NREN, PreviewYear
+from compendium_v2 import db
+from compendium_v2.db.presentation_model_enums import CarryMechanism, ConnectionMethod, ConnectivityCoverage
+from compendium_v2.db.presentation_models import NREN, AlienWave, BudgetEntry, Capacity, CentralProcurement, \
+ CertificateProviders, ChargingStructure, CommercialChargingLevel, CommercialConnectivity, ConnectedProportion, \
+ ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, \
+ CrisisExercises, \
+ DarkFibreInstalled, ECProject, EOSCListings, ExternalConnections, FibreLight, FundingSource, InstitutionURLs, \
+ MonitoringTools, NRENService, NetworkAutomation, \
+ NetworkFunctionVirtualisation, NetworkMapUrls, NonREPeers, NrenStaff, OpsAutomation, ParentOrganization, \
+ PassiveMonitoring, PertTeam, \
+ DarkFibreLease, Policy, RemoteCampuses, SecurityControls, ServiceManagement, ServiceUserTypes, SiemVendors, \
+ Standards, SubOrganization, \
+ TrafficRatio, TrafficStatistics, TrafficVolume, WeatherMap, PreviewYear
from flask import Response, request
from flask_login import current_user # type: ignore
-from sqlalchemy import select
+from sqlalchemy import select, distinct, and_
logger = logging.getLogger(__name__)
@@ -72,19 +83,465 @@ def get_data(table_class):
def fetch_data_from_table(table_model, extract_function):
- # Assuming 'table_model' has a relationship with NREN
- entries = (
- db.session.query(table_model)
- .outerjoin(NREN, table_model.nren_id == NREN.id)
- .order_by(NREN.name.asc(), table_model.year.desc())
+ distinct_years = (
+ db.session.query(distinct(table_model.year))
+ .order_by(table_model.year.asc())
+ .all()
)
- is_admin = (not current_user.is_anonymous) and current_user.is_admin
- preview = is_admin and request.args.get('preview') is not None
+ result = []
+ for distinct_year in distinct_years:
+ year = distinct_year[0]
+ entries = (
+ db.session.query(NREN, year, table_model)
+ .outerjoin(table_model, and_(NREN.id == table_model.nren_id, table_model.year == year))
+ .order_by(NREN.name.asc())
+ .all()
+ )
+ result.extend(entries)
+
+ if result is not None:
+ if table_model == ExternalConnections or table_model == RemoteCampuses:
+ return [item for entry in result for item in extract_function(entry[0], entry[1], entry[2])]
+ return [extract_function(entry[0], entry[1], entry[2]) for entry in result]
+ return []
+
+
+def extract_model_data(nren, year, entry_model):
+ if isinstance(entry_model, DarkFibreLease):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(year),
+ 'iru_or_lease': entry_model.iru_or_lease,
+ 'fibre_length_in_country': entry_model.fibre_length_in_country,
+ 'fibre_length_outside_country': entry_model.fibre_length_outside_country,
+ 'iru_duration': entry_model.iru_duration
+ }
+
+ if isinstance(entry_model, DarkFibreInstalled):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'installed': entry_model.installed,
+ 'fibre_length_in_country': entry_model.fibre_length_in_country,
+ }
+ if isinstance(entry_model, FibreLight):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'light_description': entry_model.light_description
+ }
+ if isinstance(entry_model, NetworkMapUrls):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'urls': ', '.join(entry_model.urls)
+ }
+ if isinstance(entry_model, MonitoringTools):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'tool_descriptions': ', '.join(entry_model.tool_descriptions),
+ 'netflow_processing_description': entry_model.netflow_processing_description
+ }
+ if isinstance(entry_model, PassiveMonitoring):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'monitoring': entry_model.monitoring,
+ 'method': entry_model.method.value if entry_model.method is not None else None,
+ }
+ if isinstance(entry_model, TrafficStatistics):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'traffic_statistics': entry_model.traffic_statistics,
+ 'urls': ', '.join(entry_model.urls)
+ }
+ if isinstance(entry_model, SiemVendors):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'vendor_names': ', '.join(entry_model.vendor_names)
+ }
+ if isinstance(entry_model, CertificateProviders):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'provider_names': ', '.join(entry_model.provider_names)
+ }
+ if isinstance(entry_model, WeatherMap):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'weather_map': entry_model.weather_map,
+ 'url': entry_model.url
+ }
+ if isinstance(entry_model, PertTeam):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'pert_team': entry_model.pert_team.value if entry_model.pert_team is not None else None,
+ }
+ if isinstance(entry_model, AlienWave):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'alien_wave_third_pary': entry_model.alien_wave_third_pary.value
+ if entry_model.alien_wave_third_pary is not None else None,
+ 'nr_of_alien_wave_third_party_services': entry_model.nr_of_alien_wave_third_party_services,
+ 'alien_wave_internal': entry_model.alien_wave_internal
+ }
+ if isinstance(entry_model, Capacity):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'largest_link_capacity': entry_model.largest_link_capacity,
+ 'typical_backbone_capacity': entry_model.typical_backbone_capacity,
+ }
+
+ if isinstance(entry_model, NonREPeers):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'nr_of_non_r_and_e_peers': entry_model.nr_of_non_r_and_e_peers
+ }
+ if isinstance(entry_model, TrafficRatio):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'r_and_e_percentage': entry_model.r_and_e_percentage,
+ 'commodity_percentage': entry_model.commodity_percentage,
+ }
+ if isinstance(entry_model, OpsAutomation):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'ops_automation': entry_model.ops_automation.value if entry_model.ops_automation is not None else None,
+ 'ops_automation_specifics': entry_model.ops_automation_specifics,
+ }
+ if isinstance(entry_model, NetworkFunctionVirtualisation):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'nfv': entry_model.nfv.value if entry_model.nfv is not None else None,
+ 'nfv_specifics': ', '.join(entry_model.nfv_specifics)
+ }
+ if isinstance(entry_model, NetworkAutomation):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'network_automation': entry_model.network_automation.value
+ if entry_model.network_automation is not None else None,
+ 'network_automation_specifics': ', '.join(entry_model.network_automation_specifics),
+ }
+ if isinstance(entry_model, CrisisExercises):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'exercise_descriptions': ', '.join(entry_model.exercise_descriptions),
+ }
+ if isinstance(entry_model, SecurityControls):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'security_control_descriptions': ', '.join(entry_model.security_control_descriptions),
+ }
+ if isinstance(entry_model, CentralProcurement):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'central_procurement': entry_model.central_procurement,
+ 'amount': entry_model.amount,
+ }
+ if isinstance(entry_model, ServiceManagement):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'service_management_framework': entry_model.service_management_framework,
+ 'service_level_targets': entry_model.service_management_framework
+ }
+ if isinstance(entry_model, ServiceUserTypes):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'user_category': entry_model.user_category.value,
+ 'service_category': entry_model.service_category.value
+ if entry_model.service_category is not None else None,
+ }
+ if isinstance(entry_model, EOSCListings):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'service_names': ', '.join(entry_model.service_names),
+ }
+ if isinstance(entry_model, Standards):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'audits': entry_model.audits,
+ 'audit_specifics': entry_model.audit_specifics,
+ 'business_continuity_plans': entry_model.business_continuity_plans,
+ 'business_continuity_plans_specifics': entry_model.business_continuity_plans_specifics,
+ 'crisis_management_procedure': entry_model.crisis_management_procedure,
+ }
+ if isinstance(entry_model, TrafficVolume):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'to_customers': float(entry_model.to_customers),
+ 'from_customers': float(entry_model.from_customers),
+ 'to_external': float(entry_model.to_external),
+ 'from_external': float(entry_model.from_external)
+ }
+ if isinstance(entry_model, NrenStaff):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'permanent_fte': float(entry_model.permanent_fte),
+ 'subcontracted_fte': float(entry_model.subcontracted_fte),
+ 'technical_fte': float(entry_model.technical_fte),
+ 'non_technical_fte': float(entry_model.non_technical_fte)
+ }
+ if isinstance(entry_model, Policy):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'strategic_plan': entry_model.strategic_plan,
+ 'environmental': entry_model.environmental,
+ 'equal_opportunity': entry_model.equal_opportunity,
+ 'connectivity': entry_model.connectivity,
+ 'acceptable_use': entry_model.acceptable_use,
+ 'privacy_notice': entry_model.privacy_notice,
+ 'data_protection': entry_model.data_protection,
+ 'gender_equality': entry_model.gender_equality
+ }
+ if isinstance(entry_model, SubOrganization):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'name': entry_model.organization,
+ 'role': entry_model.role
+ }
+ if isinstance(entry_model, ParentOrganization):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'name': entry_model.organization
+ }
+ if isinstance(entry_model, NRENService):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'product_name': entry_model.product_name,
+ 'additional_information': entry_model.additional_information,
+ 'official_description': entry_model.official_description,
+ 'service_name': entry_model.service.name,
+ 'service_category': entry_model.service.category.value,
+ 'service_description': entry_model.service.description
+ }
+ if isinstance(entry_model, InstitutionURLs):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'urls': entry_model.urls
+ }
+ if isinstance(entry_model, ECProject):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'project': entry_model.project
+ }
+ if isinstance(entry_model, CommercialConnectivity):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'commercial_r_and_e': entry_model.commercial_r_and_e.value
+ if entry_model.commercial_r_and_e is not None else None,
+ 'commercial_general': entry_model.commercial_general.value
+ if entry_model.commercial_general is not None else None,
+ 'commercial_collaboration': entry_model.commercial_collaboration.value
+ if entry_model.commercial_collaboration is not None else None,
+ 'commercial_service_provider': entry_model.commercial_service_provider.value
+ if entry_model.commercial_service_provider is not None else None,
+ 'university_spin_off': entry_model.university_spin_off.value
+ if entry_model.university_spin_off is not None else None,
+ }
+ if isinstance(entry_model, ConnectivityGrowth):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'user_category': entry_model.user_category.value,
+ 'growth': entry_model.growth,
+ }
+
+ if isinstance(entry_model, ConnectivityLoad):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'user_category': entry_model.user_category.value,
+ 'average_load_from_institutions': entry_model.average_load_from_institutions,
+ 'average_load_to_institutions': entry_model.average_load_to_institutions,
+ 'peak_load_from_institutions': entry_model.peak_load_from_institutions,
+ 'peak_load_to_institutions': entry_model.peak_load_to_institutions,
+ }
+ if isinstance(entry_model, ConnectionCarrier):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'user_category': entry_model.user_category.value,
+ 'carry_mechanism': entry_model.carry_mechanism.value
+ if entry_model.carry_mechanism is not None else CarryMechanism.other.value,
+ }
+
+ if isinstance(entry_model, ConnectivityLevel):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'user_category': entry_model.user_category.value,
+ 'typical_speed': entry_model.typical_speed,
+ 'highest_speed': entry_model.highest_speed,
+ 'highest_speed_proportion': entry_model.highest_speed_proportion,
+ }
+
+ if isinstance(entry_model, ConnectedProportion):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'user_category': entry_model.user_category.value
+ if entry_model.user_category is not None else None,
+ 'coverage': entry_model.coverage.value
+ if entry_model.coverage is not None else ConnectivityCoverage.unsure.value,
+ 'number_connected': entry_model.number_connected,
+ 'market_share': entry_model.market_share,
+ 'users_served': entry_model.users_served
+ }
+ if isinstance(entry_model, CommercialChargingLevel):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'collaboration': entry_model.collaboration.value
+ if entry_model.collaboration is not None else None,
+ 'service_supplier': entry_model.service_supplier.value
+ if entry_model.service_supplier is not None else None,
+ 'direct_peering': entry_model.direct_peering.value
+ if entry_model.direct_peering is not None else None,
+ }
+
+ if isinstance(entry_model, ChargingStructure):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'fee_type': entry_model.fee_type.value
+ if entry_model.fee_type is not None else None,
+ }
+ if isinstance(entry_model, FundingSource):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': entry_model.year,
+ 'client_institutions': float(entry_model.client_institutions),
+ 'european_funding': float(entry_model.european_funding),
+ 'gov_public_bodies': float(entry_model.gov_public_bodies),
+ 'commercial': float(entry_model.commercial),
+ 'other': float(entry_model.other)
+ }
+
+ if isinstance(entry_model, BudgetEntry):
+ return {
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'budget': float(entry_model.budget),
+ 'year': entry_model.year,
+ }
+ else:
+ return {
+ 'nren': nren.name,
+ 'nren_country': nren.country,
+ 'year': int(year),
+ }
- if not preview:
- preview_subquery = select(PreviewYear.year).correlate(None)
- entries = entries.filter(~table_model.year.in_(preview_subquery))
- entries = entries.all()
- return [extract_function(entry) for entry in entries]
+def extract_special_model_data(nren, year, entry_model):
+ if isinstance(entry_model, ExternalConnections):
+ flat_remote_campuses_data = []
+ for connection in entry_model.connections:
+ flat_remote_campuses_data.append({
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'link_name': connection['link_name'],
+ 'capacity': connection['capacity'],
+ 'from_organization': connection['from_organization'],
+ 'to_organization': connection['to_organization'],
+ 'interconnection_method': ConnectionMethod(connection['interconnection_method']).value
+ if connection['interconnection_method'] is not None else None
+ })
+ return flat_remote_campuses_data
+ if isinstance(entry_model, RemoteCampuses):
+ flat_remote_campuses_data = []
+ if entry_model.connections:
+ for connection in entry_model.connections:
+ flat_remote_campuses_data.append({
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'remote_campus_connectivity': entry_model.remote_campus_connectivity,
+ 'country': connection['country'],
+ 'local_r_and_e_connection': connection['local_r_and_e_connection']
+ })
+ return flat_remote_campuses_data
+ else:
+ return [{
+ 'nren': entry_model.nren.name,
+ 'nren_country': entry_model.nren.country,
+ 'year': int(entry_model.year),
+ 'remote_campus_connectivity': entry_model.remote_campus_connectivity,
+ }]
+ else:
+ return [{
+ 'nren': nren.name,
+ 'nren_country': nren.country,
+ 'year': int(year),
+ }]
diff --git a/compendium_v2/routes/data_download.py b/compendium_v2/routes/data_download.py
index a36f505afd4443f88f8902fbcb9fb1eb7be78c83..2d1c5e89fb4b00d4c9156aead186de8e22c241fc 100644
--- a/compendium_v2/routes/data_download.py
+++ b/compendium_v2/routes/data_download.py
@@ -1,29 +1,20 @@
import logging
from typing import List, Optional, Dict, Any, Sequence
-
-from compendium_v2.db.presentation_models import BudgetEntry, CommercialChargingLevel, CommercialConnectivity, \
- ConnectedProportion, ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, ECProject, \
- FundingSource, ChargingStructure, InstitutionURLs, NRENService, NrenStaff, ParentOrganization, SubOrganization, \
- TrafficVolume, Policy
+from compendium_v2.auth.session_management import admin_required
+
+from compendium_v2.db.presentation_models import AlienWave, BudgetEntry, Capacity, CentralProcurement, \
+ CertificateProviders, \
+ CommercialChargingLevel, CommercialConnectivity, \
+ ConnectedProportion, ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, CrisisExercises, \
+ DarkFibreInstalled, \
+ DarkFibreLease, ECProject, EOSCListings, ExternalConnections, FibreLight, \
+ FundingSource, ChargingStructure, InstitutionURLs, MonitoringTools, NRENService, NetworkAutomation, \
+ NetworkFunctionVirtualisation, NetworkMapUrls, NonREPeers, NrenStaff, OpsAutomation, ParentOrganization, \
+ PassiveMonitoring, PertTeam, RemoteCampuses, SecurityControls, ServiceManagement, ServiceUserTypes, SiemVendors, \
+ Standards, SubOrganization, TrafficRatio, TrafficStatistics, \
+ TrafficVolume, Policy, WeatherMap
from compendium_v2.routes import common
-from compendium_v2.routes.budget import extract_data as budget_view
-from compendium_v2.routes.charging import extract_data as charging_view
-from compendium_v2.routes.common import fetch_data_from_table
-from compendium_v2.routes.connectivity import commercial_charging_level_extract_data
-from compendium_v2.routes.connectivity import commercial_connectivity_extract_data
-from compendium_v2.routes.connectivity import connected_proportion_extract_data
-from compendium_v2.routes.connectivity import connection_carrier_extract_data
-from compendium_v2.routes.connectivity import connectivity_growth_extract_data
-from compendium_v2.routes.connectivity import connectivity_level_extract_data
-from compendium_v2.routes.connectivity import connectivity_load_extract_data
-from compendium_v2.routes.ec_projects import ec_project_extract_data
-from compendium_v2.routes.funding import extract_data as funding_view
-from compendium_v2.routes.institutions_urls import institution_extract_data
-from compendium_v2.routes.nren_services import nren_service_extract_data
-from compendium_v2.routes.organization import extract_parent_organization, extract_sub_organization
-from compendium_v2.routes.policy import policy_extract_data
-from compendium_v2.routes.staff import staff_extract_data
-from compendium_v2.routes.traffic import traffic_volume_extract_data
+from compendium_v2.routes.common import extract_model_data, extract_special_model_data, fetch_data_from_table
from flask import Blueprint
routes = Blueprint('data_download', __name__)
@@ -32,79 +23,188 @@ logger = logging.getLogger(__name__)
@routes.route('/', methods=['GET'])
@common.require_accepts_json
+@admin_required
def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
result_set: List[Optional[Dict[str, Any]]] = []
# Fetch and extract data from the BudgetEntry table and add it to the result set
- entries = fetch_data_from_table(BudgetEntry, budget_view)
+ entries = fetch_data_from_table(BudgetEntry, extract_model_data)
result_set.append({'name': 'Budget', 'data': entries})
# Fetch and extract data from the FundingSource table and add it to the result set
- entries = fetch_data_from_table(FundingSource, funding_view)
+ entries = fetch_data_from_table(FundingSource, extract_model_data)
result_set.append({'name': 'Funding Source', 'data': entries})
# Fetch and extract data from the ChargingStructure table and add it to the result set
- entries = fetch_data_from_table(ChargingStructure, charging_view)
+ entries = fetch_data_from_table(ChargingStructure, extract_model_data)
result_set.append({'name': 'Charging Structure', 'data': entries})
# Fetch and extract data from the CommercialChargingLevel table and add it to the result set
- entries = fetch_data_from_table(CommercialChargingLevel, commercial_charging_level_extract_data)
+ entries = fetch_data_from_table(CommercialChargingLevel, extract_model_data)
result_set.append({'name': 'Commercial Charging Level', 'data': entries})
# Fetch and extract data from the ConnectedProportion table and add it to the result set
- entries = fetch_data_from_table(ConnectedProportion, connected_proportion_extract_data)
+ entries = fetch_data_from_table(ConnectedProportion, extract_model_data)
result_set.append({'name': 'Connected Proportion', 'data': entries})
# Fetch and extract data from the ConnectivityLevel table and add it to the result set
- entries = fetch_data_from_table(ConnectivityLevel, connectivity_level_extract_data)
+ entries = fetch_data_from_table(ConnectivityLevel, extract_model_data)
result_set.append({'name': 'Connectivity Level', 'data': entries})
# Fetch and extract data from the ConnectionCarrier table and add it to the result set
- entries = fetch_data_from_table(ConnectionCarrier, connection_carrier_extract_data)
+ entries = fetch_data_from_table(ConnectionCarrier, extract_model_data)
result_set.append({'name': 'Connection Carrier', 'data': entries})
# Fetch and extract data from the ConnectivityLoad table and add it to the result set
- entries = fetch_data_from_table(ConnectivityLoad, connectivity_load_extract_data)
+ entries = fetch_data_from_table(ConnectivityLoad, extract_model_data)
result_set.append({'name': 'Connectivity Load', 'data': entries})
# Fetch and extract data from the ConnectivityGrowth table
- entries = fetch_data_from_table(ConnectivityGrowth, connectivity_growth_extract_data)
+ entries = fetch_data_from_table(ConnectivityGrowth, extract_model_data)
result_set.append({'name': 'Connectivity Growth', 'data': entries})
# Fetch and extract data from the CommercialConnectivity table and add it to the result set
- entries = fetch_data_from_table(CommercialConnectivity, commercial_connectivity_extract_data)
+ entries = fetch_data_from_table(CommercialConnectivity, extract_model_data)
result_set.append({'name': 'Commercial Connectivity', 'data': entries})
# Fetch and extract data from the ECProject table and add it to the result set
- entries = fetch_data_from_table(ECProject, ec_project_extract_data)
+ entries = fetch_data_from_table(ECProject, extract_model_data)
result_set.append({'name': 'EC Project', 'data': entries})
# Fetch and extract data from the InstitutionURLs table and add it to the result set
- entries = fetch_data_from_table(InstitutionURLs, institution_extract_data)
+ entries = fetch_data_from_table(InstitutionURLs, extract_model_data)
result_set.append({'name': 'Institutions', 'data': entries})
# Fetch and extract data from the NRENService table and add it to the result set
- entries = fetch_data_from_table(NRENService, nren_service_extract_data)
+ entries = fetch_data_from_table(NRENService, extract_model_data)
result_set.append({'name': 'Service', 'data': entries})
# Fetch and extract data from the ParentOrganization table and add it to the result set
- entries = fetch_data_from_table(ParentOrganization, extract_parent_organization)
+ entries = fetch_data_from_table(ParentOrganization, extract_model_data)
result_set.append({'name': 'Parent Organization', 'data': entries})
# Fetch and extract data from the SubOrganization table and add it to the result set
- entries = fetch_data_from_table(SubOrganization, extract_sub_organization)
+ entries = fetch_data_from_table(SubOrganization, extract_model_data)
result_set.append({'name': 'Sub-Organization', 'data': entries})
# Fetch and extract data from the Policy table and add it to the result set
- entries = fetch_data_from_table(Policy, policy_extract_data)
+ entries = fetch_data_from_table(Policy, extract_model_data)
result_set.append({'name': 'Policy', 'data': entries})
# Fetch and extract data from the NrenStaff table and add it to the result set
- entries = fetch_data_from_table(NrenStaff, staff_extract_data)
+ entries = fetch_data_from_table(NrenStaff, extract_model_data)
result_set.append({'name': 'Staff', 'data': entries})
# Fetch and extract data from the TrafficVolume table and add it to the result set
- entries = fetch_data_from_table(TrafficVolume, traffic_volume_extract_data)
+ entries = fetch_data_from_table(TrafficVolume, extract_model_data)
result_set.append({'name': 'Traffic Volume', 'data': entries})
+ # Fetch and extract data from the ExternalConnections table and add it to the result set
+ entries = fetch_data_from_table(ExternalConnections, extract_special_model_data)
+ result_set.append({'name': 'External Connections', 'data': entries})
+
+ # Fetch and extract data from the RemoteCampuses table and add it to the result set
+ entries = fetch_data_from_table(RemoteCampuses, extract_special_model_data)
+ result_set.append({'name': 'Remote Campuses', 'data': entries})
+
+ # Fetch and extract data from the DarkFibreLease table and add it to the result set
+ entries = fetch_data_from_table(DarkFibreLease, extract_model_data)
+ result_set.append({'name': 'Dark Fibre Lease', 'data': entries})
+
+ # Fetch and extract data from the DarkFibreInstalled table and add it to the result set
+ entries = fetch_data_from_table(DarkFibreInstalled, extract_model_data)
+ result_set.append({'name': 'Dark Fibre Installed', 'data': entries})
+
+ # Fetch and extract data from the FibreLight table and add it to the result set
+ entries = fetch_data_from_table(FibreLight, extract_model_data)
+ result_set.append({'name': 'Fibre Light', 'data': entries})
+
+ # Fetch and extract data from the NetworkMapUrls table and add it to the result set
+ entries = fetch_data_from_table(NetworkMapUrls, extract_model_data)
+ result_set.append({'name': 'Network Map Urls', 'data': entries})
+
+ # Fetch and extract data from the MonitoringTools table and add it to the result set
+ entries = fetch_data_from_table(MonitoringTools, extract_model_data)
+ result_set.append({'name': 'Monitoring Tools', 'data': entries})
+
+ # Fetch and extract data from the PassiveMonitoring table and add it to the result set
+ entries = fetch_data_from_table(PassiveMonitoring, extract_model_data)
+ result_set.append({'name': 'Passive Monitoring', 'data': entries})
+
+ # Fetch and extract data from the TrafficStatistics table and add it to the result set
+ entries = fetch_data_from_table(TrafficStatistics, extract_model_data)
+ result_set.append({'name': 'Traffic Statistics', 'data': entries})
+
+ # Fetch and extract data from the SiemVendors table and add it to the result set
+ entries = fetch_data_from_table(SiemVendors, extract_model_data)
+ result_set.append({'name': 'Siem Vendors', 'data': entries})
+
+ # Fetch and extract data from the CertificateProviders table and add it to the result set
+ entries = fetch_data_from_table(CertificateProviders, extract_model_data)
+ result_set.append({'name': 'Certificate Providers', 'data': entries})
+
+ # Fetch and extract data from the WeatherMap table and add it to the result set
+ entries = fetch_data_from_table(WeatherMap, extract_model_data)
+ result_set.append({'name': 'Weather Map', 'data': entries})
+
+ # Fetch and extract data from the AlienWave table and add it to the result set
+ entries = fetch_data_from_table(AlienWave, extract_model_data)
+ result_set.append({'name': 'Alien Wave', 'data': entries})
+
+ # Fetch and extract data from the Capacity table and add it to the result set
+ entries = fetch_data_from_table(Capacity, extract_model_data)
+ result_set.append({'name': 'Capacity', 'data': entries})
+
+ # Fetch and extract data from the NonREPeers table and add it to the result set
+ entries = fetch_data_from_table(NonREPeers, extract_model_data)
+ result_set.append({'name': 'Non R&E Peers', 'data': entries})
+
+ # Fetch and extract data from the TrafficRatio table and add it to the result set
+ entries = fetch_data_from_table(TrafficRatio, extract_model_data)
+ result_set.append({'name': 'Traffic Ratio', 'data': entries})
+
+ # Fetch and extract data from the OpsAutomation table and add it to the result set
+ entries = fetch_data_from_table(OpsAutomation, extract_model_data)
+ result_set.append({'name': 'Ops Automation', 'data': entries})
+
+ # Fetch and extract data from the NetworkFunctionVirtualisation table and add it to the result set
+ entries = fetch_data_from_table(NetworkFunctionVirtualisation, extract_model_data)
+ result_set.append({'name': 'Network Function Virtualisation', 'data': entries})
+
+ # Fetch and extract data from the NetworkAutomation table and add it to the result set
+ entries = fetch_data_from_table(NetworkAutomation, extract_model_data)
+ result_set.append({'name': 'Network Automation', 'data': entries})
+
+ # Fetch and extract data from the CrisisExercises table and add it to the result set
+ entries = fetch_data_from_table(CrisisExercises, extract_model_data)
+ result_set.append({'name': 'Crisis Exercises', 'data': entries})
+
+ # Fetch and extract data from the SecurityControls table and add it to the result set
+ entries = fetch_data_from_table(SecurityControls, extract_model_data)
+ result_set.append({'name': 'Security Controls', 'data': entries})
+
+ # Fetch and extract data from the CentralProcurement table and add it to the result set
+ entries = fetch_data_from_table(CentralProcurement, extract_model_data)
+ result_set.append({'name': 'Central Procurement', 'data': entries})
+
+ # Fetch and extract data from the CentralProcurement table and add it to the result set
+ entries = fetch_data_from_table(ServiceManagement, extract_model_data)
+ result_set.append({'name': 'Service Management', 'data': entries})
+
+ # Fetch and extract data from the ServiceUserTypes table and add it to the result set
+ entries = fetch_data_from_table(ServiceUserTypes, extract_model_data)
+ result_set.append({'name': 'Service User Types', 'data': entries})
+
+ # Fetch and extract data from the EOSCListings table and add it to the result set
+ entries = fetch_data_from_table(EOSCListings, extract_model_data)
+ result_set.append({'name': 'EOSC Listings', 'data': entries})
+
+ # Fetch and extract data from the Standards table and add it to the result set
+ entries = fetch_data_from_table(Standards, extract_model_data)
+ result_set.append({'name': 'Standards', 'data': entries})
+
+ # Fetch and extract data from the PertTeam table and add it to the result set
+ entries = fetch_data_from_table(PertTeam, extract_model_data)
+ result_set.append({'name': 'Pert Team', 'data': entries})
+
return result_set
diff --git a/setup.py b/setup.py
index 2f3ea4eb3af3a48b3ea4997489ad7a8b92377431..e470d1e539a7dbd018bf70d507c1a4d530bffba6 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='compendium-v2',
- version="0.46",
+ version="0.47",
author='GEANT',
author_email='swd@geant.org',
description='Flask and React project for displaying '