Finished release 0.47.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.47] - 2024-01-30
+- COMP-288: Mass data download network and other models
+
 ## [0.46] - 2024-01-26
 - COMP-288: Mass data download
 

compendium_v2/routes/data_download.py

 import logging
 from typing import List, Optional, Dict, Any, Sequence
-
-from compendium_v2.db.presentation_models import BudgetEntry, CommercialChargingLevel, CommercialConnectivity, \
-    ConnectedProportion, ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, ECProject, \
-    FundingSource, ChargingStructure, InstitutionURLs, NRENService, NrenStaff, ParentOrganization, SubOrganization, \
-    TrafficVolume, Policy
+from compendium_v2.auth.session_management import admin_required
+
+from compendium_v2.db.presentation_models import AlienWave, BudgetEntry, Capacity, CentralProcurement, \
+    CertificateProviders, \
+    CommercialChargingLevel, CommercialConnectivity, \
+    ConnectedProportion, ConnectionCarrier, ConnectivityGrowth, ConnectivityLevel, ConnectivityLoad, CrisisExercises, \
+    DarkFibreInstalled, \
+    DarkFibreLease, ECProject, EOSCListings, ExternalConnections, FibreLight, \
+    FundingSource, ChargingStructure, InstitutionURLs, MonitoringTools, NRENService, NetworkAutomation, \
+    NetworkFunctionVirtualisation, NetworkMapUrls, NonREPeers, NrenStaff, OpsAutomation, ParentOrganization, \
+    PassiveMonitoring, PertTeam, RemoteCampuses, SecurityControls, ServiceManagement, ServiceUserTypes, SiemVendors, \
+    Standards, SubOrganization, TrafficRatio, TrafficStatistics, \
+    TrafficVolume, Policy, WeatherMap
 from compendium_v2.routes import common
-from compendium_v2.routes.budget import extract_data as budget_view
-from compendium_v2.routes.charging import extract_data as charging_view
-from compendium_v2.routes.common import fetch_data_from_table
-from compendium_v2.routes.connectivity import commercial_charging_level_extract_data
-from compendium_v2.routes.connectivity import commercial_connectivity_extract_data
-from compendium_v2.routes.connectivity import connected_proportion_extract_data
-from compendium_v2.routes.connectivity import connection_carrier_extract_data
-from compendium_v2.routes.connectivity import connectivity_growth_extract_data
-from compendium_v2.routes.connectivity import connectivity_level_extract_data
-from compendium_v2.routes.connectivity import connectivity_load_extract_data
-from compendium_v2.routes.ec_projects import ec_project_extract_data
-from compendium_v2.routes.funding import extract_data as funding_view
-from compendium_v2.routes.institutions_urls import institution_extract_data
-from compendium_v2.routes.nren_services import nren_service_extract_data
-from compendium_v2.routes.organization import extract_parent_organization, extract_sub_organization
-from compendium_v2.routes.policy import policy_extract_data
-from compendium_v2.routes.staff import staff_extract_data
-from compendium_v2.routes.traffic import traffic_volume_extract_data
+from compendium_v2.routes.common import extract_model_data, extract_special_model_data, fetch_data_from_table
 from flask import Blueprint
 
 routes = Blueprint('data_download', __name__)
@@ -32,79 +23,188 @@ logger = logging.getLogger(__name__)
 
 @routes.route('/', methods=['GET'])
 @common.require_accepts_json
+@admin_required
 def fetch_and_combine_data() -> Sequence[Optional[Dict[str, Any]]]:
     result_set: List[Optional[Dict[str, Any]]] = []
 
     # Fetch and extract data from the BudgetEntry table and add it to the result set
-    entries = fetch_data_from_table(BudgetEntry, budget_view)
+    entries = fetch_data_from_table(BudgetEntry, extract_model_data)
     result_set.append({'name': 'Budget', 'data': entries})
 
     # Fetch and extract data from the FundingSource table and add it to the result set
-    entries = fetch_data_from_table(FundingSource, funding_view)
+    entries = fetch_data_from_table(FundingSource, extract_model_data)
     result_set.append({'name': 'Funding Source', 'data': entries})
 
     # Fetch and extract data from the ChargingStructure table and add it to the result set
-    entries = fetch_data_from_table(ChargingStructure, charging_view)
+    entries = fetch_data_from_table(ChargingStructure, extract_model_data)
     result_set.append({'name': 'Charging Structure', 'data': entries})
 
     # Fetch and extract data from the CommercialChargingLevel table and add it to the result set
-    entries = fetch_data_from_table(CommercialChargingLevel, commercial_charging_level_extract_data)
+    entries = fetch_data_from_table(CommercialChargingLevel, extract_model_data)
     result_set.append({'name': 'Commercial Charging Level', 'data': entries})
 
     # Fetch and extract data from the ConnectedProportion table and add it to the result set
-    entries = fetch_data_from_table(ConnectedProportion, connected_proportion_extract_data)
+    entries = fetch_data_from_table(ConnectedProportion, extract_model_data)
     result_set.append({'name': 'Connected Proportion', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLevel table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLevel, connectivity_level_extract_data)
+    entries = fetch_data_from_table(ConnectivityLevel, extract_model_data)
     result_set.append({'name': 'Connectivity Level', 'data': entries})
 
     # Fetch and extract data from the ConnectionCarrier table and add it to the result set
-    entries = fetch_data_from_table(ConnectionCarrier, connection_carrier_extract_data)
+    entries = fetch_data_from_table(ConnectionCarrier, extract_model_data)
     result_set.append({'name': 'Connection Carrier', 'data': entries})
 
     # Fetch and extract data from the ConnectivityLoad table and add it to the result set
-    entries = fetch_data_from_table(ConnectivityLoad, connectivity_load_extract_data)
+    entries = fetch_data_from_table(ConnectivityLoad, extract_model_data)
     result_set.append({'name': 'Connectivity Load', 'data': entries})
 
     # Fetch and extract data from the ConnectivityGrowth table
-    entries = fetch_data_from_table(ConnectivityGrowth, connectivity_growth_extract_data)
+    entries = fetch_data_from_table(ConnectivityGrowth, extract_model_data)
     result_set.append({'name': 'Connectivity Growth', 'data': entries})
 
     # Fetch and extract data from the CommercialConnectivity table and add it to the result set
-    entries = fetch_data_from_table(CommercialConnectivity, commercial_connectivity_extract_data)
+    entries = fetch_data_from_table(CommercialConnectivity, extract_model_data)
     result_set.append({'name': 'Commercial Connectivity', 'data': entries})
 
     # Fetch and extract data from the ECProject table and add it to the result set
-    entries = fetch_data_from_table(ECProject, ec_project_extract_data)
+    entries = fetch_data_from_table(ECProject, extract_model_data)
     result_set.append({'name': 'EC Project', 'data': entries})
 
     # Fetch and extract data from the InstitutionURLs table and add it to the result set
-    entries = fetch_data_from_table(InstitutionURLs, institution_extract_data)
+    entries = fetch_data_from_table(InstitutionURLs, extract_model_data)
     result_set.append({'name': 'Institutions', 'data': entries})
 
     # Fetch and extract data from the NRENService table and add it to the result set
-    entries = fetch_data_from_table(NRENService, nren_service_extract_data)
+    entries = fetch_data_from_table(NRENService, extract_model_data)
     result_set.append({'name': 'Service', 'data': entries})
 
     # Fetch and extract data from the ParentOrganization table and add it to the result set
-    entries = fetch_data_from_table(ParentOrganization, extract_parent_organization)
+    entries = fetch_data_from_table(ParentOrganization, extract_model_data)
     result_set.append({'name': 'Parent Organization', 'data': entries})
 
     # Fetch and extract data from the SubOrganization table and add it to the result set
-    entries = fetch_data_from_table(SubOrganization, extract_sub_organization)
+    entries = fetch_data_from_table(SubOrganization, extract_model_data)
     result_set.append({'name': 'Sub-Organization', 'data': entries})
 
     # Fetch and extract data from the Policy table and add it to the result set
-    entries = fetch_data_from_table(Policy, policy_extract_data)
+    entries = fetch_data_from_table(Policy, extract_model_data)
     result_set.append({'name': 'Policy', 'data': entries})
 
     # Fetch and extract data from the NrenStaff table and add it to the result set
-    entries = fetch_data_from_table(NrenStaff, staff_extract_data)
+    entries = fetch_data_from_table(NrenStaff, extract_model_data)
     result_set.append({'name': 'Staff', 'data': entries})
 
     # Fetch and extract data from the TrafficVolume table and add it to the result set
-    entries = fetch_data_from_table(TrafficVolume, traffic_volume_extract_data)
+    entries = fetch_data_from_table(TrafficVolume, extract_model_data)
     result_set.append({'name': 'Traffic Volume', 'data': entries})
 
+    # Fetch and extract data from the ExternalConnections table and add it to the result set
+    entries = fetch_data_from_table(ExternalConnections, extract_special_model_data)
+    result_set.append({'name': 'External Connections', 'data': entries})
+
+    # Fetch and extract data from the RemoteCampuses table and add it to the result set
+    entries = fetch_data_from_table(RemoteCampuses, extract_special_model_data)
+    result_set.append({'name': 'Remote Campuses', 'data': entries})
+
+    # Fetch and extract data from the DarkFibreLease table and add it to the result set
+    entries = fetch_data_from_table(DarkFibreLease, extract_model_data)
+    result_set.append({'name': 'Dark Fibre Lease', 'data': entries})
+
+    # Fetch and extract data from the DarkFibreInstalled table and add it to the result set
+    entries = fetch_data_from_table(DarkFibreInstalled, extract_model_data)
+    result_set.append({'name': 'Dark Fibre Installed', 'data': entries})
+
+    # Fetch and extract data from the FibreLight table and add it to the result set
+    entries = fetch_data_from_table(FibreLight, extract_model_data)
+    result_set.append({'name': 'Fibre Light', 'data': entries})
+
+    # Fetch and extract data from the NetworkMapUrls table and add it to the result set
+    entries = fetch_data_from_table(NetworkMapUrls, extract_model_data)
+    result_set.append({'name': 'Network Map Urls', 'data': entries})
+
+    # Fetch and extract data from the MonitoringTools table and add it to the result set
+    entries = fetch_data_from_table(MonitoringTools, extract_model_data)
+    result_set.append({'name': 'Monitoring Tools', 'data': entries})
+
+    # Fetch and extract data from the PassiveMonitoring table and add it to the result set
+    entries = fetch_data_from_table(PassiveMonitoring, extract_model_data)
+    result_set.append({'name': 'Passive Monitoring', 'data': entries})
+
+    # Fetch and extract data from the TrafficStatistics table and add it to the result set
+    entries = fetch_data_from_table(TrafficStatistics, extract_model_data)
+    result_set.append({'name': 'Traffic Statistics', 'data': entries})
+
+    # Fetch and extract data from the SiemVendors table and add it to the result set
+    entries = fetch_data_from_table(SiemVendors, extract_model_data)
+    result_set.append({'name': 'Siem Vendors', 'data': entries})
+
+    # Fetch and extract data from the CertificateProviders table and add it to the result set
+    entries = fetch_data_from_table(CertificateProviders, extract_model_data)
+    result_set.append({'name': 'Certificate Providers', 'data': entries})
+
+    # Fetch and extract data from the WeatherMap table and add it to the result set
+    entries = fetch_data_from_table(WeatherMap, extract_model_data)
+    result_set.append({'name': 'Weather Map', 'data': entries})
+
+    # Fetch and extract data from the AlienWave table and add it to the result set
+    entries = fetch_data_from_table(AlienWave, extract_model_data)
+    result_set.append({'name': 'Alien Wave', 'data': entries})
+
+    # Fetch and extract data from the Capacity table and add it to the result set
+    entries = fetch_data_from_table(Capacity, extract_model_data)
+    result_set.append({'name': 'Capacity', 'data': entries})
+
+    # Fetch and extract data from the NonREPeers table and add it to the result set
+    entries = fetch_data_from_table(NonREPeers, extract_model_data)
+    result_set.append({'name': 'Non R&E Peers', 'data': entries})
+
+    # Fetch and extract data from the TrafficRatio table and add it to the result set
+    entries = fetch_data_from_table(TrafficRatio, extract_model_data)
+    result_set.append({'name': 'Traffic Ratio', 'data': entries})
+
+    # Fetch and extract data from the OpsAutomation table and add it to the result set
+    entries = fetch_data_from_table(OpsAutomation, extract_model_data)
+    result_set.append({'name': 'Ops Automation', 'data': entries})
+
+    # Fetch and extract data from the NetworkFunctionVirtualisation table and add it to the result set
+    entries = fetch_data_from_table(NetworkFunctionVirtualisation, extract_model_data)
+    result_set.append({'name': 'Network Function Virtualisation', 'data': entries})
+
+    # Fetch and extract data from the NetworkAutomation table and add it to the result set
+    entries = fetch_data_from_table(NetworkAutomation, extract_model_data)
+    result_set.append({'name': 'Network Automation', 'data': entries})
+
+    # Fetch and extract data from the CrisisExercises table and add it to the result set
+    entries = fetch_data_from_table(CrisisExercises, extract_model_data)
+    result_set.append({'name': 'Crisis Exercises', 'data': entries})
+
+    # Fetch and extract data from the SecurityControls table and add it to the result set
+    entries = fetch_data_from_table(SecurityControls, extract_model_data)
+    result_set.append({'name': 'Security Controls', 'data': entries})
+
+    # Fetch and extract data from the CentralProcurement table and add it to the result set
+    entries = fetch_data_from_table(CentralProcurement, extract_model_data)
+    result_set.append({'name': 'Central Procurement', 'data': entries})
+
+    # Fetch and extract data from the CentralProcurement table and add it to the result set
+    entries = fetch_data_from_table(ServiceManagement, extract_model_data)
+    result_set.append({'name': 'Service Management', 'data': entries})
+
+    # Fetch and extract data from the ServiceUserTypes table and add it to the result set
+    entries = fetch_data_from_table(ServiceUserTypes, extract_model_data)
+    result_set.append({'name': 'Service User Types', 'data': entries})
+
+    # Fetch and extract data from the EOSCListings table and add it to the result set
+    entries = fetch_data_from_table(EOSCListings, extract_model_data)
+    result_set.append({'name': 'EOSC Listings', 'data': entries})
+
+    # Fetch and extract data from the Standards table and add it to the result set
+    entries = fetch_data_from_table(Standards, extract_model_data)
+    result_set.append({'name': 'Standards', 'data': entries})
+
+    # Fetch and extract data from the PertTeam table and add it to the result set
+    entries = fetch_data_from_table(PertTeam, extract_model_data)
+    result_set.append({'name': 'Pert Team', 'data': entries})
+
     return result_set

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='compendium-v2',
-    version="0.46",
+    version="0.47",
     author='GEANT',
     author_email='swd@geant.org',
     description='Flask and React project for displaying '