added datamodels for policy, connected users, and network questions

compendium_v2/background_task/parse_excel_data.py

@@ -2,7 +2,7 @@ import logging
 import openpyxl
 import os
 
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.model_enums import FeeType
 from compendium_v2.environment import setup_logging
 
 setup_logging()

compendium_v2/db/model.py

@@ -3,24 +3,44 @@ from __future__ import annotations
 
 import logging
 from decimal import Decimal
-from enum import Enum
 from typing import List, Optional
-from typing_extensions import Annotated
+from typing_extensions import Annotated, TypedDict
 
 from sqlalchemy import String, JSON
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.schema import ForeignKey
 
 from compendium_v2.db import db
+from compendium_v2.db.model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, ServiceCategory, \
+    ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, FeeType
 
 
 logger = logging.getLogger(__name__)
 
+
 str128 = Annotated[str, 128]
 str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
 str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
 int_pk = Annotated[int, mapped_column(primary_key=True)]
 int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
+user_category_pk = Annotated[UserCategory, mapped_column(primary_key=True)]
+json_str_list = Annotated[List[str], mapped_column(JSON)]
+
+ExternalConnection = TypedDict(
+    'ExternalConnection',
+    {
+        'link_name': str,
+        'capacity': Optional[Decimal],
+        'from_organization': str,
+        'to_organization': str,
+        'interconnection_method': Optional[ConnectionMethod]
+    }
+)
+
+RemoteCampus = TypedDict(
+    'RemoteCampus',
+    {'country': str, 'local_r_and_e_connection': Optional[bool]}
+)
 
 
 # Unfortunately flask-sqlalchemy doesnt fully support DeclarativeBase yet.
@@ -60,14 +80,6 @@ class FundingSource(db.Model):
     other: Mapped[Decimal]
 
 
-class FeeType(Enum):
-    flat_fee = "flat_fee"
-    usage_based_fee = "usage_based_fee"
-    combination = "combination"
-    no_charge = "no_charge"
-    other = "other"
-
-
 class ChargingStructure(db.Model):
     __tablename__ = 'charging_structure'
     nren_id: Mapped[int_pk_fkNREN]
@@ -140,8 +152,321 @@ class TrafficVolume(db.Model):
 
 class InstitutionURLs(db.Model):
     __tablename__ = 'institution_urls'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    urls: Mapped[json_str_list]
+
+
+class CentralProcurement(db.Model):
+    __tablename__ = 'central_procurement'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    central_procurement: Mapped[bool]
+    amount: Mapped[Optional[Decimal]]
+
+
+class ServiceManagement(db.Model):
+    __tablename__ = 'service_management'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    service_management_framework: Mapped[Optional[bool]]
+    service_level_targets: Mapped[Optional[bool]]
+
+
+class ServiceUserTypes(db.Model):
+    __tablename__ = 'service_user_types'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    service_category: Mapped[ServiceCategory]
+
+
+class EOSCListings(db.Model):
+    __tablename__ = 'eosc_listings'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    service_names: Mapped[json_str_list]
+
+
+class Standards(db.Model):
+    __tablename__ = 'standards'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    audits: Mapped[Optional[bool]]
+    audit_specifics: Mapped[str]
+    business_continuity_plans: Mapped[Optional[bool]]
+    business_continuity_plans_specifics: Mapped[str]
+    crisis_management_procedure: Mapped[Optional[bool]]
+
+
+class CrisisExcercises(db.Model):
+    __tablename__ = 'crisis_excercises'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    exercise_descriptions: Mapped[json_str_list]
+
+
+class SecurityControls(db.Model):
+    __tablename__ = 'security_controls'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    security_control_descriptions: Mapped[json_str_list]
+
+
+class ConnectedProportion(db.Model):
+    __tablename__ = 'connected_proportion'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    coverage: Mapped[Optional[ConnectivityCoverage]]
+    number_connected: Mapped[Optional[int]]
+    market_share: Mapped[Optional[Decimal]]
+    users_served: Mapped[Optional[int]]
+
+
+class ConnectivityLevel(db.Model):
+    __tablename__ = 'connectivity_level'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    typical_speed: Mapped[Optional[int]]
+    highest_speed: Mapped[Optional[int]]
+    highest_speed_proportion: Mapped[Optional[Decimal]]
+
+
+class ConnectionCarrier(db.Model):
+    __tablename__ = 'connection_carrier'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    carry_mechanism: Mapped[CarryMechanism]
+
+
+class ConnectivityLoad(db.Model):
+    __tablename__ = 'connectivity_load'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    average_load_from_institutions: Mapped[Optional[int]]
+    average_load_to_institutions: Mapped[Optional[int]]
+    peak_load_from_institutions: Mapped[Optional[int]]
+    peak_load_to_institutions: Mapped[Optional[int]]
+
+
+class ConnectivityGrowth(db.Model):
+    __tablename__ = 'connectivity_growth'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    user_category: Mapped[user_category_pk]
+    growth: Mapped[Decimal]
+
+
+class CommercialConnectivity(db.Model):
+    __tablename__ = 'commercial_connectivity'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    commercial_r_and_e: Mapped[Optional[CommercialConnectivityCoverage]]
+    commercial_general: Mapped[Optional[CommercialConnectivityCoverage]]
+    commercial_collaboration: Mapped[Optional[CommercialConnectivityCoverage]]
+    commercial_service_provider: Mapped[Optional[CommercialConnectivityCoverage]]
+    university_spin_off: Mapped[Optional[CommercialConnectivityCoverage]]
+
+
+class CommercialChargingLevel(db.Model):
+    __tablename__ = 'commercial_charging_level'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    collaboration: Mapped[Optional[CommarcialChargingLevel]]
+    service_supplier: Mapped[Optional[CommarcialChargingLevel]]
+    direct_peering: Mapped[Optional[CommarcialChargingLevel]]
+
+
+class RemoteCampuses(db.Model):
+    __tablename__ = 'remote_campuses'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    remote_campus_connectivity: Mapped[bool]
+    connections: Mapped[List[RemoteCampus]] = mapped_column(JSON)
+
+
+class DarkFibreLease(db.Model):
+    __tablename__ = 'dark_fibre_lease'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    iru_or_lease: Mapped[bool]
+    fibre_length_in_country: Mapped[Optional[int]]
+    fibre_length_outside_country: Mapped[Optional[int]]
+    iru_duration: Mapped[Optional[Decimal]]
+
+
+class DarkFibreInstalled(db.Model):
+    __tablename__ = 'dark_fibre_installed'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    installed: Mapped[bool]
+    fibre_length_in_country: Mapped[Optional[int]]
+
+
+class FibreLight(db.Model):
+    __tablename__ = 'fibre_light'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    light_description: Mapped[str]
+
+
+class NetworkMapUrls(db.Model):
+    __tablename__ = 'network_map_urls'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    urls: Mapped[json_str_list]
+
+
+class MonitoringTools(db.Model):
+    __tablename__ = 'monitoring_tools'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    tool_descriptions: Mapped[json_str_list]
+    netflow_processing_description: Mapped[str]
+
+
+class PassiveMonitoring(db.Model):
+    __tablename__ = 'passive_monitoring'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    monitoring: Mapped[bool]
+    method: Mapped[Optional[MonitoringMethod]]
+
+
+class TrafficStatistics(db.Model):
+    __tablename__ = 'traffic_statistics'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    traffic_statistics: Mapped[bool]
+    urls: Mapped[json_str_list]
+
+
+class SiemVendors(db.Model):
+    __tablename__ = 'siem_vendors'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    vendor_names: Mapped[json_str_list]
+
+
+class CertificateProviders(db.Model):
+    __tablename__ = 'certificate_providers'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    provider_names: Mapped[json_str_list]
+
+
+class WeatherMap(db.Model):
+    __tablename__ = 'weather_map'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    weather_map: Mapped[bool]
+    url: Mapped[str]
+
+
+class PertTeam(db.Model):
+    __tablename__ = 'pert_team'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    pert_team: Mapped[YesNoPlanned]
+
+
+class AlienWave(db.Model):
+    __tablename__ = 'alien_wave'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    alien_wave_third_pary: Mapped[Optional[YesNoPlanned]]
+    nr_of_alien_wave_third_party_services: Mapped[Optional[int]]
+    alien_wave_internal: Mapped[Optional[bool]]
+
+
+class Capacity(db.Model):
+    __tablename__ = 'capacity'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    largest_link_capacity: Mapped[Optional[Decimal]]
+    typical_backbone_capacity: Mapped[Optional[Decimal]]
+
+
+class ExternalConnections(db.Model):
+    __tablename__ = 'external_connections'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    connections: Mapped[List[ExternalConnection]] = mapped_column(JSON)
+
+
+class NonREPeers(db.Model):
+    __tablename__ = 'non_r_and_e_peers'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    nr_of_non_r_and_e_peers: Mapped[int]
+
+
+class TrafficRatio(db.Model):
+    __tablename__ = 'traffic_ratio'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    r_and_e_percentage: Mapped[int]
+    commodity_percentage: Mapped[int]
+
+
+class OpsAutomation(db.Model):
+    __tablename__ = 'ops_automation'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    ops_automation: Mapped[YesNoPlanned]
+    ops_automation_specifics: Mapped[str]
+
+
+class NetworkFunctionVirtualisation(db.Model):
+    __tablename__ = 'network_function_virtualisation'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    nfv: Mapped[YesNoPlanned]
+    nfv_specifics: Mapped[json_str_list]
+
 
+class NetworkAutomation(db.Model):
+    __tablename__ = 'network_automation'
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    urls: Mapped[List[str]] = mapped_column(JSON)
+    network_automation: Mapped[YesNoPlanned]
+    network_automation_specifics: Mapped[json_str_list]

compendium_v2/db/model_enums.py

+from enum import Enum
+
+
+class FeeType(Enum):
+    flat_fee = "flat_fee"
+    usage_based_fee = "usage_based_fee"
+    combination = "combination"
+    no_charge = "no_charge"
+    other = "other"
+
+
+class UserCategory(Enum):
+    universities = "universities"
+    further_education = "further_education"
+    secondary_schools = "secondary_schools"
+    primary_schools = "primary_schools"
+    institutes = "institutes"
+    cultural = "cultural"
+    hospitals = "hospitals"
+    government = "government"
+    iros = "iros"
+    for_profit_orgs = "for_profit_orgs"
+
+
+class ServiceCategory(Enum):
+    network_services = "network_services"
+    isp_support = "isp_support"
+    security = "security"
+    identity = "identity"
+    collaboration = "collaboration"
+    multimedia = "multimedia"
+    storage_and_hosting = "storage_and_hosting"
+    professional_services = "professional_services"
+
+
+class ConnectivityCoverage(Enum):
+    yes_incl_other = "yes_incl_other"
+    yes_national_nren = "yes_national_nren"
+    sometimes = "sometimes"
+    no_policy = "no_policy"
+    no_financial = "no_financial"
+    no_other = "no_other"
+    unsure = "unsure"
+
+
+class CarryMechanism(Enum):
+    nren_local_loops = "nren_local_loops"
+    regional_nren_backbone = "regional_nren_backbone"
+    commercial_provider_backbone = "commercial_provider_backbone"
+    man = "man"
+    other = "other"
+
+
+class CommercialConnectivityCoverage(Enum):
+    yes_incl_other = "yes_incl_other"
+    yes_national_nren = "yes_national_nren"
+    yes_if_sponsored = "yes_if_sponsored"
+    no_but_direct_peering = "no_but_direct_peering"
+    no_policy = "no_policy"
+    no_financial = "no_financial"
+    no_other = "no_other"
+
+
+class CommarcialChargingLevel(Enum):
+    higher_than_r_e_charges = "higher_than_r_e_charges"
+    same_as_r_e_charges = "same_as_r_e_charges"
+    no_charges_if_r_e_requested = "no_charges_if_r_e_requested"
+    lower_than_r_e_charges = "lower_than_r_e_charges"
+
+
+class ConnectionMethod(Enum):
+    internet_exchange = "internet_exchange"
+    open_exchange = "open_exchange"
+    direct = "direct"
+    geant = "geant"
+    other = "other"
+
+
+class YesNoPlanned(Enum):
+    yes = "yes"
+    no = "no"
+    planned = "planned"
+
+
+class MonitoringMethod(Enum):
+    span_ports = "span_ports"
+    taps = "taps"
+    both = "both"

compendium_v2/publishers/survey_publisher_2022.py

@@ -17,7 +17,7 @@ from sqlalchemy import delete, text
 from collections import defaultdict
 
 import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.model_enums import FeeType
 from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
 from compendium_v2.publishers.helpers import extract_urls

compendium_v2/publishers/survey_publisher_v2.py

 from decimal import Decimal
+from typing import List
 
 from sqlalchemy import delete, select
 
 from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
-    InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
+from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, FundingSource, \
+    InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection
+from compendium_v2.db.model_enums import FeeType
 from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
 
 
-def map_2023(nren, answers):
+def map_2023(nren, answers) -> None:
     year = 2023
 
     for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
                         NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume]:
-        db.session.execute(delete(table_class).where(table_class.year == 2023))
+        db.session.execute(delete(table_class).where(table_class.year == 2023))  # type: ignore
 
     answers = answers["data"]
     budget = answers.get("budget")
@@ -104,6 +106,22 @@ def map_2023(nren, answers):
                 urls=urls
             ))
 
+    external_connections = answers.get("external_connections")
+    if external_connections:
+        connections: List[ExternalConnection] = []
+        for connection in external_connections:
+            connections.append({
+                'link_name': connection.get('link_name', ''),
+                'capacity': connection.get('capacity'),
+                'from_organization': connection.get('from_organization', ''),
+                'to_organization': connection.get('to_organization', ''),
+                'interconnection_method': connection.get('interconnection_method')
+            })
+        db.session.add(ExternalConnections(
+            nren_id=nren.id, nren=nren, year=year,
+            connections=connections
+        ))
+
 
 def publish(year):
     responses = db.session.scalars(

test/test_survey_publisher_2022.py

 from sqlalchemy import select
 
-from compendium_v2.db import db, model
+from compendium_v2.db import db, model, model_enums
 from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
     StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
 
@@ -280,11 +280,11 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         ).all()
         assert len(charging_structures) == 3
         assert charging_structures[0].nren.name.lower() == 'nren1'
-        assert charging_structures[0].fee_type == model.FeeType.no_charge
+        assert charging_structures[0].fee_type == model_enums.FeeType.no_charge
         assert charging_structures[1].nren.name.lower() == 'nren2'
-        assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+        assert charging_structures[1].fee_type == model_enums.FeeType.usage_based_fee
         assert charging_structures[2].nren.name.lower() == 'nren3'
-        assert charging_structures[2].fee_type == model.FeeType.other
+        assert charging_structures[2].fee_type == model_enums.FeeType.other
 
         _ec_data = db.session.scalars(
             select(model.ECProject).order_by(model.ECProject.nren_id.asc())

test/test_survey_publisher_v2.py

@@ -5,7 +5,7 @@ import os
 from sqlalchemy import func, select
 
 from compendium_v2 import db
-from compendium_v2.db import model
+from compendium_v2.db import model, model_enums
 from compendium_v2.publishers.survey_publisher_v2 import map_2023
 
 
@@ -53,7 +53,7 @@ def test_v2_publisher_full(app):
         assert funding_source.other == Decimal("10")
 
         charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
-        assert charging_structure == model.FeeType.usage_based_fee
+        assert charging_structure == model_enums.FeeType.usage_based_fee
 
         staff = db.session.scalar(select(model.NrenStaff))
         assert staff.permanent_fte == Decimal("5.6")
@@ -97,3 +97,28 @@ def test_v2_publisher_full(app):
 
         client_urls = db.session.scalar(select(model.InstitutionURLs))
         assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
+
+        external_connections = db.session.scalar(select(model.ExternalConnections))
+        external_connection_list = external_connections.connections
+        assert len(external_connection_list) == 6
+        assert external_connection_list[0] == {
+            "capacity": "1",
+            "from_organization": "GEANT",
+            "interconnection_method": "geant",
+            "link_name": "GEANT",
+            "to_organization": "MREN"
+        }
+        assert external_connection_list[3] == {
+            "capacity": None,
+            "from_organization": "",
+            "interconnection_method": "other",
+            "link_name": "",
+            "to_organization": ""
+        }
+        assert external_connection_list[4] == {
+            "capacity": "1.1",
+            "from_organization": "",
+            "interconnection_method": None,
+            "link_name": "",
+            "to_organization": "",
+        }