diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index 5297ed4ed6c555265b453bf01d79d0690fa51037..513860e1165d87035ab52fc8c2ad69f1fe33e677 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -18,7 +18,6 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial
logger = logging.getLogger(__name__)
-
str128 = Annotated[str, 128]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
@@ -48,13 +47,16 @@ RemoteCampus = TypedDict(
# See https://github.com/pallets-eco/flask-sqlalchemy/issues/1140
# mypy: disable-error-code="name-defined"
+class PresentationModel(db.Model):
+ __abstract__ = True
+
-class PreviewYear(db.Model):
+class PreviewYear(PresentationModel):
__tablename__ = 'preview_year'
year: Mapped[int_pk]
-class NREN(db.Model):
+class NREN(PresentationModel):
__tablename__ = 'nren'
id: Mapped[int_pk]
name: Mapped[str128]
@@ -64,7 +66,7 @@ class NREN(db.Model):
return f'<NREN {self.id} | {self.name}>'
-class BudgetEntry(db.Model):
+class BudgetEntry(PresentationModel):
__tablename__ = 'budgets'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -72,7 +74,7 @@ class BudgetEntry(db.Model):
budget: Mapped[Decimal]
-class FundingSource(db.Model):
+class FundingSource(PresentationModel):
__tablename__ = 'funding_source'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -84,7 +86,7 @@ class FundingSource(db.Model):
other: Mapped[Decimal]
-class ChargingStructure(db.Model):
+class ChargingStructure(PresentationModel):
__tablename__ = 'charging_structure'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -92,7 +94,7 @@ class ChargingStructure(db.Model):
fee_type: Mapped[Optional[FeeType]]
-class NrenStaff(db.Model):
+class NrenStaff(PresentationModel):
__tablename__ = 'nren_staff'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -103,7 +105,7 @@ class NrenStaff(db.Model):
non_technical_fte: Mapped[Decimal]
-class ParentOrganization(db.Model):
+class ParentOrganization(PresentationModel):
__tablename__ = 'parent_organization'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -111,7 +113,7 @@ class ParentOrganization(db.Model):
organization: Mapped[str128]
-class SubOrganization(db.Model):
+class SubOrganization(PresentationModel):
__tablename__ = 'sub_organization'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -120,7 +122,7 @@ class SubOrganization(db.Model):
role: Mapped[str128_pk]
-class ECProject(db.Model):
+class ECProject(PresentationModel):
__tablename__ = 'ec_project'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -128,7 +130,7 @@ class ECProject(db.Model):
project: Mapped[str256_pk]
-class Policy(db.Model):
+class Policy(PresentationModel):
__tablename__ = 'policy'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -143,7 +145,7 @@ class Policy(db.Model):
gender_equality: Mapped[str]
-class TrafficVolume(db.Model):
+class TrafficVolume(PresentationModel):
__tablename__ = 'traffic_volume'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -154,7 +156,7 @@ class TrafficVolume(db.Model):
from_external: Mapped[Decimal]
-class InstitutionURLs(db.Model):
+class InstitutionURLs(PresentationModel):
__tablename__ = 'institution_urls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -162,7 +164,7 @@ class InstitutionURLs(db.Model):
urls: Mapped[json_str_list]
-class CentralProcurement(db.Model):
+class CentralProcurement(PresentationModel):
__tablename__ = 'central_procurement'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -171,7 +173,7 @@ class CentralProcurement(db.Model):
amount: Mapped[Optional[Decimal]]
-class ServiceManagement(db.Model):
+class ServiceManagement(PresentationModel):
__tablename__ = 'service_management'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -180,7 +182,7 @@ class ServiceManagement(db.Model):
service_level_targets: Mapped[Optional[bool]]
-class ServiceUserTypes(db.Model):
+class ServiceUserTypes(PresentationModel):
__tablename__ = 'service_user_types'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -189,7 +191,7 @@ class ServiceUserTypes(db.Model):
service_category: Mapped[ServiceCategory] = mapped_column(primary_key=True)
-class EOSCListings(db.Model):
+class EOSCListings(PresentationModel):
__tablename__ = 'eosc_listings'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -197,7 +199,7 @@ class EOSCListings(db.Model):
service_names: Mapped[json_str_list]
-class Standards(db.Model):
+class Standards(PresentationModel):
__tablename__ = 'standards'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -209,7 +211,7 @@ class Standards(db.Model):
crisis_management_procedure: Mapped[Optional[bool]]
-class CrisisExercises(db.Model):
+class CrisisExercises(PresentationModel):
__tablename__ = 'crisis_exercises'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -217,7 +219,7 @@ class CrisisExercises(db.Model):
exercise_descriptions: Mapped[json_str_list]
-class SecurityControls(db.Model):
+class SecurityControls(PresentationModel):
__tablename__ = 'security_controls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -225,7 +227,7 @@ class SecurityControls(db.Model):
security_control_descriptions: Mapped[json_str_list]
-class ConnectedProportion(db.Model):
+class ConnectedProportion(PresentationModel):
__tablename__ = 'connected_proportion'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -237,7 +239,7 @@ class ConnectedProportion(db.Model):
users_served: Mapped[Optional[int]]
-class ConnectivityLevel(db.Model):
+class ConnectivityLevel(PresentationModel):
__tablename__ = 'connectivity_level'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -248,7 +250,7 @@ class ConnectivityLevel(db.Model):
highest_speed_proportion: Mapped[Optional[Decimal]]
-class ConnectionCarrier(db.Model):
+class ConnectionCarrier(PresentationModel):
__tablename__ = 'connection_carrier'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -257,7 +259,7 @@ class ConnectionCarrier(db.Model):
carry_mechanism: Mapped[CarryMechanism]
-class ConnectivityLoad(db.Model):
+class ConnectivityLoad(PresentationModel):
__tablename__ = 'connectivity_load'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -269,7 +271,7 @@ class ConnectivityLoad(db.Model):
peak_load_to_institutions: Mapped[Optional[int]]
-class ConnectivityGrowth(db.Model):
+class ConnectivityGrowth(PresentationModel):
__tablename__ = 'connectivity_growth'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -278,7 +280,7 @@ class ConnectivityGrowth(db.Model):
growth: Mapped[Decimal]
-class CommercialConnectivity(db.Model):
+class CommercialConnectivity(PresentationModel):
__tablename__ = 'commercial_connectivity'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -290,7 +292,7 @@ class CommercialConnectivity(db.Model):
university_spin_off: Mapped[Optional[CommercialConnectivityCoverage]]
-class CommercialChargingLevel(db.Model):
+class CommercialChargingLevel(PresentationModel):
__tablename__ = 'commercial_charging_level'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -300,7 +302,7 @@ class CommercialChargingLevel(db.Model):
direct_peering: Mapped[Optional[CommercialCharges]]
-class RemoteCampuses(db.Model):
+class RemoteCampuses(PresentationModel):
__tablename__ = 'remote_campuses'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -309,7 +311,7 @@ class RemoteCampuses(db.Model):
connections: Mapped[List[RemoteCampus]] = mapped_column(JSON)
-class DarkFibreLease(db.Model):
+class DarkFibreLease(PresentationModel):
__tablename__ = 'dark_fibre_lease'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -320,7 +322,7 @@ class DarkFibreLease(db.Model):
iru_duration: Mapped[Optional[Decimal]]
-class DarkFibreInstalled(db.Model):
+class DarkFibreInstalled(PresentationModel):
__tablename__ = 'dark_fibre_installed'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -329,7 +331,7 @@ class DarkFibreInstalled(db.Model):
fibre_length_in_country: Mapped[Optional[int]]
-class FibreLight(db.Model):
+class FibreLight(PresentationModel):
__tablename__ = 'fibre_light'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -337,7 +339,7 @@ class FibreLight(db.Model):
light_description: Mapped[str]
-class NetworkMapUrls(db.Model):
+class NetworkMapUrls(PresentationModel):
__tablename__ = 'network_map_urls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -345,7 +347,7 @@ class NetworkMapUrls(db.Model):
urls: Mapped[json_str_list]
-class MonitoringTools(db.Model):
+class MonitoringTools(PresentationModel):
__tablename__ = 'monitoring_tools'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -354,7 +356,7 @@ class MonitoringTools(db.Model):
netflow_processing_description: Mapped[str]
-class PassiveMonitoring(db.Model):
+class PassiveMonitoring(PresentationModel):
__tablename__ = 'passive_monitoring'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -363,7 +365,7 @@ class PassiveMonitoring(db.Model):
method: Mapped[Optional[MonitoringMethod]]
-class TrafficStatistics(db.Model):
+class TrafficStatistics(PresentationModel):
__tablename__ = 'traffic_statistics'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -372,7 +374,7 @@ class TrafficStatistics(db.Model):
urls: Mapped[json_str_list]
-class SiemVendors(db.Model):
+class SiemVendors(PresentationModel):
__tablename__ = 'siem_vendors'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -380,7 +382,7 @@ class SiemVendors(db.Model):
vendor_names: Mapped[json_str_list]
-class CertificateProviders(db.Model):
+class CertificateProviders(PresentationModel):
__tablename__ = 'certificate_providers'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -388,7 +390,7 @@ class CertificateProviders(db.Model):
provider_names: Mapped[json_str_list]
-class WeatherMap(db.Model):
+class WeatherMap(PresentationModel):
__tablename__ = 'weather_map'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -397,7 +399,7 @@ class WeatherMap(db.Model):
url: Mapped[str]
-class PertTeam(db.Model):
+class PertTeam(PresentationModel):
__tablename__ = 'pert_team'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -405,7 +407,7 @@ class PertTeam(db.Model):
pert_team: Mapped[YesNoPlanned]
-class AlienWave(db.Model):
+class AlienWave(PresentationModel):
__tablename__ = 'alien_wave'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -415,7 +417,7 @@ class AlienWave(db.Model):
alien_wave_internal: Mapped[Optional[bool]]
-class Capacity(db.Model):
+class Capacity(PresentationModel):
__tablename__ = 'capacity'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -424,7 +426,7 @@ class Capacity(db.Model):
typical_backbone_capacity: Mapped[Optional[Decimal]]
-class ExternalConnections(db.Model):
+class ExternalConnections(PresentationModel):
__tablename__ = 'external_connections'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -432,7 +434,7 @@ class ExternalConnections(db.Model):
connections: Mapped[List[ExternalConnection]] = mapped_column(JSON)
-class NonREPeers(db.Model):
+class NonREPeers(PresentationModel):
__tablename__ = 'non_r_and_e_peers'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -440,7 +442,7 @@ class NonREPeers(db.Model):
nr_of_non_r_and_e_peers: Mapped[int]
-class TrafficRatio(db.Model):
+class TrafficRatio(PresentationModel):
__tablename__ = 'traffic_ratio'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -449,7 +451,7 @@ class TrafficRatio(db.Model):
commodity_percentage: Mapped[Decimal]
-class OpsAutomation(db.Model):
+class OpsAutomation(PresentationModel):
__tablename__ = 'ops_automation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -458,7 +460,7 @@ class OpsAutomation(db.Model):
ops_automation_specifics: Mapped[str]
-class NetworkFunctionVirtualisation(db.Model):
+class NetworkFunctionVirtualisation(PresentationModel):
__tablename__ = 'network_function_virtualisation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -467,7 +469,7 @@ class NetworkFunctionVirtualisation(db.Model):
nfv_specifics: Mapped[json_str_list]
-class NetworkAutomation(db.Model):
+class NetworkAutomation(PresentationModel):
__tablename__ = 'network_automation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
@@ -476,7 +478,7 @@ class NetworkAutomation(db.Model):
network_automation_specifics: Mapped[json_str_list]
-class Service(db.Model):
+class Service(PresentationModel):
__tablename__ = 'service'
name_key: Mapped[str128_pk]
name: Mapped[str128]
@@ -484,7 +486,7 @@ class Service(db.Model):
description: Mapped[str]
-class NRENService(db.Model):
+class NRENService(PresentationModel):
__tablename__ = 'nren_service'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
diff --git a/compendium_v2/publishers/survey_publisher.py b/compendium_v2/publishers/survey_publisher.py
index a2a0cdff0fb719aca5dee317c8955d5a63792e40..8acd5d885b2f8761dc2d53e4aae3aa3767f6243d 100644
--- a/compendium_v2/publishers/survey_publisher.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -9,546 +9,27 @@ Usage:
Used in publish_survey API in compendium_v2/routes/survey.py
"""
-from decimal import Decimal
-from typing import List
+from typing import Sequence, Dict, Any
+from sqlalchemy import select, delete
-from sqlalchemy import delete, select
from compendium_v2.db import db
-from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, \
- InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection, \
- FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings, \
- Standards, CrisisExercises, SecurityControls, ConnectedProportion, ConnectivityLevel, \
- ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity, \
- CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
- NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
- CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService
-from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
- CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
-from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
-
-
-def int_or_none(answers_dict, key):
- if key in answers_dict:
- value = answers_dict[key]
- if isinstance(value, str):
- value = value.replace(",", ".")
- return int(answers_dict[key])
- return None
-
-
-def decimal_or_none(answers_dict, key):
- if key in answers_dict:
- value = answers_dict[key]
- if isinstance(value, str):
- value = value.replace(",", ".")
- return Decimal(value)
- return None
-
-
-def decimal_or_zero(answers_dict, key):
- value = answers_dict.get(key, 0)
- if isinstance(value, str):
- value = value.replace(",", ".")
- return Decimal(value)
-
-
-def bool_or_none(answer, key=None):
- if key:
- answer = answer.get(key)
- if answer:
- return answer == "Yes"
- return None
-
-
-def _map_2023(nren, answers) -> None:
- year = 2023
-
- for table_class in [BudgetEntry, ChargingStructure, ECProject, ExternalConnections,
- InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume,
- FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings,
- Standards, CrisisExercises, SecurityControls, ConnectedProportion, ConnectivityLevel,
- ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity,
- CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
- NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
- CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio,
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService]:
- db.session.execute(delete(table_class).where(table_class.year == year, # type: ignore
- table_class.nren_id == nren.id)) # type: ignore
-
- answers = answers["data"]
- budget = answers.get("budget")
- if budget:
- # replace comma with dot for decimal, surveyjs allows commas for decimals (common in EU countries)
- budget = budget.replace(",", ".")
- db.session.add(BudgetEntry(nren_id=nren.id, nren=nren, year=year, budget=Decimal(budget)))
-
- funding_source = answers.get("income_sources")
- if funding_source:
- db.session.add(FundingSource(
- nren_id=nren.id, nren=nren, year=year,
- client_institutions=decimal_or_zero(funding_source, "client_institutions"),
- european_funding=decimal_or_zero(funding_source, "european_funding"),
- gov_public_bodies=decimal_or_zero(funding_source, "gov_public_bodies"),
- commercial=decimal_or_zero(funding_source, "commercial"),
- other=decimal_or_zero(funding_source, "other")
- ))
-
- charging = answers.get("charging_mechanism")
- if charging:
- db.session.add(ChargingStructure(nren_id=nren.id, nren=nren, year=year, fee_type=FeeType[charging]))
-
- staff_roles = answers.get("staff_roles", {})
- staff_employment_type = answers.get("staff_employment_type", {})
- if staff_roles or staff_employment_type:
- db.session.add(NrenStaff(
- nren_id=nren.id, nren=nren, year=year,
- permanent_fte=decimal_or_zero(staff_employment_type, "permanent_fte"),
- subcontracted_fte=decimal_or_zero(staff_employment_type, "subcontracted_fte"),
- technical_fte=decimal_or_zero(staff_roles, "technical_fte"),
- non_technical_fte=decimal_or_zero(staff_roles, "nontechnical_fte")
- ))
-
- has_parent = answers.get("parent_organization") == "Yes"
- parent = answers.get("parent_organization_name")
- if has_parent and parent:
- db.session.add(ParentOrganization(nren_id=nren.id, nren=nren, year=year, organization=parent))
-
- has_subs = answers.get("suborganizations") == 'Yes'
- subs = answers.get("suborganization_details")
- if has_subs and subs:
- for sub in subs:
- role = sub.get("suborganization_role", "")
- if role == "other":
- role = sub.get("suborganization_role-Comment", "")
- db.session.add(SubOrganization(
- nren_id=nren.id, nren=nren, year=year,
- organization=sub.get("suborganization_name"),
- role=role
- ))
- has_ec_projects = answers.get("ec_projects") == "Yes"
- ec_projects = answers.get("ec_project_names")
- if has_ec_projects and ec_projects:
- for ec_project in ec_projects:
- if ec_project:
- db.session.add(
- ECProject(nren_id=nren.id, nren=nren, year=year, project=ec_project.get("ec_project_name"))
- )
-
- strategy = answers.get("corporate_strategy_url", "")
- policies = answers.get("policies", {})
- if strategy or policies:
- db.session.add(Policy(
- nren_id=nren.id, nren=nren, year=year,
- strategic_plan=strategy,
- environmental=policies.get("environmental_policy", {}).get("url", ""),
- equal_opportunity=policies.get("equal_opportunity_policy", {}).get("url", ""),
- connectivity=policies.get("connectivity_policy", {}).get("url", ""),
- acceptable_use=policies.get("acceptable_use_policy", {}).get("url", ""),
- privacy_notice=policies.get("privacy_notice", {}).get("url", ""),
- data_protection=policies.get("data_protection_contact", {}).get("url", ""),
- gender_equality=policies.get("gender_equality_policy", {}).get("url", "")
- ))
-
- traffic_estimate = answers.get("traffic_estimate")
- if traffic_estimate:
- db.session.add(TrafficVolume(
- nren_id=nren.id, nren=nren, year=year,
- to_customers=decimal_or_zero(traffic_estimate, "to_customers"),
- from_customers=decimal_or_zero(traffic_estimate, "from_customers"),
- to_external=decimal_or_zero(traffic_estimate, "to_external"),
- from_external=decimal_or_zero(traffic_estimate, "from_external")
- ))
-
- institution_urls = answers.get("connected_sites_lists")
- if institution_urls:
- urls = [i.get("connected_sites_url", "") for i in institution_urls if i.get("connected_sites_url", "") != ""]
- if urls:
- db.session.add(InstitutionURLs(
- nren_id=nren.id, nren=nren, year=year,
- urls=urls
- ))
-
- central_procurement = answers.get("central_software_procurement") == "Yes"
- if central_procurement:
- central_procurement_amount = decimal_or_none(answers, "central_procurement_amount")
- else:
- central_procurement_amount = None
- db.session.add(CentralProcurement(
- nren_id=nren.id, nren=nren, year=year,
- central_procurement=central_procurement,
- amount=central_procurement_amount
- ))
-
- formal_service_management_framework = answers.get("formal_service_management_framework")
- service_level_targets = answers.get("service_level_targets")
- if formal_service_management_framework or service_level_targets:
- db.session.add(ServiceManagement(
- nren_id=nren.id, nren=nren, year=year,
- service_management_framework=bool_or_none(formal_service_management_framework),
- service_level_targets=bool_or_none(service_level_targets)
- ))
-
- service_type_matrix = answers.get("service_matrix", {})
- for user_type, service_types in service_type_matrix.items():
- user_type = UserCategory[user_type]
- service_types = service_types["service_types"]
- for service_type in service_types:
- service_type = ServiceCategory[service_type]
- db.session.add(ServiceUserTypes(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- service_category=service_type
- ))
-
- has_eosc_listings = answers.get("service_portfolio_eosc_portal") == "Yes"
- services_on_eosc_portal_list = answers.get("services_on_eosc_portal_list", [])
- eosc_list = [i.get("service_name") for i in services_on_eosc_portal_list]
- eosc_list = [i for i in eosc_list if i]
- if has_eosc_listings and eosc_list:
- db.session.add(EOSCListings(
- nren_id=nren.id, nren=nren, year=year,
- service_names=eosc_list
- ))
-
- audits = answers.get("audits")
- business_continuity_plans = answers.get("business_continuity_plans")
- crisis_management_procedure = answers.get("crisis_management_procedure")
- if audits or business_continuity_plans or crisis_management_procedure:
- db.session.add(Standards(
- nren_id=nren.id, nren=nren, year=year,
- audits=bool_or_none(audits),
- audit_specifics=answers.get("audit_specifics", ""),
- business_continuity_plans=bool_or_none(business_continuity_plans),
- business_continuity_plans_specifics=answers.get("business_continuity_plans_specifics", ""),
- crisis_management_procedure=bool_or_none(crisis_management_procedure)
- ))
-
- crisis_exercises = answers.get("crisis_exercises")
- if crisis_exercises:
- db.session.add(CrisisExercises(
- nren_id=nren.id, nren=nren, year=year,
- exercise_descriptions=crisis_exercises
- ))
-
- security_controls = answers.get("security_controls")
- if security_controls:
- if "other" in security_controls:
- security_controls.remove("other")
- security_controls.append(answers.get("security_controls-Comment", "other"))
- db.session.add(SecurityControls(
- nren_id=nren.id, nren=nren, year=year,
- security_control_descriptions=security_controls
- ))
- connectivity_proportions = answers.get("connectivity_proportions", {})
- for user_type, connectivity_proportion in connectivity_proportions.items():
- user_type = UserCategory[user_type]
- coverage = connectivity_proportion.get("covered")
- coverage = ConnectivityCoverage[coverage] if coverage else None
- number_connected = int_or_none(connectivity_proportion, "nr_connected")
- market_share = decimal_or_none(connectivity_proportion, "market_share_percentage")
- users_served = int_or_none(connectivity_proportion, "nr_of_users")
- db.session.add(ConnectedProportion(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- coverage=coverage,
- number_connected=number_connected,
- market_share=market_share,
- users_served=users_served
- ))
-
- connectivity_levels = answers.get("connectivity_level", {})
- for user_type, connectivity_level in connectivity_levels.items():
- user_type = UserCategory[user_type]
- db.session.add(ConnectivityLevel(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- typical_speed=int_or_none(connectivity_level, "typical_speed"),
- highest_speed=int_or_none(connectivity_level, "highest_speed"),
- highest_speed_proportion=decimal_or_none(connectivity_level, "highest_speed_connection_percentage")
- ))
-
- traffic_carriers = answers.get("traffic_carriers", {})
- for user_type, traffic_carrier in traffic_carriers.items():
- user_type = UserCategory[user_type]
- traffic_carrier = traffic_carrier.get("carry_mechanism")
- if traffic_carrier:
- db.session.add(ConnectionCarrier(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- carry_mechanism=CarryMechanism[traffic_carrier]
- ))
-
- traffic_loads = answers.get("traffic_load", {})
- for user_type, traffic_load in traffic_loads.items():
- user_type = UserCategory[user_type]
- db.session.add(ConnectivityLoad(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- average_load_from_institutions=int_or_none(traffic_load, "average_from_institutions_to_network"),
- average_load_to_institutions=int_or_none(traffic_load, "average_to_institutions_from_network"),
- peak_load_from_institutions=int_or_none(traffic_load, "peak_from_institutions_to_network"),
- peak_load_to_institutions=int_or_none(traffic_load, "peak_to_institutions_from_network")
- ))
-
- traffic_growths = answers.get("traffic_growth", {})
- for user_type, traffic_growth in traffic_growths.items():
- user_type = UserCategory[user_type]
- db.session.add(ConnectivityGrowth(
- nren_id=nren.id, nren=nren, year=year,
- user_category=user_type,
- growth=decimal_or_zero(traffic_growth, "growth_rate")
- ))
-
- commercial_organizations = answers.get("commercial_organizations")
- if commercial_organizations:
- c1 = commercial_organizations.get("commercial_r_e", {}).get("connection")
- c2 = commercial_organizations.get("commercial_general", {}).get("connection")
- c3 = commercial_organizations.get("commercial_collaboration", {}).get("connection")
- c4 = commercial_organizations.get("commercial_service_provider", {}).get("connection")
- c5 = commercial_organizations.get("university_spin_off", {}).get("connection")
- db.session.add(CommercialConnectivity(
- nren_id=nren.id, nren=nren, year=year,
- commercial_r_and_e=CommercialConnectivityCoverage[c1] if c1 else None,
- commercial_general=CommercialConnectivityCoverage[c2] if c2 else None,
- commercial_collaboration=CommercialConnectivityCoverage[c3] if c3 else None,
- commercial_service_provider=CommercialConnectivityCoverage[c4] if c4 else None,
- university_spin_off=CommercialConnectivityCoverage[c5] if c5 else None,
- ))
-
- commercial_charging_levels = answers.get("commercial_charging_levels")
- if commercial_charging_levels:
- c1 = commercial_charging_levels.get("collaboration", {}).get("charging_level")
- c2 = commercial_charging_levels.get("services", {}).get("charging_level")
- c3 = commercial_charging_levels.get("peering", {}).get("charging_level")
- db.session.add(CommercialChargingLevel(
- nren_id=nren.id, nren=nren, year=year,
- collaboration=CommercialCharges[c1] if c1 else None,
- service_supplier=CommercialCharges[c2] if c2 else None,
- direct_peering=CommercialCharges[c3] if c3 else None,
- ))
-
- remote_campuses = answers.get("remote_campuses")
- if remote_campuses:
- remote_campuses = remote_campuses == "Yes"
-
- if remote_campuses:
- remote_campuses_specifics = answers.get("remote_campuses_specifics", [])
- remote_campuses_specifics = [
- {"country": i.get("country", ""), "local_r_and_e_connection": bool_or_none(i, "connected")}
- for i in remote_campuses_specifics
- ]
- else:
- remote_campuses_specifics = []
- db.session.add(RemoteCampuses(
- nren_id=nren.id, nren=nren, year=year,
- remote_campus_connectivity=remote_campuses,
- connections=remote_campuses_specifics
- ))
-
- dark_fibre_lease = answers.get("dark_fibre_lease") == "Yes"
- if dark_fibre_lease:
- db.session.add(DarkFibreLease(
- nren_id=nren.id, nren=nren, year=year,
- iru_or_lease=dark_fibre_lease,
- fibre_length_in_country=int_or_none(answers, "dark_fibre_lease_kilometers_inside_country"),
- fibre_length_outside_country=int_or_none(answers, "dark_fibre_lease_kilometers_outside_country"),
- iru_duration=decimal_or_none(answers, "dark_fibre_lease_duration")
- ))
-
- dark_fibre_nren = answers.get("dark_fibre_nren") == "Yes"
- if dark_fibre_nren:
- db.session.add(DarkFibreInstalled(
- nren_id=nren.id, nren=nren, year=year,
- installed=dark_fibre_nren,
- fibre_length_in_country=int_or_none(answers, "dark_fibre_nren_kilometers_inside_country")
- ))
-
- fibre_light = answers.get("fibre_light")
- if fibre_light:
- if fibre_light == "other":
- fibre_light = answers.get("fibre_light-Comment", "other")
- db.session.add(FibreLight(
- nren_id=nren.id, nren=nren, year=year,
- light_description=fibre_light
- ))
-
- network_map_urls = answers.get("network_map_urls", [])
- urls = [i.get("network_map_url", "") for i in network_map_urls if i.get("network_map_url", "") != ""]
- if urls:
- db.session.add(NetworkMapUrls(
- nren_id=nren.id, nren=nren, year=year,
- urls=urls
- ))
-
- monitoring_tools = answers.get("monitoring_tools", [])
- netflow_vendors = answers.get("netflow_vendors", "")
- if monitoring_tools or netflow_vendors:
- if "other" in monitoring_tools:
- monitoring_tools.remove("other")
- monitoring_tools.append(answers.get("monitoring_tools-Comment", "other"))
- db.session.add(MonitoringTools(
- nren_id=nren.id, nren=nren, year=year,
- tool_descriptions=monitoring_tools,
- netflow_processing_description=netflow_vendors
- ))
-
- passive_monitoring = answers.get("passive_monitoring")
- if passive_monitoring:
- passive_monitoring = passive_monitoring == "Yes"
- passive_monitoring_tech = answers.get("passive_monitoring_tech")
- db.session.add(PassiveMonitoring(
- nren_id=nren.id, nren=nren, year=year,
- monitoring=passive_monitoring,
- method=MonitoringMethod[passive_monitoring_tech] if passive_monitoring_tech else None
- ))
-
- traffic_statistics = answers.get("traffic_statistics")
- if traffic_statistics:
- traffic_statistics = traffic_statistics == "Yes"
- urls = answers.get("traffic_statistics_urls", [])
- urls = [i.get("traffic_statistics_url", "") for i in urls if i.get("traffic_statistics_url")]
- db.session.add(TrafficStatistics(
- nren_id=nren.id, nren=nren, year=year,
- traffic_statistics=traffic_statistics,
- urls=urls
- ))
-
- siem_soc_vendor = answers.get("siem_soc_vendor")
- if siem_soc_vendor:
- if "other" in siem_soc_vendor:
- siem_soc_vendor.remove("other")
- siem_soc_vendor.append(answers.get("siem_soc_vendor-Comment", "other"))
- db.session.add(SiemVendors(
- nren_id=nren.id, nren=nren, year=year,
- vendor_names=siem_soc_vendor
- ))
-
- certificate_service = answers.get("certificate_service")
- if certificate_service:
- if "other" in certificate_service:
- certificate_service.remove("other")
- certificate_service.append(answers.get("certificate_service-Comment", "other"))
- db.session.add(CertificateProviders(
- nren_id=nren.id, nren=nren, year=year,
- provider_names=certificate_service
- ))
-
- network_weather = answers.get("network_weather")
- if network_weather:
- network_weather = network_weather == "Yes"
- db.session.add(WeatherMap(
- nren_id=nren.id, nren=nren, year=year,
- weather_map=network_weather,
- url=answers.get("network_weather_url", "")
- ))
-
- pert_team = answers.get("pert_team")
- if pert_team:
- pert_team = YesNoPlanned[pert_team.lower()]
- db.session.add(PertTeam(
- nren_id=nren.id, nren=nren, year=year,
- pert_team=pert_team
- ))
-
- alienwave_services = answers.get("alienwave_services")
- alienwave_internal = answers.get("alienwave_internal")
- if alienwave_services or alienwave_internal:
- alienwave_services = YesNoPlanned[alienwave_services.lower()] if alienwave_services else None
- alienwave_internal = alienwave_internal == "Yes" if alienwave_internal else None
- db.session.add(AlienWave(
- nren_id=nren.id, nren=nren, year=year,
- alien_wave_third_pary=alienwave_services,
- nr_of_alien_wave_third_party_services=int_or_none(answers, "alienwave_services_number"),
- alien_wave_internal=alienwave_internal
- ))
-
- max_capacity = answers.get("max_capacity")
- typical_capacity = answers.get("typical_capacity")
- if max_capacity or typical_capacity:
- db.session.add(Capacity(
- nren_id=nren.id, nren=nren, year=year,
- largest_link_capacity=decimal_or_none(answers, "max_capacity"),
- typical_backbone_capacity=decimal_or_none(answers, "typical_capacity")
- ))
-
- external_connections = answers.get("external_connections")
- if external_connections:
- connections: List[ExternalConnection] = []
- for connection in external_connections:
- connections.append({
- 'link_name': connection.get('link_name', ''),
- 'capacity': connection.get('capacity'),
- 'from_organization': connection.get('from_organization', ''),
- 'to_organization': connection.get('to_organization', ''),
- 'interconnection_method': connection.get('interconnection_method')
- })
- db.session.add(ExternalConnections(
- nren_id=nren.id, nren=nren, year=year,
- connections=connections
- ))
-
- non_r_and_e_peers = answers.get("non_r_and_e_peers")
- if non_r_and_e_peers:
- db.session.add(NonREPeers(
- nren_id=nren.id, nren=nren, year=year,
- nr_of_non_r_and_e_peers=int(non_r_and_e_peers)
- ))
-
- commodity_vs_r_e = answers.get("commodity_vs_r_e")
- if commodity_vs_r_e:
- db.session.add(TrafficRatio(
- nren_id=nren.id, nren=nren, year=year,
- r_and_e_percentage=decimal_or_zero(commodity_vs_r_e, "r_e"),
- commodity_percentage=decimal_or_zero(commodity_vs_r_e, "commodity"),
- ))
-
- operational_process_automation = answers.get("operational_process_automation")
- if operational_process_automation:
- db.session.add(OpsAutomation(
- nren_id=nren.id, nren=nren, year=year,
- ops_automation=YesNoPlanned[operational_process_automation.lower()],
- ops_automation_specifics=answers.get("operational_process_automation_tools", "")
- ))
-
- nfv = answers.get("nfv")
- if nfv:
- nfv_types = answers.get("nfv_types", [])
- if "other" in nfv_types:
- nfv_types.remove("other")
- nfv_types.append(answers.get("nfv_types-Comment", "other"))
- db.session.add(NetworkFunctionVirtualisation(
- nren_id=nren.id, nren=nren, year=year,
- nfv=YesNoPlanned[nfv.lower()],
- nfv_specifics=nfv_types
- ))
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
+from compendium_v2.publishers.year.map_2023 import map_2023
+from compendium_v2.db.presentation_models import PresentationModel
- network_automation = answers.get("network_automation")
- if network_automation:
- network_automation_tasks = answers.get("network_automation_tasks", [])
- db.session.add(NetworkAutomation(
- nren_id=nren.id, nren=nren, year=year,
- network_automation=YesNoPlanned[network_automation.lower()],
- network_automation_specifics=network_automation_tasks
- ))
- all_services = {}
- for question_name in ["services_collaboration", "services_hosting", "services_identity", "services_isp",
- "services_multimedia", "services_network", "services_professional", "services_security"]:
- all_services.update(answers.get(question_name, {}))
+def save_data(year: int, data: Dict[PresentationModel, Sequence[Dict[str, Any]]]):
+ table_classes = list(data.keys())
+ if not all(issubclass(table, db.Model) for table in table_classes):
+ raise ValueError("All tables must be subclasses of db.Model")
- for service_key, answers in all_services.items():
- offered = answers.get("offered")
- if offered == ["yes"]:
- db.session.add(NRENService(
- nren_id=nren.id, nren=nren, year=year,
- service_key=service_key,
- product_name=answers.get("name", ""),
- additional_information=answers.get("additional_information", ""),
- official_description=answers.get("description", "")
- ))
+ for table_class in table_classes:
+ db.session.execute(delete(table_class).where(table_class.year == year))
+ for model, rows in data.items():
+ db.session.bulk_insert_mappings(model, rows) # type: ignore
+ db.session.commit()
def publish(year):
@@ -558,7 +39,7 @@ def publish(year):
).unique()
question_mapping = {
- 2023: _map_2023
+ 2023: map_2023
}
mapping_function = question_mapping.get(year)
@@ -566,5 +47,6 @@ def publish(year):
if not mapping_function:
raise ValueError(f"No publishing function found for the {year} survey.")
- for response in responses:
- mapping_function(response.nren, response.answers)
+ data = mapping_function(responses)
+
+ save_data(year, data)
diff --git a/compendium_v2/publishers/utils.py b/compendium_v2/publishers/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..04b1f66d1447522d64b8e1708e111125b850b46a
--- /dev/null
+++ b/compendium_v2/publishers/utils.py
@@ -0,0 +1,34 @@
+from decimal import Decimal
+
+
+def int_or_none(answers_dict, key):
+ if key in answers_dict:
+ value = answers_dict[key]
+ if isinstance(value, str):
+ value = value.replace(",", ".")
+ return int(answers_dict[key])
+ return None
+
+
+def decimal_or_none(answers_dict, key):
+ if key in answers_dict:
+ value = answers_dict[key]
+ if isinstance(value, str):
+ value = value.replace(",", ".")
+ return Decimal(value)
+ return None
+
+
+def decimal_or_zero(answers_dict, key):
+ value = answers_dict.get(key, 0)
+ if isinstance(value, str):
+ value = value.replace(",", ".")
+ return Decimal(value)
+
+
+def bool_or_none(answer, key=None):
+ if key:
+ answer = answer.get(key)
+ if answer:
+ return answer == "Yes"
+ return None
diff --git a/compendium_v2/publishers/year/__init__.py b/compendium_v2/publishers/year/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/compendium_v2/publishers/year/map_2023.py b/compendium_v2/publishers/year/map_2023.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a17ef5f4308b951ca817664aab3204c635b2d93
--- /dev/null
+++ b/compendium_v2/publishers/year/map_2023.py
@@ -0,0 +1,826 @@
+from decimal import Decimal
+from typing import List, Dict, Any, Type
+from collections import defaultdict
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, \
+ InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection, \
+ FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings, \
+ Standards, CrisisExercises, SecurityControls, ConnectedProportion, ConnectivityLevel, \
+ ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity, \
+ CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
+ NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
+ CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService, NREN, PresentationModel
+from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
+ CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
+from compendium_v2.db.survey_models import SurveyResponse
+from compendium_v2.publishers.utils import decimal_or_zero, decimal_or_none, bool_or_none, int_or_none
+
+
+def map_budget_entry(year, nren, answers):
+ budget = answers.get("budget")
+ if budget:
+ # replace comma with dot for decimal, surveyjs allows commas for decimals (common in EU countries)
+ budget = budget.replace(",", ".")
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'budget': Decimal(budget)
+ }
+
+
+def map_fundingsource_entry(year, nren, answers):
+ funding_source = answers.get("income_sources")
+ if funding_source:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'client_institutions': decimal_or_zero(funding_source, "client_institutions"),
+ 'european_funding': decimal_or_zero(funding_source, "european_funding"),
+ 'gov_public_bodies': decimal_or_zero(funding_source, "gov_public_bodies"),
+ 'commercial': decimal_or_zero(funding_source, "commercial"),
+ 'other': decimal_or_zero(funding_source, "other")
+ }
+
+
+def map_charging_structure_entry(year, nren, answers):
+ charging = answers.get("charging_mechanism")
+ if charging:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'fee_type': FeeType[charging]
+ }
+
+
+def map_nren_staff(year, nren, answers):
+ staff_roles = answers.get("staff_roles", {})
+ staff_employment_type = answers.get("staff_employment_type", {})
+ if staff_roles or staff_employment_type:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'permanent_fte': decimal_or_zero(staff_employment_type, "permanent_fte"),
+ 'subcontracted_fte': decimal_or_zero(staff_employment_type, "subcontracted_fte"),
+ 'technical_fte': decimal_or_zero(staff_roles, "technical_fte"),
+ 'non_technical_fte': decimal_or_zero(staff_roles, "nontechnical_fte")
+ }
+
+
+def map_parent_orgs(nren: NREN, year: int, answers: Dict[str, Any]):
+ has_parent = answers.get("parent_organization") == "Yes"
+ parent = answers.get("parent_organization_name")
+ if has_parent and parent:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'organization': parent
+ }
+
+
+def map_sub_orgs(nren: NREN, year: int, answers: Dict[str, Any]):
+ has_subs = answers.get("suborganizations") == 'Yes'
+ subs = answers.get("suborganization_details")
+ if has_subs and subs:
+ for sub in subs:
+ role = sub.get("suborganization_role", "")
+ if role == "other":
+ role = sub.get("suborganization_role-Comment", "")
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'organization': sub.get("suborganization_name"),
+ 'role': role
+ }
+
+
+def map_ec_projects(nren: NREN, year: int, answers: Dict[str, Any]):
+ has_ec_projects = answers.get("ec_projects") == "Yes"
+ ec_projects = answers.get("ec_project_names")
+ if has_ec_projects and ec_projects:
+ for ec_project in ec_projects:
+ if ec_project:
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'project': ec_project.get("ec_project_name")
+ }
+
+
+def map_policy(nren: NREN, year: int, answers: Dict[str, Any]):
+ strategy = answers.get("corporate_strategy_url", "")
+ policies = answers.get("policies", {})
+ if strategy or policies:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'strategic_plan': strategy,
+ 'environmental': policies.get("environmental_policy", {}).get("url", ""),
+ 'equal_opportunity': policies.get("equal_opportunity_policy", {}).get("url", ""),
+ 'connectivity': policies.get("connectivity_policy", {}).get("url", ""),
+ 'acceptable_use': policies.get("acceptable_use_policy", {}).get("url", ""),
+ 'privacy_notice': policies.get("privacy_notice", {}).get("url", ""),
+ 'data_protection': policies.get("data_protection_contact", {}).get("url", ""),
+ 'gender_equality': policies.get("gender_equality_policy", {}).get("url", "")
+ }
+
+
+def map_traffic_volume(nren: NREN, year: int, answers: Dict[str, Any]):
+ traffic_estimate = answers.get("traffic_estimate")
+ if traffic_estimate:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'to_customers': decimal_or_zero(traffic_estimate, "to_customers"),
+ 'from_customers': decimal_or_zero(traffic_estimate, "from_customers"),
+ 'to_external': decimal_or_zero(traffic_estimate, "to_external"),
+ 'from_external': decimal_or_zero(traffic_estimate, "from_external")
+ }
+
+
+def map_institution_urls(nren: NREN, year: int, answers: Dict[str, Any]):
+ institution_urls = answers.get("connected_sites_lists")
+ if institution_urls:
+ urls = [i.get("connected_sites_url", "")
+ for i in institution_urls if i.get("connected_sites_url", "")]
+ if urls:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'urls': urls
+ }
+
+
+def map_central_procurement(nren: NREN, year: int, answers: Dict[str, Any]):
+ central_procurement = answers.get("central_software_procurement") == "Yes"
+ if central_procurement:
+ central_procurement_amount = decimal_or_none(answers, "central_procurement_amount")
+ else:
+ central_procurement_amount = None
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'central_procurement': central_procurement,
+ 'amount': central_procurement_amount
+ }
+
+
+def map_service_management(nren: NREN, year: int, answers: Dict[str, Any]):
+ formal_service_management_framework = answers.get("formal_service_management_framework")
+ service_level_targets = answers.get("service_level_targets")
+ if formal_service_management_framework or service_level_targets:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'service_management_framework': bool_or_none(formal_service_management_framework),
+ 'service_level_targets': bool_or_none(service_level_targets)
+ }
+
+
+def map_service_user_types(nren: NREN, year: int, answers: Dict[str, Any]):
+ service_type_matrix = answers.get("service_matrix", {})
+ for user_type, service_types in service_type_matrix.items():
+ user_type_enum = UserCategory[user_type]
+ service_types_list = service_types["service_types"]
+ for service_type in service_types_list:
+ service_type_enum = ServiceCategory[service_type]
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'service_category': service_type_enum
+ }
+
+
+def map_eosc_listings(nren: NREN, year: int, answers: Dict[str, Any]):
+ has_eosc_listings = answers.get("service_portfolio_eosc_portal") == "Yes"
+ services_on_eosc_portal_list = answers.get("services_on_eosc_portal_list", [])
+ eosc_list = [i.get("service_name") for i in services_on_eosc_portal_list if i.get("service_name")]
+ if has_eosc_listings and eosc_list:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'service_names': eosc_list
+ }
+
+
+def map_standards(nren: NREN, year: int, answers: Dict[str, Any]):
+ audits = answers.get("audits")
+ business_continuity_plans = answers.get("business_continuity_plans")
+ crisis_management_procedure = answers.get("crisis_management_procedure")
+ if audits or business_continuity_plans or crisis_management_procedure:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'audits': bool_or_none(audits),
+ 'audit_specifics': answers.get("audit_specifics", ""),
+ 'business_continuity_plans': bool_or_none(business_continuity_plans),
+ 'business_continuity_plans_specifics': answers.get("business_continuity_plans_specifics", ""),
+ 'crisis_management_procedure': bool_or_none(crisis_management_procedure)
+ }
+
+
+def map_crisis_exercises(nren: NREN, year: int, answers: Dict[str, Any]):
+ crisis_exercises = answers.get("crisis_exercises")
+ if crisis_exercises:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'exercise_descriptions': crisis_exercises
+ }
+
+
+def map_security_controls(nren: NREN, year: int, answers: Dict[str, Any]):
+ security_controls = answers.get("security_controls")
+ if security_controls:
+ if "other" in security_controls:
+ security_controls.remove("other")
+ security_controls.append(answers.get("security_controls-Comment", "other"))
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'security_control_descriptions': security_controls
+ }
+
+
+def map_connected_proportion(nren: NREN, year: int, answers: Dict[str, Any]):
+ connectivity_proportions = answers.get("connectivity_proportions", {})
+ for user_type, connectivity_proportion in connectivity_proportions.items():
+ user_type_enum = UserCategory[user_type]
+ coverage = connectivity_proportion.get("covered")
+ coverage_enum = ConnectivityCoverage[coverage] if coverage else None
+ number_connected = int_or_none(connectivity_proportion, "nr_connected")
+ market_share = decimal_or_none(connectivity_proportion, "market_share_percentage")
+ users_served = int_or_none(connectivity_proportion, "nr_of_users")
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'coverage': coverage_enum,
+ 'number_connected': number_connected,
+ 'market_share': market_share,
+ 'users_served': users_served
+ }
+
+
+def map_connectivity_levels(nren: NREN, year: int, answers: Dict[str, Any]):
+ connectivity_levels = answers.get("connectivity_level", {})
+ for user_type, connectivity_level in connectivity_levels.items():
+ user_type_enum = UserCategory[user_type]
+ typical_speed = int_or_none(connectivity_level, "typical_speed")
+ highest_speed = int_or_none(connectivity_level, "highest_speed")
+ highest_speed_proportion = decimal_or_none(connectivity_level, "highest_speed_connection_percentage")
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'typical_speed': typical_speed,
+ 'highest_speed': highest_speed,
+ 'highest_speed_proportion': highest_speed_proportion
+ }
+
+
+def map_connection_carriers(nren: NREN, year: int, answers: Dict[str, Any]):
+ traffic_carriers = answers.get("traffic_carriers", {})
+ for user_type, traffic_carrier in traffic_carriers.items():
+ user_type_enum = UserCategory[user_type]
+ carry_mechanism = traffic_carrier.get("carry_mechanism")
+ if carry_mechanism:
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'carry_mechanism': CarryMechanism[carry_mechanism]
+ }
+
+
+def map_connectivity_loads(nren: NREN, year: int, answers: Dict[str, Any]):
+ traffic_loads = answers.get("traffic_load", {})
+ for user_type, traffic_load in traffic_loads.items():
+ user_type_enum = UserCategory[user_type]
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'average_load_from_institutions': int_or_none(traffic_load, "average_from_institutions_to_network"),
+ 'average_load_to_institutions': int_or_none(traffic_load, "average_to_institutions_from_network"),
+ 'peak_load_from_institutions': int_or_none(traffic_load, "peak_from_institutions_to_network"),
+ 'peak_load_to_institutions': int_or_none(traffic_load, "peak_to_institutions_from_network")
+ }
+
+
+def map_connectivity_growth(nren: NREN, year: int, answers: Dict[str, Any]):
+ traffic_growths = answers.get("traffic_growth", {})
+ for user_type, traffic_growth in traffic_growths.items():
+ user_type_enum = UserCategory[user_type]
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'user_category': user_type_enum,
+ 'growth': decimal_or_zero(traffic_growth, "growth_rate")
+ }
+
+
+def map_commercial_connectivity(nren: NREN, year: int, answers: Dict[str, Any]):
+ commercial_organizations = answers.get("commercial_organizations")
+ if commercial_organizations:
+ c1 = commercial_organizations.get("commercial_r_e", {}).get("connection")
+ c2 = commercial_organizations.get("commercial_general", {}).get("connection")
+ c3 = commercial_organizations.get("commercial_collaboration", {}).get("connection")
+ c4 = commercial_organizations.get("commercial_service_provider", {}).get("connection")
+ c5 = commercial_organizations.get("university_spin_off", {}).get("connection")
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'commercial_r_and_e': CommercialConnectivityCoverage[c1] if c1 else None,
+ 'commercial_general': CommercialConnectivityCoverage[c2] if c2 else None,
+ 'commercial_collaboration': CommercialConnectivityCoverage[c3] if c3 else None,
+ 'commercial_service_provider': CommercialConnectivityCoverage[c4] if c4 else None,
+ 'university_spin_off': CommercialConnectivityCoverage[c5] if c5 else None,
+ }
+
+
+def map_commercial_charging_levels(nren: NREN, year: int, answers: Dict[str, Any]):
+ commercial_charging_levels = answers.get("commercial_charging_levels")
+ if commercial_charging_levels:
+ c1 = commercial_charging_levels.get("collaboration", {}).get("charging_level")
+ c2 = commercial_charging_levels.get("services", {}).get("charging_level")
+ c3 = commercial_charging_levels.get("peering", {}).get("charging_level")
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'collaboration': CommercialCharges[c1] if c1 else None,
+ 'service_supplier': CommercialCharges[c2] if c2 else None,
+ 'direct_peering': CommercialCharges[c3] if c3 else None,
+ }
+
+
+def map_remote_campuses(nren: NREN, year: int, answers: Dict[str, Any]):
+ remote_campuses = answers.get("remote_campuses")
+ if remote_campuses:
+ remote_campuses = remote_campuses == "Yes"
+ remote_campuses_specifics = []
+ if remote_campuses:
+ remote_campuses_specifics = [
+ {"country": i.get("country", ""), "local_r_and_e_connection": bool_or_none(i, "connected")}
+ for i in answers.get("remote_campuses_specifics", [])
+ ]
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'remote_campus_connectivity': remote_campuses,
+ 'connections': remote_campuses_specifics
+ }
+
+
+def map_dark_fibre_lease(nren: NREN, year: int, answers: Dict[str, Any]):
+ dark_fibre_lease = answers.get("dark_fibre_lease") == "Yes"
+ if dark_fibre_lease:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'iru_or_lease': dark_fibre_lease,
+ 'fibre_length_in_country': int_or_none(answers, "dark_fibre_lease_kilometers_inside_country"),
+ 'fibre_length_outside_country': int_or_none(answers, "dark_fibre_lease_kilometers_outside_country"),
+ 'iru_duration': decimal_or_none(answers, "dark_fibre_lease_duration")
+ }
+
+
+def map_dark_fibre_installed(nren: NREN, year: int, answers: Dict[str, Any]):
+ dark_fibre_nren = answers.get("dark_fibre_nren") == "Yes"
+ if dark_fibre_nren:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'installed': dark_fibre_nren,
+ 'fibre_length_in_country': int_or_none(answers, "dark_fibre_nren_kilometers_inside_country")
+ }
+
+
+def map_fibre_light(nren: NREN, year: int, answers: Dict[str, Any]):
+ fibre_light = answers.get("fibre_light")
+ if fibre_light:
+ if fibre_light == "other":
+ fibre_light = answers.get("fibre_light-Comment", "other")
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'light_description': fibre_light
+ }
+
+
+def map_monitoring_tools(nren: NREN, year: int, answers: Dict[str, Any]):
+ monitoring_tools = answers.get("monitoring_tools", [])
+ netflow_vendors = answers.get("netflow_vendors", "")
+ if monitoring_tools or netflow_vendors:
+ if "other" in monitoring_tools:
+ monitoring_tools.remove("other")
+ monitoring_tools.append(answers.get("monitoring_tools-Comment", "other"))
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'tool_descriptions': monitoring_tools,
+ 'netflow_processing_description': netflow_vendors
+ }
+
+
+def map_passive_monitoring(nren: NREN, year: int, answers: Dict[str, Any]):
+ passive_monitoring = answers.get("passive_monitoring")
+ if passive_monitoring:
+ passive_monitoring = passive_monitoring == "Yes"
+ passive_monitoring_tech = answers.get("passive_monitoring_tech")
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'monitoring': passive_monitoring,
+ 'method': MonitoringMethod[passive_monitoring_tech] if passive_monitoring_tech else None
+ }
+
+
+def map_traffic_statistics(nren: NREN, year: int, answers: Dict[str, Any]):
+ traffic_statistics = answers.get("traffic_statistics")
+ if traffic_statistics:
+ traffic_statistics = traffic_statistics == "Yes"
+ urls = answers.get("traffic_statistics_urls", [])
+ urls = [i.get("traffic_statistics_url", "") for i in urls if i.get("traffic_statistics_url")]
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'traffic_statistics': traffic_statistics,
+ 'urls': urls
+ }
+
+
+def map_siem_vendors(nren: NREN, year: int, answers: Dict[str, Any]):
+ siem_soc_vendor = answers.get("siem_soc_vendor")
+ if siem_soc_vendor:
+ if "other" in siem_soc_vendor:
+ siem_soc_vendor.remove("other")
+ siem_soc_vendor.append(answers.get("siem_soc_vendor-Comment", "other"))
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'vendor_names': siem_soc_vendor
+ }
+
+
+def map_network_map_urls(nren: NREN, year: int, answers: Dict[str, Any]):
+ network_map_urls = answers.get("network_map_urls", [])
+ urls = [i.get("network_map_url", "") for i in network_map_urls if i.get("network_map_url", "")]
+ if urls:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'urls': urls
+ }
+
+
+def map_certificate_providers(nren: NREN, year: int, answers: Dict[str, Any]):
+ certificate_service = answers.get("certificate_service")
+ if certificate_service:
+ if "other" in certificate_service:
+ certificate_service.remove("other")
+ certificate_service.append(answers.get("certificate_service-Comment", "other"))
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'provider_names': certificate_service
+ }
+
+
+def map_weathermap_entry(nren: NREN, year: int, answers: Dict[str, Any]):
+ network_weather = answers.get("network_weather")
+ if network_weather:
+ network_weather = network_weather == "Yes"
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'weather_map': network_weather,
+ 'url': answers.get("network_weather_url", "")
+ }
+
+
+def map_pert_team(nren: NREN, year: int, answers: Dict[str, Any]):
+ pert_team = answers.get("pert_team")
+ if pert_team:
+ pert_team = YesNoPlanned[pert_team.lower()]
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'pert_team': pert_team
+ }
+
+
+def map_alienwave_entry(nren: NREN, year: int, answers: Dict[str, Any]):
+ alienwave_services = answers.get("alienwave_services")
+ alienwave_internal = answers.get("alienwave_internal")
+ if alienwave_services or alienwave_internal:
+ alienwave_services = YesNoPlanned[alienwave_services.lower()] if alienwave_services else None
+ alienwave_internal = alienwave_internal == "Yes" if alienwave_internal else None
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'alien_wave_third_pary': alienwave_services,
+ 'nr_of_alien_wave_third_party_services': int_or_none(answers, "alienwave_services_number"),
+ 'alien_wave_internal': alienwave_internal
+ }
+
+
+def map_capacity_entry(nren: NREN, year: int, answers: Dict[str, Any]):
+ max_capacity = answers.get("max_capacity")
+ typical_capacity = answers.get("typical_capacity")
+ if max_capacity or typical_capacity:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'largest_link_capacity': decimal_or_none(answers, "max_capacity"),
+ 'typical_backbone_capacity': decimal_or_none(answers, "typical_capacity")
+ }
+
+
+def map_external_connections(nren: NREN, year: int, answers: Dict[str, Any]):
+ external_connections = answers.get("external_connections")
+ if external_connections:
+ connections: List[ExternalConnection] = [{
+ 'link_name': connection.get('link_name', ''),
+ 'capacity': connection.get('capacity'),
+ 'from_organization': connection.get('from_organization', ''),
+ 'to_organization': connection.get('to_organization', ''),
+ 'interconnection_method': connection.get('interconnection_method')
+ } for connection in external_connections]
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'connections': connections
+ }
+
+
+def map_non_re_peers(nren: NREN, year: int, answers: Dict[str, Any]):
+ non_r_and_e_peers = answers.get("non_r_and_e_peers")
+ if non_r_and_e_peers:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'nr_of_non_r_and_e_peers': int(non_r_and_e_peers)
+ }
+
+
+def map_traffic_ratio(nren: NREN, year: int, answers: Dict[str, Any]):
+ commodity_vs_r_e = answers.get("commodity_vs_r_e")
+ if commodity_vs_r_e:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'r_and_e_percentage': decimal_or_zero(commodity_vs_r_e, "r_e"),
+ 'commodity_percentage': decimal_or_zero(commodity_vs_r_e, "commodity"),
+ }
+
+
+def map_ops_automation(nren: NREN, year: int, answers: Dict[str, Any]):
+ operational_process_automation = answers.get("operational_process_automation")
+ if operational_process_automation:
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'ops_automation': YesNoPlanned[operational_process_automation.lower()],
+ 'ops_automation_specifics': answers.get("operational_process_automation_tools", "")
+ }
+
+
+def map_network_function_virtualisation(nren: NREN, year: int, answers: Dict[str, Any]):
+ nfv = answers.get("nfv")
+ if nfv:
+ nfv_types = answers.get("nfv_types", [])
+ if "other" in nfv_types:
+ nfv_types.remove("other")
+ nfv_types.append(answers.get("nfv_types-Comment", "other"))
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'nfv': YesNoPlanned[nfv.lower()],
+ 'nfv_specifics': nfv_types
+ }
+
+
+def map_network_automation(nren: NREN, year: int, answers: Dict[str, Any]):
+ network_automation = answers.get("network_automation")
+ if network_automation:
+ network_automation_tasks = answers.get("network_automation_tasks", [])
+ return {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'network_automation': YesNoPlanned[network_automation.lower()],
+ 'network_automation_specifics': network_automation_tasks
+ }
+
+
+def map_nren_services(nren: NREN, year: int, answers: Dict[str, Any]):
+ all_services = {}
+ for question_name in [
+ "services_collaboration", "services_hosting", "services_identity", "services_isp",
+ "services_multimedia", "services_network", "services_professional", "services_security"
+ ]:
+ all_services.update(answers.get(question_name, {}))
+
+ for service_key, service_answers in all_services.items():
+ offered = service_answers.get("offered")
+ if offered == ["yes"]:
+ yield {
+ 'nren_id': nren.id,
+ 'year': year,
+ 'service_key': service_key,
+ 'product_name': service_answers.get("name", ""),
+ 'additional_information': service_answers.get("additional_information", ""),
+ 'official_description': service_answers.get("description", "")
+ }
+
+
+def map_2023(responses: List[SurveyResponse]) -> Dict[Type[PresentationModel], List[Dict]]:
+ year = 2023
+
+ result = defaultdict(list) # {model: [data]}
+
+ for response in responses:
+ nren = response.nren
+ answers = response.answers['data']
+
+ budget = map_budget_entry(year, nren, answers)
+ if budget:
+ result[BudgetEntry].append(budget)
+
+ funding = map_fundingsource_entry(year, nren, answers)
+ if funding:
+ result[FundingSource].append(funding)
+
+ charging = map_charging_structure_entry(year, nren, answers)
+ if charging:
+ result[ChargingStructure].append(charging)
+
+ nren_staff = map_nren_staff(year, nren, answers)
+ if nren_staff:
+ result[NrenStaff].append(nren_staff)
+
+ parent = map_parent_orgs(nren, year, answers)
+ if parent:
+ result[ParentOrganization].append(parent)
+
+ subs = map_sub_orgs(nren, year, answers)
+ for sub in subs:
+ result[SubOrganization].append(sub)
+
+ ec_projects = map_ec_projects(nren, year, answers)
+ for ec_project in ec_projects:
+ result[ECProject].append(ec_project)
+
+ policy = map_policy(nren, year, answers)
+ if policy:
+ result[Policy].append(policy)
+
+ traffic_volume = map_traffic_volume(nren, year, answers)
+ if traffic_volume:
+ result[TrafficVolume].append(traffic_volume)
+
+ institution_urls = map_institution_urls(nren, year, answers)
+ if institution_urls:
+ result[InstitutionURLs].append(institution_urls)
+
+ central_procurement = map_central_procurement(nren, year, answers)
+ if central_procurement:
+ result[CentralProcurement].append(central_procurement)
+
+ service_management = map_service_management(nren, year, answers)
+ if service_management:
+ result[ServiceManagement].append(service_management)
+
+ service_user_types = map_service_user_types(nren, year, answers)
+ for service_user_type in service_user_types:
+ result[ServiceUserTypes].append(service_user_type)
+
+ eosc_listings = map_eosc_listings(nren, year, answers)
+ if eosc_listings:
+ result[EOSCListings].append(eosc_listings)
+
+ standards = map_standards(nren, year, answers)
+ if standards:
+ result[Standards].append(standards)
+
+ crisis_exercises = map_crisis_exercises(nren, year, answers)
+ if crisis_exercises:
+ result[CrisisExercises].append(crisis_exercises)
+
+ security_controls = map_security_controls(nren, year, answers)
+ if security_controls:
+ result[SecurityControls].append(security_controls)
+
+ connected_proportions = map_connected_proportion(nren, year, answers)
+ for connected_proportion in connected_proportions:
+ result[ConnectedProportion].append(connected_proportion)
+
+ connectivity_levels = map_connectivity_levels(nren, year, answers)
+ for connectivity_level in connectivity_levels:
+ result[ConnectivityLevel].append(connectivity_level)
+
+ connection_carriers = map_connection_carriers(nren, year, answers)
+ for connection_carrier in connection_carriers:
+ result[ConnectionCarrier].append(connection_carrier)
+
+ connectivity_loads = map_connectivity_loads(nren, year, answers)
+ for connectivity_load in connectivity_loads:
+ result[ConnectivityLoad].append(connectivity_load)
+
+ connectivity_growths = map_connectivity_growth(nren, year, answers)
+ for connectivity_growth in connectivity_growths:
+ result[ConnectivityGrowth].append(connectivity_growth)
+
+ commercial_connectivities = map_commercial_connectivity(nren, year, answers)
+ for commercial_connectivity in commercial_connectivities:
+ result[CommercialConnectivity].append(commercial_connectivity)
+
+ commercial_charging_levels = map_commercial_charging_levels(nren, year, answers)
+ for commercial_charging_level in commercial_charging_levels:
+ result[CommercialChargingLevel].append(commercial_charging_level)
+
+ remote_campuses = map_remote_campuses(nren, year, answers)
+ if remote_campuses:
+ result[RemoteCampuses].append(remote_campuses)
+
+ dark_fibre_lease = map_dark_fibre_lease(nren, year, answers)
+ if dark_fibre_lease:
+ result[DarkFibreLease].append(dark_fibre_lease)
+
+ dark_fibre_installed = map_dark_fibre_installed(nren, year, answers)
+ if dark_fibre_installed:
+ result[DarkFibreInstalled].append(dark_fibre_installed)
+
+ fibre_light = map_fibre_light(nren, year, answers)
+ if fibre_light:
+ result[FibreLight].append(fibre_light)
+
+ network_map_urls = map_network_map_urls(nren, year, answers)
+ if network_map_urls:
+ result[NetworkMapUrls].append(network_map_urls)
+
+ monitoring_tools = map_monitoring_tools(nren, year, answers)
+ if monitoring_tools:
+ result[MonitoringTools].append(monitoring_tools)
+
+ passive_monitoring = map_passive_monitoring(nren, year, answers)
+ if passive_monitoring:
+ result[PassiveMonitoring].append(passive_monitoring)
+
+ traffic_statistics = map_traffic_statistics(nren, year, answers)
+ if traffic_statistics:
+ result[TrafficStatistics].append(traffic_statistics)
+
+ siem_vendors = map_siem_vendors(nren, year, answers)
+ if siem_vendors:
+ result[SiemVendors].append(siem_vendors)
+
+ certificate_providers = map_certificate_providers(nren, year, answers)
+ if certificate_providers:
+ result[CertificateProviders].append(certificate_providers)
+
+ weathermap = map_weathermap_entry(nren, year, answers)
+ if weathermap:
+ result[WeatherMap].append(weathermap)
+
+ pert_team = map_pert_team(nren, year, answers)
+ if pert_team:
+ result[PertTeam].append(pert_team)
+
+ alienwave = map_alienwave_entry(nren, year, answers)
+ if alienwave:
+ result[AlienWave].append(alienwave)
+
+ capacity = map_capacity_entry(nren, year, answers)
+ if capacity:
+ result[Capacity].append(capacity)
+
+ external_connections = map_external_connections(nren, year, answers)
+ if external_connections:
+ result[ExternalConnections].append(external_connections)
+
+ non_re_peers = map_non_re_peers(nren, year, answers)
+ if non_re_peers:
+ result[NonREPeers].append(non_re_peers)
+
+ traffic_ratio = map_traffic_ratio(nren, year, answers)
+ if traffic_ratio:
+ result[TrafficRatio].append(traffic_ratio)
+
+ ops_automation = map_ops_automation(nren, year, answers)
+ if ops_automation:
+ result[OpsAutomation].append(ops_automation)
+
+ nfv = map_network_function_virtualisation(nren, year, answers)
+ if nfv:
+ result[NetworkFunctionVirtualisation].append(nfv)
+
+ network_automation = map_network_automation(nren, year, answers)
+ if network_automation:
+ result[NetworkAutomation].append(network_automation)
+
+ nren_services = map_nren_services(nren, year, answers)
+ for nren_service in nren_services:
+ result[NRENService].append(nren_service)
+
+ return result
diff --git a/compendium_v2/publishers/year/map_2024.py b/compendium_v2/publishers/year/map_2024.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/test/test_survey_publisher.py b/test/test_survey_publisher.py
index 140356e22e7db89e5e68a3dc0e3f7b9d6d24999a..5b9eb9d3c4a92d12522a2ee7f73157a9a9be38c6 100644
--- a/test/test_survey_publisher.py
+++ b/test/test_survey_publisher.py
@@ -6,7 +6,8 @@ from sqlalchemy import func, select
from compendium_v2 import db
from compendium_v2.db import presentation_model_enums as model_enums, presentation_models
-from compendium_v2.publishers.survey_publisher import _map_2023
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus, SurveyStatus
+from compendium_v2.publishers.survey_publisher import publish
JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
@@ -17,13 +18,13 @@ def test_v2_publisher_empty(app):
with app.app_context():
nren = presentation_models.NREN(name='name', country='country')
+ db.session.add(nren)
+ survey = Survey(year=2023, survey={}, status=SurveyStatus.open)
+ response = SurveyResponse(survey=survey, nren=nren, answers={"data": data}, status=ResponseStatus.completed)
+ db.session.add(survey)
+ db.session.add(response)
db.session.commit()
- with app.app_context():
- _map_2023(nren, {"data": data})
- db.session.commit()
-
- with app.app_context():
budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count == 0
# the main thing is actually that it doesnt crash
@@ -35,13 +36,15 @@ def test_v2_publisher_full(app):
with app.app_context():
nren = presentation_models.NREN(name='name', country='country')
+ db.session.add(nren)
+ survey = Survey(year=2023, survey={}, status=SurveyStatus.open)
+ response = SurveyResponse(survey=survey, nren=nren, answers={"data": data}, status=ResponseStatus.completed)
+ db.session.add(survey)
+ db.session.add(response)
db.session.commit()
- with app.app_context():
- _map_2023(nren, {"data": data})
- db.session.commit()
+ publish(2023)
- with app.app_context():
budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
assert budget == Decimal("124.76")