diff --git a/compendium_v2/db/presentation_model_enums.py b/compendium_v2/db/presentation_model_enums.py
index bda21ddfdbdf3a1523bc5ba0f570f59a9a06582e..3003d4f99cf12e6ec5f827e66f9cc66d276b7edf 100644
--- a/compendium_v2/db/presentation_model_enums.py
+++ b/compendium_v2/db/presentation_model_enums.py
@@ -61,7 +61,7 @@ class CommercialConnectivityCoverage(Enum):
no_other = "no_other"
-class CommarcialChargingLevel(Enum):
+class CommercialCharges(Enum):
higher_than_r_e_charges = "higher_than_r_e_charges"
same_as_r_e_charges = "same_as_r_e_charges"
no_charges_if_r_e_requested = "no_charges_if_r_e_requested"
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index de57349364b55690b7a03322816f814f9f1a700d..9c18348bedd094da8aa1501df596eb8df2e68793 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -11,7 +11,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
from compendium_v2.db import db
-from compendium_v2.db.presentation_model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, \
+from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, UserCategory, \
ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, \
FeeType, ServiceCategory
@@ -183,7 +183,7 @@ class ServiceUserTypes(db.Model):
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
user_category: Mapped[user_category_pk]
- service_category: Mapped[ServiceCategory]
+ service_category: Mapped[ServiceCategory] = mapped_column(primary_key=True)
class EOSCListings(db.Model):
@@ -292,9 +292,9 @@ class CommercialChargingLevel(db.Model):
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
- collaboration: Mapped[Optional[CommarcialChargingLevel]]
- service_supplier: Mapped[Optional[CommarcialChargingLevel]]
- direct_peering: Mapped[Optional[CommarcialChargingLevel]]
+ collaboration: Mapped[Optional[CommercialCharges]]
+ service_supplier: Mapped[Optional[CommercialCharges]]
+ direct_peering: Mapped[Optional[CommercialCharges]]
class RemoteCampuses(db.Model):
@@ -442,8 +442,8 @@ class TrafficRatio(db.Model):
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
- r_and_e_percentage: Mapped[int]
- commodity_percentage: Mapped[int]
+ r_and_e_percentage: Mapped[Decimal]
+ commodity_percentage: Mapped[Decimal]
class OpsAutomation(db.Model):
diff --git a/compendium_v2/migrations/versions/87e1e35051a0_fixes_for_new_presentation_models.py b/compendium_v2/migrations/versions/87e1e35051a0_fixes_for_new_presentation_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b08a3e6037bdf256cd0412bbf2113002397fa7f
--- /dev/null
+++ b/compendium_v2/migrations/versions/87e1e35051a0_fixes_for_new_presentation_models.py
@@ -0,0 +1,85 @@
+"""fixes for new presentation models
+
+Revision ID: 87e1e35051a0
+Revises: 8ff7260ad48f
+Create Date: 2023-09-17 15:24:21.873533
+
+"""
+
+# flake8: noqa
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '87e1e35051a0'
+down_revision = '8ff7260ad48f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commercialcharges').create(op.get_bind())
+ op.drop_table('service_user_types')
+
+ op.create_table(
+ 'service_user_types',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('service_category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_service_user_types_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', 'service_category', name=op.f('pk_service_user_types'))
+ )
+
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN collaboration TYPE commercialcharges USING collaboration::text::commercialcharges")
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN service_supplier TYPE commercialcharges USING service_supplier::text::commercialcharges")
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN direct_peering TYPE commercialcharges USING direct_peering::text::commercialcharges")
+
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').drop(op.get_bind())
+
+ with op.batch_alter_table('traffic_ratio', schema=None) as batch_op:
+ batch_op.alter_column('r_and_e_percentage',
+ existing_type=sa.INTEGER(),
+ type_=sa.Numeric(),
+ existing_nullable=False)
+ batch_op.alter_column('commodity_percentage',
+ existing_type=sa.INTEGER(),
+ type_=sa.Numeric(),
+ existing_nullable=False)
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table('traffic_ratio', schema=None) as batch_op:
+ batch_op.alter_column('commodity_percentage',
+ existing_type=sa.Numeric(),
+ type_=sa.INTEGER(),
+ existing_nullable=False)
+ batch_op.alter_column('r_and_e_percentage',
+ existing_type=sa.Numeric(),
+ type_=sa.INTEGER(),
+ existing_nullable=False)
+
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').create(op.get_bind())
+
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN collaboration TYPE commarcialcharginglevel USING collaboration::text::commarcialcharginglevel")
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN service_supplier TYPE commarcialcharginglevel USING service_supplier::text::commarcialcharginglevel")
+ op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN direct_peering TYPE commarcialcharginglevel USING direct_peering::text::commarcialcharginglevel")
+
+ op.drop_table('service_user_types')
+
+ op.create_table(
+ 'service_user_types',
+ sa.Column('nren_id', sa.INTEGER(), autoincrement=False, nullable=False),
+ sa.Column('year', sa.INTEGER(), autoincrement=False, nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), autoincrement=False, nullable=False),
+ sa.Column('service_category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), autoincrement=False, nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name='fk_service_user_types_nren_id_nren'),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name='pk_service_user_types')
+ )
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commercialcharges').drop(op.get_bind())
+ # ### end Alembic commands ###
diff --git a/compendium_v2/publishers/survey_publisher.py b/compendium_v2/publishers/survey_publisher.py
index 6946aa10847f85e81394666dce49219fa91dc0c4..10a50baf7e32a762aaa251cd79fbac3b65d4fddf 100644
--- a/compendium_v2/publishers/survey_publisher.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -17,17 +17,55 @@ from sqlalchemy import delete, select
from compendium_v2.db import db
from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, \
InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection, \
- FundingSource
-from compendium_v2.db.presentation_model_enums import FeeType
+ FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings, \
+ Standards, CrisisExcercises, SecurityControls, ConnectedProportion, ConnectivityLevel, \
+ ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity, \
+ CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
+ NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
+ CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation
+from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
+ CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
+def int_or_none(answers_dict, key):
+ if key in answers_dict:
+ return int(answers_dict[key])
+ return None
+
+
+def decimal_or_none(answers_dict, key):
+ if key in answers_dict:
+ return Decimal(answers_dict[key])
+ return None
+
+
+def decimal_or_zero(answers_dict, key):
+ return Decimal(answers_dict.get(key, 0))
+
+
+def bool_or_none(answer, key=None):
+ if key:
+ answer = answer.get(key)
+ if answer:
+ return answer == "Yes"
+ return None
+
+
def _map_2023(nren, answers) -> None:
year = 2023
- for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
- NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume]:
- db.session.execute(delete(table_class).where(table_class.year == 2023)) # type: ignore
+ for table_class in [BudgetEntry, ChargingStructure, ECProject, ExternalConnections,
+ InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume,
+ FundingSource, CentralProcurement, ServiceManagement, ServiceUserTypes, EOSCListings,
+ Standards, CrisisExcercises, SecurityControls, ConnectedProportion, ConnectivityLevel,
+ ConnectionCarrier, ConnectivityLoad, ConnectivityGrowth, CommercialConnectivity,
+ CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
+ NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
+ CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio,
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation]:
+ db.session.execute(delete(table_class).where(table_class.year == year)) # type: ignore
answers = answers["data"]
budget = answers.get("budget")
@@ -38,11 +76,11 @@ def _map_2023(nren, answers) -> None:
if funding_source:
db.session.add(FundingSource(
nren_id=nren.id, nren=nren, year=year,
- client_institutions=Decimal(funding_source.get("client_institutions", 0)),
- european_funding=Decimal(funding_source.get("european_funding", 0)),
- gov_public_bodies=Decimal(funding_source.get("gov_public_bodies", 0)),
- commercial=Decimal(funding_source.get("commercial", 0)),
- other=Decimal(funding_source.get("other", 0))
+ client_institutions=decimal_or_zero(funding_source, "client_institutions"),
+ european_funding=decimal_or_zero(funding_source, "european_funding"),
+ gov_public_bodies=decimal_or_zero(funding_source, "gov_public_bodies"),
+ commercial=decimal_or_zero(funding_source, "commercial"),
+ other=decimal_or_zero(funding_source, "other")
))
charging = answers.get("charging_mechanism")
@@ -54,10 +92,10 @@ def _map_2023(nren, answers) -> None:
if staff_roles or staff_employment_type:
db.session.add(NrenStaff(
nren_id=nren.id, nren=nren, year=year,
- permanent_fte=Decimal(staff_employment_type.get("permanent_fte", 0)),
- subcontracted_fte=Decimal(staff_employment_type.get("subcontracted_fte", 0)),
- technical_fte=Decimal(staff_roles.get("technical_fte", 0)),
- non_technical_fte=Decimal(staff_roles.get("nontechnical_fte", 0))
+ permanent_fte=decimal_or_zero(staff_employment_type, "permanent_fte"),
+ subcontracted_fte=decimal_or_zero(staff_employment_type, "subcontracted_fte"),
+ technical_fte=decimal_or_zero(staff_roles, "technical_fte"),
+ non_technical_fte=decimal_or_zero(staff_roles, "nontechnical_fte")
))
parent = answers.get("parent_organization_name")
@@ -103,10 +141,10 @@ def _map_2023(nren, answers) -> None:
if traffic_estimate:
db.session.add(TrafficVolume(
nren_id=nren.id, nren=nren, year=year,
- to_customers=Decimal(traffic_estimate.get("to_customers")),
- from_customers=Decimal(traffic_estimate.get("from_customers")),
- to_external=Decimal(traffic_estimate.get("to_external")),
- from_external=Decimal(traffic_estimate.get("from_external")),
+ to_customers=decimal_or_zero(traffic_estimate, "to_customers"),
+ from_customers=decimal_or_zero(traffic_estimate, "from_customers"),
+ to_external=decimal_or_zero(traffic_estimate, "to_external"),
+ from_external=decimal_or_zero(traffic_estimate, "from_external")
))
institution_urls = answers.get("connected_sites_lists")
@@ -118,6 +156,306 @@ def _map_2023(nren, answers) -> None:
urls=urls
))
+ central_procurement = answers.get("central_software_procurement")
+ if central_procurement:
+ central_procurement = central_procurement == "Yes"
+ db.session.add(CentralProcurement(
+ nren_id=nren.id, nren=nren, year=year,
+ central_procurement=central_procurement,
+ amount=decimal_or_none(answers, "central_procurement_amount")
+ ))
+
+ formal_service_management_framework = answers.get("formal_service_management_framework")
+ service_level_targets = answers.get("service_level_targets")
+ if formal_service_management_framework or service_level_targets:
+ db.session.add(ServiceManagement(
+ nren_id=nren.id, nren=nren, year=year,
+ service_management_framework=bool_or_none(formal_service_management_framework),
+ service_level_targets=bool_or_none(service_level_targets)
+ ))
+
+ service_type_matrix = answers.get("service_matrix", {})
+ for user_type, service_types in service_type_matrix.items():
+ user_type = UserCategory[user_type]
+ service_types = service_types["service_types"]
+ for service_type in service_types:
+ service_type = ServiceCategory[service_type]
+ db.session.add(ServiceUserTypes(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ service_category=service_type
+ ))
+
+ services_on_eosc_portal_list = answers.get("services_on_eosc_portal_list", [])
+ eosc_list = [i.get("service_name") for i in services_on_eosc_portal_list]
+ eosc_list = [i for i in eosc_list if i]
+ if eosc_list:
+ db.session.add(EOSCListings(
+ nren_id=nren.id, nren=nren, year=year,
+ service_names=eosc_list
+ ))
+
+ audits = answers.get("audits")
+ business_continuity_plans = answers.get("business_continuity_plans")
+ crisis_management_procedure = answers.get("crisis_management_procedure")
+ if audits or business_continuity_plans or crisis_management_procedure:
+ db.session.add(Standards(
+ nren_id=nren.id, nren=nren, year=year,
+ audits=bool_or_none(audits),
+ audit_specifics=answers.get("audit_specifics", ""),
+ business_continuity_plans=bool_or_none(business_continuity_plans),
+ business_continuity_plans_specifics=answers.get("business_continuity_plans_specifics", ""),
+ crisis_management_procedure=bool_or_none(crisis_management_procedure)
+ ))
+
+ crisis_exercises = answers.get("crisis_exercises")
+ if crisis_exercises:
+ db.session.add(CrisisExcercises(
+ nren_id=nren.id, nren=nren, year=year,
+ exercise_descriptions=crisis_exercises
+ ))
+
+ security_controls = answers.get("security_controls")
+ if security_controls:
+ if "other" in security_controls:
+ security_controls.remove("other")
+ security_controls.append(answers.get("security_controls-Comment", "other"))
+ db.session.add(SecurityControls(
+ nren_id=nren.id, nren=nren, year=year,
+ security_control_descriptions=security_controls
+ ))
+
+ connectivity_proportions = answers.get("connectivity_proportions", {})
+ for user_type, connectivity_proportion in connectivity_proportions.items():
+ user_type = UserCategory[user_type]
+ coverage = connectivity_proportion.get("covered")
+ coverage = ConnectivityCoverage[coverage] if coverage else None
+ number_connected = int_or_none(connectivity_proportion, "nr_connected")
+ market_share = decimal_or_none(connectivity_proportion, "market_share_percentage")
+ users_served = int_or_none(connectivity_proportion, "nr_of_users")
+ db.session.add(ConnectedProportion(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ coverage=coverage,
+ number_connected=number_connected,
+ market_share=market_share,
+ users_served=users_served
+ ))
+
+ connectivity_levels = answers.get("connectivity_level", {})
+ for user_type, connectivity_level in connectivity_levels.items():
+ user_type = UserCategory[user_type]
+ db.session.add(ConnectivityLevel(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ typical_speed=int_or_none(connectivity_level, "typical_speed"),
+ highest_speed=int_or_none(connectivity_level, "highest_speed"),
+ highest_speed_proportion=decimal_or_none(connectivity_level, "highest_speed_connection_percentage")
+ ))
+
+ traffic_carriers = answers.get("traffic_carriers", {})
+ for user_type, traffic_carrier in traffic_carriers.items():
+ user_type = UserCategory[user_type]
+ traffic_carrier = traffic_carrier.get("carry_mechanism")
+ if traffic_carrier:
+ db.session.add(ConnectionCarrier(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ carry_mechanism=CarryMechanism[traffic_carrier]
+ ))
+
+ traffic_loads = answers.get("traffic_load", {})
+ for user_type, traffic_load in traffic_loads.items():
+ user_type = UserCategory[user_type]
+ db.session.add(ConnectivityLoad(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ average_load_from_institutions=int_or_none(traffic_load, "average_from_institutions_to_network"),
+ average_load_to_institutions=int_or_none(traffic_load, "average_to_institutions_from_network"),
+ peak_load_from_institutions=int_or_none(traffic_load, "peak_from_institutions_to_network"),
+ peak_load_to_institutions=int_or_none(traffic_load, "peak_to_institutions_from_network")
+ ))
+
+ traffic_growths = answers.get("traffic_growth", {})
+ for user_type, traffic_growth in traffic_growths.items():
+ user_type = UserCategory[user_type]
+ db.session.add(ConnectivityGrowth(
+ nren_id=nren.id, nren=nren, year=year,
+ user_category=user_type,
+ growth=decimal_or_zero(traffic_growth, "growth_rate")
+ ))
+
+ commercial_organizations = answers.get("commercial_organizations")
+ if commercial_organizations:
+ c1 = commercial_organizations.get("commercial_r_e", {}).get("connection")
+ c2 = commercial_organizations.get("commercial_general", {}).get("connection")
+ c3 = commercial_organizations.get("commercial_collaboration", {}).get("connection")
+ c4 = commercial_organizations.get("commercial_service_provider", {}).get("connection")
+ c5 = commercial_organizations.get("university_spin_off", {}).get("connection")
+ db.session.add(CommercialConnectivity(
+ nren_id=nren.id, nren=nren, year=year,
+ commercial_r_and_e=CommercialConnectivityCoverage[c1] if c1 else None,
+ commercial_general=CommercialConnectivityCoverage[c2] if c2 else None,
+ commercial_collaboration=CommercialConnectivityCoverage[c3] if c3 else None,
+ commercial_service_provider=CommercialConnectivityCoverage[c4] if c4 else None,
+ university_spin_off=CommercialConnectivityCoverage[c5] if c5 else None,
+ ))
+
+ commercial_charging_levels = answers.get("commercial_charging_levels")
+ if commercial_charging_levels:
+ c1 = commercial_charging_levels.get("collaboration", {}).get("charging_level")
+ c2 = commercial_charging_levels.get("services", {}).get("charging_level")
+ c3 = commercial_charging_levels.get("peering", {}).get("charging_level")
+ db.session.add(CommercialChargingLevel(
+ nren_id=nren.id, nren=nren, year=year,
+ collaboration=CommercialCharges[c1] if c1 else None,
+ service_supplier=CommercialCharges[c2] if c2 else None,
+ direct_peering=CommercialCharges[c3] if c3 else None,
+ ))
+
+ remote_campuses = answers.get("remote_campuses")
+ if remote_campuses:
+ remote_campuses = remote_campuses == "Yes"
+ remote_campuses_specifics = answers.get("remote_campuses_specifics", [])
+ remote_campuses_specifics = [
+ {"country": i.get("country", ""), "local_r_and_e_connection": bool_or_none(i, "connected")}
+ for i in remote_campuses_specifics
+ ]
+ db.session.add(RemoteCampuses(
+ nren_id=nren.id, nren=nren, year=year,
+ remote_campus_connectivity=remote_campuses,
+ connections=remote_campuses_specifics
+ ))
+
+ dark_fibre_lease = answers.get("dark_fibre_lease")
+ if dark_fibre_lease:
+ dark_fibre_lease = dark_fibre_lease == "Yes"
+ db.session.add(DarkFibreLease(
+ nren_id=nren.id, nren=nren, year=year,
+ iru_or_lease=dark_fibre_lease,
+ fibre_length_in_country=int_or_none(answers, "dark_fibre_lease_kilometers_inside_country"),
+ fibre_length_outside_country=int_or_none(answers, "dark_fibre_lease_kilometers_outside_country"),
+ iru_duration=decimal_or_none(answers, "dark_fibre_lease_duration")
+ ))
+
+ dark_fibre_nren = answers.get("dark_fibre_nren")
+ if dark_fibre_nren:
+ dark_fibre_nren = dark_fibre_nren == "Yes"
+ db.session.add(DarkFibreInstalled(
+ nren_id=nren.id, nren=nren, year=year,
+ installed=dark_fibre_nren,
+ fibre_length_in_country=int_or_none(answers, "dark_fibre_nren_kilometers_inside_country")
+ ))
+
+ fibre_light = answers.get("fibre_light")
+ if fibre_light:
+ if fibre_light == "other":
+ fibre_light = answers.get("fibre_light-Comment", "other")
+ db.session.add(FibreLight(
+ nren_id=nren.id, nren=nren, year=year,
+ light_description=fibre_light
+ ))
+
+ network_map_urls = answers.get("network_map_urls")
+ if network_map_urls:
+ urls = [i.get("network_map_url", "") for i in network_map_urls if i.get("network_map_url", "") != ""]
+ if urls:
+ db.session.add(NetworkMapUrls(
+ nren_id=nren.id, nren=nren, year=year,
+ urls=urls
+ ))
+
+ monitoring_tools = answers.get("monitoring_tools", [])
+ netflow_vendors = answers.get("netflow_vendors", "")
+ if monitoring_tools or netflow_vendors:
+ if "other" in monitoring_tools:
+ monitoring_tools.remove("other")
+ monitoring_tools.append(answers.get("monitoring_tools-Comment", "other"))
+ db.session.add(MonitoringTools(
+ nren_id=nren.id, nren=nren, year=year,
+ tool_descriptions=monitoring_tools,
+ netflow_processing_description=netflow_vendors
+ ))
+
+ passive_monitoring = answers.get("passive_monitoring")
+ if passive_monitoring:
+ passive_monitoring = passive_monitoring == "Yes"
+ passive_monitoring_tech = answers.get("passive_monitoring_tech")
+ db.session.add(PassiveMonitoring(
+ nren_id=nren.id, nren=nren, year=year,
+ monitoring=passive_monitoring,
+ method=MonitoringMethod[passive_monitoring_tech] if passive_monitoring_tech else None
+ ))
+
+ traffic_statistics = answers.get("traffic_statistics")
+ if traffic_statistics:
+ traffic_statistics = traffic_statistics == "Yes"
+ urls = answers.get("traffic_statistics_urls")
+ urls = [i.get("traffic_statistics_url", "") for i in urls if i.get("traffic_statistics_url")]
+ db.session.add(TrafficStatistics(
+ nren_id=nren.id, nren=nren, year=year,
+ traffic_statistics=traffic_statistics,
+ urls=urls
+ ))
+
+ siem_soc_vendor = answers.get("siem_soc_vendor")
+ if siem_soc_vendor:
+ if "other" in siem_soc_vendor:
+ siem_soc_vendor.remove("other")
+ siem_soc_vendor.append(answers.get("siem_soc_vendor-Comment", "other"))
+ db.session.add(SiemVendors(
+ nren_id=nren.id, nren=nren, year=year,
+ vendor_names=siem_soc_vendor
+ ))
+
+ certificate_service = answers.get("certificate_service")
+ if certificate_service:
+ if "other" in certificate_service:
+ certificate_service.remove("other")
+ certificate_service.append(answers.get("certificate_service-Comment", "other"))
+ db.session.add(CertificateProviders(
+ nren_id=nren.id, nren=nren, year=year,
+ provider_names=certificate_service
+ ))
+
+ network_weather = answers.get("network_weather")
+ if network_weather:
+ network_weather = network_weather == "Yes"
+ db.session.add(WeatherMap(
+ nren_id=nren.id, nren=nren, year=year,
+ weather_map=network_weather,
+ url=answers.get("network_weather_url", "")
+ ))
+
+ pert_team = answers.get("pert_team")
+ if pert_team:
+ pert_team = YesNoPlanned[pert_team.lower()]
+ db.session.add(PertTeam(
+ nren_id=nren.id, nren=nren, year=year,
+ pert_team=pert_team
+ ))
+
+ alienwave_services = answers.get("alienwave_services")
+ alienwave_internal = answers.get("alienwave_internal")
+ if alienwave_services or alienwave_internal:
+ alienwave_services = YesNoPlanned[alienwave_services.lower()] if alienwave_services else None
+ alienwave_internal = alienwave_internal == "Yes" if alienwave_internal else None
+ db.session.add(AlienWave(
+ nren_id=nren.id, nren=nren, year=year,
+ alien_wave_third_pary=alienwave_services,
+ nr_of_alien_wave_third_party_services=int_or_none(answers, "alienwave_services_number"),
+ alien_wave_internal=alienwave_internal
+ ))
+
+ max_capacity = answers.get("max_capacity")
+ typical_capacity = answers.get("typical_capacity")
+ if max_capacity or typical_capacity:
+ db.session.add(Capacity(
+ nren_id=nren.id, nren=nren, year=year,
+ largest_link_capacity=decimal_or_none(answers, "max_capacity"),
+ typical_backbone_capacity=decimal_or_none(answers, "typical_capacity")
+ ))
+
external_connections = answers.get("external_connections")
if external_connections:
connections: List[ExternalConnection] = []
@@ -134,6 +472,50 @@ def _map_2023(nren, answers) -> None:
connections=connections
))
+ non_r_and_e_peers = answers.get("non_r_and_e_peers")
+ if non_r_and_e_peers:
+ db.session.add(NonREPeers(
+ nren_id=nren.id, nren=nren, year=year,
+ nr_of_non_r_and_e_peers=int(non_r_and_e_peers)
+ ))
+
+ commodity_vs_r_e = answers.get("commodity_vs_r_e")
+ if commodity_vs_r_e:
+ db.session.add(TrafficRatio(
+ nren_id=nren.id, nren=nren, year=year,
+ r_and_e_percentage=decimal_or_zero(commodity_vs_r_e, "r_e"),
+ commodity_percentage=decimal_or_zero(commodity_vs_r_e, "commodity"),
+ ))
+
+ operational_process_automation = answers.get("operational_process_automation")
+ if operational_process_automation:
+ db.session.add(OpsAutomation(
+ nren_id=nren.id, nren=nren, year=year,
+ ops_automation=YesNoPlanned[operational_process_automation.lower()],
+ ops_automation_specifics=answers.get("operational_process_automation_tools", "")
+ ))
+
+ nfv = answers.get("nfv")
+ if nfv:
+ nfv_types = answers.get("nfv_types", [])
+ if "other" in nfv_types:
+ nfv_types.remove("other")
+ nfv_types.append(answers.get("nfv_types-Comment", "other"))
+ db.session.add(NetworkFunctionVirtualisation(
+ nren_id=nren.id, nren=nren, year=year,
+ nfv=YesNoPlanned[nfv.lower()],
+ nfv_specifics=nfv_types
+ ))
+
+ network_automation = answers.get("network_automation")
+ if network_automation:
+ network_automation_tasks = answers.get("network_automation_tasks", [])
+ db.session.add(NetworkAutomation(
+ nren_id=nren.id, nren=nren, year=year,
+ network_automation=YesNoPlanned[network_automation.lower()],
+ network_automation_specifics=network_automation_tasks
+ ))
+
def publish(year):
responses = db.session.scalars(
diff --git a/compendium_v2/publishers/survey_publisher_old_db_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
index 70ddd0508bf559f18743c4f1dcb23b3cb41837de..f071d75d001076811d8320a06f5b06a63f6dc9e1 100644
--- a/compendium_v2/publishers/survey_publisher_old_db_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -1,6 +1,6 @@
"""
survey_publisher_old_db_2022
-=========================
+============================
This module loads the survey data from 2022 from the old survey database into presentation_models.
Registered as click cli command when installing compendium-v2.
diff --git a/test/test_survey_publisher.py b/test/test_survey_publisher.py
index e6cc54f68f434b937fb9158b4046cfac5ef2f671..3e66d6f0605d1a62195f49355dc3bd70bbe61c65 100644
--- a/test/test_survey_publisher.py
+++ b/test/test_survey_publisher.py
@@ -5,7 +5,7 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import presentation_model_enums, presentation_models
+from compendium_v2.db import presentation_model_enums as model_enums, presentation_models
from compendium_v2.publishers.survey_publisher import _map_2023
@@ -53,7 +53,7 @@ def test_v2_publisher_full(app):
assert funding_source.other == Decimal("10")
charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
- assert charging_structure == presentation_model_enums.FeeType.usage_based_fee
+ assert charging_structure == model_enums.FeeType.usage_based_fee
staff = db.session.scalar(select(presentation_models.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
@@ -102,6 +102,163 @@ def test_v2_publisher_full(app):
client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
+ procurement = db.session.scalar(select(presentation_models.CentralProcurement))
+ assert procurement.central_procurement
+ assert procurement.amount == Decimal("57676")
+
+ service_management = db.session.scalar(select(presentation_models.ServiceManagement))
+ assert service_management.service_management_framework
+ assert not service_management.service_level_targets
+
+ service_user_types = [s for s in db.session.scalars(select(presentation_models.ServiceUserTypes))]
+ user_categories = set([s.user_category for s in service_user_types])
+ assert len(user_categories) == 10
+ institute_services = [
+ s for s in service_user_types if s.user_category == model_enums.UserCategory.institutes
+ ]
+ assert len(institute_services) == 7
+
+ eosc_listings = db.session.scalar(select(presentation_models.EOSCListings))
+ assert eosc_listings.service_names == ["dy", "rrr"]
+
+ standards = db.session.scalar(select(presentation_models.Standards))
+ assert standards.audits
+ assert standards.audit_specifics == "isooo 557"
+ assert standards.business_continuity_plans
+ assert standards.business_continuity_plans_specifics == "no"
+ assert not standards.crisis_management_procedure
+
+ crisis_excercises = db.session.scalar(select(presentation_models.CrisisExcercises))
+ assert crisis_excercises.exercise_descriptions == [
+ "geant_workshops", "national_excercises", "tabletop_exercises", "other_excercises",
+ "none", "simulation_excercises", "real_crisis"
+ ]
+
+ security_controls = db.session.scalar(select(presentation_models.SecurityControls))
+ assert security_controls.security_control_descriptions == [
+ "monitoring", "acl", "anti_virus", "firewall", "anti_spam", "ddos_mitigation",
+ "ips_ids", "segmentation", "integrity_checking", "vgjh"
+ ]
+
+ connected_proportion = [s for s in db.session.scalars(select(presentation_models.ConnectedProportion))]
+ assert len(connected_proportion) == 10
+ uni = [c for c in connected_proportion if c.user_category == model_enums.UserCategory.universities]
+ assert len(uni) == 1
+ assert uni[0].coverage == model_enums.ConnectivityCoverage.yes_national_nren
+ assert uni[0].number_connected == 19
+ assert uni[0].market_share == Decimal("1")
+ assert uni[0].users_served == 20000
+
+ connectivity_level = [s for s in db.session.scalars(select(presentation_models.ConnectivityLevel))]
+ assert len(connectivity_level) == 10
+ gov = [c for c in connectivity_level if c.user_category == model_enums.UserCategory.government]
+ assert len(gov) == 1
+ assert gov[0].typical_speed == 100
+ assert gov[0].highest_speed == 100
+ assert gov[0].highest_speed_proportion == Decimal("100")
+
+ carrier = [s for s in db.session.scalars(select(presentation_models.ConnectionCarrier))]
+ assert len(carrier) == 10
+ profit = [c for c in carrier if c.user_category == model_enums.UserCategory.for_profit_orgs]
+ assert len(profit) == 1
+ assert profit[0].carry_mechanism == model_enums.CarryMechanism.commercial_provider_backbone
+ primary = [c for c in carrier if c.user_category == model_enums.UserCategory.primary_schools]
+ assert len(primary) == 1
+ assert primary[0].carry_mechanism == model_enums.CarryMechanism.man
+ hospital = [c for c in carrier if c.user_category == model_enums.UserCategory.hospitals]
+ assert len(hospital) == 1
+ assert hospital[0].carry_mechanism == model_enums.CarryMechanism.nren_local_loops
+
+ load = [s for s in db.session.scalars(select(presentation_models.ConnectivityLoad))]
+ assert len(load) == 10
+ cultural = [c for c in load if c.user_category == model_enums.UserCategory.cultural]
+ assert len(cultural) == 1
+ assert cultural[0].average_load_from_institutions == 10
+ assert cultural[0].average_load_to_institutions == 10
+ assert cultural[0].peak_load_from_institutions == 50
+ assert cultural[0].peak_load_to_institutions == 10
+
+ growth = [s for s in db.session.scalars(select(presentation_models.ConnectivityGrowth))]
+ assert len(growth) == 10
+ cultural = [c for c in growth if c.user_category == model_enums.UserCategory.cultural]
+ assert len(cultural) == 1
+ assert cultural[0].growth == Decimal("50")
+
+ commercial = db.session.scalar(select(presentation_models.CommercialConnectivity))
+ assert commercial.commercial_r_and_e == model_enums.CommercialConnectivityCoverage.no_but_direct_peering
+ assert commercial.commercial_general == model_enums.CommercialConnectivityCoverage.no_policy
+ assert commercial.commercial_collaboration == model_enums.CommercialConnectivityCoverage.yes_incl_other
+ assert commercial.commercial_service_provider == model_enums.CommercialConnectivityCoverage.no_other
+ assert commercial.university_spin_off == model_enums.CommercialConnectivityCoverage.no_financial
+
+ commercial_charging = db.session.scalar(select(presentation_models.CommercialChargingLevel))
+ assert commercial_charging.collaboration == model_enums.CommercialCharges.no_charges_if_r_e_requested
+ assert commercial_charging.service_supplier == model_enums.CommercialCharges.same_as_r_e_charges
+ assert commercial_charging.direct_peering == model_enums.CommercialCharges.higher_than_r_e_charges
+
+ remote_campuses = db.session.scalar(select(presentation_models.RemoteCampuses))
+ assert remote_campuses.remote_campus_connectivity
+ assert remote_campuses.connections == [
+ {"country": "fyuyg", "local_r_and_e_connection": False},
+ {"country": "g", "local_r_and_e_connection": None}
+ ]
+
+ dark_fibre_lease = db.session.scalar(select(presentation_models.DarkFibreLease))
+ assert dark_fibre_lease.iru_or_lease
+ assert dark_fibre_lease.fibre_length_in_country == 405
+ assert dark_fibre_lease.fibre_length_outside_country == 0
+ assert dark_fibre_lease.iru_duration == Decimal("10")
+
+ dark_fibre_installed = db.session.scalar(select(presentation_models.DarkFibreInstalled))
+ assert dark_fibre_installed.installed
+ assert dark_fibre_installed.fibre_length_in_country == 1
+
+ fibre_light = db.session.scalar(select(presentation_models.FibreLight))
+ assert fibre_light.light_description == "jj"
+
+ network_map_urls = db.session.scalar(select(presentation_models.NetworkMapUrls))
+ assert network_map_urls.urls == ["http://netmon.ucg.ac.me/mantra/"]
+
+ monitoring_tools = db.session.scalar(select(presentation_models.MonitoringTools))
+ assert monitoring_tools.tool_descriptions == [
+ "status_dashboard", "historical_traffic_volumes", "netflow_analysis", "looking_glass", "hiui"
+ ]
+ assert monitoring_tools.netflow_processing_description == "ghhhh"
+
+ passive_monitoring = db.session.scalar(select(presentation_models.PassiveMonitoring))
+ assert passive_monitoring.monitoring
+ assert passive_monitoring.method == model_enums.MonitoringMethod.both
+
+ traffic_statistics = db.session.scalar(select(presentation_models.TrafficStatistics))
+ assert traffic_statistics.traffic_statistics
+ assert traffic_statistics.urls == [
+ "http://netmon.cis.ac.me/cacti/graph_view.php?action=tree&tree_id=36&leaf_id=1430"
+ ]
+
+ siem_vendors = db.session.scalar(select(presentation_models.SiemVendors))
+ assert siem_vendors.vendor_names == ["Exabeam", "Splunk", "IBM Qradar", "LogRythm", "Securonix", "figuoho"]
+
+ certificate_providers = db.session.scalar(select(presentation_models.CertificateProviders))
+ assert certificate_providers.provider_names == [
+ "TCS", "Digicert", "Sectigo", "GlobalSign", "GeoDaddy", "GeoTrust", "Entrust Datacard", "earer"
+ ]
+
+ weather_map = db.session.scalar(select(presentation_models.WeatherMap))
+ assert weather_map.weather_map
+ assert weather_map.url == "http://weather.com"
+
+ pert_team = db.session.scalar(select(presentation_models.PertTeam))
+ assert pert_team.pert_team == model_enums.YesNoPlanned.planned
+
+ alien_wave = db.session.scalar(select(presentation_models.AlienWave))
+ assert alien_wave.alien_wave_third_pary == model_enums.YesNoPlanned.yes
+ assert alien_wave.nr_of_alien_wave_third_party_services == 66
+ assert not alien_wave.alien_wave_internal
+
+ capacity = db.session.scalar(select(presentation_models.Capacity))
+ assert capacity.largest_link_capacity == Decimal("1")
+ assert capacity.typical_backbone_capacity == Decimal("567")
+
external_connections = db.session.scalar(select(presentation_models.ExternalConnections))
external_connection_list = external_connections.connections
assert len(external_connection_list) == 6
@@ -126,3 +283,26 @@ def test_v2_publisher_full(app):
"link_name": "",
"to_organization": "",
}
+
+ non_r_e_peers = db.session.scalar(select(presentation_models.NonREPeers))
+ assert non_r_e_peers.nr_of_non_r_and_e_peers == 0
+
+ traffic_ratio = db.session.scalar(select(presentation_models.TrafficRatio))
+ assert traffic_ratio.r_and_e_percentage == Decimal("1")
+ assert traffic_ratio.commodity_percentage == Decimal("99")
+
+ ops_automation = db.session.scalar(select(presentation_models.OpsAutomation))
+ assert ops_automation.ops_automation == model_enums.YesNoPlanned.yes
+ assert ops_automation.ops_automation_specifics == "ghioil;"
+
+ nfv = db.session.scalar(select(presentation_models.NetworkFunctionVirtualisation))
+ assert nfv.nfv == model_enums.YesNoPlanned.planned
+ assert nfv.nfv_specifics == [
+ "routers", "firewalls", "load_balancers", "vpn_concentrators", "rtghjk"
+ ]
+
+ network_automation = db.session.scalar(select(presentation_models.NetworkAutomation))
+ assert network_automation.network_automation == model_enums.YesNoPlanned.planned
+ assert network_automation.network_automation_specifics == [
+ "config_management", "provisioning", "data_collection", "compliance", "reporting", "troubleshooting"
+ ]