From d2981688f32b41d9ed94e11b85dbbf129ce59128 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Thu, 14 Sep 2023 15:57:03 +0200
Subject: [PATCH 1/2] added datamodels for policy, connected users, and network
questions
---
.../background_task/parse_excel_data.py | 2 +-
compendium_v2/db/model.py | 347 ++++++++++++++++-
compendium_v2/db/model_enums.py | 88 +++++
.../8ff7260ad48f_add_presentation_models.py | 359 ++++++++++++++++++
.../publishers/survey_publisher_2022.py | 2 +-
.../publishers/survey_publisher_v2.py | 26 +-
test/test_survey_publisher_2022.py | 8 +-
test/test_survey_publisher_v2.py | 29 +-
8 files changed, 838 insertions(+), 23 deletions(-)
create mode 100644 compendium_v2/db/model_enums.py
create mode 100644 compendium_v2/migrations/versions/8ff7260ad48f_add_presentation_models.py
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/background_task/parse_excel_data.py
index aba3ef6f..69612055 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/background_task/parse_excel_data.py
@@ -2,7 +2,7 @@ import logging
import openpyxl
import os
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.model_enums import FeeType
from compendium_v2.environment import setup_logging
setup_logging()
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/model.py
index a9d92279..5464188b 100644
--- a/compendium_v2/db/model.py
+++ b/compendium_v2/db/model.py
@@ -3,24 +3,44 @@ from __future__ import annotations
import logging
from decimal import Decimal
-from enum import Enum
from typing import List, Optional
-from typing_extensions import Annotated
+from typing_extensions import Annotated, TypedDict
from sqlalchemy import String, JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
from compendium_v2.db import db
+from compendium_v2.db.model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, ServiceCategory, \
+ ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, FeeType
logger = logging.getLogger(__name__)
+
str128 = Annotated[str, 128]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
int_pk = Annotated[int, mapped_column(primary_key=True)]
int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
+user_category_pk = Annotated[UserCategory, mapped_column(primary_key=True)]
+json_str_list = Annotated[List[str], mapped_column(JSON)]
+
+ExternalConnection = TypedDict(
+ 'ExternalConnection',
+ {
+ 'link_name': str,
+ 'capacity': Optional[Decimal],
+ 'from_organization': str,
+ 'to_organization': str,
+ 'interconnection_method': Optional[ConnectionMethod]
+ }
+)
+
+RemoteCampus = TypedDict(
+ 'RemoteCampus',
+ {'country': str, 'local_r_and_e_connection': Optional[bool]}
+)
# Unfortunately flask-sqlalchemy doesnt fully support DeclarativeBase yet.
@@ -60,14 +80,6 @@ class FundingSource(db.Model):
other: Mapped[Decimal]
-class FeeType(Enum):
- flat_fee = "flat_fee"
- usage_based_fee = "usage_based_fee"
- combination = "combination"
- no_charge = "no_charge"
- other = "other"
-
-
class ChargingStructure(db.Model):
__tablename__ = 'charging_structure'
nren_id: Mapped[int_pk_fkNREN]
@@ -140,8 +152,321 @@ class TrafficVolume(db.Model):
class InstitutionURLs(db.Model):
__tablename__ = 'institution_urls'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ urls: Mapped[json_str_list]
+
+
+class CentralProcurement(db.Model):
+ __tablename__ = 'central_procurement'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ central_procurement: Mapped[bool]
+ amount: Mapped[Optional[Decimal]]
+
+
+class ServiceManagement(db.Model):
+ __tablename__ = 'service_management'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ service_management_framework: Mapped[Optional[bool]]
+ service_level_targets: Mapped[Optional[bool]]
+
+
+class ServiceUserTypes(db.Model):
+ __tablename__ = 'service_user_types'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ service_category: Mapped[ServiceCategory]
+
+
+class EOSCListings(db.Model):
+ __tablename__ = 'eosc_listings'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ service_names: Mapped[json_str_list]
+
+
+class Standards(db.Model):
+ __tablename__ = 'standards'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ audits: Mapped[Optional[bool]]
+ audit_specifics: Mapped[str]
+ business_continuity_plans: Mapped[Optional[bool]]
+ business_continuity_plans_specifics: Mapped[str]
+ crisis_management_procedure: Mapped[Optional[bool]]
+
+
+class CrisisExcercises(db.Model):
+ __tablename__ = 'crisis_excercises'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ exercise_descriptions: Mapped[json_str_list]
+
+
+class SecurityControls(db.Model):
+ __tablename__ = 'security_controls'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ security_control_descriptions: Mapped[json_str_list]
+
+
+class ConnectedProportion(db.Model):
+ __tablename__ = 'connected_proportion'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ coverage: Mapped[Optional[ConnectivityCoverage]]
+ number_connected: Mapped[Optional[int]]
+ market_share: Mapped[Optional[Decimal]]
+ users_served: Mapped[Optional[int]]
+
+
+class ConnectivityLevel(db.Model):
+ __tablename__ = 'connectivity_level'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ typical_speed: Mapped[Optional[int]]
+ highest_speed: Mapped[Optional[int]]
+ highest_speed_proportion: Mapped[Optional[Decimal]]
+
+
+class ConnectionCarrier(db.Model):
+ __tablename__ = 'connection_carrier'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ carry_mechanism: Mapped[CarryMechanism]
+
+
+class ConnectivityLoad(db.Model):
+ __tablename__ = 'connectivity_load'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ average_load_from_institutions: Mapped[Optional[int]]
+ average_load_to_institutions: Mapped[Optional[int]]
+ peak_load_from_institutions: Mapped[Optional[int]]
+ peak_load_to_institutions: Mapped[Optional[int]]
+
+
+class ConnectivityGrowth(db.Model):
+ __tablename__ = 'connectivity_growth'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ user_category: Mapped[user_category_pk]
+ growth: Mapped[Decimal]
+
+
+class CommercialConnectivity(db.Model):
+ __tablename__ = 'commercial_connectivity'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ commercial_r_and_e: Mapped[Optional[CommercialConnectivityCoverage]]
+ commercial_general: Mapped[Optional[CommercialConnectivityCoverage]]
+ commercial_collaboration: Mapped[Optional[CommercialConnectivityCoverage]]
+ commercial_service_provider: Mapped[Optional[CommercialConnectivityCoverage]]
+ university_spin_off: Mapped[Optional[CommercialConnectivityCoverage]]
+
+
+class CommercialChargingLevel(db.Model):
+ __tablename__ = 'commercial_charging_level'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ collaboration: Mapped[Optional[CommarcialChargingLevel]]
+ service_supplier: Mapped[Optional[CommarcialChargingLevel]]
+ direct_peering: Mapped[Optional[CommarcialChargingLevel]]
+
+
+class RemoteCampuses(db.Model):
+ __tablename__ = 'remote_campuses'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ remote_campus_connectivity: Mapped[bool]
+ connections: Mapped[List[RemoteCampus]] = mapped_column(JSON)
+
+
+class DarkFibreLease(db.Model):
+ __tablename__ = 'dark_fibre_lease'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ iru_or_lease: Mapped[bool]
+ fibre_length_in_country: Mapped[Optional[int]]
+ fibre_length_outside_country: Mapped[Optional[int]]
+ iru_duration: Mapped[Optional[Decimal]]
+
+
+class DarkFibreInstalled(db.Model):
+ __tablename__ = 'dark_fibre_installed'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ installed: Mapped[bool]
+ fibre_length_in_country: Mapped[Optional[int]]
+
+
+class FibreLight(db.Model):
+ __tablename__ = 'fibre_light'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ light_description: Mapped[str]
+
+
+class NetworkMapUrls(db.Model):
+ __tablename__ = 'network_map_urls'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ urls: Mapped[json_str_list]
+
+
+class MonitoringTools(db.Model):
+ __tablename__ = 'monitoring_tools'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ tool_descriptions: Mapped[json_str_list]
+ netflow_processing_description: Mapped[str]
+
+
+class PassiveMonitoring(db.Model):
+ __tablename__ = 'passive_monitoring'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ monitoring: Mapped[bool]
+ method: Mapped[Optional[MonitoringMethod]]
+
+
+class TrafficStatistics(db.Model):
+ __tablename__ = 'traffic_statistics'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ traffic_statistics: Mapped[bool]
+ urls: Mapped[json_str_list]
+
+
+class SiemVendors(db.Model):
+ __tablename__ = 'siem_vendors'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ vendor_names: Mapped[json_str_list]
+
+
+class CertificateProviders(db.Model):
+ __tablename__ = 'certificate_providers'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ provider_names: Mapped[json_str_list]
+
+
+class WeatherMap(db.Model):
+ __tablename__ = 'weather_map'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ weather_map: Mapped[bool]
+ url: Mapped[str]
+
+
+class PertTeam(db.Model):
+ __tablename__ = 'pert_team'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ pert_team: Mapped[YesNoPlanned]
+
+
+class AlienWave(db.Model):
+ __tablename__ = 'alien_wave'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ alien_wave_third_pary: Mapped[Optional[YesNoPlanned]]
+ nr_of_alien_wave_third_party_services: Mapped[Optional[int]]
+ alien_wave_internal: Mapped[Optional[bool]]
+
+
+class Capacity(db.Model):
+ __tablename__ = 'capacity'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ largest_link_capacity: Mapped[Optional[Decimal]]
+ typical_backbone_capacity: Mapped[Optional[Decimal]]
+
+
+class ExternalConnections(db.Model):
+ __tablename__ = 'external_connections'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ connections: Mapped[List[ExternalConnection]] = mapped_column(JSON)
+
+
+class NonREPeers(db.Model):
+ __tablename__ = 'non_r_and_e_peers'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ nr_of_non_r_and_e_peers: Mapped[int]
+
+
+class TrafficRatio(db.Model):
+ __tablename__ = 'traffic_ratio'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ r_and_e_percentage: Mapped[int]
+ commodity_percentage: Mapped[int]
+
+
+class OpsAutomation(db.Model):
+ __tablename__ = 'ops_automation'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ ops_automation: Mapped[YesNoPlanned]
+ ops_automation_specifics: Mapped[str]
+
+
+class NetworkFunctionVirtualisation(db.Model):
+ __tablename__ = 'network_function_virtualisation'
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ nfv: Mapped[YesNoPlanned]
+ nfv_specifics: Mapped[json_str_list]
+
+class NetworkAutomation(db.Model):
+ __tablename__ = 'network_automation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
- urls: Mapped[List[str]] = mapped_column(JSON)
+ network_automation: Mapped[YesNoPlanned]
+ network_automation_specifics: Mapped[json_str_list]
diff --git a/compendium_v2/db/model_enums.py b/compendium_v2/db/model_enums.py
new file mode 100644
index 00000000..bda21ddf
--- /dev/null
+++ b/compendium_v2/db/model_enums.py
@@ -0,0 +1,88 @@
+from enum import Enum
+
+
+class FeeType(Enum):
+ flat_fee = "flat_fee"
+ usage_based_fee = "usage_based_fee"
+ combination = "combination"
+ no_charge = "no_charge"
+ other = "other"
+
+
+class UserCategory(Enum):
+ universities = "universities"
+ further_education = "further_education"
+ secondary_schools = "secondary_schools"
+ primary_schools = "primary_schools"
+ institutes = "institutes"
+ cultural = "cultural"
+ hospitals = "hospitals"
+ government = "government"
+ iros = "iros"
+ for_profit_orgs = "for_profit_orgs"
+
+
+class ServiceCategory(Enum):
+ network_services = "network_services"
+ isp_support = "isp_support"
+ security = "security"
+ identity = "identity"
+ collaboration = "collaboration"
+ multimedia = "multimedia"
+ storage_and_hosting = "storage_and_hosting"
+ professional_services = "professional_services"
+
+
+class ConnectivityCoverage(Enum):
+ yes_incl_other = "yes_incl_other"
+ yes_national_nren = "yes_national_nren"
+ sometimes = "sometimes"
+ no_policy = "no_policy"
+ no_financial = "no_financial"
+ no_other = "no_other"
+ unsure = "unsure"
+
+
+class CarryMechanism(Enum):
+ nren_local_loops = "nren_local_loops"
+ regional_nren_backbone = "regional_nren_backbone"
+ commercial_provider_backbone = "commercial_provider_backbone"
+ man = "man"
+ other = "other"
+
+
+class CommercialConnectivityCoverage(Enum):
+ yes_incl_other = "yes_incl_other"
+ yes_national_nren = "yes_national_nren"
+ yes_if_sponsored = "yes_if_sponsored"
+ no_but_direct_peering = "no_but_direct_peering"
+ no_policy = "no_policy"
+ no_financial = "no_financial"
+ no_other = "no_other"
+
+
+class CommarcialChargingLevel(Enum):
+ higher_than_r_e_charges = "higher_than_r_e_charges"
+ same_as_r_e_charges = "same_as_r_e_charges"
+ no_charges_if_r_e_requested = "no_charges_if_r_e_requested"
+ lower_than_r_e_charges = "lower_than_r_e_charges"
+
+
+class ConnectionMethod(Enum):
+ internet_exchange = "internet_exchange"
+ open_exchange = "open_exchange"
+ direct = "direct"
+ geant = "geant"
+ other = "other"
+
+
+class YesNoPlanned(Enum):
+ yes = "yes"
+ no = "no"
+ planned = "planned"
+
+
+class MonitoringMethod(Enum):
+ span_ports = "span_ports"
+ taps = "taps"
+ both = "both"
diff --git a/compendium_v2/migrations/versions/8ff7260ad48f_add_presentation_models.py b/compendium_v2/migrations/versions/8ff7260ad48f_add_presentation_models.py
new file mode 100644
index 00000000..67384a3f
--- /dev/null
+++ b/compendium_v2/migrations/versions/8ff7260ad48f_add_presentation_models.py
@@ -0,0 +1,359 @@
+"""add presentation models
+
+Revision ID: 8ff7260ad48f
+Revises: e17f9e6c90ab
+Create Date: 2023-09-14 15:09:36.301058
+
+"""
+
+# flake8: noqa
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '8ff7260ad48f'
+down_revision = 'e17f9e6c90ab'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ sa.Enum('yes', 'no', 'planned', name='yesnoplanned').create(op.get_bind())
+ sa.Enum('span_ports', 'taps', 'both', name='monitoringmethod').create(op.get_bind())
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').create(op.get_bind())
+ sa.Enum('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage').create(op.get_bind())
+ sa.Enum('nren_local_loops', 'regional_nren_backbone', 'commercial_provider_backbone', 'man', 'other', name='carrymechanism').create(op.get_bind())
+ sa.Enum('yes_incl_other', 'yes_national_nren', 'sometimes', 'no_policy', 'no_financial', 'no_other', 'unsure', name='connectivitycoverage').create(op.get_bind())
+ sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').create(op.get_bind())
+ sa.Enum('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory').create(op.get_bind())
+ op.create_table('alien_wave',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('alien_wave_third_pary', postgresql.ENUM('yes', 'no', 'planned', name='yesnoplanned', create_type=False), nullable=True),
+ sa.Column('nr_of_alien_wave_third_party_services', sa.Integer(), nullable=True),
+ sa.Column('alien_wave_internal', sa.Boolean(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_alien_wave_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_alien_wave'))
+ )
+ op.create_table('capacity',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('largest_link_capacity', sa.Numeric(), nullable=True),
+ sa.Column('typical_backbone_capacity', sa.Numeric(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_capacity_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_capacity'))
+ )
+ op.create_table('central_procurement',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('central_procurement', sa.Boolean(), nullable=False),
+ sa.Column('amount', sa.Numeric(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_central_procurement_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_central_procurement'))
+ )
+ op.create_table('certificate_providers',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('provider_names', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_certificate_providers_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_certificate_providers'))
+ )
+ op.create_table('commercial_charging_level',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('collaboration', postgresql.ENUM('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel', create_type=False), nullable=True),
+ sa.Column('service_supplier', postgresql.ENUM('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel', create_type=False), nullable=True),
+ sa.Column('direct_peering', postgresql.ENUM('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel', create_type=False), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_commercial_charging_level_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_commercial_charging_level'))
+ )
+ op.create_table('commercial_connectivity',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('commercial_r_and_e', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage', create_type=False), nullable=True),
+ sa.Column('commercial_general', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage', create_type=False), nullable=True),
+ sa.Column('commercial_collaboration', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage', create_type=False), nullable=True),
+ sa.Column('commercial_service_provider', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage', create_type=False), nullable=True),
+ sa.Column('university_spin_off', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage', create_type=False), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_commercial_connectivity_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_commercial_connectivity'))
+ )
+ op.create_table('connected_proportion',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('coverage', postgresql.ENUM('yes_incl_other', 'yes_national_nren', 'sometimes', 'no_policy', 'no_financial', 'no_other', 'unsure', name='connectivitycoverage', create_type=False), nullable=True),
+ sa.Column('number_connected', sa.Integer(), nullable=True),
+ sa.Column('market_share', sa.Numeric(), nullable=True),
+ sa.Column('users_served', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_connected_proportion_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_connected_proportion'))
+ )
+ op.create_table('connection_carrier',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('carry_mechanism', postgresql.ENUM('nren_local_loops', 'regional_nren_backbone', 'commercial_provider_backbone', 'man', 'other', name='carrymechanism', create_type=False), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_connection_carrier_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_connection_carrier'))
+ )
+ op.create_table('connectivity_growth',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('growth', sa.Numeric(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_connectivity_growth_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_connectivity_growth'))
+ )
+ op.create_table('connectivity_level',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('typical_speed', sa.Integer(), nullable=True),
+ sa.Column('highest_speed', sa.Integer(), nullable=True),
+ sa.Column('highest_speed_proportion', sa.Numeric(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_connectivity_level_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_connectivity_level'))
+ )
+ op.create_table('connectivity_load',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('average_load_from_institutions', sa.Integer(), nullable=True),
+ sa.Column('average_load_to_institutions', sa.Integer(), nullable=True),
+ sa.Column('peak_load_from_institutions', sa.Integer(), nullable=True),
+ sa.Column('peak_load_to_institutions', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_connectivity_load_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_connectivity_load'))
+ )
+ op.create_table('crisis_excercises',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('exercise_descriptions', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_crisis_excercises_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_crisis_excercises'))
+ )
+ op.create_table('dark_fibre_installed',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('installed', sa.Boolean(), nullable=False),
+ sa.Column('fibre_length_in_country', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_dark_fibre_installed_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_dark_fibre_installed'))
+ )
+ op.create_table('dark_fibre_lease',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('iru_or_lease', sa.Boolean(), nullable=False),
+ sa.Column('fibre_length_in_country', sa.Integer(), nullable=True),
+ sa.Column('fibre_length_outside_country', sa.Integer(), nullable=True),
+ sa.Column('iru_duration', sa.Numeric(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_dark_fibre_lease_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_dark_fibre_lease'))
+ )
+ op.create_table('eosc_listings',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('service_names', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_eosc_listings_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_eosc_listings'))
+ )
+ op.create_table('external_connections',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('connections', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_external_connections_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_external_connections'))
+ )
+ op.create_table('fibre_light',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('light_description', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_fibre_light_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_fibre_light'))
+ )
+ op.create_table('monitoring_tools',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('tool_descriptions', sa.JSON(), nullable=False),
+ sa.Column('netflow_processing_description', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_monitoring_tools_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_monitoring_tools'))
+ )
+ op.create_table('network_automation',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('network_automation', postgresql.ENUM('yes', 'no', 'planned', name='yesnoplanned', create_type=False), nullable=False),
+ sa.Column('network_automation_specifics', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_network_automation_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_network_automation'))
+ )
+ op.create_table('network_function_virtualisation',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('nfv', postgresql.ENUM('yes', 'no', 'planned', name='yesnoplanned', create_type=False), nullable=False),
+ sa.Column('nfv_specifics', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_network_function_virtualisation_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_network_function_virtualisation'))
+ )
+ op.create_table('network_map_urls',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('urls', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_network_map_urls_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_network_map_urls'))
+ )
+ op.create_table('non_r_and_e_peers',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('nr_of_non_r_and_e_peers', sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_non_r_and_e_peers_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_non_r_and_e_peers'))
+ )
+ op.create_table('ops_automation',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('ops_automation', postgresql.ENUM('yes', 'no', 'planned', name='yesnoplanned', create_type=False), nullable=False),
+ sa.Column('ops_automation_specifics', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_ops_automation_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_ops_automation'))
+ )
+ op.create_table('passive_monitoring',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('monitoring', sa.Boolean(), nullable=False),
+ sa.Column('method', postgresql.ENUM('span_ports', 'taps', 'both', name='monitoringmethod', create_type=False), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_passive_monitoring_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_passive_monitoring'))
+ )
+ op.create_table('pert_team',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('pert_team', postgresql.ENUM('yes', 'no', 'planned', name='yesnoplanned', create_type=False), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_pert_team_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_pert_team'))
+ )
+ op.create_table('remote_campuses',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('remote_campus_connectivity', sa.Boolean(), nullable=False),
+ sa.Column('connections', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_remote_campuses_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_remote_campuses'))
+ )
+ op.create_table('security_controls',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('security_control_descriptions', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_security_controls_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_security_controls'))
+ )
+ op.create_table('service_management',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('service_management_framework', sa.Boolean(), nullable=True),
+ sa.Column('service_level_targets', sa.Boolean(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_service_management_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_service_management'))
+ )
+ op.create_table('service_user_types',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+ sa.Column('service_category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_service_user_types_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name=op.f('pk_service_user_types'))
+ )
+ op.create_table('siem_vendors',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('vendor_names', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_siem_vendors_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_siem_vendors'))
+ )
+ op.create_table('standards',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('audits', sa.Boolean(), nullable=True),
+ sa.Column('audit_specifics', sa.String(), nullable=False),
+ sa.Column('business_continuity_plans', sa.Boolean(), nullable=True),
+ sa.Column('business_continuity_plans_specifics', sa.String(), nullable=False),
+ sa.Column('crisis_management_procedure', sa.Boolean(), nullable=True),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_standards_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_standards'))
+ )
+ op.create_table('traffic_ratio',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('r_and_e_percentage', sa.Integer(), nullable=False),
+ sa.Column('commodity_percentage', sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_ratio_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_ratio'))
+ )
+ op.create_table('traffic_statistics',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('traffic_statistics', sa.Boolean(), nullable=False),
+ sa.Column('urls', sa.JSON(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_statistics_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_statistics'))
+ )
+ op.create_table('weather_map',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('weather_map', sa.Boolean(), nullable=False),
+ sa.Column('url', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_weather_map_nren_id_nren')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_weather_map'))
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('weather_map')
+ op.drop_table('traffic_statistics')
+ op.drop_table('traffic_ratio')
+ op.drop_table('standards')
+ op.drop_table('siem_vendors')
+ op.drop_table('service_user_types')
+ op.drop_table('service_management')
+ op.drop_table('security_controls')
+ op.drop_table('remote_campuses')
+ op.drop_table('pert_team')
+ op.drop_table('passive_monitoring')
+ op.drop_table('ops_automation')
+ op.drop_table('non_r_and_e_peers')
+ op.drop_table('network_map_urls')
+ op.drop_table('network_function_virtualisation')
+ op.drop_table('network_automation')
+ op.drop_table('monitoring_tools')
+ op.drop_table('fibre_light')
+ op.drop_table('external_connections')
+ op.drop_table('eosc_listings')
+ op.drop_table('dark_fibre_lease')
+ op.drop_table('dark_fibre_installed')
+ op.drop_table('crisis_excercises')
+ op.drop_table('connectivity_load')
+ op.drop_table('connectivity_level')
+ op.drop_table('connectivity_growth')
+ op.drop_table('connection_carrier')
+ op.drop_table('connected_proportion')
+ op.drop_table('commercial_connectivity')
+ op.drop_table('commercial_charging_level')
+ op.drop_table('certificate_providers')
+ op.drop_table('central_procurement')
+ op.drop_table('capacity')
+ op.drop_table('alien_wave')
+ sa.Enum('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory').drop(op.get_bind())
+ sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').drop(op.get_bind())
+ sa.Enum('yes_incl_other', 'yes_national_nren', 'sometimes', 'no_policy', 'no_financial', 'no_other', 'unsure', name='connectivitycoverage').drop(op.get_bind())
+ sa.Enum('nren_local_loops', 'regional_nren_backbone', 'commercial_provider_backbone', 'man', 'other', name='carrymechanism').drop(op.get_bind())
+ sa.Enum('yes_incl_other', 'yes_national_nren', 'yes_if_sponsored', 'no_but_direct_peering', 'no_policy', 'no_financial', 'no_other', name='commercialconnectivitycoverage').drop(op.get_bind())
+ sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').drop(op.get_bind())
+ sa.Enum('span_ports', 'taps', 'both', name='monitoringmethod').drop(op.get_bind())
+ sa.Enum('yes', 'no', 'planned', name='yesnoplanned').drop(op.get_bind())
+ # ### end Alembic commands ###
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_2022.py
index 39038d1e..7251db54 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_2022.py
@@ -17,7 +17,7 @@ from sqlalchemy import delete, text
from collections import defaultdict
import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.model_enums import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher_v2.py
index ebede055..5bb12b78 100644
--- a/compendium_v2/publishers/survey_publisher_v2.py
+++ b/compendium_v2/publishers/survey_publisher_v2.py
@@ -1,19 +1,21 @@
from decimal import Decimal
+from typing import List
from sqlalchemy import delete, select
from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
- InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
+from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, FundingSource, \
+ InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection
+from compendium_v2.db.model_enums import FeeType
from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
-def map_2023(nren, answers):
+def map_2023(nren, answers) -> None:
year = 2023
for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume]:
- db.session.execute(delete(table_class).where(table_class.year == 2023))
+ db.session.execute(delete(table_class).where(table_class.year == 2023)) # type: ignore
answers = answers["data"]
budget = answers.get("budget")
@@ -104,6 +106,22 @@ def map_2023(nren, answers):
urls=urls
))
+ external_connections = answers.get("external_connections")
+ if external_connections:
+ connections: List[ExternalConnection] = []
+ for connection in external_connections:
+ connections.append({
+ 'link_name': connection.get('link_name', ''),
+ 'capacity': connection.get('capacity'),
+ 'from_organization': connection.get('from_organization', ''),
+ 'to_organization': connection.get('to_organization', ''),
+ 'interconnection_method': connection.get('interconnection_method')
+ })
+ db.session.add(ExternalConnections(
+ nren_id=nren.id, nren=nren, year=year,
+ connections=connections
+ ))
+
def publish(year):
responses = db.session.scalars(
diff --git a/test/test_survey_publisher_2022.py b/test/test_survey_publisher_2022.py
index e8eaa8b4..8f6d1616 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_survey_publisher_2022.py
@@ -1,6 +1,6 @@
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, model, model_enums
from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
@@ -280,11 +280,11 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == model.FeeType.no_charge
+ assert charging_structures[0].fee_type == model_enums.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+ assert charging_structures[1].fee_type == model_enums.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == model.FeeType.other
+ assert charging_structures[2].fee_type == model_enums.FeeType.other
_ec_data = db.session.scalars(
select(model.ECProject).order_by(model.ECProject.nren_id.asc())
diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher_v2.py
index 68b4b2d0..d0a0a8cb 100644
--- a/test/test_survey_publisher_v2.py
+++ b/test/test_survey_publisher_v2.py
@@ -5,7 +5,7 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import model
+from compendium_v2.db import model, model_enums
from compendium_v2.publishers.survey_publisher_v2 import map_2023
@@ -53,7 +53,7 @@ def test_v2_publisher_full(app):
assert funding_source.other == Decimal("10")
charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
- assert charging_structure == model.FeeType.usage_based_fee
+ assert charging_structure == model_enums.FeeType.usage_based_fee
staff = db.session.scalar(select(model.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
@@ -97,3 +97,28 @@ def test_v2_publisher_full(app):
client_urls = db.session.scalar(select(model.InstitutionURLs))
assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
+
+ external_connections = db.session.scalar(select(model.ExternalConnections))
+ external_connection_list = external_connections.connections
+ assert len(external_connection_list) == 6
+ assert external_connection_list[0] == {
+ "capacity": "1",
+ "from_organization": "GEANT",
+ "interconnection_method": "geant",
+ "link_name": "GEANT",
+ "to_organization": "MREN"
+ }
+ assert external_connection_list[3] == {
+ "capacity": None,
+ "from_organization": "",
+ "interconnection_method": "other",
+ "link_name": "",
+ "to_organization": ""
+ }
+ assert external_connection_list[4] == {
+ "capacity": "1.1",
+ "from_organization": "",
+ "interconnection_method": None,
+ "link_name": "",
+ "to_organization": "",
+ }
--
GitLab
From e94f9a88960474f6b5256c788d127f9fcad83fca Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Fri, 15 Sep 2023 14:57:37 +0200
Subject: [PATCH 2/2] better name for enum file
---
.../db/{model_enums.py => presentation_model_enums.py} | 0
compendium_v2/db/presentation_models.py | 5 +++--
compendium_v2/publishers/excel_parser.py | 2 +-
compendium_v2/publishers/survey_publisher.py | 2 +-
compendium_v2/publishers/survey_publisher_old_db_2022.py | 2 +-
test/test_db_survey_publisher_2022.py | 8 ++++----
test/test_survey_publisher.py | 4 ++--
7 files changed, 12 insertions(+), 11 deletions(-)
rename compendium_v2/db/{model_enums.py => presentation_model_enums.py} (100%)
diff --git a/compendium_v2/db/model_enums.py b/compendium_v2/db/presentation_model_enums.py
similarity index 100%
rename from compendium_v2/db/model_enums.py
rename to compendium_v2/db/presentation_model_enums.py
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index 5464188b..de573493 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -11,8 +11,9 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
from compendium_v2.db import db
-from compendium_v2.db.model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, ServiceCategory, \
- ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, FeeType
+from compendium_v2.db.presentation_model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, \
+ ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, \
+ FeeType, ServiceCategory
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/publishers/excel_parser.py b/compendium_v2/publishers/excel_parser.py
index 1e964be8..41f7c014 100644
--- a/compendium_v2/publishers/excel_parser.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -2,7 +2,7 @@ import logging
import openpyxl
import os
-from compendium_v2.db.model_enums import FeeType
+from compendium_v2.db.presentation_model_enums import FeeType
from compendium_v2.environment import setup_logging
setup_logging()
diff --git a/compendium_v2/publishers/survey_publisher.py b/compendium_v2/publishers/survey_publisher.py
index 604d2723..6946aa10 100644
--- a/compendium_v2/publishers/survey_publisher.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -18,7 +18,7 @@ from compendium_v2.db import db
from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, ExternalConnections, \
InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume, ExternalConnection, \
FundingSource
-from compendium_v2.db.model_enums import FeeType
+from compendium_v2.db.presentation_model_enums import FeeType
from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
diff --git a/compendium_v2/publishers/survey_publisher_old_db_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
index 322e06e1..c4c1bb86 100644
--- a/compendium_v2/publishers/survey_publisher_old_db_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -17,7 +17,7 @@ from sqlalchemy import delete, text
from collections import defaultdict
import compendium_v2
-from compendium_v2.db.model_enums import FeeType
+from compendium_v2.db.presentation_model_enums import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
diff --git a/test/test_db_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py
index bcec6c98..1d0fbcd1 100644
--- a/test/test_db_survey_publisher_2022.py
+++ b/test/test_db_survey_publisher_2022.py
@@ -1,6 +1,6 @@
from sqlalchemy import select
-from compendium_v2.db import db, presentation_models, model_enums
+from compendium_v2.db import db, presentation_model_enums, presentation_models
from compendium_v2.publishers.survey_publisher_old_db_2022 import _cli, FundingSource, \
StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
@@ -282,11 +282,11 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == model_enums.FeeType.no_charge
+ assert charging_structures[0].fee_type == presentation_model_enums.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == model_enums.FeeType.usage_based_fee
+ assert charging_structures[1].fee_type == presentation_model_enums.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == model_enums.FeeType.other
+ assert charging_structures[2].fee_type == presentation_model_enums.FeeType.other
_ec_data = db.session.scalars(
select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc())
diff --git a/test/test_survey_publisher.py b/test/test_survey_publisher.py
index b1ca4534..e6cc54f6 100644
--- a/test/test_survey_publisher.py
+++ b/test/test_survey_publisher.py
@@ -5,7 +5,7 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import presentation_models, model_enums
+from compendium_v2.db import presentation_model_enums, presentation_models
from compendium_v2.publishers.survey_publisher import _map_2023
@@ -53,7 +53,7 @@ def test_v2_publisher_full(app):
assert funding_source.other == Decimal("10")
charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
- assert charging_structure == model_enums.FeeType.usage_based_fee
+ assert charging_structure == presentation_model_enums.FeeType.usage_based_fee
staff = db.session.scalar(select(presentation_models.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
--
GitLab