Merge branch 'feature/COMP-285_add_new_datamodel_to_publisher' into 'develop'

add all policy, connected users, and network models to the publisher

See merge request !81

compendium_v2/db/presentation_model_enums.py

@@ -61,7 +61,7 @@ class CommercialConnectivityCoverage(Enum):
     no_other = "no_other"
 
 
-class CommarcialChargingLevel(Enum):
+class CommercialCharges(Enum):
     higher_than_r_e_charges = "higher_than_r_e_charges"
     same_as_r_e_charges = "same_as_r_e_charges"
     no_charges_if_r_e_requested = "no_charges_if_r_e_requested"

compendium_v2/db/presentation_models.py

@@ -11,7 +11,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
 from sqlalchemy.schema import ForeignKey
 
 from compendium_v2.db import db
-from compendium_v2.db.presentation_model_enums import CarryMechanism, CommarcialChargingLevel, UserCategory, \
+from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, UserCategory, \
     ConnectivityCoverage, ConnectionMethod, YesNoPlanned, MonitoringMethod, CommercialConnectivityCoverage, \
     FeeType, ServiceCategory
 
@@ -183,7 +183,7 @@ class ServiceUserTypes(db.Model):
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
     user_category: Mapped[user_category_pk]
-    service_category: Mapped[ServiceCategory]
+    service_category: Mapped[ServiceCategory] = mapped_column(primary_key=True)
 
 
 class EOSCListings(db.Model):
@@ -292,9 +292,9 @@ class CommercialChargingLevel(db.Model):
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    collaboration: Mapped[Optional[CommarcialChargingLevel]]
-    service_supplier: Mapped[Optional[CommarcialChargingLevel]]
-    direct_peering: Mapped[Optional[CommarcialChargingLevel]]
+    collaboration: Mapped[Optional[CommercialCharges]]
+    service_supplier: Mapped[Optional[CommercialCharges]]
+    direct_peering: Mapped[Optional[CommercialCharges]]
 
 
 class RemoteCampuses(db.Model):
@@ -442,8 +442,8 @@ class TrafficRatio(db.Model):
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    r_and_e_percentage: Mapped[int]
-    commodity_percentage: Mapped[int]
+    r_and_e_percentage: Mapped[Decimal]
+    commodity_percentage: Mapped[Decimal]
 
 
 class OpsAutomation(db.Model):

compendium_v2/migrations/versions/87e1e35051a0_fixes_for_new_presentation_models.py

+"""fixes for new presentation models
+
+Revision ID: 87e1e35051a0
+Revises: 8ff7260ad48f
+Create Date: 2023-09-17 15:24:21.873533
+
+"""
+
+# flake8: noqa
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '87e1e35051a0'
+down_revision = '8ff7260ad48f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commercialcharges').create(op.get_bind())
+    op.drop_table('service_user_types')
+
+    op.create_table(
+        'service_user_types',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), nullable=False),
+        sa.Column('service_category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_service_user_types_nren_id_nren')),
+        sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', 'service_category', name=op.f('pk_service_user_types'))
+    )
+
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN collaboration TYPE commercialcharges USING collaboration::text::commercialcharges")
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN service_supplier TYPE commercialcharges USING service_supplier::text::commercialcharges")
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN direct_peering TYPE commercialcharges USING direct_peering::text::commercialcharges")
+
+    sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').drop(op.get_bind())
+
+    with op.batch_alter_table('traffic_ratio', schema=None) as batch_op:
+        batch_op.alter_column('r_and_e_percentage',
+               existing_type=sa.INTEGER(),
+               type_=sa.Numeric(),
+               existing_nullable=False)
+        batch_op.alter_column('commodity_percentage',
+               existing_type=sa.INTEGER(),
+               type_=sa.Numeric(),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('traffic_ratio', schema=None) as batch_op:
+        batch_op.alter_column('commodity_percentage',
+               existing_type=sa.Numeric(),
+               type_=sa.INTEGER(),
+               existing_nullable=False)
+        batch_op.alter_column('r_and_e_percentage',
+               existing_type=sa.Numeric(),
+               type_=sa.INTEGER(),
+               existing_nullable=False)
+
+    sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commarcialcharginglevel').create(op.get_bind())
+
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN collaboration TYPE commarcialcharginglevel USING collaboration::text::commarcialcharginglevel")
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN service_supplier TYPE commarcialcharginglevel USING service_supplier::text::commarcialcharginglevel")
+    op.execute("ALTER TABLE commercial_charging_level ALTER COLUMN direct_peering TYPE commarcialcharginglevel USING direct_peering::text::commarcialcharginglevel")
+
+    op.drop_table('service_user_types')
+
+    op.create_table(
+        'service_user_types',
+        sa.Column('nren_id', sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column('year', sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column('user_category', postgresql.ENUM('universities', 'further_education', 'secondary_schools', 'primary_schools', 'institutes', 'cultural', 'hospitals', 'government', 'iros', 'for_profit_orgs', name='usercategory', create_type=False), autoincrement=False, nullable=False),
+        sa.Column('service_category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), autoincrement=False, nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name='fk_service_user_types_nren_id_nren'),
+        sa.PrimaryKeyConstraint('nren_id', 'year', 'user_category', name='pk_service_user_types')
+    )
+    sa.Enum('higher_than_r_e_charges', 'same_as_r_e_charges', 'no_charges_if_r_e_requested', 'lower_than_r_e_charges', name='commercialcharges').drop(op.get_bind())
+    # ### end Alembic commands ###

compendium_v2/publishers/survey_publisher_old_db_2022.py

 """
 survey_publisher_old_db_2022
-=========================
+============================
 
 This module loads the survey data from 2022 from the old survey database into presentation_models.
 Registered as click cli command when installing compendium-v2.

test/test_survey_publisher.py

@@ -5,7 +5,7 @@ import os
 from sqlalchemy import func, select
 
 from compendium_v2 import db
-from compendium_v2.db import presentation_model_enums, presentation_models
+from compendium_v2.db import presentation_model_enums as model_enums, presentation_models
 from compendium_v2.publishers.survey_publisher import _map_2023
 
 
@@ -53,7 +53,7 @@ def test_v2_publisher_full(app):
         assert funding_source.other == Decimal("10")
 
         charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
-        assert charging_structure == presentation_model_enums.FeeType.usage_based_fee
+        assert charging_structure == model_enums.FeeType.usage_based_fee
 
         staff = db.session.scalar(select(presentation_models.NrenStaff))
         assert staff.permanent_fte == Decimal("5.6")
@@ -102,6 +102,163 @@ def test_v2_publisher_full(app):
         client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
         assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
 
+        procurement = db.session.scalar(select(presentation_models.CentralProcurement))
+        assert procurement.central_procurement
+        assert procurement.amount == Decimal("57676")
+
+        service_management = db.session.scalar(select(presentation_models.ServiceManagement))
+        assert service_management.service_management_framework
+        assert not service_management.service_level_targets
+
+        service_user_types = [s for s in db.session.scalars(select(presentation_models.ServiceUserTypes))]
+        user_categories = set([s.user_category for s in service_user_types])
+        assert len(user_categories) == 10
+        institute_services = [
+            s for s in service_user_types if s.user_category == model_enums.UserCategory.institutes
+        ]
+        assert len(institute_services) == 7
+
+        eosc_listings = db.session.scalar(select(presentation_models.EOSCListings))
+        assert eosc_listings.service_names == ["dy", "rrr"]
+
+        standards = db.session.scalar(select(presentation_models.Standards))
+        assert standards.audits
+        assert standards.audit_specifics == "isooo  557"
+        assert standards.business_continuity_plans
+        assert standards.business_continuity_plans_specifics == "no"
+        assert not standards.crisis_management_procedure
+
+        crisis_excercises = db.session.scalar(select(presentation_models.CrisisExcercises))
+        assert crisis_excercises.exercise_descriptions == [
+            "geant_workshops", "national_excercises", "tabletop_exercises", "other_excercises",
+            "none", "simulation_excercises", "real_crisis"
+        ]
+
+        security_controls = db.session.scalar(select(presentation_models.SecurityControls))
+        assert security_controls.security_control_descriptions == [
+            "monitoring", "acl", "anti_virus", "firewall", "anti_spam", "ddos_mitigation",
+            "ips_ids", "segmentation", "integrity_checking", "vgjh"
+        ]
+
+        connected_proportion = [s for s in db.session.scalars(select(presentation_models.ConnectedProportion))]
+        assert len(connected_proportion) == 10
+        uni = [c for c in connected_proportion if c.user_category == model_enums.UserCategory.universities]
+        assert len(uni) == 1
+        assert uni[0].coverage == model_enums.ConnectivityCoverage.yes_national_nren
+        assert uni[0].number_connected == 19
+        assert uni[0].market_share == Decimal("1")
+        assert uni[0].users_served == 20000
+
+        connectivity_level = [s for s in db.session.scalars(select(presentation_models.ConnectivityLevel))]
+        assert len(connectivity_level) == 10
+        gov = [c for c in connectivity_level if c.user_category == model_enums.UserCategory.government]
+        assert len(gov) == 1
+        assert gov[0].typical_speed == 100
+        assert gov[0].highest_speed == 100
+        assert gov[0].highest_speed_proportion == Decimal("100")
+
+        carrier = [s for s in db.session.scalars(select(presentation_models.ConnectionCarrier))]
+        assert len(carrier) == 10
+        profit = [c for c in carrier if c.user_category == model_enums.UserCategory.for_profit_orgs]
+        assert len(profit) == 1
+        assert profit[0].carry_mechanism == model_enums.CarryMechanism.commercial_provider_backbone
+        primary = [c for c in carrier if c.user_category == model_enums.UserCategory.primary_schools]
+        assert len(primary) == 1
+        assert primary[0].carry_mechanism == model_enums.CarryMechanism.man
+        hospital = [c for c in carrier if c.user_category == model_enums.UserCategory.hospitals]
+        assert len(hospital) == 1
+        assert hospital[0].carry_mechanism == model_enums.CarryMechanism.nren_local_loops
+
+        load = [s for s in db.session.scalars(select(presentation_models.ConnectivityLoad))]
+        assert len(load) == 10
+        cultural = [c for c in load if c.user_category == model_enums.UserCategory.cultural]
+        assert len(cultural) == 1
+        assert cultural[0].average_load_from_institutions == 10
+        assert cultural[0].average_load_to_institutions == 10
+        assert cultural[0].peak_load_from_institutions == 50
+        assert cultural[0].peak_load_to_institutions == 10
+
+        growth = [s for s in db.session.scalars(select(presentation_models.ConnectivityGrowth))]
+        assert len(growth) == 10
+        cultural = [c for c in growth if c.user_category == model_enums.UserCategory.cultural]
+        assert len(cultural) == 1
+        assert cultural[0].growth == Decimal("50")
+
+        commercial = db.session.scalar(select(presentation_models.CommercialConnectivity))
+        assert commercial.commercial_r_and_e == model_enums.CommercialConnectivityCoverage.no_but_direct_peering
+        assert commercial.commercial_general == model_enums.CommercialConnectivityCoverage.no_policy
+        assert commercial.commercial_collaboration == model_enums.CommercialConnectivityCoverage.yes_incl_other
+        assert commercial.commercial_service_provider == model_enums.CommercialConnectivityCoverage.no_other
+        assert commercial.university_spin_off == model_enums.CommercialConnectivityCoverage.no_financial
+
+        commercial_charging = db.session.scalar(select(presentation_models.CommercialChargingLevel))
+        assert commercial_charging.collaboration == model_enums.CommercialCharges.no_charges_if_r_e_requested
+        assert commercial_charging.service_supplier == model_enums.CommercialCharges.same_as_r_e_charges
+        assert commercial_charging.direct_peering == model_enums.CommercialCharges.higher_than_r_e_charges
+
+        remote_campuses = db.session.scalar(select(presentation_models.RemoteCampuses))
+        assert remote_campuses.remote_campus_connectivity
+        assert remote_campuses.connections == [
+            {"country": "fyuyg", "local_r_and_e_connection": False},
+            {"country": "g", "local_r_and_e_connection": None}
+        ]
+
+        dark_fibre_lease = db.session.scalar(select(presentation_models.DarkFibreLease))
+        assert dark_fibre_lease.iru_or_lease
+        assert dark_fibre_lease.fibre_length_in_country == 405
+        assert dark_fibre_lease.fibre_length_outside_country == 0
+        assert dark_fibre_lease.iru_duration == Decimal("10")
+
+        dark_fibre_installed = db.session.scalar(select(presentation_models.DarkFibreInstalled))
+        assert dark_fibre_installed.installed
+        assert dark_fibre_installed.fibre_length_in_country == 1
+
+        fibre_light = db.session.scalar(select(presentation_models.FibreLight))
+        assert fibre_light.light_description == "jj"
+
+        network_map_urls = db.session.scalar(select(presentation_models.NetworkMapUrls))
+        assert network_map_urls.urls == ["http://netmon.ucg.ac.me/mantra/"]
+
+        monitoring_tools = db.session.scalar(select(presentation_models.MonitoringTools))
+        assert monitoring_tools.tool_descriptions == [
+            "status_dashboard", "historical_traffic_volumes", "netflow_analysis", "looking_glass", "hiui"
+        ]
+        assert monitoring_tools.netflow_processing_description == "ghhhh"
+
+        passive_monitoring = db.session.scalar(select(presentation_models.PassiveMonitoring))
+        assert passive_monitoring.monitoring
+        assert passive_monitoring.method == model_enums.MonitoringMethod.both
+
+        traffic_statistics = db.session.scalar(select(presentation_models.TrafficStatistics))
+        assert traffic_statistics.traffic_statistics
+        assert traffic_statistics.urls == [
+            "http://netmon.cis.ac.me/cacti/graph_view.php?action=tree&tree_id=36&leaf_id=1430"
+        ]
+
+        siem_vendors = db.session.scalar(select(presentation_models.SiemVendors))
+        assert siem_vendors.vendor_names == ["Exabeam", "Splunk", "IBM Qradar", "LogRythm", "Securonix", "figuoho"]
+
+        certificate_providers = db.session.scalar(select(presentation_models.CertificateProviders))
+        assert certificate_providers.provider_names == [
+            "TCS", "Digicert", "Sectigo", "GlobalSign", "GeoDaddy", "GeoTrust", "Entrust Datacard", "earer"
+        ]
+
+        weather_map = db.session.scalar(select(presentation_models.WeatherMap))
+        assert weather_map.weather_map
+        assert weather_map.url == "http://weather.com"
+
+        pert_team = db.session.scalar(select(presentation_models.PertTeam))
+        assert pert_team.pert_team == model_enums.YesNoPlanned.planned
+
+        alien_wave = db.session.scalar(select(presentation_models.AlienWave))
+        assert alien_wave.alien_wave_third_pary == model_enums.YesNoPlanned.yes
+        assert alien_wave.nr_of_alien_wave_third_party_services == 66
+        assert not alien_wave.alien_wave_internal
+
+        capacity = db.session.scalar(select(presentation_models.Capacity))
+        assert capacity.largest_link_capacity == Decimal("1")
+        assert capacity.typical_backbone_capacity == Decimal("567")
+
         external_connections = db.session.scalar(select(presentation_models.ExternalConnections))
         external_connection_list = external_connections.connections
         assert len(external_connection_list) == 6
@@ -126,3 +283,26 @@ def test_v2_publisher_full(app):
             "link_name": "",
             "to_organization": "",
         }
+
+        non_r_e_peers = db.session.scalar(select(presentation_models.NonREPeers))
+        assert non_r_e_peers.nr_of_non_r_and_e_peers == 0
+
+        traffic_ratio = db.session.scalar(select(presentation_models.TrafficRatio))
+        assert traffic_ratio.r_and_e_percentage == Decimal("1")
+        assert traffic_ratio.commodity_percentage == Decimal("99")
+
+        ops_automation = db.session.scalar(select(presentation_models.OpsAutomation))
+        assert ops_automation.ops_automation == model_enums.YesNoPlanned.yes
+        assert ops_automation.ops_automation_specifics == "ghioil;"
+
+        nfv = db.session.scalar(select(presentation_models.NetworkFunctionVirtualisation))
+        assert nfv.nfv == model_enums.YesNoPlanned.planned
+        assert nfv.nfv_specifics == [
+            "routers", "firewalls", "load_balancers", "vpn_concentrators", "rtghjk"
+        ]
+
+        network_automation = db.session.scalar(select(presentation_models.NetworkAutomation))
+        assert network_automation.network_automation == model_enums.YesNoPlanned.planned
+        assert network_automation.network_automation_specifics == [
+            "config_management", "provisioning", "data_collection", "compliance", "reporting", "troubleshooting"
+        ]