service matrix prototype

MANIFEST.in

@@ -4,5 +4,5 @@ include compendium_v2/templates/survey-index.html
 include compendium_v2/migrations/alembic.ini
 recursive-include compendium_v2/migrations/versions *
 recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/resources *
+recursive-include compendium_v2/resources *
 recursive-exclude test *

compendium_v2/conversion/mapping.py

+from compendium_v2.db.presentation_models import ServiceCategory
+
 ANSWERS_2022_QUERY = """
 SELECT question_id, value
 FROM answers a
@@ -664,3 +666,14 @@ SERVICES_MAPPING = {
     'SaaS': 'services_hosting:saas',
     'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
 }
+
+SERVICE_CATEGORY_MAPPING = {
+    'services_collaboration': ServiceCategory.collaboration.value,
+    'services_identity': ServiceCategory.identity.value,
+    'services_isp': ServiceCategory.isp_support.value,
+    'services_multimedia': ServiceCategory.multimedia.value,
+    'services_network': ServiceCategory.network_services.value,
+    'services_professional': ServiceCategory.professional_services.value,
+    'services_security': ServiceCategory.security.value,
+    'services_hosting': ServiceCategory.storage_and_hosting.value,
+}

compendium_v2/db/presentation_models.py

@@ -471,3 +471,25 @@ class NetworkAutomation(db.Model):
     year: Mapped[int_pk]
     network_automation: Mapped[YesNoPlanned]
     network_automation_specifics: Mapped[json_str_list]
+
+
+class Service(db.Model):
+    __tablename__ = 'service'
+
+    name_key: Mapped[str128_pk]
+    name: Mapped[str128]
+    category: Mapped[ServiceCategory]
+    description: Mapped[str]
+
+
+class NRENService(db.Model):
+    __tablename__ = 'nren_service'
+
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
+    service: Mapped[Service] = relationship(lazy='joined')
+    product_name: Mapped[str128]
+    additional_information: Mapped[str]
+    official_description: Mapped[str]

compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py

+"""added Service and NRENService tables
+
+Revision ID: 1fbc4582c0ab
+Revises: 87e1e35051a0
+Create Date: 2023-09-19 22:23:36.448210
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '1fbc4582c0ab'
+down_revision = '87e1e35051a0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'service',
+        sa.Column('name_key', sa.String(length=128), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('category', postgresql.ENUM(
+            'network_services', 'isp_support', 'security', 'identity',
+            'collaboration', 'multimedia', 'storage_and_hosting',
+            'professional_services', name='servicecategory',
+            create_type=False), nullable=False
+        ),
+        sa.Column('description', sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint('name_key', name=op.f('pk_service'))
+    )
+    op.create_table(
+        'nren_service',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('service_key', sa.String(length=128), nullable=False),
+        sa.Column('product_name', sa.String(), nullable=False),
+        sa.Column('additional_information', sa.String(), nullable=False),
+        sa.Column('official_description', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
+        sa.ForeignKeyConstraint(
+            ['service_key'], ['service.name_key'], name=op.f('fk_nren_service_service_key_service')
+        ),
+        sa.PrimaryKeyConstraint('nren_id', 'year', 'service_key', name=op.f('pk_nren_service'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('nren_service')
+    op.drop_table('service')
+    # ### end Alembic commands ###

compendium_v2/publishers/excel_parser.py

 import logging
+
 import openpyxl
-import os
 
-from compendium_v2.db.presentation_model_enums import FeeType
+from compendium_v2.conversion import mapping
+from compendium_v2.db.presentation_models import FeeType
 from compendium_v2.environment import setup_logging
+from compendium_v2.resources import get_resource_file_path
 
 setup_logging()
 
 logger = logging.getLogger(__name__)
 
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_ORGANISATION = get_resource_file_path("2021_Organisation_DataSeries.xlsx")
+EXCEL_FILE_NETWORKS = get_resource_file_path("2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_NREN_SERVICES = get_resource_file_path("NREN-Services-prefills_2023_Recovered.xlsx")
 
 
 def fetch_budget_excel_data():
     # load the xlsx file
     sheet_name = "1. Budget"
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     ws = wb[sheet_name]
@@ -36,7 +39,7 @@ def fetch_budget_excel_data():
 
 def fetch_funding_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "2. Income Sources"
@@ -119,7 +122,7 @@ def fetch_funding_excel_data():
 
 def fetch_charging_structure_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "3. Charging mechanism"
@@ -186,7 +189,7 @@ def fetch_charging_structure_excel_data():
 
 def fetch_staffing_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "4. Staff"
@@ -234,7 +237,7 @@ def fetch_staffing_excel_data():
 
 def fetch_staff_function_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "5. Staff by Function"
@@ -302,7 +305,7 @@ def fetch_staff_function_excel_data():
 
 def fetch_ecproject_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "7. EC Projects"
@@ -334,7 +337,7 @@ def fetch_ecproject_excel_data():
 
 def fetch_organization_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "Organization"
@@ -352,7 +355,7 @@ def fetch_organization_excel_data():
 
 def fetch_traffic_excel_data():
     # load the xlsx file
-    wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
+    wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
 
     # select the active worksheet
     sheet_name = "Estimated_Traffic TByte"
@@ -392,3 +395,48 @@ def fetch_traffic_excel_data():
     yield from create_points_for_year(2020, 14)
     yield from create_points_for_year(2021, 8)
     yield from create_points_for_year(2022, 2)
+
+
+def fetch_nren_services_excel_data():
+    wb = openpyxl.load_workbook(EXCEL_FILE_NREN_SERVICES, data_only=True, read_only=True)
+    ws = wb["Sheet1"]
+    rows = list(ws.rows)
+
+    titles = rows[0]
+
+    nren_service_data_columns = {}
+
+    def normalize_nren_name(n: str) -> str:
+        n = n.split(' ')[0].upper()
+        return {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(n, n)
+
+    for i in range(0, 131):
+        if titles[i].value:
+            name = normalize_nren_name(titles[i].value)
+            nren_service_data_columns[name] = i
+
+    for nren_name, start_column in nren_service_data_columns.items():
+        for row_index in range(2, 61):
+            row = rows[row_index]
+            service_name = row[0].value
+            if row[start_column].value and row[start_column].value.upper() == 'YES':
+                product_name = ''
+                additional_information = ''
+                if row[start_column + 1].value:
+                    product_name = row[start_column + 1].value
+                if row[start_column + 2].value:
+                    additional_information = row[start_column + 2].value
+
+                service_category, service_name_key = mapping.SERVICES_MAPPING[service_name].split(':')
+                service_category = mapping.SERVICE_CATEGORY_MAPPING[service_category]
+
+                yield {
+                    'nren_name': nren_name,
+                    'service_name': service_name,
+                    'service_category': service_category,
+                    'service_name_key': service_name_key,
+                    'year': 2022,
+                    'product_name': product_name.strip(),
+                    'additional_information': additional_information.strip(),
+                    'official_description': '',
+                }

compendium_v2/publishers/survey_publisher.py

@@ -23,7 +23,7 @@ from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure,
     CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
     NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
     CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
-    OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation
+    OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService
 from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
     CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
 from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
@@ -64,7 +64,7 @@ def _map_2023(nren, answers) -> None:
                         CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
                         NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
                         CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio,
-                        OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation]:
+                        OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService]:
         db.session.execute(delete(table_class).where(table_class.year == year))  # type: ignore
 
     answers = answers["data"]
@@ -516,6 +516,22 @@ def _map_2023(nren, answers) -> None:
             network_automation_specifics=network_automation_tasks
         ))
 
+    all_services = {}
+    for question_name in ["services_collaboration", "services_hosting", "services_identity", "services_isp",
+                          "services_multimedia", "services_network", "services_professional", "services_security"]:
+        all_services.update(answers.get(question_name, {}))
+
+    for service_key, answers in all_services.items():
+        offered = answers.get("offered")
+        if offered == ["yes"]:
+            db.session.add(NRENService(
+                nren_id=nren.id, nren=nren, year=year,
+                service_key=service_key,
+                product_name=answers.get("name", ""),
+                additional_information=answers.get("additional_information", ""),
+                official_description=answers.get("description", "")
+            ))
+
 
 def publish(year):
     responses = db.session.scalars(

compendium_v2/publishers/survey_publisher_legacy_excel.py

@@ -8,18 +8,21 @@ Registered as click cli command when installing compendium-v2.
 
 """
 from __future__ import annotations
+
+import json
 import logging
 import math
-import click
 
+import click
 from sqlalchemy import select
 
 import compendium_v2
-from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
 from compendium_v2.db import db, presentation_models
-from compendium_v2.survey_db import model as survey_model
+from compendium_v2.environment import setup_logging
 from compendium_v2.publishers import helpers, excel_parser
+from compendium_v2.resources import get_resource_file_path
+from compendium_v2.survey_db import model as survey_model
 
 setup_logging()
 
@@ -224,7 +227,49 @@ def db_traffic_volume_migration(nren_dict):
     db.session.commit()
 
 
-def _cli(config, app):
+def db_services_migration():
+    with open(get_resource_file_path('nren_services.json')) as f:
+        services = json.load(f)
+        for name_key, record in services.items():
+            service = presentation_models.Service(
+                name_key=name_key,
+                name=record['name'],
+                category=record['category'],
+                description=record['description'],
+            )
+            db.session.merge(service)
+        db.session.commit()
+
+
+def db_nren_services_migration(nren_dict):
+    services = [s for s in db.session.scalars(select(presentation_models.Service))]
+
+    for service_info in excel_parser.fetch_nren_services_excel_data():
+        [service] = [s for s in services if s.name_key == service_info['service_name_key']]
+
+        abbrev = service_info['nren_name']
+        if abbrev not in nren_dict:
+            logger.warning(f'{abbrev} unknown. Skipping.')
+            continue
+
+        nren = nren_dict[abbrev]
+        nren_service = presentation_models.NRENService(
+            nren=nren,
+            nren_id=nren.id,
+            year=service_info['year'],
+            service=service,
+            service_key=service.name_key,
+            product_name=service_info['product_name'],
+            additional_information=service_info['additional_information'],
+            official_description=service_info['official_description']
+        )
+
+        db.session.merge(nren_service)
+
+    db.session.commit()
+
+
+def _cli(app):
     with app.app_context():
         nren_dict = helpers.get_uppercase_nren_dict()
         db_budget_migration(nren_dict)
@@ -234,6 +279,8 @@ def _cli(config, app):
         db_ecprojects_migration(nren_dict)
         db_organizations_migration(nren_dict)
         db_traffic_volume_migration(nren_dict)
+        db_services_migration()
+        db_nren_services_migration(nren_dict)
 
 
 @click.command()
@@ -245,7 +292,7 @@ def cli(config):
 
     app = compendium_v2._create_app_with_db(app_config)
     print("survey-publisher-v1 starting")
-    _cli(app_config, app)
+    _cli(app)
 
 
 if __name__ == "__main__":

compendium_v2/resources/__init__.py

+from pathlib import Path
+
+
+def get_resource_file_path(filename: str) -> Path:
+    resource_dir = Path(__file__).resolve().parent
+    return resource_dir / filename

compendium_v2/resources/nren_services.json

+{
+  "anti-spam": {
+    "name": "Anti-spam solution",
+    "category": "security",
+    "description": "Anti-spam solutions for detecting and eliminating viruses and spam mails."
+  },
+  "class-registration": {
+    "name": "Class registration tool",
+    "category": "collaboration",
+    "description": "Software for teachers to register students for classes etc."
+  },
+  "cloud-service-end-user": {
+    "name": "Cloud storage (end user)",
+    "category": "storage_and_hosting",
+    "description": "Browser based virtual storage service for individuals."
+  },
+  "connectivity": {
+    "name": "IP connectivity",
+    "category": "network_services",
+    "description": "Basic IP connectivity services inc. R+E and commodity internet"
+  },
+  "consultancy": {
+    "name": "Consultancy/training",
+    "category": "professional_services",
+    "description": "Training and consultancy services provided by the NREN."
+  },
+  "content-delivery-hosting": {
+    "name": "Content delivery hosting",
+    "category": "storage_and_hosting",
+    "description": "Hosting of content delivery servers e.g. Akamai."
+  },
+  "content-management-system": {
+    "name": "CMS",
+    "category": "collaboration",
+    "description": "Provision of software systems for website authoring, collaboration and administration tools."
+  },
+  "csirt": {
+    "name": "CERT/CSIRT",
+    "category": "security",
+    "description": "A single point of contact for users to deal with computer security incidents and prevention."
+  },
+  "ddos-prevention": {
+    "name": "DDos mitigation",
+    "category": "security",
+    "description": "Tools and techniques for mitigating Distributed Denial of Service attacks."
+  },
+  "domain-registration": {
+    "name": "Domain name registration",
+    "category": "isp_support",
+    "description": "Administration/registration of top and second level domain names."
+  },
+  "eduroam-wifi": {
+    "name": "Eduroam",
+    "category": "identity",
+    "description": "Inter-WLAN service to facilitate easy and secure Internet access for roaming educational users."
+  },
+  "email-services": {
+    "name": "Email server hosting",
+    "category": "collaboration",
+    "description": "NREN hosted email servers."
+  },
+  "filesender": {
+    "name": "Filesender",
+    "category": "storage_and_hosting",
+    "description": "Web based application that allows authenticated users to securely and easily send arbitrarily large files."
+  },
+  "firewall-on-demand": {
+    "name": "Firewall-on-demand",
+    "category": "security",
+    "description": "Provision of a dynamic firewall services to mitigate against DDOS attacks. "
+  },
+  "home-vpn": {
+    "name": "Remote access VPN service",
+    "category": "network_services",
+    "description": "Remote access and site-to-site secure VPN."
+  },
+  "aai": {
+    "name": "Hosted campus AAI",
+    "category": "identity",
+    "description": "Hosting of an Identity Provider service on behalf of connected institutions to authenticate users."
+  },
+  "instant-messaging": {
+    "name": "Instant messaging",
+    "category": "collaboration",
+    "description": "Commercial or NREN own online chat service which offers real-time text transmission over the Internet."
+  },
+  "interfederation": {
+    "name": "Interfederation",
+    "category": "identity",
+    "description": "Participation in an inter-federation (i.e. eduGAIN, KALMAR)."
+  },
+  "internet-radio-tv": {
+    "name": "TV/radio streaming",
+    "category": "multimedia",
+    "description": "Internet and radio streaming services."
+  },
+  "internship-database": {
+    "name": "Database services",
+    "category": "collaboration",
+    "description": "Provision of tools or services to create or host databases."
+  },
+  "ip-address-allocation": {
+    "name": "IP address allocation/LIR",
+    "category": "isp_support",
+    "description": "Local Internet Registry service for assigning of IP address space."
+  },
+  "ipv6": {
+    "name": "IPv6",
+    "category": "network_services",
+    "description": "The new version of the Internet Protocol (IP) that should eventually replace the current IP (version 4)."
+  },
+  "ix-operation": {
+    "name": "National IX operation",
+    "category": "isp_support",
+    "description": "Operation of a national Internet Exchange."
+  },
+  "journal-library-access": {
+    "name": "Journal access",
+    "category": "collaboration",
+    "description": "Access to academic journals."
+  },
+  "lambda": {
+    "name": "Optical wavelength",
+    "category": "network_services",
+    "description": "Layer 1 optical channel for provision of dedicated capacity to demanding users."
+  },
+  "mailing-lists": {
+    "name": "Mailing lists",
+    "category": "collaboration",
+    "description": "Service for operation of electronic discussion lists."
+  },
+  "mobile-data": {
+    "name": "3G/4G data service",
+    "category": "network_services",
+    "description": "Provision of mobile broadband and phone contracts to users."
+  },
+  "multicast": {
+    "name": "Multicast",
+    "category": "network_services",
+    "description": "Extension to the IP protocol which allows individual packets to be sent to multiple hosts on the Internet."
+  },
+  "news-service": {
+    "name": "Netnews/USENET service",
+    "category": "storage_and_hosting",
+    "description": "Netnews/USENET news feed service."
+  },
+  "online-payment": {
+    "name": "Online payment connectivity",
+    "category": "identity",
+    "description": "Connectivity for chip and pin payment terminals."
+  },
+  "open-lightpath-exchange": {
+    "name": "Open Lightpath Exchange",
+    "category": "network_services",
+    "description": "Provision of an Open Lightpath exchange for users to connect to other parties."
+  },
+  "pert": {
+    "name": "PERT",
+    "category": "network_services",
+    "description": "Team supporting resolution of end-to-end performance problems for networked applications."
+  },
+  "plagarism-detection": {
+    "name": "Plagarism detection",
+    "category": "collaboration",
+    "description": "Provision of software for use by teachers etc to detect plagarism."
+  },
+  "point-to-point-circuit-vpn": {
+    "name": "Virtual circuit/VPN",
+    "category": "network_services",
+    "description": "Virtual point to point circuits or VPNs delivered as a service to users."
+  },
+  "procurement": {
+    "name": "Procurement/brokerage",
+    "category": "professional_services",
+    "description": "Procurement services, inc. negotiating agreements and framework agreements."
+  },
+  "project-collaboration-toolkit": {
+    "name": "Project collaboration tools",
+    "category": "collaboration",
+    "description": "Packaged services for virtual project groups e.g. mailing lists, storage, web meetings, wiki."
+  },
+  "quality-of-service": {
+    "name": "Quality of Service",
+    "category": "network_services",
+    "description": "Preferential service to specific applications or classes of applications."
+  },
+  "scheduling-tool": {
+    "name": "Scheduling tool",
+    "category": "collaboration",
+    "description": "Provision of tools to users for scheduling appointments or classes."
+  },
+  "sdn-testbed": {
+    "name": "SDN testbed",
+    "category": "network_services",
+    "description": "Software defined networking testbed available to users."
+  },
+  "sms-messaging": {
+    "name": "SMS messaging",
+    "category": "collaboration",
+    "description": "Service for users to send or receive SMS message by email."
+  },
+  "software-development": {
+    "name": "Software development",
+    "category": "professional_services",
+    "description": "Software development service for users."
+  },
+  "software-licenses": {
+    "name": "Software licenses",
+    "category": "professional_services",
+    "description": "Provision of software for organisational or institutional purchase."
+  },
+  "storage-co-location": {
+    "name": "Housing/co-location",
+    "category": "storage_and_hosting",
+    "description": "Hosting of user equipment in a managed data centre."
+  },
+  "survey-tool": {
+    "name": "Survey/polling tool",
+    "category": "collaboration",
+    "description": "Provision of applications for creating surveys or polls."
+  },
+  "system-backup": {
+    "name": "Hot standby",
+    "category": "storage_and_hosting",
+    "description": "Failover protection for primary web servers."
+  },
+  "timeserver-ntp": {
+    "name": "NTP service",
+    "category": "isp_support",
+    "description": "Allows the synchronization of computer clocks over  the Internet."
+  },
+  "video-portal": {
+    "name": "Provision of content portal/s to users for hosting and viewing multi-media content.",
+    "category": "multimedia",
+    "description": "Multi-media content portal"
+  },
+  "videoconferencing": {
+    "name": "Event recording/streaming",
+    "category": "multimedia",
+    "description": "Provision of equipment and/or software to support event streaming/recording."
+  },
+  "virtual-learning-environment": {
+    "name": "VLE",
+    "category": "collaboration",
+    "description": "Online e-learning education system that  provides virtual access to resources used in teaching.."
+  },
+  "virtual-machines-iaas": {
+    "name": "Virtual machines/IaaS",
+    "category": "storage_and_hosting",
+    "description": "Access to virtual computing resources."
+  },
+  "voip": {
+    "name": "VoIP",
+    "category": "collaboration",
+    "description": "Service to deliver voice communications and multimedia sessions over Internet Protocol (IP) networks."
+  },
+  "vulnerability-testing": {
+    "name": "Vulnerability scanning",
+    "category": "security",
+    "description": "Vulnerability service that allows users to scan their own IP networks for security holes."
+  },
+  "web-conferencing": {
+    "name": "Web/desktop conferencing",
+    "category": "multimedia",
+    "description": "Video conferencing service to desktops and hand-held devices using software."
+  },
+  "web-design-production": {
+    "name": "Web development",
+    "category": "professional_services",
+    "description": "Web development service provided to NREN users."
+  },
+  "web-email-hosting": {
+    "name": "Web hosting",
+    "category": "storage_and_hosting",
+    "description": "Service to provide space on central web servers for users to publish their website."
+  },
+  "dns-server": {
+    "name": "DNS hosting",
+    "category": "storage_and_hosting",
+    "description": "Hosting of primary and secondary DNS servers."
+  },
+  "dissemination": {
+    "name": "Dissemination",
+    "category": "professional_services",
+    "description": "Dissemination of information to users e.g. newsletters and magazines."
+  },
+  "network-monitoring": {
+    "name": "Network monitoring",
+    "category": "network_services",
+    "description": "Network information system that shows the current and past performance of the network."
+  },
+  "disaster-recovery": {
+    "name": "Disaster recovery",
+    "category": "storage_and_hosting",
+    "description": "Off site backup services."
+  },
+  "user-monitoring": {
+    "name": "Network troubleshooting",
+    "category": "network_services",
+    "description": "Enables users at connected institutions to monitor Internet services in real time."
+  },
+  "netflow": {
+    "name": "Netflow tool",
+    "category": "network_services",
+    "description": "Network protocol for collecting IP traffic information and monitoring network traffic."
+  },
+  "managed-router": {
+    "name": "Managed router service",
+    "category": "network_services",
+    "description": "Remote network router support to institutions."
+  },
+  "intrusion": {
+    "name": "Intrusion detection",
+    "category": "security",
+    "description": "Systems for detecting and preventing Intrusions (IDS/IPS)."
+  },
+  "security-audit": {
+    "name": "Security auditing",
+    "category": "security",
+    "description": "Carrying out vulnerability assesments and security reviews of user systems and resources on their behalf."
+  },
+  "web-filtering": {
+    "name": "Web filtering",
+    "category": "security",
+    "description": "Centralised web content filtering service for protection against access to inappropriate content."
+  },
+  "pgp-key": {
+    "name": "PGP key server",
+    "category": "security",
+    "description": "Operation of PGP key server."
+  },
+  "identifier-reg": {
+    "name": "Identifier registry",
+    "category": "security",
+    "description": "Registering of unique and automatically-processable identifiers in the form of text or numeric strings."
+  },
+  "post-production": {
+    "name": "Media post production",
+    "category": "multimedia",
+    "description": "Work undertaken after the process of recording (or production) e.g. editing, synchronisation."
+  },
+  "e-portfolio": {
+    "name": "e-portfolio service",
+    "category": "collaboration",
+    "description": "Functions to create and share user professional and career e-portfolios."
+  },
+  "user-conference": {
+    "name": "User conferences",
+    "category": "professional_services",
+    "description": "Hosting of regular user conferences."
+  },
+  "user-portal": {
+    "name": "User portals",
+    "category": "professional_services",
+    "description": "User portal for service management and monitoring."
+  },
+  "admin-tools": {
+    "name": "Finance/admin systems",
+    "category": "professional_services",
+    "description": "Provision of ICT systems used in finance and administration."
+  },
+  "nameserver": {
+    "name": "Nameserver services",
+    "category": "isp_support",
+    "description": "Operation of nameservers and maintenance of DNS information on behalf of users."
+  },
+  "saas": {
+    "name": "SaaS",
+    "category": "storage_and_hosting",
+    "description": "Software as a service e.g. GoogleApps for Education."
+  }
+}
\ No newline at end of file

compendium_v2/routes/api.py

@@ -16,6 +16,7 @@ from compendium_v2.routes.nren import routes as nren_routes
 from compendium_v2.routes.response import routes as response_routes
 from compendium_v2.routes.traffic import routes as traffic_routes
 from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
+from compendium_v2.routes.nren_services import routes as nren_services_routes
 
 routes = Blueprint('compendium-v2-api', __name__)
 routes.register_blueprint(budget_routes, url_prefix='/budget')
@@ -31,6 +32,7 @@ routes.register_blueprint(nren_routes, url_prefix='/nren')
 routes.register_blueprint(response_routes, url_prefix='/response')
 routes.register_blueprint(traffic_routes, url_prefix='/traffic')
 routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
+routes.register_blueprint(nren_services_routes, url_prefix='/nren-services')
 
 logger = logging.getLogger(__name__)
 

compendium_v2/routes/institutions_urls.py

 from typing import Any
 
 from flask import Blueprint, jsonify
-from sqlalchemy import select
 
-from compendium_v2.db import db
-from compendium_v2.db.presentation_models import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import InstitutionURLs
 from compendium_v2.routes import common
 
 routes = Blueprint('institutions-urls', __name__)
@@ -40,7 +38,10 @@ def institutions_urls_view() -> Any:
     or organizations connected to the NREN.
     Many NRENs have one or more pages on their website listing such user institutions.
 
-    response will be formatted as per INSTITUTION_URLS_RESPONSE_SCHEMA
+    response will be formatted as:
+
+    .. asjson::
+        compendium_v2.routes.institutions_urls.INSTITUTION_URLS_RESPONSE_SCHEMA
 
     :return:
     """
@@ -54,8 +55,7 @@ def institutions_urls_view() -> Any:
         }
 
     entries = []
-    records = db.session.scalars(
-        select(InstitutionURLs).join(NREN).order_by(NREN.name.asc(), InstitutionURLs.year.desc()))
+    records = common.get_data(InstitutionURLs)
 
     for entry in records:
         entries.append(_extract_data(entry))

compendium_v2/routes/nren_services.py

+from typing import Any
+
+from flask import Blueprint, jsonify
+
+from compendium_v2.db.presentation_models import NRENService
+from compendium_v2.routes import common
+
+routes = Blueprint('nren-services', __name__)
+
+NREN_SERVICES_RESPONSE_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+    'definitions': {
+        'nren_services': {
+            'type': 'object',
+            'properties': {
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'year': {'type': 'integer'},
+                'product_name': {'type': 'string'},
+                'additional_information': {'type': 'string'},
+                'official_description': {'type': 'string'},
+                'service_name': {'type': 'string'},
+                'service_category': {'type': 'string'},
+                'service_description': {'type': 'string'}
+            },
+            'required': ['nren', 'nren_country', 'year', 'product_name', 'additional_information',
+                         'official_description', 'service_name', 'service_category', 'service_description'],
+            'additionalProperties': False
+        }
+    },
+    'type': 'array',
+    'items': {'$ref': '#/definitions/nren_services'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def nren_service_view() -> Any:
+    """
+    handler for /api/nren-services/ requests
+    Endpoint for getting the service matrix that shows which NREN uses which services.
+
+    response will be formatted as:
+
+    .. asjson::
+        compendium_v2.routes.nren_services.NREN_SERVICES_RESPONSE_SCHEMA
+
+    :return:
+    """
+
+    def _extract_data(nren_service: NRENService) -> dict:
+        return {
+            'nren': nren_service.nren.name,
+            'nren_country': nren_service.nren.country,
+            'year': nren_service.year,
+            'product_name': nren_service.product_name,
+            'additional_information': nren_service.additional_information,
+            'official_description': nren_service.official_description,
+            'service_name': nren_service.service.name,
+            'service_category': nren_service.service.category.value,
+            'service_description': nren_service.service.description
+        }
+
+    entries = [_extract_data(entry) for entry in common.get_data(NRENService)]
+
+    return jsonify(entries)

test/conftest.py

@@ -7,6 +7,7 @@ from sqlalchemy import select
 from flask_login import LoginManager  # type: ignore
 import compendium_v2
 from compendium_v2.db import db, presentation_models, survey_models
+from compendium_v2.db.presentation_model_enums import ServiceCategory
 from compendium_v2.survey_db import model as survey_db_model
 from compendium_v2.auth.session_management import setup_login_manager, User, ROLES
 
@@ -408,3 +409,60 @@ def test_institution_urls_data(app):
         created_nrens = _create_and_save_nrens(unique_nren_names)
         _create_and_save_institution_urls(predefined_nrens_and_years, created_nrens)
         db.session.commit()
+
+
+@pytest.fixture
+def test_nren_services_data(app):
+    def _create_and_save_nrens(nren_names):
+        nrens = {}
+        for nren_name in nren_names:
+            nren_instance = presentation_models.NREN(name=nren_name, country='country')
+            nrens[nren_name] = nren_instance
+            db.session.add(nren_instance)
+        return nrens
+
+    def _create_and_save_services(services):
+        services_map = {}
+        for service in services:
+            service_instance = presentation_models.Service(
+                name=service[0],
+                name_key=service[1],
+                description='description',
+                category=service[2],
+            )
+            services_map[service[1]] = service_instance
+            db.session.add(service_instance)
+
+        return services_map
+
+    def _create_and_save_nren_services(nren_services, nrens, services):
+        for nren_name, year, service_name, service_name_key, service_category in nren_services:
+            nren_instance = nrens[nren_name]
+            service_instance = services[service_name_key]
+
+            institution_urls_model = presentation_models.NRENService(
+                nren=nren_instance,
+                year=year,
+                service=service_instance,
+                product_name="brand name",
+                additional_information="extra info",
+                official_description="description"
+            )
+
+            db.session.add(institution_urls_model)
+
+    with app.app_context():
+        predefined_nrens_services = [
+            ('nren1', 2019, 'service1', 'service1_key', ServiceCategory.network_services.value),
+            ('nren1', 2020, 'service1', 'service1_key', ServiceCategory.network_services.value),
+            ('nren1', 2021, 'service1', 'service1_key', ServiceCategory.network_services.value),
+            ('nren2', 2019, 'service2', 'service2_key', ServiceCategory.professional_services.value),
+            ('nren2', 2021, 'service2', 'service2_key', ServiceCategory.professional_services.value)
+        ]
+
+        unique_nren_names = {nren[0] for nren in predefined_nrens_services}
+        unique_services = {nren[2:] for nren in predefined_nrens_services}
+        created_nrens = _create_and_save_nrens(unique_nren_names)
+        created_services = _create_and_save_services(unique_services)
+        _create_and_save_nren_services(predefined_nrens_services, created_nrens, created_services)
+        db.session.commit()

test/test_nren_services.py

+import json
+import jsonschema
+
+from compendium_v2.routes.nren_services import NREN_SERVICES_RESPONSE_SCHEMA
+
+
+def test_nren_services(client, test_nren_services_data):
+    rv = client.get(
+        '/api/nren-services/',
+        headers={'Accept': ['application/json']})
+    assert rv.status_code == 200
+    result = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(result, NREN_SERVICES_RESPONSE_SCHEMA)
+    assert result

test/test_survey_publisher.py

@@ -306,3 +306,15 @@ def test_v2_publisher_full(app):
         assert network_automation.network_automation_specifics == [
             "config_management", "provisioning", "data_collection", "compliance", "reporting", "troubleshooting"
         ]
+
+        services = [s for s in db.session.scalars(select(presentation_models.NRENService))]
+        assert len(services) == 59
+        virtual_learning_environment = [s for s in services if s.service_key == "virtual-learning-environment"]
+        assert len(virtual_learning_environment) == 1
+        assert virtual_learning_environment[0].additional_information == "zd"
+        csirt = [s for s in services if s.service_key == "csirt"]
+        assert len(csirt) == 1
+        assert csirt[0].official_description == "zdf"
+        connectivity = [s for s in services if s.service_key == "connectivity"]
+        assert len(connectivity) == 1
+        assert connectivity[0].product_name == "zfz"

test/test_survey_publisher_legacy_excel.py

@@ -9,15 +9,15 @@ from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
 EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
 
 
-def test_publisher(app_with_survey_db, mocker, dummy_config):
-    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
+def test_excel_publisher(app_with_survey_db, mocker):
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE)
 
     with app_with_survey_db.app_context():
         nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
         db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
         db.session.commit()
 
-    _cli(dummy_config, app_with_survey_db)
+    _cli(app_with_survey_db)
 
     with app_with_survey_db.app_context():
         budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
@@ -94,3 +94,13 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert len(asnet2021) == 1
         assert asnet2021[0].organization\
             == 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia'
+
+        service_data = db.session.scalars(select(presentation_models.Service)).all()
+        assert len(service_data) == 74
+
+        nren_service_data = db.session.scalars(select(presentation_models.NRENService)).all()
+        # test a random entry
+        sikt2022 = [x for x in nren_service_data
+                    if x.nren.name == 'SIKT' and x.year == 2022 and x.service.name == 'Journal access']
+        assert len(sikt2022) == 1
+        assert sikt2022[0].additional_information.startswith("Sikt negotiates license a")