From da1f649e0a3ecbe20e5bcef8eeacdf41a6cb5ea4 Mon Sep 17 00:00:00 2001
From: Mohammad Torkashvand <mohammad.torkashvand@geant.org>
Date: Fri, 15 Sep 2023 17:25:35 +0200
Subject: [PATCH 1/7] service matrix prototype
---
compendium_v2/conversion/mapping.py | 13 +
compendium_v2/db/presentation_models.py | 28 +-
...ab_added_service_and_nrenservice_tables.py | 50 +++
compendium_v2/publishers/excel_parser.py | 73 +++-
.../survey_publisher_legacy_excel.py | 45 ++-
compendium_v2/resources/__init__.py | 6 +
compendium_v2/resources/nren_services.json | 372 ++++++++++++++++++
compendium_v2/routes/api.py | 2 +
compendium_v2/routes/nren_services.py | 58 +++
test/conftest.py | 55 +++
test/test_nren_services.py | 14 +
test/test_survey_publisher_legacy_excel.py | 5 +-
...y => test_survey_publisher_old_db_2022.py} | 0
13 files changed, 704 insertions(+), 17 deletions(-)
create mode 100644 compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
create mode 100644 compendium_v2/resources/__init__.py
create mode 100644 compendium_v2/resources/nren_services.json
create mode 100644 compendium_v2/routes/nren_services.py
create mode 100644 test/test_nren_services.py
rename test/{test_db_survey_publisher_2022.py => test_survey_publisher_old_db_2022.py} (100%)
diff --git a/compendium_v2/conversion/mapping.py b/compendium_v2/conversion/mapping.py
index 1258da6c..4a4f7df8 100644
--- a/compendium_v2/conversion/mapping.py
+++ b/compendium_v2/conversion/mapping.py
@@ -1,3 +1,5 @@
+from compendium_v2.db.presentation_models import ServiceCategory
+
ANSWERS_2022_QUERY = """
SELECT question_id, value
FROM answers a
@@ -664,3 +666,14 @@ SERVICES_MAPPING = {
'SaaS': 'services_hosting:saas',
'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
}
+
+SERVICE_CATEGORY_MAPPING = {
+ 'services_collaboration': ServiceCategory.collaboration.value,
+ 'services_identity': ServiceCategory.identity.value,
+ 'services_isp': ServiceCategory.isp_support.value,
+ 'services_multimedia': ServiceCategory.multimedia.value,
+ 'services_network': ServiceCategory.network_services.value,
+ 'services_professional': ServiceCategory.professional_services.value,
+ 'services_security': ServiceCategory.security.value,
+ 'services_hosting': ServiceCategory.storage_and_hosting.value,
+}
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index de573493..81de9bc1 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -6,7 +6,7 @@ from decimal import Decimal
from typing import List, Optional
from typing_extensions import Annotated, TypedDict
-from sqlalchemy import String, JSON
+from sqlalchemy import String, JSON, Text
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
@@ -22,10 +22,13 @@ logger = logging.getLogger(__name__)
str128 = Annotated[str, 128]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
+str128_unique = Annotated[str, mapped_column(String(128), unique=True)]
int_pk = Annotated[int, mapped_column(primary_key=True)]
+str_text = Annotated[str, mapped_column(Text)]
int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
user_category_pk = Annotated[UserCategory, mapped_column(primary_key=True)]
json_str_list = Annotated[List[str], mapped_column(JSON)]
+int_pk_fkService = Annotated[int, mapped_column(ForeignKey("service.id"), primary_key=True)]
ExternalConnection = TypedDict(
'ExternalConnection',
@@ -471,3 +474,26 @@ class NetworkAutomation(db.Model):
year: Mapped[int_pk]
network_automation: Mapped[YesNoPlanned]
network_automation_specifics: Mapped[json_str_list]
+
+
+class Service(db.Model):
+ __tablename__ = 'service'
+
+ id: Mapped[int_pk]
+ name_key: Mapped[str128_unique]
+ name: Mapped[str128]
+ category: Mapped[ServiceCategory]
+ description: Mapped[str_text]
+
+
+class NRENService(db.Model):
+ __tablename__ = 'nren_service'
+
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ service_id: Mapped[int_pk_fkService]
+ service: Mapped[Service] = relationship(lazy='joined')
+ product_name: Mapped[str128]
+ additional_information: Mapped[str_text]
+ official_description: Mapped[str_text]
diff --git a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
new file mode 100644
index 00000000..1c16c6a9
--- /dev/null
+++ b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
@@ -0,0 +1,50 @@
+"""added Service and NRENService tables
+
+Revision ID: 1fbc4582c0ab
+Revises: e17f9e6c90ab
+Create Date: 2023-09-15 16:54:34.700341
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '1fbc4582c0ab'
+down_revision = 'e17f9e6c90ab'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').create(op.get_bind())
+ op.create_table('service',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name_key', sa.String(length=128), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
+ sa.Column('description', sa.Text(), nullable=False),
+ sa.PrimaryKeyConstraint('id', name=op.f('pk_service')),
+ sa.UniqueConstraint('name_key', name=op.f('uq_service_name_key'))
+ )
+ op.create_table('nren_service',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('service_id', sa.Integer(), nullable=False),
+ sa.Column('product_name', sa.String(), nullable=False),
+ sa.Column('additional_information', sa.Text(), nullable=False),
+ sa.Column('official_description', sa.Text(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
+ sa.ForeignKeyConstraint(['service_id'], ['service.id'], name=op.f('fk_nren_service_service_id_service')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'service_id', name=op.f('pk_nren_service'))
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('nren_service')
+ op.drop_table('service')
+ sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').drop(op.get_bind())
+ # ### end Alembic commands ###
diff --git a/compendium_v2/publishers/excel_parser.py b/compendium_v2/publishers/excel_parser.py
index 41f7c014..b842950b 100644
--- a/compendium_v2/publishers/excel_parser.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -1,22 +1,26 @@
import logging
+from typing import Dict
+
import openpyxl
-import os
-from compendium_v2.db.presentation_model_enums import FeeType
+from compendium_v2.conversion import mapping
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
+from compendium_v2.resources import get_resource_file_path
setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_ORGANISATION = get_resource_file_path("2021_Organisation_DataSeries.xlsx")
+EXCEL_FILE_NETWORKS = get_resource_file_path("2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_NREN_SERVICES = get_resource_file_path("NREN-Services-prefills_2023_Recovered.xlsx")
def fetch_budget_excel_data():
# load the xlsx file
sheet_name = "1. Budget"
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
ws = wb[sheet_name]
@@ -36,7 +40,7 @@ def fetch_budget_excel_data():
def fetch_funding_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "2. Income Sources"
@@ -119,7 +123,7 @@ def fetch_funding_excel_data():
def fetch_charging_structure_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "3. Charging mechanism"
@@ -186,7 +190,7 @@ def fetch_charging_structure_excel_data():
def fetch_staffing_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "4. Staff"
@@ -234,7 +238,7 @@ def fetch_staffing_excel_data():
def fetch_staff_function_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "5. Staff by Function"
@@ -302,7 +306,7 @@ def fetch_staff_function_excel_data():
def fetch_ecproject_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "7. EC Projects"
@@ -334,7 +338,7 @@ def fetch_ecproject_excel_data():
def fetch_organization_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Organization"
@@ -352,7 +356,7 @@ def fetch_organization_excel_data():
def fetch_traffic_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Estimated_Traffic TByte"
@@ -392,3 +396,48 @@ def fetch_traffic_excel_data():
yield from create_points_for_year(2020, 14)
yield from create_points_for_year(2021, 8)
yield from create_points_for_year(2022, 2)
+
+
+def fetch_nren_services_excel_data() -> Dict[str, str]:
+ wb = openpyxl.load_workbook(EXCEL_FILE_NREN_SERVICES, data_only=True, read_only=True)
+ ws = wb["Sheet1"]
+ rows = list(ws.rows)
+
+ titles = rows[0]
+
+ nren_service_data_columns = {}
+
+ def normalize_nren_name(n: str) -> str:
+ n = n.split(' ')[0].upper()
+ return {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(n, n)
+
+ for i in range(0, 131):
+ if titles[i].value:
+ name = normalize_nren_name(titles[i].value)
+ nren_service_data_columns[name] = i
+
+ for nren_name, start_column in nren_service_data_columns.items():
+ for row_index in range(2, 61):
+ row = rows[row_index]
+ service_name = row[0].value
+ if row[start_column].value and row[start_column].value.upper() == 'YES':
+ product_name = ''
+ additional_information = ''
+ if row[start_column + 1].value:
+ product_name = row[start_column + 1].value
+ if row[start_column + 2].value:
+ additional_information = row[start_column + 2].value
+
+ service_category, service_name_key = mapping.SERVICES_MAPPING[service_name].split(':')
+ service_category = mapping.SERVICE_CATEGORY_MAPPING[service_category]
+
+ yield {
+ 'nren_name': nren_name,
+ 'service_name': service_name,
+ 'service_category': service_category,
+ 'service_name_key': service_name_key,
+ 'year': 2022,
+ 'product_name': product_name.strip(),
+ 'additional_information': additional_information.strip(),
+ 'official_description': '',
+ }
diff --git a/compendium_v2/publishers/survey_publisher_legacy_excel.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
index 6503b79f..8f41b196 100644
--- a/compendium_v2/publishers/survey_publisher_legacy_excel.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -8,18 +8,21 @@ Registered as click cli command when installing compendium-v2.
"""
from __future__ import annotations
+
+import json
import logging
import math
-import click
+import click
from sqlalchemy import select
import compendium_v2
-from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.db import db, presentation_models
-from compendium_v2.survey_db import model as survey_model
+from compendium_v2.environment import setup_logging
from compendium_v2.publishers import helpers, excel_parser
+from compendium_v2.resources import get_resource_file_path
+from compendium_v2.survey_db import model as survey_model
setup_logging()
@@ -224,6 +227,40 @@ def db_traffic_volume_migration(nren_dict):
db.session.commit()
+def db_services_migration():
+ with open(get_resource_file_path('nren_services.json')) as f:
+ services = json.load(f)
+ for name_key, record in services.items():
+ service = presentation_models.Service(
+ name_key=name_key,
+ name=record['name'],
+ category=record['category'],
+ description=record['description'],
+ )
+ db.session.merge(service)
+ db.session.commit()
+
+
+def db_nren_services_migration(nren_dict):
+ for service_info in excel_parser.fetch_nren_services_excel_data():
+ service = db.session.query(presentation_models.Service).filter_by(
+ name_key=service_info['service_name_key']).first()
+
+ nren_service = presentation_models.NRENService(
+ nren=nren_dict[service_info['nren_name']],
+ nren_id=nren_dict[service_info['nren_name']].id,
+ year=service_info['year'],
+ service=service,
+ product_name=service_info['product_name'],
+ additional_information=service_info['additional_information'],
+ official_description=service_info['official_description']
+ )
+
+ db.session.merge(nren_service)
+
+ db.session.commit()
+
+
def _cli(config, app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
@@ -234,6 +271,8 @@ def _cli(config, app):
db_ecprojects_migration(nren_dict)
db_organizations_migration(nren_dict)
db_traffic_volume_migration(nren_dict)
+ db_services_migration()
+ db_nren_services_migration(nren_dict)
@click.command()
diff --git a/compendium_v2/resources/__init__.py b/compendium_v2/resources/__init__.py
new file mode 100644
index 00000000..e2499aeb
--- /dev/null
+++ b/compendium_v2/resources/__init__.py
@@ -0,0 +1,6 @@
+from pathlib import Path
+
+
+def get_resource_file_path(filename: str) -> Path:
+ resource_dir = Path(__file__).resolve().parent
+ return resource_dir / filename
diff --git a/compendium_v2/resources/nren_services.json b/compendium_v2/resources/nren_services.json
new file mode 100644
index 00000000..29d14104
--- /dev/null
+++ b/compendium_v2/resources/nren_services.json
@@ -0,0 +1,372 @@
+{
+ "anti-spam": {
+ "name": "Anti-spam solution",
+ "category": "security",
+ "description": "Anti-spam solutions for detecting and eliminating viruses and spam mails."
+ },
+ "class-registration": {
+ "name": "Class registration tool",
+ "category": "collaboration",
+ "description": "Software for teachers to register students for classes etc."
+ },
+ "cloud-service-end-user": {
+ "name": "Cloud storage (end user)",
+ "category": "storage_and_hosting",
+ "description": "Browser based virtual storage service for individuals."
+ },
+ "connectivity": {
+ "name": "IP connectivity",
+ "category": "network_services",
+ "description": "Basic IP connectivity services inc. R+E and commodity internet"
+ },
+ "consultancy": {
+ "name": "Consultancy/training",
+ "category": "professional_services",
+ "description": "Training and consultancy services provided by the NREN."
+ },
+ "content-delivery-hosting": {
+ "name": "Content delivery hosting",
+ "category": "storage_and_hosting",
+ "description": "Hosting of content delivery servers e.g. Akamai."
+ },
+ "content-management-system": {
+ "name": "CMS",
+ "category": "collaboration",
+ "description": "Provision of software systems for website authoring, collaboration and administration tools."
+ },
+ "csirt": {
+ "name": "CERT/CSIRT",
+ "category": "security",
+ "description": "A single point of contact for users to deal with computer security incidents and prevention."
+ },
+ "ddos-prevention": {
+ "name": "DDos mitigation",
+ "category": "security",
+ "description": "Tools and techniques for mitigating Distributed Denial of Service attacks."
+ },
+ "domain-registration": {
+ "name": "Domain name registration",
+ "category": "isp_support",
+ "description": "Administration/registration of top and second level domain names."
+ },
+ "eduroam-wifi": {
+ "name": "Eduroam",
+ "category": "identity",
+ "description": "Inter-WLAN service to facilitate easy and secure Internet access for roaming educational users."
+ },
+ "email-services": {
+ "name": "Email server hosting",
+ "category": "collaboration",
+ "description": "NREN hosted email servers."
+ },
+ "filesender": {
+ "name": "Filesender",
+ "category": "storage_and_hosting",
+ "description": "Web based application that allows authenticated users to securely and easily send arbitrarily large files."
+ },
+ "firewall-on-demand": {
+ "name": "Firewall-on-demand",
+ "category": "security",
+ "description": "Provision of a dynamic firewall services to mitigate against DDOS attacks. "
+ },
+ "home-vpn": {
+ "name": "Remote access VPN service",
+ "category": "network_services",
+ "description": "Remote access and site-to-site secure VPN."
+ },
+ "aai": {
+ "name": "Hosted campus AAI",
+ "category": "identity",
+ "description": "Hosting of an Identity Provider service on behalf of connected institutions to authenticate users."
+ },
+ "instant-messaging": {
+ "name": "Instant messaging",
+ "category": "collaboration",
+ "description": "Commercial or NREN own online chat service which offers real-time text transmission over the Internet."
+ },
+ "interfederation": {
+ "name": "Interfederation",
+ "category": "identity",
+ "description": "Participation in an inter-federation (i.e. eduGAIN, KALMAR)."
+ },
+ "internet-radio-tv": {
+ "name": "TV/radio streaming",
+ "category": "multimedia",
+ "description": "Internet and radio streaming services."
+ },
+ "internship-database": {
+ "name": "Database services",
+ "category": "collaboration",
+ "description": "Provision of tools or services to create or host databases."
+ },
+ "ip-address-allocation": {
+ "name": "IP address allocation/LIR",
+ "category": "isp_support",
+ "description": "Local Internet Registry service for assigning of IP address space."
+ },
+ "ipv6": {
+ "name": "IPv6",
+ "category": "network_services",
+ "description": "The new version of the Internet Protocol (IP) that should eventually replace the current IP (version 4)."
+ },
+ "ix-operation": {
+ "name": "National IX operation",
+ "category": "isp_support",
+ "description": "Operation of a national Internet Exchange."
+ },
+ "journal-library-access": {
+ "name": "Journal access",
+ "category": "collaboration",
+ "description": "Access to academic journals."
+ },
+ "lambda": {
+ "name": "Optical wavelength",
+ "category": "network_services",
+ "description": "Layer 1 optical channel for provision of dedicated capacity to demanding users."
+ },
+ "mailing-lists": {
+ "name": "Mailing lists",
+ "category": "collaboration",
+ "description": "Service for operation of electronic discussion lists."
+ },
+ "mobile-data": {
+ "name": "3G/4G data service",
+ "category": "network_services",
+ "description": "Provision of mobile broadband and phone contracts to users."
+ },
+ "multicast": {
+ "name": "Multicast",
+ "category": "network_services",
+ "description": "Extension to the IP protocol which allows individual packets to be sent to multiple hosts on the Internet."
+ },
+ "news-service": {
+ "name": "Netnews/USENET service",
+ "category": "storage_and_hosting",
+ "description": "Netnews/USENET news feed service."
+ },
+ "online-payment": {
+ "name": "Online payment connectivity",
+ "category": "identity",
+ "description": "Connectivity for chip and pin payment terminals."
+ },
+ "open-lightpath-exchange": {
+ "name": "Open Lightpath Exchange",
+ "category": "network_services",
+ "description": "Provision of an Open Lightpath exchange for users to connect to other parties."
+ },
+ "pert": {
+ "name": "PERT",
+ "category": "network_services",
+ "description": "Team supporting resolution of end-to-end performance problems for networked applications."
+ },
+ "plagarism-detection": {
+ "name": "Plagarism detection",
+ "category": "collaboration",
+ "description": "Provision of software for use by teachers etc to detect plagarism."
+ },
+ "point-to-point-circuit-vpn": {
+ "name": "Virtual circuit/VPN",
+ "category": "network_services",
+ "description": "Virtual point to point circuits or VPNs delivered as a service to users."
+ },
+ "procurement": {
+ "name": "Procurement/brokerage",
+ "category": "professional_services",
+ "description": "Procurement services, inc. negotiating agreements and framework agreements."
+ },
+ "project-collaboration-toolkit": {
+ "name": "Project collaboration tools",
+ "category": "collaboration",
+ "description": "Packaged services for virtual project groups e.g. mailing lists, storage, web meetings, wiki."
+ },
+ "quality-of-service": {
+ "name": "Quality of Service",
+ "category": "network_services",
+ "description": "Preferential service to specific applications or classes of applications."
+ },
+ "scheduling-tool": {
+ "name": "Scheduling tool",
+ "category": "collaboration",
+ "description": "Provision of tools to users for scheduling appointments or classes."
+ },
+ "sdn-testbed": {
+ "name": "SDN testbed",
+ "category": "network_services",
+ "description": "Software defined networking testbed available to users."
+ },
+ "sms-messaging": {
+ "name": "SMS messaging",
+ "category": "collaboration",
+ "description": "Service for users to send or receive SMS message by email."
+ },
+ "software-development": {
+ "name": "Software development",
+ "category": "professional_services",
+ "description": "Software development service for users."
+ },
+ "software-licenses": {
+ "name": "Software licenses",
+ "category": "professional_services",
+ "description": "Provision of software for organisational or institutional purchase."
+ },
+ "storage-co-location": {
+ "name": "Housing/co-location",
+ "category": "storage_and_hosting",
+ "description": "Hosting of user equipment in a managed data centre."
+ },
+ "survey-tool": {
+ "name": "Survey/polling tool",
+ "category": "collaboration",
+ "description": "Provision of applications for creating surveys or polls."
+ },
+ "system-backup": {
+ "name": "Hot standby",
+ "category": "storage_and_hosting",
+ "description": "Failover protection for primary web servers."
+ },
+ "timeserver-ntp": {
+ "name": "NTP service",
+ "category": "isp_support",
+ "description": "Allows the synchronization of computer clocks over the Internet."
+ },
+ "video-portal": {
+ "name": "Provision of content portal/s to users for hosting and viewing multi-media content.",
+ "category": "multimedia",
+ "description": "Multi-media content portal"
+ },
+ "videoconferencing": {
+ "name": "Event recording/streaming",
+ "category": "multimedia",
+ "description": "Provision of equipment and/or software to support event streaming/recording."
+ },
+ "virtual-learning-environment": {
+ "name": "VLE",
+ "category": "collaboration",
+ "description": "Online e-learning education system that provides virtual access to resources used in teaching.."
+ },
+ "virtual-machines-iaas": {
+ "name": "Virtual machines/IaaS",
+ "category": "storage_and_hosting",
+ "description": "Access to virtual computing resources."
+ },
+ "voip": {
+ "name": "VoIP",
+ "category": "collaboration",
+ "description": "Service to deliver voice communications and multimedia sessions over Internet Protocol (IP) networks."
+ },
+ "vulnerability-testing": {
+ "name": "Vulnerability scanning",
+ "category": "security",
+ "description": "Vulnerability service that allows users to scan their own IP networks for security holes."
+ },
+ "web-conferencing": {
+ "name": "Web/desktop conferencing",
+ "category": "multimedia",
+ "description": "Video conferencing service to desktops and hand-held devices using software."
+ },
+ "web-design-production": {
+ "name": "Web development",
+ "category": "professional_services",
+ "description": "Web development service provided to NREN users."
+ },
+ "web-email-hosting": {
+ "name": "Web hosting",
+ "category": "storage_and_hosting",
+ "description": "Service to provide space on central web servers for users to publish their website."
+ },
+ "dns-server": {
+ "name": "DNS hosting",
+ "category": "storage_and_hosting",
+ "description": "Hosting of primary and secondary DNS servers."
+ },
+ "dissemination": {
+ "name": "Dissemination",
+ "category": "professional_services",
+ "description": "Dissemination of information to users e.g. newsletters and magazines."
+ },
+ "network-monitoring": {
+ "name": "Network monitoring",
+ "category": "network_services",
+ "description": "Network information system that shows the current and past performance of the network."
+ },
+ "disaster-recovery": {
+ "name": "Disaster recovery",
+ "category": "storage_and_hosting",
+ "description": "Off site backup services."
+ },
+ "user-monitoring": {
+ "name": "Network troubleshooting",
+ "category": "network_services",
+ "description": "Enables users at connected institutions to monitor Internet services in real time."
+ },
+ "netflow": {
+ "name": "Netflow tool",
+ "category": "network_services",
+ "description": "Network protocol for collecting IP traffic information and monitoring network traffic."
+ },
+ "managed-router": {
+ "name": "Managed router service",
+ "category": "network_services",
+ "description": "Remote network router support to institutions."
+ },
+ "intrusion": {
+ "name": "Intrusion detection",
+ "category": "security",
+ "description": "Systems for detecting and preventing Intrusions (IDS/IPS)."
+ },
+ "security-audit": {
+ "name": "Security auditing",
+ "category": "security",
+ "description": "Carrying out vulnerability assesments and security reviews of user systems and resources on their behalf."
+ },
+ "web-filtering": {
+ "name": "Web filtering",
+ "category": "security",
+ "description": "Centralised web content filtering service for protection against access to inappropriate content."
+ },
+ "pgp-key": {
+ "name": "PGP key server",
+ "category": "security",
+ "description": "Operation of PGP key server."
+ },
+ "identifier-reg": {
+ "name": "Identifier registry",
+ "category": "security",
+ "description": "Registering of unique and automatically-processable identifiers in the form of text or numeric strings."
+ },
+ "post-production": {
+ "name": "Media post production",
+ "category": "multimedia",
+ "description": "Work undertaken after the process of recording (or production) e.g. editing, synchronisation."
+ },
+ "e-portfolio": {
+ "name": "e-portfolio service",
+ "category": "collaboration",
+ "description": "Functions to create and share user professional and career e-portfolios."
+ },
+ "user-conference": {
+ "name": "User conferences",
+ "category": "professional_services",
+ "description": "Hosting of regular user conferences."
+ },
+ "user-portal": {
+ "name": "User portals",
+ "category": "professional_services",
+ "description": "User portal for service management and monitoring."
+ },
+ "admin-tools": {
+ "name": "Finance/admin systems",
+ "category": "professional_services",
+ "description": "Provision of ICT systems used in finance and administration."
+ },
+ "nameserver": {
+ "name": "Nameserver services",
+ "category": "isp_support",
+ "description": "Operation of nameservers and maintenance of DNS information on behalf of users."
+ },
+ "saas": {
+ "name": "SaaS",
+ "category": "storage_and_hosting",
+ "description": "Software as a service e.g. GoogleApps for Education."
+ }
+}
\ No newline at end of file
diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py
index 9af16ad2..cc7ac6e0 100644
--- a/compendium_v2/routes/api.py
+++ b/compendium_v2/routes/api.py
@@ -16,6 +16,7 @@ from compendium_v2.routes.nren import routes as nren_routes
from compendium_v2.routes.response import routes as response_routes
from compendium_v2.routes.traffic import routes as traffic_routes
from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
+from compendium_v2.routes.nren_services import routes as nren_services_routes
routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget')
@@ -31,6 +32,7 @@ routes.register_blueprint(nren_routes, url_prefix='/nren')
routes.register_blueprint(response_routes, url_prefix='/response')
routes.register_blueprint(traffic_routes, url_prefix='/traffic')
routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
+routes.register_blueprint(nren_services_routes, url_prefix='/nren-services')
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/routes/nren_services.py b/compendium_v2/routes/nren_services.py
new file mode 100644
index 00000000..fde064d8
--- /dev/null
+++ b/compendium_v2/routes/nren_services.py
@@ -0,0 +1,58 @@
+from typing import Any
+
+from flask import Blueprint, jsonify
+from sqlalchemy import select
+
+from compendium_v2.db import db
+from compendium_v2.db.presentation_models import NREN, InstitutionURLs, NRENService, Service
+from compendium_v2.routes import common
+
+routes = Blueprint('nren-services', __name__)
+
+NREN_SERVICES_RESPONSE_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'nren_services': {
+ 'type': 'object',
+ 'properties': {
+ 'nren': {'type': 'string'},
+ 'nren_country': {'type': 'string'},
+ 'year': {'type': 'integer'},
+ 'urls': {'type': 'array', 'items': {'type': 'string'}}
+ },
+ 'required': ['nren', 'nren_country', 'year', 'urls'], # TODO
+ 'additionalProperties': False
+ }
+ },
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/nren_services'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def nren_service_view() -> Any:
+ """
+ handler for /api/nren-services/ requests
+ Endpoint for getting the service matrix that shows which NREN uses which services.
+
+ This endpoint retrieves the URLs of webpages that list the institutions
+ or organizations connected to the NREN.
+ Many NRENs have one or more pages on their website listing such user institutions.
+
+ response will be formatted as per NREN_SERVICES_RESPONSE_SCHEMA
+
+ :return:
+ """
+
+ def _extract_data(institution: InstitutionURLs) -> dict:
+ return {
+ 'nren': institution.nren.name,
+ 'nren_country': institution.nren.country,
+ 'year': institution.year,
+ # TODO
+ }
+
+ entries = []
+ # TODO
+ return jsonify(entries)
diff --git a/test/conftest.py b/test/conftest.py
index 591fe482..ff66f316 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -7,6 +7,7 @@ from sqlalchemy import select
from flask_login import LoginManager # type: ignore
import compendium_v2
from compendium_v2.db import db, presentation_models, survey_models
+from compendium_v2.db.presentation_model_enums import ServiceCategory
from compendium_v2.survey_db import model as survey_db_model
from compendium_v2.auth.session_management import setup_login_manager, User, ROLES
@@ -408,3 +409,57 @@ def test_institution_urls_data(app):
created_nrens = _create_and_save_nrens(unique_nren_names)
_create_and_save_institution_urls(predefined_nrens_and_years, created_nrens)
db.session.commit()
+
+
+@pytest.fixture
+def test_nren_services_data(app):
+ def _create_and_save_nrens(nren_names):
+ nrens = {}
+ for nren_name in nren_names:
+ nren_instance = presentation_models.NREN(name=nren_name, country='country')
+ nrens[nren_name] = nren_instance
+ db.session.add(nren_instance)
+ return nrens
+
+ def _create_and_save_services(services):
+ services_map = {}
+ for service in services:
+ service_instance = presentation_models.Service(
+ name=service[2],
+ name_key=service[3],
+ description='description',
+ category=service[4],
+ )
+ services_map[service[3]] = service_instance
+ db.session.add(service_instance)
+
+ return services_map
+
+ def _create_and_save_nren_services(nren_services, nrens, services):
+ for nren_name, year, service_name, service_name_key, service_category in nren_services:
+ nren_instance = nrens[nren_name]
+ service_instance = services[service_name_key]
+
+ institution_urls_model = presentation_models.NRENService(
+ nren=nren_instance,
+ year=year,
+ service=service_instance,
+ )
+
+ db.session.add(institution_urls_model)
+
+ with app.app_context():
+ predefined_nrens_services = [
+ ('nren1', 2019, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren1', 2020, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren1', 2021, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren2', 2019, 'service2', 'service2_key', ServiceCategory.professional_services.value),
+ ('nren2', 2021, 'service2', 'service2_key', ServiceCategory.professional_services.value)
+ ]
+
+ unique_nren_names = {nren[0] for nren in predefined_nrens_services}
+ unique_services = {nren[1:] for nren in predefined_nrens_services}
+ created_nrens = _create_and_save_nrens(unique_nren_names)
+ created_services = _create_and_save_services(unique_services)
+ _create_and_save_nren_services(predefined_nrens_services, created_nrens, created_services)
+ db.session.commit()
diff --git a/test/test_nren_services.py b/test/test_nren_services.py
new file mode 100644
index 00000000..00ba6aba
--- /dev/null
+++ b/test/test_nren_services.py
@@ -0,0 +1,14 @@
+import json
+import jsonschema
+
+from compendium_v2.routes.institutions_urls import INSTITUTION_URLS_RESPONSE_SCHEMA
+
+
+def test_institutions_urls_response(client, test_nren_services_data):
+ rv = client.get(
+ '/api/nren-services/',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ result = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(result, INSTITUTION_URLS_RESPONSE_SCHEMA)
+ assert result
diff --git a/test/test_survey_publisher_legacy_excel.py b/test/test_survey_publisher_legacy_excel.py
index 5a20fc06..4f23686a 100644
--- a/test/test_survey_publisher_legacy_excel.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -10,7 +10,7 @@ EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_
def test_publisher(app_with_survey_db, mocker, dummy_config):
- mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
+ mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE)
with app_with_survey_db.app_context():
nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
@@ -94,3 +94,6 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert len(asnet2021) == 1
assert asnet2021[0].organization\
== 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia'
+
+ service_data = db.session.scalars(select(presentation_models.Service)).all()
+ assert len(service_data) == 100
\ No newline at end of file
diff --git a/test/test_db_survey_publisher_2022.py b/test/test_survey_publisher_old_db_2022.py
similarity index 100%
rename from test/test_db_survey_publisher_2022.py
rename to test/test_survey_publisher_old_db_2022.py
--
GitLab
From f2b9459914c59430f5ab3d03b9cc4bd134916de6 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Tue, 19 Sep 2023 15:54:07 +0200
Subject: [PATCH 2/7] fix migration script after merging
---
.../1fbc4582c0ab_added_service_and_nrenservice_tables.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
index 1c16c6a9..c3dfe8fe 100644
--- a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
+++ b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
@@ -1,7 +1,7 @@
"""added Service and NRENService tables
Revision ID: 1fbc4582c0ab
-Revises: e17f9e6c90ab
+Revises: 87e1e35051a0
Create Date: 2023-09-15 16:54:34.700341
"""
@@ -11,14 +11,13 @@ from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '1fbc4582c0ab'
-down_revision = 'e17f9e6c90ab'
+down_revision = '87e1e35051a0'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
- sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').create(op.get_bind())
op.create_table('service',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name_key', sa.String(length=128), nullable=False),
@@ -46,5 +45,4 @@ def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('nren_service')
op.drop_table('service')
- sa.Enum('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory').drop(op.get_bind())
# ### end Alembic commands ###
--
GitLab
From ae70cf84b3e19b0f3b96f83c17c47af85ea04129 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Tue, 19 Sep 2023 22:42:50 +0200
Subject: [PATCH 3/7] change datamodel a bit and apply some fixes to the
publisher
---
compendium_v2/db/presentation_models.py | 14 +++++--------
...ab_added_service_and_nrenservice_tables.py | 18 ++++++++---------
.../survey_publisher_legacy_excel.py | 20 +++++++++++++------
test/test_survey_publisher_legacy_excel.py | 13 +++++++++---
4 files changed, 37 insertions(+), 28 deletions(-)
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index d6f02456..f5edb05a 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -22,13 +22,10 @@ logger = logging.getLogger(__name__)
str128 = Annotated[str, 128]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
-str128_unique = Annotated[str, mapped_column(String(128), unique=True)]
int_pk = Annotated[int, mapped_column(primary_key=True)]
-str_text = Annotated[str, mapped_column(Text)]
int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
user_category_pk = Annotated[UserCategory, mapped_column(primary_key=True)]
json_str_list = Annotated[List[str], mapped_column(JSON)]
-int_pk_fkService = Annotated[int, mapped_column(ForeignKey("service.id"), primary_key=True)]
ExternalConnection = TypedDict(
'ExternalConnection',
@@ -479,11 +476,10 @@ class NetworkAutomation(db.Model):
class Service(db.Model):
__tablename__ = 'service'
- id: Mapped[int_pk]
- name_key: Mapped[str128_unique]
+ name_key: Mapped[str128_pk]
name: Mapped[str128]
category: Mapped[ServiceCategory]
- description: Mapped[str_text]
+ description: Mapped[str]
class NRENService(db.Model):
@@ -492,8 +488,8 @@ class NRENService(db.Model):
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
- service_id: Mapped[int_pk_fkService]
+ service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
service: Mapped[Service] = relationship(lazy='joined')
product_name: Mapped[str128]
- additional_information: Mapped[str_text]
- official_description: Mapped[str_text]
+ additional_information: Mapped[str]
+ official_description: Mapped[str]
diff --git a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
index c3dfe8fe..7bcc398e 100644
--- a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
+++ b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
@@ -2,7 +2,7 @@
Revision ID: 1fbc4582c0ab
Revises: 87e1e35051a0
-Create Date: 2023-09-15 16:54:34.700341
+Create Date: 2023-09-19 22:23:36.448210
"""
from alembic import op
@@ -19,24 +19,22 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('service',
- sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name_key', sa.String(length=128), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
- sa.Column('description', sa.Text(), nullable=False),
- sa.PrimaryKeyConstraint('id', name=op.f('pk_service')),
- sa.UniqueConstraint('name_key', name=op.f('uq_service_name_key'))
+ sa.Column('description', sa.String(), nullable=False),
+ sa.PrimaryKeyConstraint('name_key', name=op.f('pk_service'))
)
op.create_table('nren_service',
sa.Column('nren_id', sa.Integer(), nullable=False),
sa.Column('year', sa.Integer(), nullable=False),
- sa.Column('service_id', sa.Integer(), nullable=False),
+ sa.Column('service_key', sa.String(length=128), nullable=False),
sa.Column('product_name', sa.String(), nullable=False),
- sa.Column('additional_information', sa.Text(), nullable=False),
- sa.Column('official_description', sa.Text(), nullable=False),
+ sa.Column('additional_information', sa.String(), nullable=False),
+ sa.Column('official_description', sa.String(), nullable=False),
sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
- sa.ForeignKeyConstraint(['service_id'], ['service.id'], name=op.f('fk_nren_service_service_id_service')),
- sa.PrimaryKeyConstraint('nren_id', 'year', 'service_id', name=op.f('pk_nren_service'))
+ sa.ForeignKeyConstraint(['service_key'], ['service.name_key'], name=op.f('fk_nren_service_service_key_service')),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'service_key', name=op.f('pk_nren_service'))
)
# ### end Alembic commands ###
diff --git a/compendium_v2/publishers/survey_publisher_legacy_excel.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
index 8f41b196..dfb6b6b5 100644
--- a/compendium_v2/publishers/survey_publisher_legacy_excel.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -242,15 +242,23 @@ def db_services_migration():
def db_nren_services_migration(nren_dict):
+ services = [s for s in db.session.scalars(select(presentation_models.Service))]
+
for service_info in excel_parser.fetch_nren_services_excel_data():
- service = db.session.query(presentation_models.Service).filter_by(
- name_key=service_info['service_name_key']).first()
+ [service] = [s for s in services if s.name_key == service_info['service_name_key']]
+
+ abbrev = service_info['nren_name']
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+ nren = nren_dict[abbrev]
nren_service = presentation_models.NRENService(
- nren=nren_dict[service_info['nren_name']],
- nren_id=nren_dict[service_info['nren_name']].id,
+ nren=nren,
+ nren_id=nren.id,
year=service_info['year'],
service=service,
+ service_key=service.name_key,
product_name=service_info['product_name'],
additional_information=service_info['additional_information'],
official_description=service_info['official_description']
@@ -261,7 +269,7 @@ def db_nren_services_migration(nren_dict):
db.session.commit()
-def _cli(config, app):
+def _cli(app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
db_budget_migration(nren_dict)
@@ -284,7 +292,7 @@ def cli(config):
app = compendium_v2._create_app_with_db(app_config)
print("survey-publisher-v1 starting")
- _cli(app_config, app)
+ _cli(app)
if __name__ == "__main__":
diff --git a/test/test_survey_publisher_legacy_excel.py b/test/test_survey_publisher_legacy_excel.py
index 4f23686a..b066767e 100644
--- a/test/test_survey_publisher_legacy_excel.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -9,7 +9,7 @@ from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
-def test_publisher(app_with_survey_db, mocker, dummy_config):
+def test_excel_publisher(app_with_survey_db, mocker):
mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE)
with app_with_survey_db.app_context():
@@ -17,7 +17,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
- _cli(dummy_config, app_with_survey_db)
+ _cli(app_with_survey_db)
with app_with_survey_db.app_context():
budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
@@ -96,4 +96,11 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
== 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia'
service_data = db.session.scalars(select(presentation_models.Service)).all()
- assert len(service_data) == 100
\ No newline at end of file
+ assert len(service_data) == 74
+
+ nren_service_data = db.session.scalars(select(presentation_models.NRENService)).all()
+ # test a random entry
+ sikt2022 = [x for x in nren_service_data
+ if x.nren.name == 'SIKT' and x.year == 2022 and x.service.name == 'Journal access']
+ assert len(sikt2022) == 1
+ assert sikt2022[0].additional_information.startswith("Sikt negotiates license a")
--
GitLab
From c910ab456a1e3c0656788dff1a08a003897a6a2a Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Tue, 19 Sep 2023 23:18:34 +0200
Subject: [PATCH 4/7] fix tox issues
---
compendium_v2/db/presentation_models.py | 2 +-
...ab_added_service_and_nrenservice_tables.py | 41 +++++++++++--------
compendium_v2/publishers/excel_parser.py | 3 +-
compendium_v2/routes/nren_services.py | 23 +++++------
test/test_nren_services.py | 4 +-
5 files changed, 41 insertions(+), 32 deletions(-)
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index f5edb05a..a25200e7 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -6,7 +6,7 @@ from decimal import Decimal
from typing import List, Optional
from typing_extensions import Annotated, TypedDict
-from sqlalchemy import String, JSON, Text
+from sqlalchemy import String, JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
diff --git a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
index 7bcc398e..1c3dc396 100644
--- a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
+++ b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
@@ -18,23 +18,32 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
- op.create_table('service',
- sa.Column('name_key', sa.String(length=128), nullable=False),
- sa.Column('name', sa.String(), nullable=False),
- sa.Column('category', postgresql.ENUM('network_services', 'isp_support', 'security', 'identity', 'collaboration', 'multimedia', 'storage_and_hosting', 'professional_services', name='servicecategory', create_type=False), nullable=False),
- sa.Column('description', sa.String(), nullable=False),
- sa.PrimaryKeyConstraint('name_key', name=op.f('pk_service'))
+ op.create_table(
+ 'service',
+ sa.Column('name_key', sa.String(length=128), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('category', postgresql.ENUM(
+ 'network_services', 'isp_support', 'security', 'identity',
+ 'collaboration', 'multimedia', 'storage_and_hosting',
+ 'professional_services', name='servicecategory',
+ create_type=False), nullable=False
+ ),
+ sa.Column('description', sa.String(), nullable=False),
+ sa.PrimaryKeyConstraint('name_key', name=op.f('pk_service'))
)
- op.create_table('nren_service',
- sa.Column('nren_id', sa.Integer(), nullable=False),
- sa.Column('year', sa.Integer(), nullable=False),
- sa.Column('service_key', sa.String(length=128), nullable=False),
- sa.Column('product_name', sa.String(), nullable=False),
- sa.Column('additional_information', sa.String(), nullable=False),
- sa.Column('official_description', sa.String(), nullable=False),
- sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
- sa.ForeignKeyConstraint(['service_key'], ['service.name_key'], name=op.f('fk_nren_service_service_key_service')),
- sa.PrimaryKeyConstraint('nren_id', 'year', 'service_key', name=op.f('pk_nren_service'))
+ op.create_table(
+ 'nren_service',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('service_key', sa.String(length=128), nullable=False),
+ sa.Column('product_name', sa.String(), nullable=False),
+ sa.Column('additional_information', sa.String(), nullable=False),
+ sa.Column('official_description', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
+ sa.ForeignKeyConstraint(
+ ['service_key'], ['service.name_key'], name=op.f('fk_nren_service_service_key_service')
+ ),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'service_key', name=op.f('pk_nren_service'))
)
# ### end Alembic commands ###
diff --git a/compendium_v2/publishers/excel_parser.py b/compendium_v2/publishers/excel_parser.py
index b842950b..ce1efff4 100644
--- a/compendium_v2/publishers/excel_parser.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -1,5 +1,4 @@
import logging
-from typing import Dict
import openpyxl
@@ -398,7 +397,7 @@ def fetch_traffic_excel_data():
yield from create_points_for_year(2022, 2)
-def fetch_nren_services_excel_data() -> Dict[str, str]:
+def fetch_nren_services_excel_data():
wb = openpyxl.load_workbook(EXCEL_FILE_NREN_SERVICES, data_only=True, read_only=True)
ws = wb["Sheet1"]
rows = list(ws.rows)
diff --git a/compendium_v2/routes/nren_services.py b/compendium_v2/routes/nren_services.py
index fde064d8..25c58aaf 100644
--- a/compendium_v2/routes/nren_services.py
+++ b/compendium_v2/routes/nren_services.py
@@ -1,10 +1,10 @@
from typing import Any
from flask import Blueprint, jsonify
-from sqlalchemy import select
+# from sqlalchemy import select
-from compendium_v2.db import db
-from compendium_v2.db.presentation_models import NREN, InstitutionURLs, NRENService, Service
+# from compendium_v2.db import db
+# from compendium_v2.db.presentation_models import NREN, InstitutionURLs, NRENService, Service
from compendium_v2.routes import common
routes = Blueprint('nren-services', __name__)
@@ -45,14 +45,13 @@ def nren_service_view() -> Any:
:return:
"""
- def _extract_data(institution: InstitutionURLs) -> dict:
- return {
- 'nren': institution.nren.name,
- 'nren_country': institution.nren.country,
- 'year': institution.year,
- # TODO
- }
+ # def _extract_data(institution: InstitutionURLs) -> dict:
+ # return {
+ # 'nren': institution.nren.name,
+ # 'nren_country': institution.nren.country,
+ # 'year': institution.year,
+ # # TODO
+ # }
- entries = []
# TODO
- return jsonify(entries)
+ return jsonify([])
diff --git a/test/test_nren_services.py b/test/test_nren_services.py
index 00ba6aba..2e778b4e 100644
--- a/test/test_nren_services.py
+++ b/test/test_nren_services.py
@@ -1,10 +1,12 @@
import json
import jsonschema
+import pytest
from compendium_v2.routes.institutions_urls import INSTITUTION_URLS_RESPONSE_SCHEMA
-def test_institutions_urls_response(client, test_nren_services_data):
+@pytest.mark.skip(reason="wip")
+def test_nren_services(client, test_nren_services_data):
rv = client.get(
'/api/nren-services/',
headers={'Accept': ['application/json']})
--
GitLab
From 01a5560a0670ca7b64f9ed1bdb461cfb29d11ed8 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Wed, 20 Sep 2023 15:45:32 +0200
Subject: [PATCH 5/7] complete nren services api endpoint
---
compendium_v2/routes/institutions_urls.py | 12 +++---
compendium_v2/routes/nren_services.py | 45 ++++++++++++++---------
test/conftest.py | 13 ++++---
test/test_nren_services.py | 6 +--
4 files changed, 43 insertions(+), 33 deletions(-)
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 8af1bbb6..45c71287 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -1,10 +1,8 @@
from typing import Any
from flask import Blueprint, jsonify
-from sqlalchemy import select
-from compendium_v2.db import db
-from compendium_v2.db.presentation_models import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
@@ -40,7 +38,10 @@ def institutions_urls_view() -> Any:
or organizations connected to the NREN.
Many NRENs have one or more pages on their website listing such user institutions.
- response will be formatted as per INSTITUTION_URLS_RESPONSE_SCHEMA
+ response will be formatted as:
+
+ .. asjson::
+ compendium_v2.routes.institutions_urls.INSTITUTION_URLS_RESPONSE_SCHEMA
:return:
"""
@@ -54,8 +55,7 @@ def institutions_urls_view() -> Any:
}
entries = []
- records = db.session.scalars(
- select(InstitutionURLs).join(NREN).order_by(NREN.name.asc(), InstitutionURLs.year.desc()))
+ records = common.get_data(InstitutionURLs)
for entry in records:
entries.append(_extract_data(entry))
diff --git a/compendium_v2/routes/nren_services.py b/compendium_v2/routes/nren_services.py
index 25c58aaf..f4ed08b9 100644
--- a/compendium_v2/routes/nren_services.py
+++ b/compendium_v2/routes/nren_services.py
@@ -1,10 +1,8 @@
from typing import Any
from flask import Blueprint, jsonify
-# from sqlalchemy import select
-# from compendium_v2.db import db
-# from compendium_v2.db.presentation_models import NREN, InstitutionURLs, NRENService, Service
+from compendium_v2.db.presentation_models import NRENService
from compendium_v2.routes import common
routes = Blueprint('nren-services', __name__)
@@ -18,9 +16,15 @@ NREN_SERVICES_RESPONSE_SCHEMA = {
'nren': {'type': 'string'},
'nren_country': {'type': 'string'},
'year': {'type': 'integer'},
- 'urls': {'type': 'array', 'items': {'type': 'string'}}
+ 'product_name': {'type': 'string'},
+ 'additional_information': {'type': 'string'},
+ 'official_description': {'type': 'string'},
+ 'service_name': {'type': 'string'},
+ 'service_category': {'type': 'string'},
+ 'service_description': {'type': 'string'}
},
- 'required': ['nren', 'nren_country', 'year', 'urls'], # TODO
+ 'required': ['nren', 'nren_country', 'year', 'product_name', 'additional_information',
+ 'official_description', 'service_name', 'service_category', 'service_description'],
'additionalProperties': False
}
},
@@ -36,22 +40,27 @@ def nren_service_view() -> Any:
handler for /api/nren-services/ requests
Endpoint for getting the service matrix that shows which NREN uses which services.
- This endpoint retrieves the URLs of webpages that list the institutions
- or organizations connected to the NREN.
- Many NRENs have one or more pages on their website listing such user institutions.
+ response will be formatted as:
- response will be formatted as per NREN_SERVICES_RESPONSE_SCHEMA
+ .. asjson::
+ compendium_v2.routes.nren_services.NREN_SERVICES_RESPONSE_SCHEMA
:return:
"""
- # def _extract_data(institution: InstitutionURLs) -> dict:
- # return {
- # 'nren': institution.nren.name,
- # 'nren_country': institution.nren.country,
- # 'year': institution.year,
- # # TODO
- # }
+ def _extract_data(nren_service: NRENService) -> dict:
+ return {
+ 'nren': nren_service.nren.name,
+ 'nren_country': nren_service.nren.country,
+ 'year': nren_service.year,
+ 'product_name': nren_service.product_name,
+ 'additional_information': nren_service.additional_information,
+ 'official_description': nren_service.official_description,
+ 'service_name': nren_service.service.name,
+ 'service_category': nren_service.service.category.value,
+ 'service_description': nren_service.service.description
+ }
+
+ entries = [_extract_data(entry) for entry in common.get_data(NRENService)]
- # TODO
- return jsonify([])
+ return jsonify(entries)
diff --git a/test/conftest.py b/test/conftest.py
index ff66f316..7bb203ae 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -425,12 +425,12 @@ def test_nren_services_data(app):
services_map = {}
for service in services:
service_instance = presentation_models.Service(
- name=service[2],
- name_key=service[3],
+ name=service[0],
+ name_key=service[1],
description='description',
- category=service[4],
+ category=service[2],
)
- services_map[service[3]] = service_instance
+ services_map[service[1]] = service_instance
db.session.add(service_instance)
return services_map
@@ -444,6 +444,9 @@ def test_nren_services_data(app):
nren=nren_instance,
year=year,
service=service_instance,
+ product_name="brand name",
+ additional_information="extra info",
+ official_description="description"
)
db.session.add(institution_urls_model)
@@ -458,7 +461,7 @@ def test_nren_services_data(app):
]
unique_nren_names = {nren[0] for nren in predefined_nrens_services}
- unique_services = {nren[1:] for nren in predefined_nrens_services}
+ unique_services = {nren[2:] for nren in predefined_nrens_services}
created_nrens = _create_and_save_nrens(unique_nren_names)
created_services = _create_and_save_services(unique_services)
_create_and_save_nren_services(predefined_nrens_services, created_nrens, created_services)
diff --git a/test/test_nren_services.py b/test/test_nren_services.py
index 2e778b4e..ce832e6e 100644
--- a/test/test_nren_services.py
+++ b/test/test_nren_services.py
@@ -1,16 +1,14 @@
import json
import jsonschema
-import pytest
-from compendium_v2.routes.institutions_urls import INSTITUTION_URLS_RESPONSE_SCHEMA
+from compendium_v2.routes.nren_services import NREN_SERVICES_RESPONSE_SCHEMA
-@pytest.mark.skip(reason="wip")
def test_nren_services(client, test_nren_services_data):
rv = client.get(
'/api/nren-services/',
headers={'Accept': ['application/json']})
assert rv.status_code == 200
result = json.loads(rv.data.decode('utf-8'))
- jsonschema.validate(result, INSTITUTION_URLS_RESPONSE_SCHEMA)
+ jsonschema.validate(result, NREN_SERVICES_RESPONSE_SCHEMA)
assert result
--
GitLab
From 44dd5fdc8456c9189b5199a0e1244edf7f789f65 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Wed, 20 Sep 2023 16:34:15 +0200
Subject: [PATCH 6/7] survey publisher (v2) for the service matrix
---
compendium_v2/publishers/survey_publisher.py | 20 ++++++++++++++++++--
test/test_survey_publisher.py | 12 ++++++++++++
2 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/compendium_v2/publishers/survey_publisher.py b/compendium_v2/publishers/survey_publisher.py
index 10a50baf..ccd4b094 100644
--- a/compendium_v2/publishers/survey_publisher.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -23,7 +23,7 @@ from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure,
CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService
from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
@@ -64,7 +64,7 @@ def _map_2023(nren, answers) -> None:
CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio,
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation]:
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService]:
db.session.execute(delete(table_class).where(table_class.year == year)) # type: ignore
answers = answers["data"]
@@ -516,6 +516,22 @@ def _map_2023(nren, answers) -> None:
network_automation_specifics=network_automation_tasks
))
+ all_services = {}
+ for question_name in ["services_collaboration", "services_hosting", "services_identity", "services_isp",
+ "services_multimedia", "services_network", "services_professional", "services_security"]:
+ all_services.update(answers.get(question_name, {}))
+
+ for service_key, answers in all_services.items():
+ offered = answers.get("offered")
+ if offered == ["yes"]:
+ db.session.add(NRENService(
+ nren_id=nren.id, nren=nren, year=year,
+ service_key=service_key,
+ product_name=answers.get("name", ""),
+ additional_information=answers.get("additional_information", ""),
+ official_description=answers.get("description", "")
+ ))
+
def publish(year):
responses = db.session.scalars(
diff --git a/test/test_survey_publisher.py b/test/test_survey_publisher.py
index 3e66d6f0..2889eebb 100644
--- a/test/test_survey_publisher.py
+++ b/test/test_survey_publisher.py
@@ -306,3 +306,15 @@ def test_v2_publisher_full(app):
assert network_automation.network_automation_specifics == [
"config_management", "provisioning", "data_collection", "compliance", "reporting", "troubleshooting"
]
+
+ services = [s for s in db.session.scalars(select(presentation_models.NRENService))]
+ assert len(services) == 59
+ virtual_learning_environment = [s for s in services if s.service_key == "virtual-learning-environment"]
+ assert len(virtual_learning_environment) == 1
+ assert virtual_learning_environment[0].additional_information == "zd"
+ csirt = [s for s in services if s.service_key == "csirt"]
+ assert len(csirt) == 1
+ assert csirt[0].official_description == "zdf"
+ connectivity = [s for s in services if s.service_key == "connectivity"]
+ assert len(connectivity) == 1
+ assert connectivity[0].product_name == "zfz"
--
GitLab
From 453cfeb5b323748364b3b9312a5b561550d695ad Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Wed, 20 Sep 2023 16:41:50 +0200
Subject: [PATCH 7/7] ensure the required data files are part of the package
---
MANIFEST.in | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 58874ea5..19e22890 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,5 +4,5 @@ include compendium_v2/templates/survey-index.html
include compendium_v2/migrations/alembic.ini
recursive-include compendium_v2/migrations/versions *
recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/resources *
+recursive-include compendium_v2/resources *
recursive-exclude test *
--
GitLab