diff --git a/MANIFEST.in b/MANIFEST.in
index 58874ea53d89bcd754171ef8f9c65be2a37b38ad..19e228904bea02449a42b500c5cda214201794b6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,5 +4,5 @@ include compendium_v2/templates/survey-index.html
include compendium_v2/migrations/alembic.ini
recursive-include compendium_v2/migrations/versions *
recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/resources *
+recursive-include compendium_v2/resources *
recursive-exclude test *
diff --git a/compendium_v2/conversion/mapping.py b/compendium_v2/conversion/mapping.py
index 1258da6c95a6489910d00980879bf13b2f1d9b41..4a4f7df82496f7f64a7fb3e1c557e925417f2ddb 100644
--- a/compendium_v2/conversion/mapping.py
+++ b/compendium_v2/conversion/mapping.py
@@ -1,3 +1,5 @@
+from compendium_v2.db.presentation_models import ServiceCategory
+
ANSWERS_2022_QUERY = """
SELECT question_id, value
FROM answers a
@@ -664,3 +666,14 @@ SERVICES_MAPPING = {
'SaaS': 'services_hosting:saas',
'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
}
+
+SERVICE_CATEGORY_MAPPING = {
+ 'services_collaboration': ServiceCategory.collaboration.value,
+ 'services_identity': ServiceCategory.identity.value,
+ 'services_isp': ServiceCategory.isp_support.value,
+ 'services_multimedia': ServiceCategory.multimedia.value,
+ 'services_network': ServiceCategory.network_services.value,
+ 'services_professional': ServiceCategory.professional_services.value,
+ 'services_security': ServiceCategory.security.value,
+ 'services_hosting': ServiceCategory.storage_and_hosting.value,
+}
diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py
index 9c18348bedd094da8aa1501df596eb8df2e68793..a25200e7a21a3b0551b75e09b36994939bc76df3 100644
--- a/compendium_v2/db/presentation_models.py
+++ b/compendium_v2/db/presentation_models.py
@@ -471,3 +471,25 @@ class NetworkAutomation(db.Model):
year: Mapped[int_pk]
network_automation: Mapped[YesNoPlanned]
network_automation_specifics: Mapped[json_str_list]
+
+
+class Service(db.Model):
+ __tablename__ = 'service'
+
+ name_key: Mapped[str128_pk]
+ name: Mapped[str128]
+ category: Mapped[ServiceCategory]
+ description: Mapped[str]
+
+
+class NRENService(db.Model):
+ __tablename__ = 'nren_service'
+
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
+ service: Mapped[Service] = relationship(lazy='joined')
+ product_name: Mapped[str128]
+ additional_information: Mapped[str]
+ official_description: Mapped[str]
diff --git a/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c3dc3964133f843a7842756766aa77106c101cd
--- /dev/null
+++ b/compendium_v2/migrations/versions/1fbc4582c0ab_added_service_and_nrenservice_tables.py
@@ -0,0 +1,55 @@
+"""added Service and NRENService tables
+
+Revision ID: 1fbc4582c0ab
+Revises: 87e1e35051a0
+Create Date: 2023-09-19 22:23:36.448210
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '1fbc4582c0ab'
+down_revision = '87e1e35051a0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table(
+ 'service',
+ sa.Column('name_key', sa.String(length=128), nullable=False),
+ sa.Column('name', sa.String(), nullable=False),
+ sa.Column('category', postgresql.ENUM(
+ 'network_services', 'isp_support', 'security', 'identity',
+ 'collaboration', 'multimedia', 'storage_and_hosting',
+ 'professional_services', name='servicecategory',
+ create_type=False), nullable=False
+ ),
+ sa.Column('description', sa.String(), nullable=False),
+ sa.PrimaryKeyConstraint('name_key', name=op.f('pk_service'))
+ )
+ op.create_table(
+ 'nren_service',
+ sa.Column('nren_id', sa.Integer(), nullable=False),
+ sa.Column('year', sa.Integer(), nullable=False),
+ sa.Column('service_key', sa.String(length=128), nullable=False),
+ sa.Column('product_name', sa.String(), nullable=False),
+ sa.Column('additional_information', sa.String(), nullable=False),
+ sa.Column('official_description', sa.String(), nullable=False),
+ sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_nren_service_nren_id_nren')),
+ sa.ForeignKeyConstraint(
+ ['service_key'], ['service.name_key'], name=op.f('fk_nren_service_service_key_service')
+ ),
+ sa.PrimaryKeyConstraint('nren_id', 'year', 'service_key', name=op.f('pk_nren_service'))
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('nren_service')
+ op.drop_table('service')
+ # ### end Alembic commands ###
diff --git a/compendium_v2/publishers/excel_parser.py b/compendium_v2/publishers/excel_parser.py
index 41f7c0149b3f4cb9a89c30e42dbf9304a336cb25..ce1efff48e133c142f4d888ec3186bed05a5734a 100644
--- a/compendium_v2/publishers/excel_parser.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -1,22 +1,25 @@
import logging
+
import openpyxl
-import os
-from compendium_v2.db.presentation_model_enums import FeeType
+from compendium_v2.conversion import mapping
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
+from compendium_v2.resources import get_resource_file_path
setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_ORGANISATION = get_resource_file_path("2021_Organisation_DataSeries.xlsx")
+EXCEL_FILE_NETWORKS = get_resource_file_path("2022_Networks_DataSeries.xlsx")
+EXCEL_FILE_NREN_SERVICES = get_resource_file_path("NREN-Services-prefills_2023_Recovered.xlsx")
def fetch_budget_excel_data():
# load the xlsx file
sheet_name = "1. Budget"
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
ws = wb[sheet_name]
@@ -36,7 +39,7 @@ def fetch_budget_excel_data():
def fetch_funding_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "2. Income Sources"
@@ -119,7 +122,7 @@ def fetch_funding_excel_data():
def fetch_charging_structure_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "3. Charging mechanism"
@@ -186,7 +189,7 @@ def fetch_charging_structure_excel_data():
def fetch_staffing_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "4. Staff"
@@ -234,7 +237,7 @@ def fetch_staffing_excel_data():
def fetch_staff_function_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "5. Staff by Function"
@@ -302,7 +305,7 @@ def fetch_staff_function_excel_data():
def fetch_ecproject_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "7. EC Projects"
@@ -334,7 +337,7 @@ def fetch_ecproject_excel_data():
def fetch_organization_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_ORGANISATION, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Organization"
@@ -352,7 +355,7 @@ def fetch_organization_excel_data():
def fetch_traffic_excel_data():
# load the xlsx file
- wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
+ wb = openpyxl.load_workbook(EXCEL_FILE_NETWORKS, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Estimated_Traffic TByte"
@@ -392,3 +395,48 @@ def fetch_traffic_excel_data():
yield from create_points_for_year(2020, 14)
yield from create_points_for_year(2021, 8)
yield from create_points_for_year(2022, 2)
+
+
+def fetch_nren_services_excel_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE_NREN_SERVICES, data_only=True, read_only=True)
+ ws = wb["Sheet1"]
+ rows = list(ws.rows)
+
+ titles = rows[0]
+
+ nren_service_data_columns = {}
+
+ def normalize_nren_name(n: str) -> str:
+ n = n.split(' ')[0].upper()
+ return {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(n, n)
+
+ for i in range(0, 131):
+ if titles[i].value:
+ name = normalize_nren_name(titles[i].value)
+ nren_service_data_columns[name] = i
+
+ for nren_name, start_column in nren_service_data_columns.items():
+ for row_index in range(2, 61):
+ row = rows[row_index]
+ service_name = row[0].value
+ if row[start_column].value and row[start_column].value.upper() == 'YES':
+ product_name = ''
+ additional_information = ''
+ if row[start_column + 1].value:
+ product_name = row[start_column + 1].value
+ if row[start_column + 2].value:
+ additional_information = row[start_column + 2].value
+
+ service_category, service_name_key = mapping.SERVICES_MAPPING[service_name].split(':')
+ service_category = mapping.SERVICE_CATEGORY_MAPPING[service_category]
+
+ yield {
+ 'nren_name': nren_name,
+ 'service_name': service_name,
+ 'service_category': service_category,
+ 'service_name_key': service_name_key,
+ 'year': 2022,
+ 'product_name': product_name.strip(),
+ 'additional_information': additional_information.strip(),
+ 'official_description': '',
+ }
diff --git a/compendium_v2/publishers/survey_publisher.py b/compendium_v2/publishers/survey_publisher.py
index 10a50baf7e32a762aaa251cd79fbac3b65d4fddf..ccd4b0940343d79fa4d35d4ce97bb5aaebaee602 100644
--- a/compendium_v2/publishers/survey_publisher.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -23,7 +23,7 @@ from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure,
CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight, \
NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors, \
CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio, \
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService
from compendium_v2.db.presentation_model_enums import CarryMechanism, CommercialCharges, YesNoPlanned, \
CommercialConnectivityCoverage, ConnectivityCoverage, FeeType, MonitoringMethod, ServiceCategory, UserCategory
from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
@@ -64,7 +64,7 @@ def _map_2023(nren, answers) -> None:
CommercialChargingLevel, RemoteCampuses, DarkFibreLease, DarkFibreInstalled, FibreLight,
NetworkMapUrls, MonitoringTools, PassiveMonitoring, TrafficStatistics, SiemVendors,
CertificateProviders, WeatherMap, PertTeam, AlienWave, Capacity, NonREPeers, TrafficRatio,
- OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation]:
+ OpsAutomation, NetworkFunctionVirtualisation, NetworkAutomation, NRENService]:
db.session.execute(delete(table_class).where(table_class.year == year)) # type: ignore
answers = answers["data"]
@@ -516,6 +516,22 @@ def _map_2023(nren, answers) -> None:
network_automation_specifics=network_automation_tasks
))
+ all_services = {}
+ for question_name in ["services_collaboration", "services_hosting", "services_identity", "services_isp",
+ "services_multimedia", "services_network", "services_professional", "services_security"]:
+ all_services.update(answers.get(question_name, {}))
+
+ for service_key, answers in all_services.items():
+ offered = answers.get("offered")
+ if offered == ["yes"]:
+ db.session.add(NRENService(
+ nren_id=nren.id, nren=nren, year=year,
+ service_key=service_key,
+ product_name=answers.get("name", ""),
+ additional_information=answers.get("additional_information", ""),
+ official_description=answers.get("description", "")
+ ))
+
def publish(year):
responses = db.session.scalars(
diff --git a/compendium_v2/publishers/survey_publisher_legacy_excel.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
index 6503b79faeda6e296c9fe2bf043658e95f4b8c5b..dfb6b6b51574f2b78cc93d77536d1d16103209d7 100644
--- a/compendium_v2/publishers/survey_publisher_legacy_excel.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -8,18 +8,21 @@ Registered as click cli command when installing compendium-v2.
"""
from __future__ import annotations
+
+import json
import logging
import math
-import click
+import click
from sqlalchemy import select
import compendium_v2
-from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.db import db, presentation_models
-from compendium_v2.survey_db import model as survey_model
+from compendium_v2.environment import setup_logging
from compendium_v2.publishers import helpers, excel_parser
+from compendium_v2.resources import get_resource_file_path
+from compendium_v2.survey_db import model as survey_model
setup_logging()
@@ -224,7 +227,49 @@ def db_traffic_volume_migration(nren_dict):
db.session.commit()
-def _cli(config, app):
+def db_services_migration():
+ with open(get_resource_file_path('nren_services.json')) as f:
+ services = json.load(f)
+ for name_key, record in services.items():
+ service = presentation_models.Service(
+ name_key=name_key,
+ name=record['name'],
+ category=record['category'],
+ description=record['description'],
+ )
+ db.session.merge(service)
+ db.session.commit()
+
+
+def db_nren_services_migration(nren_dict):
+ services = [s for s in db.session.scalars(select(presentation_models.Service))]
+
+ for service_info in excel_parser.fetch_nren_services_excel_data():
+ [service] = [s for s in services if s.name_key == service_info['service_name_key']]
+
+ abbrev = service_info['nren_name']
+ if abbrev not in nren_dict:
+ logger.warning(f'{abbrev} unknown. Skipping.')
+ continue
+
+ nren = nren_dict[abbrev]
+ nren_service = presentation_models.NRENService(
+ nren=nren,
+ nren_id=nren.id,
+ year=service_info['year'],
+ service=service,
+ service_key=service.name_key,
+ product_name=service_info['product_name'],
+ additional_information=service_info['additional_information'],
+ official_description=service_info['official_description']
+ )
+
+ db.session.merge(nren_service)
+
+ db.session.commit()
+
+
+def _cli(app):
with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict()
db_budget_migration(nren_dict)
@@ -234,6 +279,8 @@ def _cli(config, app):
db_ecprojects_migration(nren_dict)
db_organizations_migration(nren_dict)
db_traffic_volume_migration(nren_dict)
+ db_services_migration()
+ db_nren_services_migration(nren_dict)
@click.command()
@@ -245,7 +292,7 @@ def cli(config):
app = compendium_v2._create_app_with_db(app_config)
print("survey-publisher-v1 starting")
- _cli(app_config, app)
+ _cli(app)
if __name__ == "__main__":
diff --git a/compendium_v2/resources/__init__.py b/compendium_v2/resources/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2499aeb09afcf692f1532633362b5e0d8dd3647
--- /dev/null
+++ b/compendium_v2/resources/__init__.py
@@ -0,0 +1,6 @@
+from pathlib import Path
+
+
+def get_resource_file_path(filename: str) -> Path:
+ resource_dir = Path(__file__).resolve().parent
+ return resource_dir / filename
diff --git a/compendium_v2/resources/nren_services.json b/compendium_v2/resources/nren_services.json
new file mode 100644
index 0000000000000000000000000000000000000000..29d14104ef2d6dfed5d9d90b4c8c7cb917d29020
--- /dev/null
+++ b/compendium_v2/resources/nren_services.json
@@ -0,0 +1,372 @@
+{
+ "anti-spam": {
+ "name": "Anti-spam solution",
+ "category": "security",
+ "description": "Anti-spam solutions for detecting and eliminating viruses and spam mails."
+ },
+ "class-registration": {
+ "name": "Class registration tool",
+ "category": "collaboration",
+ "description": "Software for teachers to register students for classes etc."
+ },
+ "cloud-service-end-user": {
+ "name": "Cloud storage (end user)",
+ "category": "storage_and_hosting",
+ "description": "Browser based virtual storage service for individuals."
+ },
+ "connectivity": {
+ "name": "IP connectivity",
+ "category": "network_services",
+ "description": "Basic IP connectivity services inc. R+E and commodity internet"
+ },
+ "consultancy": {
+ "name": "Consultancy/training",
+ "category": "professional_services",
+ "description": "Training and consultancy services provided by the NREN."
+ },
+ "content-delivery-hosting": {
+ "name": "Content delivery hosting",
+ "category": "storage_and_hosting",
+ "description": "Hosting of content delivery servers e.g. Akamai."
+ },
+ "content-management-system": {
+ "name": "CMS",
+ "category": "collaboration",
+ "description": "Provision of software systems for website authoring, collaboration and administration tools."
+ },
+ "csirt": {
+ "name": "CERT/CSIRT",
+ "category": "security",
+ "description": "A single point of contact for users to deal with computer security incidents and prevention."
+ },
+ "ddos-prevention": {
+ "name": "DDos mitigation",
+ "category": "security",
+ "description": "Tools and techniques for mitigating Distributed Denial of Service attacks."
+ },
+ "domain-registration": {
+ "name": "Domain name registration",
+ "category": "isp_support",
+ "description": "Administration/registration of top and second level domain names."
+ },
+ "eduroam-wifi": {
+ "name": "Eduroam",
+ "category": "identity",
+ "description": "Inter-WLAN service to facilitate easy and secure Internet access for roaming educational users."
+ },
+ "email-services": {
+ "name": "Email server hosting",
+ "category": "collaboration",
+ "description": "NREN hosted email servers."
+ },
+ "filesender": {
+ "name": "Filesender",
+ "category": "storage_and_hosting",
+ "description": "Web based application that allows authenticated users to securely and easily send arbitrarily large files."
+ },
+ "firewall-on-demand": {
+ "name": "Firewall-on-demand",
+ "category": "security",
+ "description": "Provision of a dynamic firewall services to mitigate against DDOS attacks. "
+ },
+ "home-vpn": {
+ "name": "Remote access VPN service",
+ "category": "network_services",
+ "description": "Remote access and site-to-site secure VPN."
+ },
+ "aai": {
+ "name": "Hosted campus AAI",
+ "category": "identity",
+ "description": "Hosting of an Identity Provider service on behalf of connected institutions to authenticate users."
+ },
+ "instant-messaging": {
+ "name": "Instant messaging",
+ "category": "collaboration",
+ "description": "Commercial or NREN own online chat service which offers real-time text transmission over the Internet."
+ },
+ "interfederation": {
+ "name": "Interfederation",
+ "category": "identity",
+ "description": "Participation in an inter-federation (i.e. eduGAIN, KALMAR)."
+ },
+ "internet-radio-tv": {
+ "name": "TV/radio streaming",
+ "category": "multimedia",
+ "description": "Internet and radio streaming services."
+ },
+ "internship-database": {
+ "name": "Database services",
+ "category": "collaboration",
+ "description": "Provision of tools or services to create or host databases."
+ },
+ "ip-address-allocation": {
+ "name": "IP address allocation/LIR",
+ "category": "isp_support",
+ "description": "Local Internet Registry service for assigning of IP address space."
+ },
+ "ipv6": {
+ "name": "IPv6",
+ "category": "network_services",
+ "description": "The new version of the Internet Protocol (IP) that should eventually replace the current IP (version 4)."
+ },
+ "ix-operation": {
+ "name": "National IX operation",
+ "category": "isp_support",
+ "description": "Operation of a national Internet Exchange."
+ },
+ "journal-library-access": {
+ "name": "Journal access",
+ "category": "collaboration",
+ "description": "Access to academic journals."
+ },
+ "lambda": {
+ "name": "Optical wavelength",
+ "category": "network_services",
+ "description": "Layer 1 optical channel for provision of dedicated capacity to demanding users."
+ },
+ "mailing-lists": {
+ "name": "Mailing lists",
+ "category": "collaboration",
+ "description": "Service for operation of electronic discussion lists."
+ },
+ "mobile-data": {
+ "name": "3G/4G data service",
+ "category": "network_services",
+ "description": "Provision of mobile broadband and phone contracts to users."
+ },
+ "multicast": {
+ "name": "Multicast",
+ "category": "network_services",
+ "description": "Extension to the IP protocol which allows individual packets to be sent to multiple hosts on the Internet."
+ },
+ "news-service": {
+ "name": "Netnews/USENET service",
+ "category": "storage_and_hosting",
+ "description": "Netnews/USENET news feed service."
+ },
+ "online-payment": {
+ "name": "Online payment connectivity",
+ "category": "identity",
+ "description": "Connectivity for chip and pin payment terminals."
+ },
+ "open-lightpath-exchange": {
+ "name": "Open Lightpath Exchange",
+ "category": "network_services",
+ "description": "Provision of an Open Lightpath exchange for users to connect to other parties."
+ },
+ "pert": {
+ "name": "PERT",
+ "category": "network_services",
+ "description": "Team supporting resolution of end-to-end performance problems for networked applications."
+ },
+ "plagarism-detection": {
+ "name": "Plagarism detection",
+ "category": "collaboration",
+ "description": "Provision of software for use by teachers etc to detect plagarism."
+ },
+ "point-to-point-circuit-vpn": {
+ "name": "Virtual circuit/VPN",
+ "category": "network_services",
+ "description": "Virtual point to point circuits or VPNs delivered as a service to users."
+ },
+ "procurement": {
+ "name": "Procurement/brokerage",
+ "category": "professional_services",
+ "description": "Procurement services, inc. negotiating agreements and framework agreements."
+ },
+ "project-collaboration-toolkit": {
+ "name": "Project collaboration tools",
+ "category": "collaboration",
+ "description": "Packaged services for virtual project groups e.g. mailing lists, storage, web meetings, wiki."
+ },
+ "quality-of-service": {
+ "name": "Quality of Service",
+ "category": "network_services",
+ "description": "Preferential service to specific applications or classes of applications."
+ },
+ "scheduling-tool": {
+ "name": "Scheduling tool",
+ "category": "collaboration",
+ "description": "Provision of tools to users for scheduling appointments or classes."
+ },
+ "sdn-testbed": {
+ "name": "SDN testbed",
+ "category": "network_services",
+ "description": "Software defined networking testbed available to users."
+ },
+ "sms-messaging": {
+ "name": "SMS messaging",
+ "category": "collaboration",
+ "description": "Service for users to send or receive SMS message by email."
+ },
+ "software-development": {
+ "name": "Software development",
+ "category": "professional_services",
+ "description": "Software development service for users."
+ },
+ "software-licenses": {
+ "name": "Software licenses",
+ "category": "professional_services",
+ "description": "Provision of software for organisational or institutional purchase."
+ },
+ "storage-co-location": {
+ "name": "Housing/co-location",
+ "category": "storage_and_hosting",
+ "description": "Hosting of user equipment in a managed data centre."
+ },
+ "survey-tool": {
+ "name": "Survey/polling tool",
+ "category": "collaboration",
+ "description": "Provision of applications for creating surveys or polls."
+ },
+ "system-backup": {
+ "name": "Hot standby",
+ "category": "storage_and_hosting",
+ "description": "Failover protection for primary web servers."
+ },
+ "timeserver-ntp": {
+ "name": "NTP service",
+ "category": "isp_support",
+ "description": "Allows the synchronization of computer clocks over the Internet."
+ },
+ "video-portal": {
+ "name": "Provision of content portal/s to users for hosting and viewing multi-media content.",
+ "category": "multimedia",
+ "description": "Multi-media content portal"
+ },
+ "videoconferencing": {
+ "name": "Event recording/streaming",
+ "category": "multimedia",
+ "description": "Provision of equipment and/or software to support event streaming/recording."
+ },
+ "virtual-learning-environment": {
+ "name": "VLE",
+ "category": "collaboration",
+ "description": "Online e-learning education system that provides virtual access to resources used in teaching.."
+ },
+ "virtual-machines-iaas": {
+ "name": "Virtual machines/IaaS",
+ "category": "storage_and_hosting",
+ "description": "Access to virtual computing resources."
+ },
+ "voip": {
+ "name": "VoIP",
+ "category": "collaboration",
+ "description": "Service to deliver voice communications and multimedia sessions over Internet Protocol (IP) networks."
+ },
+ "vulnerability-testing": {
+ "name": "Vulnerability scanning",
+ "category": "security",
+ "description": "Vulnerability service that allows users to scan their own IP networks for security holes."
+ },
+ "web-conferencing": {
+ "name": "Web/desktop conferencing",
+ "category": "multimedia",
+ "description": "Video conferencing service to desktops and hand-held devices using software."
+ },
+ "web-design-production": {
+ "name": "Web development",
+ "category": "professional_services",
+ "description": "Web development service provided to NREN users."
+ },
+ "web-email-hosting": {
+ "name": "Web hosting",
+ "category": "storage_and_hosting",
+ "description": "Service to provide space on central web servers for users to publish their website."
+ },
+ "dns-server": {
+ "name": "DNS hosting",
+ "category": "storage_and_hosting",
+ "description": "Hosting of primary and secondary DNS servers."
+ },
+ "dissemination": {
+ "name": "Dissemination",
+ "category": "professional_services",
+ "description": "Dissemination of information to users e.g. newsletters and magazines."
+ },
+ "network-monitoring": {
+ "name": "Network monitoring",
+ "category": "network_services",
+ "description": "Network information system that shows the current and past performance of the network."
+ },
+ "disaster-recovery": {
+ "name": "Disaster recovery",
+ "category": "storage_and_hosting",
+ "description": "Off site backup services."
+ },
+ "user-monitoring": {
+ "name": "Network troubleshooting",
+ "category": "network_services",
+ "description": "Enables users at connected institutions to monitor Internet services in real time."
+ },
+ "netflow": {
+ "name": "Netflow tool",
+ "category": "network_services",
+ "description": "Network protocol for collecting IP traffic information and monitoring network traffic."
+ },
+ "managed-router": {
+ "name": "Managed router service",
+ "category": "network_services",
+ "description": "Remote network router support to institutions."
+ },
+ "intrusion": {
+ "name": "Intrusion detection",
+ "category": "security",
+ "description": "Systems for detecting and preventing Intrusions (IDS/IPS)."
+ },
+ "security-audit": {
+ "name": "Security auditing",
+ "category": "security",
+ "description": "Carrying out vulnerability assesments and security reviews of user systems and resources on their behalf."
+ },
+ "web-filtering": {
+ "name": "Web filtering",
+ "category": "security",
+ "description": "Centralised web content filtering service for protection against access to inappropriate content."
+ },
+ "pgp-key": {
+ "name": "PGP key server",
+ "category": "security",
+ "description": "Operation of PGP key server."
+ },
+ "identifier-reg": {
+ "name": "Identifier registry",
+ "category": "security",
+ "description": "Registering of unique and automatically-processable identifiers in the form of text or numeric strings."
+ },
+ "post-production": {
+ "name": "Media post production",
+ "category": "multimedia",
+ "description": "Work undertaken after the process of recording (or production) e.g. editing, synchronisation."
+ },
+ "e-portfolio": {
+ "name": "e-portfolio service",
+ "category": "collaboration",
+ "description": "Functions to create and share user professional and career e-portfolios."
+ },
+ "user-conference": {
+ "name": "User conferences",
+ "category": "professional_services",
+ "description": "Hosting of regular user conferences."
+ },
+ "user-portal": {
+ "name": "User portals",
+ "category": "professional_services",
+ "description": "User portal for service management and monitoring."
+ },
+ "admin-tools": {
+ "name": "Finance/admin systems",
+ "category": "professional_services",
+ "description": "Provision of ICT systems used in finance and administration."
+ },
+ "nameserver": {
+ "name": "Nameserver services",
+ "category": "isp_support",
+ "description": "Operation of nameservers and maintenance of DNS information on behalf of users."
+ },
+ "saas": {
+ "name": "SaaS",
+ "category": "storage_and_hosting",
+ "description": "Software as a service e.g. GoogleApps for Education."
+ }
+}
\ No newline at end of file
diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py
index 9af16ad2ced857457ba14daa49cf8ab841a9c81b..cc7ac6e0d5c8530611cea3acae7a38fb052f4979 100644
--- a/compendium_v2/routes/api.py
+++ b/compendium_v2/routes/api.py
@@ -16,6 +16,7 @@ from compendium_v2.routes.nren import routes as nren_routes
from compendium_v2.routes.response import routes as response_routes
from compendium_v2.routes.traffic import routes as traffic_routes
from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
+from compendium_v2.routes.nren_services import routes as nren_services_routes
routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget')
@@ -31,6 +32,7 @@ routes.register_blueprint(nren_routes, url_prefix='/nren')
routes.register_blueprint(response_routes, url_prefix='/response')
routes.register_blueprint(traffic_routes, url_prefix='/traffic')
routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
+routes.register_blueprint(nren_services_routes, url_prefix='/nren-services')
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 8af1bbb67cd51df9a922ca0be240ba4278856713..45c71287bb1ed025810df7377e0d2c4f39a6a352 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -1,10 +1,8 @@
from typing import Any
from flask import Blueprint, jsonify
-from sqlalchemy import select
-from compendium_v2.db import db
-from compendium_v2.db.presentation_models import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
@@ -40,7 +38,10 @@ def institutions_urls_view() -> Any:
or organizations connected to the NREN.
Many NRENs have one or more pages on their website listing such user institutions.
- response will be formatted as per INSTITUTION_URLS_RESPONSE_SCHEMA
+ response will be formatted as:
+
+ .. asjson::
+ compendium_v2.routes.institutions_urls.INSTITUTION_URLS_RESPONSE_SCHEMA
:return:
"""
@@ -54,8 +55,7 @@ def institutions_urls_view() -> Any:
}
entries = []
- records = db.session.scalars(
- select(InstitutionURLs).join(NREN).order_by(NREN.name.asc(), InstitutionURLs.year.desc()))
+ records = common.get_data(InstitutionURLs)
for entry in records:
entries.append(_extract_data(entry))
diff --git a/compendium_v2/routes/nren_services.py b/compendium_v2/routes/nren_services.py
new file mode 100644
index 0000000000000000000000000000000000000000..f4ed08b9a7242e9111e68988e06b29c27abd4fa6
--- /dev/null
+++ b/compendium_v2/routes/nren_services.py
@@ -0,0 +1,66 @@
+from typing import Any
+
+from flask import Blueprint, jsonify
+
+from compendium_v2.db.presentation_models import NRENService
+from compendium_v2.routes import common
+
+routes = Blueprint('nren-services', __name__)
+
+NREN_SERVICES_RESPONSE_SCHEMA = {
+ '$schema': 'http://json-schema.org/draft-07/schema#',
+ 'definitions': {
+ 'nren_services': {
+ 'type': 'object',
+ 'properties': {
+ 'nren': {'type': 'string'},
+ 'nren_country': {'type': 'string'},
+ 'year': {'type': 'integer'},
+ 'product_name': {'type': 'string'},
+ 'additional_information': {'type': 'string'},
+ 'official_description': {'type': 'string'},
+ 'service_name': {'type': 'string'},
+ 'service_category': {'type': 'string'},
+ 'service_description': {'type': 'string'}
+ },
+ 'required': ['nren', 'nren_country', 'year', 'product_name', 'additional_information',
+ 'official_description', 'service_name', 'service_category', 'service_description'],
+ 'additionalProperties': False
+ }
+ },
+ 'type': 'array',
+ 'items': {'$ref': '#/definitions/nren_services'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def nren_service_view() -> Any:
+ """
+ handler for /api/nren-services/ requests
+ Endpoint for getting the service matrix that shows which NREN uses which services.
+
+ response will be formatted as:
+
+ .. asjson::
+ compendium_v2.routes.nren_services.NREN_SERVICES_RESPONSE_SCHEMA
+
+ :return:
+ """
+
+ def _extract_data(nren_service: NRENService) -> dict:
+ return {
+ 'nren': nren_service.nren.name,
+ 'nren_country': nren_service.nren.country,
+ 'year': nren_service.year,
+ 'product_name': nren_service.product_name,
+ 'additional_information': nren_service.additional_information,
+ 'official_description': nren_service.official_description,
+ 'service_name': nren_service.service.name,
+ 'service_category': nren_service.service.category.value,
+ 'service_description': nren_service.service.description
+ }
+
+ entries = [_extract_data(entry) for entry in common.get_data(NRENService)]
+
+ return jsonify(entries)
diff --git a/test/conftest.py b/test/conftest.py
index 591fe482257bc5bf6e96244309c9f7ed484dc57f..7bb203ae7cbed1d9b25d4ada1f0c3281b4cd509c 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -7,6 +7,7 @@ from sqlalchemy import select
from flask_login import LoginManager # type: ignore
import compendium_v2
from compendium_v2.db import db, presentation_models, survey_models
+from compendium_v2.db.presentation_model_enums import ServiceCategory
from compendium_v2.survey_db import model as survey_db_model
from compendium_v2.auth.session_management import setup_login_manager, User, ROLES
@@ -408,3 +409,60 @@ def test_institution_urls_data(app):
created_nrens = _create_and_save_nrens(unique_nren_names)
_create_and_save_institution_urls(predefined_nrens_and_years, created_nrens)
db.session.commit()
+
+
+@pytest.fixture
+def test_nren_services_data(app):
+ def _create_and_save_nrens(nren_names):
+ nrens = {}
+ for nren_name in nren_names:
+ nren_instance = presentation_models.NREN(name=nren_name, country='country')
+ nrens[nren_name] = nren_instance
+ db.session.add(nren_instance)
+ return nrens
+
+ def _create_and_save_services(services):
+ services_map = {}
+ for service in services:
+ service_instance = presentation_models.Service(
+ name=service[0],
+ name_key=service[1],
+ description='description',
+ category=service[2],
+ )
+ services_map[service[1]] = service_instance
+ db.session.add(service_instance)
+
+ return services_map
+
+ def _create_and_save_nren_services(nren_services, nrens, services):
+ for nren_name, year, service_name, service_name_key, service_category in nren_services:
+ nren_instance = nrens[nren_name]
+ service_instance = services[service_name_key]
+
+ institution_urls_model = presentation_models.NRENService(
+ nren=nren_instance,
+ year=year,
+ service=service_instance,
+ product_name="brand name",
+ additional_information="extra info",
+ official_description="description"
+ )
+
+ db.session.add(institution_urls_model)
+
+ with app.app_context():
+ predefined_nrens_services = [
+ ('nren1', 2019, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren1', 2020, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren1', 2021, 'service1', 'service1_key', ServiceCategory.network_services.value),
+ ('nren2', 2019, 'service2', 'service2_key', ServiceCategory.professional_services.value),
+ ('nren2', 2021, 'service2', 'service2_key', ServiceCategory.professional_services.value)
+ ]
+
+ unique_nren_names = {nren[0] for nren in predefined_nrens_services}
+ unique_services = {nren[2:] for nren in predefined_nrens_services}
+ created_nrens = _create_and_save_nrens(unique_nren_names)
+ created_services = _create_and_save_services(unique_services)
+ _create_and_save_nren_services(predefined_nrens_services, created_nrens, created_services)
+ db.session.commit()
diff --git a/test/test_nren_services.py b/test/test_nren_services.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce832e6e1147c5f973e70c2baccea0434e326995
--- /dev/null
+++ b/test/test_nren_services.py
@@ -0,0 +1,14 @@
+import json
+import jsonschema
+
+from compendium_v2.routes.nren_services import NREN_SERVICES_RESPONSE_SCHEMA
+
+
+def test_nren_services(client, test_nren_services_data):
+ rv = client.get(
+ '/api/nren-services/',
+ headers={'Accept': ['application/json']})
+ assert rv.status_code == 200
+ result = json.loads(rv.data.decode('utf-8'))
+ jsonschema.validate(result, NREN_SERVICES_RESPONSE_SCHEMA)
+ assert result
diff --git a/test/test_survey_publisher.py b/test/test_survey_publisher.py
index 3e66d6f0605d1a62195f49355dc3bd70bbe61c65..2889eebbc26d928de2af57e3df5d346fd1aee742 100644
--- a/test/test_survey_publisher.py
+++ b/test/test_survey_publisher.py
@@ -306,3 +306,15 @@ def test_v2_publisher_full(app):
assert network_automation.network_automation_specifics == [
"config_management", "provisioning", "data_collection", "compliance", "reporting", "troubleshooting"
]
+
+ services = [s for s in db.session.scalars(select(presentation_models.NRENService))]
+ assert len(services) == 59
+ virtual_learning_environment = [s for s in services if s.service_key == "virtual-learning-environment"]
+ assert len(virtual_learning_environment) == 1
+ assert virtual_learning_environment[0].additional_information == "zd"
+ csirt = [s for s in services if s.service_key == "csirt"]
+ assert len(csirt) == 1
+ assert csirt[0].official_description == "zdf"
+ connectivity = [s for s in services if s.service_key == "connectivity"]
+ assert len(connectivity) == 1
+ assert connectivity[0].product_name == "zfz"
diff --git a/test/test_survey_publisher_legacy_excel.py b/test/test_survey_publisher_legacy_excel.py
index 5a20fc063e843f3d930ce463034f5f910762bf5b..b066767e579ce05fe14f1dcba005b7e0af2276ff 100644
--- a/test/test_survey_publisher_legacy_excel.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -9,15 +9,15 @@ from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
-def test_publisher(app_with_survey_db, mocker, dummy_config):
- mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
+def test_excel_publisher(app_with_survey_db, mocker):
+ mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE)
with app_with_survey_db.app_context():
nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
- _cli(dummy_config, app_with_survey_db)
+ _cli(app_with_survey_db)
with app_with_survey_db.app_context():
budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
@@ -94,3 +94,13 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert len(asnet2021) == 1
assert asnet2021[0].organization\
== 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia'
+
+ service_data = db.session.scalars(select(presentation_models.Service)).all()
+ assert len(service_data) == 74
+
+ nren_service_data = db.session.scalars(select(presentation_models.NRENService)).all()
+ # test a random entry
+ sikt2022 = [x for x in nren_service_data
+ if x.nren.name == 'SIKT' and x.year == 2022 and x.service.name == 'Journal access']
+ assert len(sikt2022) == 1
+ assert sikt2022[0].additional_information.startswith("Sikt negotiates license a")
diff --git a/test/test_db_survey_publisher_2022.py b/test/test_survey_publisher_old_db_2022.py
similarity index 100%
rename from test/test_db_survey_publisher_2022.py
rename to test/test_survey_publisher_old_db_2022.py