diff --git a/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx b/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..fdc64ed6d40d85dc137be94e356c845167467f90
Binary files /dev/null and b/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx differ
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
index f28b69fdf40e85824262518e0e2ee8bf45efadd5..9c0e9f91dcf86639d8a812680e9d39a078d7be0e 100644
--- a/compendium_v2/conversion/conversion.py
+++ b/compendium_v2/conversion/conversion.py
@@ -5,12 +5,15 @@ conversion
This module loads the survey data from 2022 from the survey database
and stores the data in the json structure of the new survey, so that
it can be used to prefill the 2023 survey.
+It also loads an xlsx file with the data for the services questions.
"""
import logging
import click
import json
+import os
+import openpyxl
from sqlalchemy import delete, text, select
import compendium_v2
@@ -27,6 +30,8 @@ setup_logging()
logger = logging.getLogger('conversion')
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills 2023_Recovered.xlsx")
+
def query_nren(nren_id: int):
query = mapping.ANSWERS_2022_QUERY.format(nren_id)
@@ -106,7 +111,46 @@ def convert_answers(answers):
return {"data": data}
+def load_service_data():
+ wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+ ws = wb["Sheet1"]
+ rows = list(ws.rows)
+
+ titles = rows[0]
+
+ nren_service_data_columns = {}
+ for i in range(0, 131):
+ if titles[i].value:
+ name = titles[i].value.split(' ')[0].upper()
+ name = {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(name, name)
+ nren_service_data_columns[name] = i
+
+ nren_service_data = {}
+
+ for nren_name, start_column in nren_service_data_columns.items():
+ nren_service_data[nren_name] = {}
+ for row_index in range(2, 61):
+ row = rows[row_index]
+ service_name = row[0].value
+ if row[start_column].value and row[start_column].value.upper() == 'YES':
+ question_data = {"offered": ["yes"]}
+ if row[start_column + 1].value:
+ question_data["name"] = row[start_column + 1].value
+ if row[start_column + 2].value:
+ question_data["additional_information"] = row[start_column + 2].value
+
+ question_key_base = mapping.SERVICES_MAPPING[service_name]
+ question_name, subquestion_name = question_key_base.split(':')
+ question_dict = nren_service_data[nren_name].setdefault(question_name, {})
+ question_dict[subquestion_name] = question_data
+
+ return nren_service_data
+
+
def _cli(app):
+
+ nren_service_data = load_service_data()
+
with app.app_context():
nren_surveys = {}
@@ -115,6 +159,10 @@ def _cli(app):
survey_db_nren_id = mapping.NREN_IDS[nren.name]
nren_surveys[nren] = query_nren(survey_db_nren_id)
+ for nren_name in nren_service_data.keys():
+ if nren_name not in [n.name.upper() for n in nren_surveys.keys()]:
+ raise Exception('NREN in excel not found in source dataset!')
+
db.session.execute(delete(SurveyResponse).where(
SurveyResponse.survey_year == 2022
))
@@ -123,6 +171,7 @@ def _cli(app):
for nren, answers in nren_surveys.items():
survey_dict = convert_answers(answers)
+ survey_dict["data"].update(nren_service_data.get(nren.name.upper(), {}))
survey_dict["page"] = 0
survey_dict["verification_status"] = {}
response = SurveyResponse(
diff --git a/compendium_v2/conversion/mapping.py b/compendium_v2/conversion/mapping.py
index 00c69e2ab26e9382041d6478a8848cc5e0b21ddf..4b9c704bacb9b8d9bc082139cb55c66de3329f51 100644
--- a/compendium_v2/conversion/mapping.py
+++ b/compendium_v2/conversion/mapping.py
@@ -602,3 +602,65 @@ VALUE_TO_CODE_MAPPING = {
"Troubleshooting": "troubleshooting"
}
}
+
+SERVICES_MAPPING = {
+ 'e-portfolio service': 'services_collaboration:e-portfolio',
+ 'Journal access': 'services_collaboration:journal-library-access',
+ 'Mailing lists': 'services_collaboration:mailing-lists',
+ 'Project collaboration': 'services_collaboration:project-collaboration-toolkit',
+ 'Scheduling tool': 'services_collaboration:scheduling-tool',
+ 'Survey/polling tool': 'services_collaboration:survey-tool',
+ 'VLE': 'services_collaboration:virtual-learning-environment',
+ 'VoIP': 'services_collaboration:voip',
+ 'Web hosting': 'services_collaboration:web-email-hosting',
+ 'Eduroam': 'services_identity:eduroam-wifi',
+ 'Hosted campus AAI': 'services_identity:aai',
+ 'Interfederation': 'services_identity:interfederation',
+ 'Domain name registration': 'services_isp:domain-registration',
+ 'IP address allocation': 'services_isp:ip-address-allocation',
+ 'Nameserver services': 'services_isp:nameserver',
+ 'National IX operation': 'services_isp:ix-operation',
+ 'NTP service': 'services_isp:timeserver-ntp',
+ 'Event recording/streaming': 'services_multimedia:videoconferencing',
+ 'Provision of content portal': 'services_multimedia:video-portal',
+ 'TV/radio streaming': 'services_multimedia:internet-radio-tv',
+ 'Web/desktop conferencing': 'services_multimedia:web-conferencing',
+ 'IP Connectivity': 'services_network:connectivity',
+ 'IPv6': 'services_network:ipv6',
+ 'Managed router service': 'services_network:managed-router',
+ 'Multicast': 'services_network:multicast',
+ 'Netflow tool': 'services_network:netflow',
+ 'Network monitoring': 'services_network:network-monitoring',
+ 'Open Lightpath Exchange': 'services_network:open-lightpath-exchange',
+ 'Optical wavelength': 'services_network:lambda',
+ 'PERT': 'services_network:pert',
+ 'Quality of Service': 'services_network:quality-of-service',
+ 'Remote access VPN server': 'services_network:home-vpn',
+ 'Virtual circuit/VPN': 'services_network:point-to-point-circuit-vpn',
+ 'Consultancy/training': 'services_professional:consultancy',
+ 'Dissemination': 'services_professional:dissemination',
+ 'Procurement/brokerage': 'services_professional:procurement',
+ 'Software licenses': 'services_professional:software-licenses',
+ 'User conferences': 'services_professional:user-conference',
+ 'User portals': 'services_professional:user-portal',
+ 'Anti-spam solution': 'services_security:anti-spam',
+ 'CERT/CSIRT': 'services_security:csirt',
+ 'DDoS mitigation': 'services_security:ddos-prevention',
+ 'Firewall-on-demand': 'services_security:firewall-on-demand',
+ 'Identifier Registry': 'services_collaboration:identifier-reg',
+ 'Intrusion detection': 'services_security:intrusion',
+ 'Network troubleshooting': 'services_network:user-monitoring',
+ 'PGP key server': 'services_security:pgp-key',
+ 'Security auditing': 'services_security:security-audit',
+ 'Vulnerability scanning': 'services_security:vulnerability-testing',
+ 'Web filtering': 'services_security:web-filtering',
+ 'Cloud storage (end user)': 'services_hosting:cloud-service-end-user',
+ 'Content delivery hosting': 'services_hosting:content-delivery-hosting',
+ 'Disaster recovery': 'services_hosting:disaster-recovery',
+ 'DNS hosting': 'services_hosting:dns-server',
+ 'Email server hosting': 'services_hosting:email-services',
+ 'Filesender': 'services_hosting:filesender',
+ 'Housing/co-location': 'services_hosting:storage-co-location',
+ 'SaaS': 'services_hosting:saas',
+ 'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
+}
diff --git a/test/test_conversion.py b/test/test_conversion.py
index 4c6f8805771561b17f1a5cf289e29547397bb09f..0f48050554fadf9b9df037cee1d5e76dfc809d89 100644
--- a/test/test_conversion.py
+++ b/test/test_conversion.py
@@ -4,7 +4,7 @@ from sqlalchemy.orm import lazyload
from compendium_v2.db import db
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus
-from compendium_v2.conversion.conversion import _cli, convert_answers
+from compendium_v2.conversion.conversion import _cli, convert_answers, load_service_data
def mock_convert_answers(_):
@@ -15,6 +15,10 @@ def mock_query_nren(_):
return {16455: "answer1"}
+def mock_load_service_data():
+ return {}
+
+
def test_queries(app_with_survey_db, mocker):
with app_with_survey_db.app_context():
@@ -24,6 +28,7 @@ def test_queries(app_with_survey_db, mocker):
mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
+ mocker.patch('compendium_v2.conversion.conversion.load_service_data', mock_load_service_data)
_cli(app_with_survey_db)
@@ -67,3 +72,19 @@ def test_conversion():
}
}
}
+
+
+def test_load_service_data():
+ service_data_dict = load_service_data()
+ assert len(service_data_dict.keys()) == 42
+ assert len(service_data_dict['ARNES'].keys()) == 8
+ assert len(service_data_dict['ARNES']['services_identity']) == 3
+ assert len(service_data_dict['SURF'].keys()) == 7
+ assert len(service_data_dict['SURF']['services_network']) == 10
+ assert service_data_dict['SURF']['services_security']['security-audit'] == {
+ 'offered': ['yes'],
+ 'name': 'SURFaudit',
+ 'additional_information': 'The Standard for Information Security in Higher Education is the basis of '
+ 'SURFaudit’s self-assessment.\n\nSURFaudit Security Standard for Higher '
+ 'Education is derived from the ISO/IEC 27002:2013 standard'
+ }