Skip to content
Snippets Groups Projects
Commit 90d3a187 authored by Remco Tukker's avatar Remco Tukker
Browse files

Merge branch 'feature/COMP-255_services_prefills' into 'develop'

Conversion of services data

See merge request !66
parents 8b1d3c33 6e8812aa
No related branches found
No related tags found
1 merge request!66Conversion of services data
File added
...@@ -5,12 +5,15 @@ conversion ...@@ -5,12 +5,15 @@ conversion
This module loads the survey data from 2022 from the survey database This module loads the survey data from 2022 from the survey database
and stores the data in the json structure of the new survey, so that and stores the data in the json structure of the new survey, so that
it can be used to prefill the 2023 survey. it can be used to prefill the 2023 survey.
It also loads an xlsx file with the data for the services questions.
""" """
import logging import logging
import click import click
import json import json
import os
import openpyxl
from sqlalchemy import delete, text, select from sqlalchemy import delete, text, select
import compendium_v2 import compendium_v2
...@@ -27,6 +30,8 @@ setup_logging() ...@@ -27,6 +30,8 @@ setup_logging()
logger = logging.getLogger('conversion') logger = logging.getLogger('conversion')
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills 2023_Recovered.xlsx")
def query_nren(nren_id: int): def query_nren(nren_id: int):
query = mapping.ANSWERS_2022_QUERY.format(nren_id) query = mapping.ANSWERS_2022_QUERY.format(nren_id)
...@@ -106,7 +111,46 @@ def convert_answers(answers): ...@@ -106,7 +111,46 @@ def convert_answers(answers):
return {"data": data} return {"data": data}
def load_service_data():
wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
ws = wb["Sheet1"]
rows = list(ws.rows)
titles = rows[0]
nren_service_data_columns = {}
for i in range(0, 131):
if titles[i].value:
name = titles[i].value.split(' ')[0].upper()
name = {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(name, name)
nren_service_data_columns[name] = i
nren_service_data = {}
for nren_name, start_column in nren_service_data_columns.items():
nren_service_data[nren_name] = {}
for row_index in range(2, 61):
row = rows[row_index]
service_name = row[0].value
if row[start_column].value and row[start_column].value.upper() == 'YES':
question_data = {"offered": ["yes"]}
if row[start_column + 1].value:
question_data["name"] = row[start_column + 1].value
if row[start_column + 2].value:
question_data["additional_information"] = row[start_column + 2].value
question_key_base = mapping.SERVICES_MAPPING[service_name]
question_name, subquestion_name = question_key_base.split(':')
question_dict = nren_service_data[nren_name].setdefault(question_name, {})
question_dict[subquestion_name] = question_data
return nren_service_data
def _cli(app): def _cli(app):
nren_service_data = load_service_data()
with app.app_context(): with app.app_context():
nren_surveys = {} nren_surveys = {}
...@@ -115,6 +159,10 @@ def _cli(app): ...@@ -115,6 +159,10 @@ def _cli(app):
survey_db_nren_id = mapping.NREN_IDS[nren.name] survey_db_nren_id = mapping.NREN_IDS[nren.name]
nren_surveys[nren] = query_nren(survey_db_nren_id) nren_surveys[nren] = query_nren(survey_db_nren_id)
for nren_name in nren_service_data.keys():
if nren_name not in [n.name.upper() for n in nren_surveys.keys()]:
raise Exception('NREN in excel not found in source dataset!')
db.session.execute(delete(SurveyResponse).where( db.session.execute(delete(SurveyResponse).where(
SurveyResponse.survey_year == 2022 SurveyResponse.survey_year == 2022
)) ))
...@@ -123,6 +171,7 @@ def _cli(app): ...@@ -123,6 +171,7 @@ def _cli(app):
for nren, answers in nren_surveys.items(): for nren, answers in nren_surveys.items():
survey_dict = convert_answers(answers) survey_dict = convert_answers(answers)
survey_dict["data"].update(nren_service_data.get(nren.name.upper(), {}))
survey_dict["page"] = 0 survey_dict["page"] = 0
survey_dict["verification_status"] = {} survey_dict["verification_status"] = {}
response = SurveyResponse( response = SurveyResponse(
......
...@@ -602,3 +602,65 @@ VALUE_TO_CODE_MAPPING = { ...@@ -602,3 +602,65 @@ VALUE_TO_CODE_MAPPING = {
"Troubleshooting": "troubleshooting" "Troubleshooting": "troubleshooting"
} }
} }
SERVICES_MAPPING = {
'e-portfolio service': 'services_collaboration:e-portfolio',
'Journal access': 'services_collaboration:journal-library-access',
'Mailing lists': 'services_collaboration:mailing-lists',
'Project collaboration': 'services_collaboration:project-collaboration-toolkit',
'Scheduling tool': 'services_collaboration:scheduling-tool',
'Survey/polling tool': 'services_collaboration:survey-tool',
'VLE': 'services_collaboration:virtual-learning-environment',
'VoIP': 'services_collaboration:voip',
'Web hosting': 'services_collaboration:web-email-hosting',
'Eduroam': 'services_identity:eduroam-wifi',
'Hosted campus AAI': 'services_identity:aai',
'Interfederation': 'services_identity:interfederation',
'Domain name registration': 'services_isp:domain-registration',
'IP address allocation': 'services_isp:ip-address-allocation',
'Nameserver services': 'services_isp:nameserver',
'National IX operation': 'services_isp:ix-operation',
'NTP service': 'services_isp:timeserver-ntp',
'Event recording/streaming': 'services_multimedia:videoconferencing',
'Provision of content portal': 'services_multimedia:video-portal',
'TV/radio streaming': 'services_multimedia:internet-radio-tv',
'Web/desktop conferencing': 'services_multimedia:web-conferencing',
'IP Connectivity': 'services_network:connectivity',
'IPv6': 'services_network:ipv6',
'Managed router service': 'services_network:managed-router',
'Multicast': 'services_network:multicast',
'Netflow tool': 'services_network:netflow',
'Network monitoring': 'services_network:network-monitoring',
'Open Lightpath Exchange': 'services_network:open-lightpath-exchange',
'Optical wavelength': 'services_network:lambda',
'PERT': 'services_network:pert',
'Quality of Service': 'services_network:quality-of-service',
'Remote access VPN server': 'services_network:home-vpn',
'Virtual circuit/VPN': 'services_network:point-to-point-circuit-vpn',
'Consultancy/training': 'services_professional:consultancy',
'Dissemination': 'services_professional:dissemination',
'Procurement/brokerage': 'services_professional:procurement',
'Software licenses': 'services_professional:software-licenses',
'User conferences': 'services_professional:user-conference',
'User portals': 'services_professional:user-portal',
'Anti-spam solution': 'services_security:anti-spam',
'CERT/CSIRT': 'services_security:csirt',
'DDoS mitigation': 'services_security:ddos-prevention',
'Firewall-on-demand': 'services_security:firewall-on-demand',
'Identifier Registry': 'services_collaboration:identifier-reg',
'Intrusion detection': 'services_security:intrusion',
'Network troubleshooting': 'services_network:user-monitoring',
'PGP key server': 'services_security:pgp-key',
'Security auditing': 'services_security:security-audit',
'Vulnerability scanning': 'services_security:vulnerability-testing',
'Web filtering': 'services_security:web-filtering',
'Cloud storage (end user)': 'services_hosting:cloud-service-end-user',
'Content delivery hosting': 'services_hosting:content-delivery-hosting',
'Disaster recovery': 'services_hosting:disaster-recovery',
'DNS hosting': 'services_hosting:dns-server',
'Email server hosting': 'services_hosting:email-services',
'Filesender': 'services_hosting:filesender',
'Housing/co-location': 'services_hosting:storage-co-location',
'SaaS': 'services_hosting:saas',
'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
}
...@@ -4,7 +4,7 @@ from sqlalchemy.orm import lazyload ...@@ -4,7 +4,7 @@ from sqlalchemy.orm import lazyload
from compendium_v2.db import db from compendium_v2.db import db
from compendium_v2.db.model import NREN from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus
from compendium_v2.conversion.conversion import _cli, convert_answers from compendium_v2.conversion.conversion import _cli, convert_answers, load_service_data
def mock_convert_answers(_): def mock_convert_answers(_):
...@@ -15,6 +15,10 @@ def mock_query_nren(_): ...@@ -15,6 +15,10 @@ def mock_query_nren(_):
return {16455: "answer1"} return {16455: "answer1"}
def mock_load_service_data():
return {}
def test_queries(app_with_survey_db, mocker): def test_queries(app_with_survey_db, mocker):
with app_with_survey_db.app_context(): with app_with_survey_db.app_context():
...@@ -24,6 +28,7 @@ def test_queries(app_with_survey_db, mocker): ...@@ -24,6 +28,7 @@ def test_queries(app_with_survey_db, mocker):
mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers) mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren) mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
mocker.patch('compendium_v2.conversion.conversion.load_service_data', mock_load_service_data)
_cli(app_with_survey_db) _cli(app_with_survey_db)
...@@ -67,3 +72,19 @@ def test_conversion(): ...@@ -67,3 +72,19 @@ def test_conversion():
} }
} }
} }
def test_load_service_data():
service_data_dict = load_service_data()
assert len(service_data_dict.keys()) == 42
assert len(service_data_dict['ARNES'].keys()) == 8
assert len(service_data_dict['ARNES']['services_identity']) == 3
assert len(service_data_dict['SURF'].keys()) == 7
assert len(service_data_dict['SURF']['services_network']) == 10
assert service_data_dict['SURF']['services_security']['security-audit'] == {
'offered': ['yes'],
'name': 'SURFaudit',
'additional_information': 'The Standard for Information Security in Higher Education is the basis of '
'SURFaudit’s self-assessment.\n\nSURFaudit Security Standard for Higher '
'Education is derived from the ISO/IEC 27002:2013 standard'
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment