diff --git a/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx b/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..fdc64ed6d40d85dc137be94e356c845167467f90 Binary files /dev/null and b/compendium_v2/conversion/NREN-Services-prefills 2023_Recovered.xlsx differ diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py index f28b69fdf40e85824262518e0e2ee8bf45efadd5..687964286f87b4a04ed468730457c8465edce011 100644 --- a/compendium_v2/conversion/conversion.py +++ b/compendium_v2/conversion/conversion.py @@ -10,7 +10,9 @@ it can be used to prefill the 2023 survey. import logging import click import json +import os +import openpyxl from sqlalchemy import delete, text, select import compendium_v2 @@ -27,6 +29,8 @@ setup_logging() logger = logging.getLogger('conversion') +EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills 2023_Recovered.xlsx") + def query_nren(nren_id: int): query = mapping.ANSWERS_2022_QUERY.format(nren_id) @@ -107,8 +111,44 @@ def convert_answers(answers): def _cli(app): + + wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True) + ws = wb["Sheet1"] + rows = list(ws.rows) + + titles = rows[0] + + nren_service_data_columns = {} + for i in range(0, 131): + if titles[i].value: + name = titles[i].value.split(' ')[0].upper() + name = {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(name, name) + nren_service_data_columns[name] = i + + nren_service_data = {} + + for nren_name, start_column in nren_service_data_columns.items(): + for row_index in range(2, 61): + row = rows[row_index] + service_name = row[0].value + question_key_base = mapping.SERVICES_MAPPING[service_name] + nren_service_data[nren_name] = { + 'question_key_base': question_key_base, + 'offered': row[start_column].value, + 'service_name': row[start_column + 1].value, + 'additional': row[start_column + 2].value + } + with app.app_context(): + nren_names = set() + for nren in db.session.scalars(select(NREN)): + nren_names.add(nren.name.upper()) + for nren_name in nren_service_data.keys(): + if nren_name not in nren_names: + raise Exception('NREN in excel not found in db!') + # TODO also check if its right that we dont have ANAS data.. + nren_surveys = {} for nren in db.session.scalars(select(NREN)): @@ -125,6 +165,7 @@ def _cli(app): survey_dict = convert_answers(answers) survey_dict["page"] = 0 survey_dict["verification_status"] = {} + # TODO add service data to the dict response = SurveyResponse( nren=nren, nren_id=nren.id, diff --git a/compendium_v2/conversion/mapping.py b/compendium_v2/conversion/mapping.py index 00c69e2ab26e9382041d6478a8848cc5e0b21ddf..4b9c704bacb9b8d9bc082139cb55c66de3329f51 100644 --- a/compendium_v2/conversion/mapping.py +++ b/compendium_v2/conversion/mapping.py @@ -602,3 +602,65 @@ VALUE_TO_CODE_MAPPING = { "Troubleshooting": "troubleshooting" } } + +SERVICES_MAPPING = { + 'e-portfolio service': 'services_collaboration:e-portfolio', + 'Journal access': 'services_collaboration:journal-library-access', + 'Mailing lists': 'services_collaboration:mailing-lists', + 'Project collaboration': 'services_collaboration:project-collaboration-toolkit', + 'Scheduling tool': 'services_collaboration:scheduling-tool', + 'Survey/polling tool': 'services_collaboration:survey-tool', + 'VLE': 'services_collaboration:virtual-learning-environment', + 'VoIP': 'services_collaboration:voip', + 'Web hosting': 'services_collaboration:web-email-hosting', + 'Eduroam': 'services_identity:eduroam-wifi', + 'Hosted campus AAI': 'services_identity:aai', + 'Interfederation': 'services_identity:interfederation', + 'Domain name registration': 'services_isp:domain-registration', + 'IP address allocation': 'services_isp:ip-address-allocation', + 'Nameserver services': 'services_isp:nameserver', + 'National IX operation': 'services_isp:ix-operation', + 'NTP service': 'services_isp:timeserver-ntp', + 'Event recording/streaming': 'services_multimedia:videoconferencing', + 'Provision of content portal': 'services_multimedia:video-portal', + 'TV/radio streaming': 'services_multimedia:internet-radio-tv', + 'Web/desktop conferencing': 'services_multimedia:web-conferencing', + 'IP Connectivity': 'services_network:connectivity', + 'IPv6': 'services_network:ipv6', + 'Managed router service': 'services_network:managed-router', + 'Multicast': 'services_network:multicast', + 'Netflow tool': 'services_network:netflow', + 'Network monitoring': 'services_network:network-monitoring', + 'Open Lightpath Exchange': 'services_network:open-lightpath-exchange', + 'Optical wavelength': 'services_network:lambda', + 'PERT': 'services_network:pert', + 'Quality of Service': 'services_network:quality-of-service', + 'Remote access VPN server': 'services_network:home-vpn', + 'Virtual circuit/VPN': 'services_network:point-to-point-circuit-vpn', + 'Consultancy/training': 'services_professional:consultancy', + 'Dissemination': 'services_professional:dissemination', + 'Procurement/brokerage': 'services_professional:procurement', + 'Software licenses': 'services_professional:software-licenses', + 'User conferences': 'services_professional:user-conference', + 'User portals': 'services_professional:user-portal', + 'Anti-spam solution': 'services_security:anti-spam', + 'CERT/CSIRT': 'services_security:csirt', + 'DDoS mitigation': 'services_security:ddos-prevention', + 'Firewall-on-demand': 'services_security:firewall-on-demand', + 'Identifier Registry': 'services_collaboration:identifier-reg', + 'Intrusion detection': 'services_security:intrusion', + 'Network troubleshooting': 'services_network:user-monitoring', + 'PGP key server': 'services_security:pgp-key', + 'Security auditing': 'services_security:security-audit', + 'Vulnerability scanning': 'services_security:vulnerability-testing', + 'Web filtering': 'services_security:web-filtering', + 'Cloud storage (end user)': 'services_hosting:cloud-service-end-user', + 'Content delivery hosting': 'services_hosting:content-delivery-hosting', + 'Disaster recovery': 'services_hosting:disaster-recovery', + 'DNS hosting': 'services_hosting:dns-server', + 'Email server hosting': 'services_hosting:email-services', + 'Filesender': 'services_hosting:filesender', + 'Housing/co-location': 'services_hosting:storage-co-location', + 'SaaS': 'services_hosting:saas', + 'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas', +}