added data and mapping config for the services data

compendium_v2/conversion/conversion.py

@@ -10,7 +10,9 @@ it can be used to prefill the 2023 survey.
 import logging
 import click
 import json
+import os
 
+import openpyxl
 from sqlalchemy import delete, text, select
 
 import compendium_v2
@@ -27,6 +29,8 @@ setup_logging()
 
 logger = logging.getLogger('conversion')
 
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills 2023_Recovered.xlsx")
+
 
 def query_nren(nren_id: int):
     query = mapping.ANSWERS_2022_QUERY.format(nren_id)
@@ -107,8 +111,44 @@ def convert_answers(answers):
 
 
 def _cli(app):
+
+    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+    ws = wb["Sheet1"]
+    rows = list(ws.rows)
+
+    titles = rows[0]
+
+    nren_service_data_columns = {}
+    for i in range(0, 131):
+        if titles[i].value:
+            name = titles[i].value.split(' ')[0].upper()
+            name = {'KIFÜ': 'KIFU', 'AZSCIENCENET': 'ANAS', 'PSNC': 'PIONIER'}.get(name, name)
+            nren_service_data_columns[name] = i
+
+    nren_service_data = {}
+
+    for nren_name, start_column in nren_service_data_columns.items():
+        for row_index in range(2, 61):
+            row = rows[row_index]
+            service_name = row[0].value
+            question_key_base = mapping.SERVICES_MAPPING[service_name]
+            nren_service_data[nren_name] = {
+                'question_key_base': question_key_base,
+                'offered': row[start_column].value,
+                'service_name': row[start_column + 1].value,
+                'additional': row[start_column + 2].value
+            }
+
     with app.app_context():
 
+        nren_names = set()
+        for nren in db.session.scalars(select(NREN)):
+            nren_names.add(nren.name.upper())
+        for nren_name in nren_service_data.keys():
+            if nren_name not in nren_names:
+                raise Exception('NREN in excel not found in db!')
+                # TODO also check if its right that we dont have ANAS data..
+
         nren_surveys = {}
 
         for nren in db.session.scalars(select(NREN)):
@@ -125,6 +165,7 @@ def _cli(app):
             survey_dict = convert_answers(answers)
             survey_dict["page"] = 0
             survey_dict["verification_status"] = {}
+            # TODO add service data to the dict
             response = SurveyResponse(
                 nren=nren,
                 nren_id=nren.id,

compendium_v2/conversion/mapping.py

@@ -602,3 +602,65 @@ VALUE_TO_CODE_MAPPING = {
         "Troubleshooting": "troubleshooting"
     }
 }
+
+SERVICES_MAPPING = {
+    'e-portfolio service': 'services_collaboration:e-portfolio',
+    'Journal access': 'services_collaboration:journal-library-access',
+    'Mailing lists': 'services_collaboration:mailing-lists',
+    'Project collaboration': 'services_collaboration:project-collaboration-toolkit',
+    'Scheduling tool': 'services_collaboration:scheduling-tool',
+    'Survey/polling tool': 'services_collaboration:survey-tool',
+    'VLE': 'services_collaboration:virtual-learning-environment',
+    'VoIP': 'services_collaboration:voip',
+    'Web hosting': 'services_collaboration:web-email-hosting',
+    'Eduroam': 'services_identity:eduroam-wifi',
+    'Hosted campus AAI': 'services_identity:aai',
+    'Interfederation': 'services_identity:interfederation',
+    'Domain name registration': 'services_isp:domain-registration',
+    'IP address allocation': 'services_isp:ip-address-allocation',
+    'Nameserver services': 'services_isp:nameserver',
+    'National IX operation': 'services_isp:ix-operation',
+    'NTP service': 'services_isp:timeserver-ntp',
+    'Event recording/streaming': 'services_multimedia:videoconferencing',
+    'Provision of content portal': 'services_multimedia:video-portal',
+    'TV/radio streaming': 'services_multimedia:internet-radio-tv',
+    'Web/desktop conferencing': 'services_multimedia:web-conferencing',
+    'IP Connectivity': 'services_network:connectivity',
+    'IPv6': 'services_network:ipv6',
+    'Managed router service': 'services_network:managed-router',
+    'Multicast': 'services_network:multicast',
+    'Netflow tool': 'services_network:netflow',
+    'Network monitoring': 'services_network:network-monitoring',
+    'Open Lightpath Exchange': 'services_network:open-lightpath-exchange',
+    'Optical wavelength': 'services_network:lambda',
+    'PERT': 'services_network:pert',
+    'Quality of Service': 'services_network:quality-of-service',
+    'Remote access VPN server': 'services_network:home-vpn',
+    'Virtual circuit/VPN': 'services_network:point-to-point-circuit-vpn',
+    'Consultancy/training': 'services_professional:consultancy',
+    'Dissemination': 'services_professional:dissemination',
+    'Procurement/brokerage': 'services_professional:procurement',
+    'Software licenses': 'services_professional:software-licenses',
+    'User conferences': 'services_professional:user-conference',
+    'User portals': 'services_professional:user-portal',
+    'Anti-spam solution': 'services_security:anti-spam',
+    'CERT/CSIRT': 'services_security:csirt',
+    'DDoS mitigation': 'services_security:ddos-prevention',
+    'Firewall-on-demand': 'services_security:firewall-on-demand',
+    'Identifier Registry': 'services_collaboration:identifier-reg',
+    'Intrusion detection': 'services_security:intrusion',
+    'Network troubleshooting': 'services_network:user-monitoring',
+    'PGP key server': 'services_security:pgp-key',
+    'Security auditing': 'services_security:security-audit',
+    'Vulnerability scanning': 'services_security:vulnerability-testing',
+    'Web filtering': 'services_security:web-filtering',
+    'Cloud storage (end user)': 'services_hosting:cloud-service-end-user',
+    'Content delivery hosting': 'services_hosting:content-delivery-hosting',
+    'Disaster recovery': 'services_hosting:disaster-recovery',
+    'DNS hosting': 'services_hosting:dns-server',
+    'Email server hosting': 'services_hosting:email-services',
+    'Filesender': 'services_hosting:filesender',
+    'Housing/co-location': 'services_hosting:storage-co-location',
+    'SaaS': 'services_hosting:saas',
+    'Virtual machines/IaaS': 'services_hosting:virtual-machines-iaas',
+}