Fix tests for legacy publisher

compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py

@@ -12,6 +12,111 @@ from compendium_v2.db.survey_models import SurveyResponse, Survey, SurveyStatus,
 from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db import fetch_data as fetch_data_db
 from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_excel import fetch_data as fetch_data_excel
 
+VALID_KEYS_2024_SURVEY = {
+    "abbreviation_english",
+    "abbreviation_national_languages",
+    "alienwave_internal",
+    "alienwave_services",
+    "alienwave_services_number",
+    "audit_specifics",
+    "audits",
+    "budget",
+    "business_continuity_plans",
+    "business_continuity_plans_specifics",
+    "central_procurement_amount",
+    "central_software_procurement",
+    "certificate_service",
+    "certificate_service-Comment",
+    "charging_mechanism",
+    "city",
+    "commercial_charging_levels",
+    "commercial_organizations",
+    "commodity_vs_r_e",
+    "connected_sites_lists",
+    "connected_users_comments",
+    "connectivity_level",
+    "connectivity_proportions",
+    "corporate_strategy",
+    "corporate_strategy_url",
+    "country",
+    "crisis_exercises",
+    "crisis_management_procedure",
+    "dark_fibre_lease",
+    "dark_fibre_lease_duration",
+    "dark_fibre_lease_kilometers_inside_country",
+    "dark_fibre_lease_kilometers_outside_country",
+    "dark_fibre_nren",
+    "dark_fibre_nren_kilometers_inside_country",
+    "ec_project_names",
+    "ec_projects",
+    "email_address",
+    "external_connections",
+    "fibre_light",
+    "fibre_light-Comment",
+    "formal_service_management_framework",
+    "full_name_english",
+    "full_name_national_languages",
+    "income_sources",
+    "max_capacity",
+    "monitoring_tools",
+    "monitoring_tools-Comment",
+    "netflow_vendors",
+    "network_automation",
+    "network_automation_tasks",
+    "network_comments",
+    "network_map_urls",
+    "network_weather",
+    "network_weather_url",
+    "nfv",
+    "nfv_types",
+    "nfv_types-Comment",
+    "non_r_and_e_peers",
+    "operational_process_automation",
+    "operational_process_automation_tools",
+    "organization_comments",
+    "parent_organization",
+    "parent_organization_name",
+    "passive_monitoring",
+    "passive_monitoring_tech",
+    "pert_team",
+    "phone_number",
+    "policies",
+    "policy_comments",
+    "postal_code",
+    "remote_campuses",
+    "remote_campuses_specifics",
+    "security_controls",
+    "security_controls-Comment",
+    "service_comments",
+    "service_level_targets",
+    "service_matrix",
+    "service_portfolio_eosc_portal",
+    "services_collaboration",
+    "services_hosting",
+    "services_identity",
+    "services_isp",
+    "services_multimedia",
+    "services_network",
+    "services_on_eosc_portal_list",
+    "services_professional",
+    "services_security",
+    "siem_soc_vendor",
+    "siem_soc_vendor-Comment",
+    "staff_employment_type",
+    "staff_roles",
+    "street_name_and_number",
+    "suborganization_details",
+    "suborganizations",
+    "traffic_carriers",
+    "traffic_estimate",
+    "traffic_growth",
+    "traffic_load",
+    "traffic_statistics",
+    "traffic_statistics_urls",
+    "typical_capacity",
+    "website"
+}
+
 
 def insert_survey_data(survey_2024: Survey, nren: NREN, year: int, answer: Dict[str, Any]):
     # we're basing the generated survey on the 2024 survey, so we need to make sure that exists
@@ -88,22 +193,9 @@ def ensure_string_tree(tree: Dict[str, Any]):
     return tree
 
 
-@click.command()
-@click.option('--config', type=click.STRING, default='config.json')
-def cli(config):
-    app_config = load(open(config, 'r'))
-
-    app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
+def _cli(app):
 
-    app = compendium_v2._create_app_with_db(app_config)
-    print("survey-publisher-legacy starting")
     with app.app_context():
-        survey_2024 = db.session.query(SurveyResponse).filter(SurveyResponse.survey_year == 2024).all()
-
-        data_2024 = [resp.answers['data'] for resp in survey_2024 if 'data' in resp.answers]
-
-        valid_keys_2024_survey = set(chain(*[a.keys() for a in data_2024]))
-
         nren_map = defaultdict(lambda: defaultdict(lambda: {'data': {}}))
 
         survey_2022 = db.session.query(SurveyResponse).filter(SurveyResponse.survey_year == 2022).all()
@@ -114,7 +206,7 @@ def cli(config):
             nren_map[resp.nren][resp.survey_year]['data'] = resp.answers['data']
 
         for data_key, nren, nren_id, year, value in chain(fetch_data_excel(), fetch_data_db()):
-            if data_key not in valid_keys_2024_survey:
+            if data_key not in VALID_KEYS_2024_SURVEY:
                 print(f'Invalid data key: {data_key} for NREN: {nren} ({nren_id}) in year {year}')
 
             # overwrite the data if we have a new value
@@ -132,8 +224,22 @@ def cli(config):
                     # from 2023 onwards it's all handled by the new survey system.
                     continue
                 data = ensure_string_tree(data)
+                if not data or not data['data']:
+                    continue
                 insert_survey_data(survey_2024, nren, year, data)
 
 
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+    app_config = load(open(config, 'r'))
+
+    app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
+
+    app = compendium_v2._create_app_with_db(app_config)
+    print("survey-publisher-legacy starting")
+    _cli(app)
+
+
 if __name__ == "__main__":
     cli()

test/test_legacy_survey_generation.py

+import os
+import openpyxl
+import json
+
+from compendium_v2.db import db, presentation_models
+from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db import FundingSource, \
+    StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
+from compendium_v2.db.survey_models import Survey, SurveyStatus, SurveyResponse
+from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy import _cli, VALID_KEYS_2024_SURVEY
+
+
+def _test_data(filename):
+    data_filename = os.path.join(os.path.dirname(__file__), 'data', filename)
+
+    with open(data_filename, "r") as f:
+        return json.load(f)
+
+
+def org_data(question):
+    """
+    This function defines test data for the org questions.
+
+    The following data is defined for the appropriate questions as modeled in Compendium:
+
+    nren1,CYNET-CSIRT,cert team
+    nren1,DFN-CERT,CERT
+    nren2,Educampus Services,MIS shared services for third level.
+    nren3,VilniusTech,Technical centre
+    nren3,KU,Technical centre
+    nren3,VDU,Technical centre
+    nren3,VU,Technical centre
+    nren3,org_data
+org_data
+org_data
+org_data
+org_dataKTU,"NOC, administrative authority"
+    """
+
+    if question == OrgQuestion.PARENT_ORG_NAME:
+        return [
+            ('nren1', 'Org1'),
+            ('nren3', 'Org3'),
+        ]
+
+    if str(question.name).endswith('1_NAME'):
+        return [
+            ('nren1', 'CYNET-CSIRT'),
+            ('nren2', 'Educampus Services'),
+            ('nren3', 'VilniusTech'),
+        ]
+
+    if str(question.name).endswith('2_NAME'):
+        return [
+            ('nren1', 'DFN-CERT'),
+            ('nren3', 'KU'),
+        ]
+
+    if str(question.name).endswith('3_NAME'):
+        return [
+            ('nren3', 'VDU'),
+        ]
+
+    if str(question.name).endswith('4_NAME'):
+        return [
+            ('nren3', 'VU'),
+        ]
+
+    if str(question.name).endswith('5_NAME'):
+        return [
+            ('nren3', 'KTU'),
+        ]
+
+    if str(question.name).endswith('1_CHOICE'):
+        return [
+            ('nren1', 'other'),
+            ('nren2', 'other'),
+            ('nren3', 'Technical centre'),
+        ]
+
+    if str(question.name).endswith('2_CHOICE'):
+        return [
+            ('nren1', 'other'),
+            ('nren3', 'Technical centre'),
+        ]
+
+    if str(question.name).endswith('3_CHOICE'):
+        return [
+            ('nren3', 'Technical centre'),
+        ]
+
+    if str(question.name).endswith('4_CHOICE'):
+        return [
+            ('nren3', 'Technical centre'),
+        ]
+
+    if str(question.name).endswith('5_CHOICE'):
+        return [
+            ('nren3', 'other'),
+        ]
+
+    if str(question.name).endswith('1_ROLE'):
+        return [
+            ('nren1', 'cert team'),
+            ('nren2', 'MIS shared services for third level.')
+        ]
+
+    if str(question.name).endswith('2_ROLE'):
+        return [
+            ('nren1', 'CERT'),
+        ]
+
+    if str(question.name).endswith('3_ROLE'):
+        return []
+
+    if str(question.name).endswith('4_ROLE'):
+        return []
+
+    if str(question.name).endswith('5_ROLE'):
+        return [
+            ('nren3', 'NOC, administrative authority')
+        ]
+
+
+def setup_survey_db(app_with_survey_db, mocker):
+    global org_data
+
+    def get_rows_as_tuples(*args, **kwargs):
+        return [
+            ('nren1', '100'),
+            ('nren2', '200'),
+            ('nren3', '300'),
+            ('nren4', 'abcd')
+        ]
+
+    def funding_source_data():
+        yield FundingSource.CLIENT_INSTITUTIONS, [
+            ('nren1', '10'),
+            ('nren2', '80'),
+            ('nren3', '30'),
+        ]
+        yield FundingSource.EUROPEAN_FUNDING, [
+            ('nren1', '50'),
+            ('nren2', '20'),
+            ('nren3', '30'),
+        ]
+        yield FundingSource.OTHER, [
+            ('nren1', '40'),
+            ('nren2', 'abc'),
+            ('nren3', '30'),
+        ]
+
+    def question_data(question):
+        if question == StaffQuestion.NON_TECHNICAL_FTE:
+            return [
+                ('nren1', '10'),
+                ('nren2', '80'),
+                ('nren3', '30'),
+            ]
+
+        if question == StaffQuestion.TECHNICAL_FTE:
+            return [
+                ('nren1', '50'),
+                ('nren2', '20'),
+                ('nren3', '30'),
+            ]
+
+        if question == StaffQuestion.PERMANENT_FTE:
+            return [
+                ('nren1', '60'),
+                ('nren2', 'abc'),
+                ('nren3', '30'),
+            ]
+
+        if question == StaffQuestion.SUBCONTRACTED_FTE:
+            return [
+                ('nren1', '0'),
+                ('nren2', '0'),
+                ('nren3', '0'),
+            ]
+        if question in OrgQuestion:
+            return org_data(question)
+
+        if question == ChargingStructure.charging_structure:
+            return [
+                ('nren1', 'We do not charge them directly'),
+                ('nren2', 'We charge a usage-based fee'),
+                ('nren3', 'Other'),
+            ]
+
+        if question in ECQuestion:
+            return [
+                ('nren1', '[""'),
+                ('nren2', '["project1", "project2"]'),
+                ('nren3', '["project3"]'),
+            ]
+
+    def question_id_data(question_id, year):
+        if question_id in [
+                16469, 16064, 15720, 15305, 14910, 16471, 16066, 15722, 15307, 14912, 16473, 16378,
+                16475, 16068, 15724, 15309, 14914, 16477, 16070, 15726, 15311, 14916, 16479, 16072, 15728, 15575,
+                16481, 16074, 15730, 15577, 16761]:
+            return [
+                ('nren1', f'www.nren.com/somepolicy{year}.pdf'),
+                ('nren2', 'policyemail@nren.com'),
+                ('nren3', 'n.a. online'),
+            ]
+
+    def institutions_urls_data(question_id):
+        if question_id == 16507:
+            return [
+                (87483, 'ANA', 2013, "http://www.rash.al/index.php/network/points-of-presence-pop"),
+                (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"),
+            ]
+        else:
+            return []
+
+    mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_budget',
+                 get_rows_as_tuples)
+    mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_funding_sources',
+                 funding_source_data)
+    mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_question',
+                 question_data)
+    mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_question_id',
+                 question_id_data)
+    mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.recursive_query',
+                 institutions_urls_data)
+
+
+def test_legacy_publisher(app_with_survey_db, mocker, nren_services):
+    setup_survey_db(app_with_survey_db, mocker)
+    nren_services(app_with_survey_db)
+    EXCEL_FILE_ORGANISATION = openpyxl.load_workbook(os.path.join(os.path.dirname(
+        __file__), "data", "2021_Organisation_DataSeries.xlsx"), data_only=True, read_only=True)
+    EXCEL_FILE_USERS = openpyxl.load_workbook(os.path.join(os.path.dirname(
+        __file__), "data", "2022_Connected_Users_DataSeries.xlsx"), data_only=True, read_only=True)
+    EXCEL_FILE_NETWORKS = openpyxl.load_workbook(os.path.join(os.path.dirname(
+        __file__), "data", "2022_Networks_DataSeries.xlsx"), data_only=True, read_only=True)
+    EXCEL_FILE_NREN_SERVICES = openpyxl.load_workbook(os.path.join(os.path.dirname(
+        __file__), "data", "NREN-Services-prefills_2023_Recovered.xlsx"), data_only=True, read_only=True)
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE_ORGANISATION)
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_USERS', EXCEL_FILE_USERS)
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NETWORKS', EXCEL_FILE_NETWORKS)
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NREN_SERVICES', EXCEL_FILE_NREN_SERVICES)
+
+    survey_2024 = _test_data('survey_model_2024_testdata.json')
+
+    nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
+                  'SIKT', 'LAT', 'RASH', 'AzScienceNet', 'GRNET', 'CSC', 'PSNC']
+    with app_with_survey_db.app_context():
+        db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
+        db.session.commit()
+
+    with app_with_survey_db.app_context():
+        survey = Survey(year=2024, survey=survey_2024, status=SurveyStatus.published)
+        db.session.add(survey)
+        db.session.commit()
+
+    _cli(app_with_survey_db)
+
+    with app_with_survey_db.app_context():
+        responses = db.session.query(SurveyResponse).all()
+        assert len(responses) == 78
+
+        nrens = set([resp.nren.name for resp in responses])
+
+        assert len(nrens) == len(nren_names) - 2  # Nren4 and University of Malta have no valid data
+
+        for response in responses:
+            data = response.answers['data']
+            assert data
+            for key in data:
+                assert key in VALID_KEYS_2024_SURVEY

tox.ini

@@ -10,6 +10,8 @@ exclude = .tox,compendium-frontend,survey-frontend,compendium-survey-creator
 
 [coverage:run]
 concurrency = multiprocessing,thread
+; it's legacy stuff, ignore it for coverage
+omit = */survey_publisher_legacy_db* 
 
 [testenv]
 passenv = XDG_CACHE_HOME