diff --git a/Changelog.md b/Changelog.md index c3179f4b5a9eff5133eb0bd7a41218978b115095..938f2cebb998464086adc5a79ea4d2f33c0e5a3f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [0.80] - 2025-01-21 +- Add missing legacy_publisher module due to missing init file + ## [0.79] - 2025-01-21 - COMP-281: Add delete user functionality to the user management page - General improvements to the user management page diff --git a/compendium_v2/publishers/legacy_publisher/__init__.py b/compendium_v2/publishers/legacy_publisher/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py b/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py index 33ce1c94a8fe354555e4aff3ac63bf0f61131529..696609c0dfc449abe5c8daf0072580583d700770 100644 --- a/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py +++ b/compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py @@ -12,6 +12,111 @@ from compendium_v2.db.survey_models import SurveyResponse, Survey, SurveyStatus, from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db import fetch_data as fetch_data_db from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_excel import fetch_data as fetch_data_excel +VALID_KEYS_2024_SURVEY = { + "abbreviation_english", + "abbreviation_national_languages", + "alienwave_internal", + "alienwave_services", + "alienwave_services_number", + "audit_specifics", + "audits", + "budget", + "business_continuity_plans", + "business_continuity_plans_specifics", + "central_procurement_amount", + "central_software_procurement", + "certificate_service", + "certificate_service-Comment", + "charging_mechanism", + "city", + "commercial_charging_levels", + "commercial_organizations", + "commodity_vs_r_e", + "connected_sites_lists", + "connected_users_comments", + "connectivity_level", + "connectivity_proportions", + "corporate_strategy", + "corporate_strategy_url", + "country", + "crisis_exercises", + "crisis_management_procedure", + "dark_fibre_lease", + "dark_fibre_lease_duration", + "dark_fibre_lease_kilometers_inside_country", + "dark_fibre_lease_kilometers_outside_country", + "dark_fibre_nren", + "dark_fibre_nren_kilometers_inside_country", + "ec_project_names", + "ec_projects", + "email_address", + "external_connections", + "fibre_light", + "fibre_light-Comment", + "formal_service_management_framework", + "full_name_english", + "full_name_national_languages", + "income_sources", + "max_capacity", + "monitoring_tools", + "monitoring_tools-Comment", + "netflow_vendors", + "network_automation", + "network_automation_tasks", + "network_comments", + "network_map_urls", + "network_weather", + "network_weather_url", + "nfv", + "nfv_types", + "nfv_types-Comment", + "non_r_and_e_peers", + "operational_process_automation", + "operational_process_automation_tools", + "organization_comments", + "parent_organization", + "parent_organization_name", + "passive_monitoring", + "passive_monitoring_tech", + "pert_team", + "phone_number", + "policies", + "policy_comments", + "postal_code", + "remote_campuses", + "remote_campuses_specifics", + "security_controls", + "security_controls-Comment", + "service_comments", + "service_level_targets", + "service_matrix", + "service_portfolio_eosc_portal", + "services_collaboration", + "services_hosting", + "services_identity", + "services_isp", + "services_multimedia", + "services_network", + "services_on_eosc_portal_list", + "services_professional", + "services_security", + "siem_soc_vendor", + "siem_soc_vendor-Comment", + "staff_employment_type", + "staff_roles", + "street_name_and_number", + "suborganization_details", + "suborganizations", + "traffic_carriers", + "traffic_estimate", + "traffic_growth", + "traffic_load", + "traffic_statistics", + "traffic_statistics_urls", + "typical_capacity", + "website" +} + def insert_survey_data(survey_2024: Survey, nren: NREN, year: int, answer: Dict[str, Any]): # we're basing the generated survey on the 2024 survey, so we need to make sure that exists @@ -88,22 +193,9 @@ def ensure_string_tree(tree: Dict[str, Any]): return tree -@click.command() -@click.option('--config', type=click.STRING, default='config.json') -def cli(config): - app_config = load(open(config, 'r')) - - app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']} +def _cli(app): - app = compendium_v2._create_app_with_db(app_config) - print("survey-publisher-legacy starting") with app.app_context(): - survey_2024 = db.session.query(SurveyResponse).filter(SurveyResponse.survey_year == 2024).all() - - data_2024 = [resp.answers['data'] for resp in survey_2024 if 'data' in resp.answers] - - valid_keys_2024_survey = set(chain(*[a.keys() for a in data_2024])) - nren_map = defaultdict(lambda: defaultdict(lambda: {'data': {}})) survey_2022 = db.session.query(SurveyResponse).filter(SurveyResponse.survey_year == 2022).all() @@ -114,7 +206,7 @@ def cli(config): nren_map[resp.nren][resp.survey_year]['data'] = resp.answers['data'] for data_key, nren, nren_id, year, value in chain(fetch_data_excel(), fetch_data_db()): - if data_key not in valid_keys_2024_survey: + if data_key not in VALID_KEYS_2024_SURVEY: print(f'Invalid data key: {data_key} for NREN: {nren} ({nren_id}) in year {year}') # overwrite the data if we have a new value @@ -132,8 +224,22 @@ def cli(config): # from 2023 onwards it's all handled by the new survey system. continue data = ensure_string_tree(data) + if not data or not data['data']: + continue insert_survey_data(survey_2024, nren, year, data) +@click.command() +@click.option('--config', type=click.STRING, default='config.json') +def cli(config): + app_config = load(open(config, 'r')) + + app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']} + + app = compendium_v2._create_app_with_db(app_config) + print("survey-publisher-legacy starting") + _cli(app) + + if __name__ == "__main__": cli() diff --git a/setup.py b/setup.py index 6775135cfa627a29e2d6f73d9bdcf1c3822df901..c5e3ec076f3551ece99f98ef4f540d7f37737241 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='compendium-v2', - version="0.79", + version="0.80", author='GEANT', author_email='swd@geant.org', description='Flask and React project for displaying ' diff --git a/test/data/survey_model_2024_testdata.json b/test/data/survey_model_2024_testdata.json new file mode 100644 index 0000000000000000000000000000000000000000..50efb32312904edb37c04f2c540b08cbc70d7d57 --- /dev/null +++ b/test/data/survey_model_2024_testdata.json @@ -0,0 +1,2782 @@ +{ + "title": "Compendium", + "logoPosition": "right", + "pages": [ + { + "name": "organization", + "elements": [ + { + "type": "panel", + "name": "panel1", + "elements": [ + { + "type": "text", + "name": "budget", + "title": "What is your NREN's budget for {surveyyear} in million euro?", + "description": "Note that this question has changed slightly and now asks for the year we are still in. If your budget is not per calendar year, please provide figures for the budget that covers the largest part of {surveyyear} including GEANT subsidy. When responding to this question, please include all parts of your organisation serving your users, bearing in mind the services you deliver, as referenced later in the survey.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "type": "multipletext", + "name": "income_sources", + "title": "Estimate (in % of income) the sources of your NREN-related income for {surveyyear}.", + "description": "European Funding should include GÉANT funding.", + "validators": [ + { + "type": "expression", + "text": "The percentages should add up to 100", + "expression": "(sum({income_sources.client_institutions}, {income_sources.gov_public_bodies}, {income_sources.european_funding}, {income_sources.commercial}, {income_sources.other}) = 100) or ({income_sources} empty)" + } + ], + "items": [ + { + "name": "client_institutions", + "title": "Client institutions (universities/schools/research institutes/commercial/other)", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "gov_public_bodies", + "title": "Government/public bodies", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "european_funding", + "title": "European funding", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "commercial", + "title": "Commercial services (e.g. domain reg, security)", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "other", + "title": "Other", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + } + ] + }, + { + "type": "dropdown", + "name": "charging_mechanism", + "title": "How do you charge your client institutions?", + "choices": [ + { + "value": "no_charge", + "text": "We do not charge them directly" + }, + { + "value": "flat_fee", + "text": "We charge a flat fee, based on bandwidth" + }, + { + "value": "usage_based_fee", + "text": "We charge a usage-based fee" + }, + { + "value": "combination", + "text": "We use a combination of flat fee and usage-based fee" + }, + { + "value": "other", + "text": "Other" + } + ] + } + ], + "title": "Budget, Income and Billing" + }, + { + "type": "panel", + "name": "staff", + "elements": [ + { + "type": "multipletext", + "name": "staff_employment_type", + "title": "What is the number of staff engaged in the NREN activities? (In Full Time Equivalent (FTEs))", + "description": "When responding to this question, please include all parts of your organisation serving your users, bearing in mind the services you deliver, as referenced later in the survey.", + "items": [ + { + "name": "permanent_fte", + "title": "Permanent staff", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "subcontracted_fte", + "title": "Subcontracted staff", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + } + ] + }, + { + "type": "multipletext", + "name": "staff_roles", + "title": "How much FTE is devoted to roles in the following functional areas?", + "items": [ + { + "name": "technical_fte", + "title": "Technical roles", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "nontechnical_fte", + "title": "Non-technical roles", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + } + ] + }, + { + "type": "radiogroup", + "name": "parent_organization", + "title": "Is your NREN part of a larger organisation (e.g. ministry, university)?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "parent_organization_name", + "visibleIf": "{parent_organization} = 'Yes'", + "indent": 1, + "title": "What is the name of this larger organisation?" + }, + { + "type": "radiogroup", + "name": "suborganizations", + "title": "Does your NREN have sub-organisations?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "suborganization_details", + "visibleIf": "{suborganizations} = 'Yes'", + "indent": 1, + "title": "Please fill in the details:", + "columns": [ + { + "name": "suborganization_name", + "title": "Name of your sub-organisation?", + "cellType": "text", + "isRequired": true + }, + { + "name": "suborganization_role", + "title": "Role of your sub-organisation?", + "cellType": "dropdown", + "choices": [ + { + "value": "idc", + "text": "IDC federation" + }, + { + "value": "hpc", + "text": "HPC centre" + } + ], + "showOtherItem": true + } + ], + "rowCount": 1, + "maxRowCount": 30 + }, + { + "type": "radiogroup", + "name": "ec_projects", + "title": "Other than GÉANT, is your NREN involved in other EC projects?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "ec_project_names", + "visibleIf": "{ec_projects} = 'Yes'", + "indent": 1, + "title": "Please list the name of the projects", + "description": "Use one line per project.", + "columns": [ + { + "name": "ec_project_name", + "title": "Project name", + "cellType": "text" + } + ], + "rowCount": 1 + } + ], + "title": "Staff and Projects" + }, + { + "type": "panel", + "name": "basic_information", + "elements": [ + { + "type": "text", + "name": "full_name_english", + "title": "Full name of the organisation (in English):" + }, + { + "type": "text", + "name": "full_name_national_languages", + "title": "Full name of the organisation in the national language(s):", + "description": "Please use the Latin alphabet." + }, + { + "type": "text", + "name": "abbreviation_english", + "title": "Abbreviation in English (if applicable):", + "description": "Please always provide this information." + }, + { + "type": "text", + "name": "abbreviation_national_languages", + "title": "Abbreviation in the national language(s) (if applicable):", + "description": "Please always provide this information. Please use the Latin alphabet." + }, + { + "type": "text", + "name": "street_name_and_number", + "title": "Number and street name:" + }, + { + "type": "text", + "name": "city", + "title": "Postal town (county):" + }, + { + "type": "text", + "name": "postal_code", + "title": "Postal Code:" + }, + { + "type": "text", + "name": "country", + "title": "Country (in English):" + }, + { + "type": "text", + "name": "phone_number", + "title": "Phone number:", + "inputType": "tel" + }, + { + "type": "text", + "name": "email_address", + "title": "General email address:", + "validators": [ + { + "type": "email" + } + ], + "inputType": "email" + }, + { + "type": "text", + "name": "website", + "title": "Website:", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + } + ], + "title": "Basic Information" + }, + { + "type": "comment", + "name": "organization_comments", + "title": "Comments regarding this section:" + } + ], + "title": "Organisation" + }, + { + "name": "standards_and_policies", + "elements": [ + { + "type": "panel", + "name": "policy", + "elements": [ + { + "type": "radiogroup", + "name": "corporate_strategy", + "title": "Have you made any updates to your corporate strategy over the last year?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "corporate_strategy_url", + "title": "Please provide the URL for your latest (corporate) strategic plan?", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + }, + { + "type": "matrixdropdown", + "name": "policies", + "title": "Does your NREN have the following policies?", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "available", + "title": "Policy available", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "url", + "title": "Please provide the URL", + "cellType": "text", + "visibleIf": "{row.available} = ['yes']", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "environmental_policy", + "text": "Environmental Policy" + }, + { + "value": "equal_opportunity_policy", + "text": "Equality Opportunity Policy" + }, + { + "value": "gender_equality_policy", + "text": "Gender Equality Plan" + }, + { + "value": "connectivity_policy", + "text": "Connectivity Policy" + }, + { + "value": "acceptable_use_policy", + "text": "Acceptable Use Policy (AUP)" + }, + { + "value": "privacy_notice", + "text": "Organisational Privacy Policy" + }, + { + "value": "data_protection_contact", + "text": "Dedicated contact for data protection, privacy or GDPR queries" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "radiogroup", + "name": "central_software_procurement", + "title": "Do you centrally procure software for your customers?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "central_procurement_amount", + "visibleIf": "{central_software_procurement} = 'yes'", + "indent": 1, + "title": "What is the total amount (in Euro) that you procured in {surveyyear} or {previousyear}/{surveyyear} on behalf of your customers?", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "radiogroup", + "name": "formal_service_management_framework", + "title": "Does your NREN operate a formal service management framework for all of your services?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "radiogroup", + "name": "service_level_targets", + "title": "Are Service Level Targets available for your NREN services?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdropdown", + "name": "service_matrix", + "title": "Which service types are available for which user types?", + "columns": [ + { + "name": "service_types", + "title": "Service types", + "cellType": "checkbox", + "showInMultipleColumns": true, + "choices": [ + { + "value": "network_services", + "text": "Network services" + }, + { + "value": "isp_support", + "text": "ISP support" + }, + { + "value": "security", + "text": "Security" + }, + { + "value": "identity", + "text": "Identity/T&I" + }, + { + "value": "collaboration", + "text": "Collaboration" + }, + { + "value": "multimedia", + "text": "Multimedia" + }, + { + "value": "storage_and_hosting", + "text": "Storage and Hosting" + }, + { + "value": "professional_services", + "text": "Professional services" + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + }, + { + "type": "radiogroup", + "name": "service_portfolio_eosc_portal", + "title": "Are any of the services in your service portfolio listed on the EOSC portal?", + "readOnly": true, + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "services_on_eosc_portal_list", + "visibleIf": "{service_portfolio_eosc_portal} = 'Yes'", + "readOnly": true, + "indent": 1, + "title": "Can you list them?", + "columns": [ + { + "name": "service_name", + "title": "Service name", + "cellType": "text" + } + ], + "rowCount": 1, + "maxRowCount": 50 + } + ], + "title": "Policy & Portfolio" + }, + { + "type": "panel", + "name": "standards", + "elements": [ + { + "type": "radiogroup", + "name": "audits", + "title": "Do you have external or internal audits of your information security management systems e.g. risk management and policies?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "audit_specifics", + "visibleIf": "{audits} = 'yes'", + "indent": 1, + "title": "Please specify (for example a certified security auditor on ISO 27001 is performing the audits):" + }, + { + "type": "radiogroup", + "name": "business_continuity_plans", + "title": "Do you have Business Continuity plans in place to ensure business continuation and operations?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "business_continuity_plans_specifics", + "visibleIf": "{business_continuity_plans} = 'yes'", + "indent": 1, + "title": "Please specify if you comply with any international standard and if you test the continuity plans regularly." + }, + { + "type": "radiogroup", + "name": "crisis_management_procedure", + "title": "Does your NREN have a formal crisis management procedure?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "checkbox", + "name": "crisis_exercises", + "title": "Does your NREN run or participate in operational security measures and other exercises including crisis awareness to train and test employees?", + "description": "Multiple answers possible, in the last 12 months", + "choices": [ + { + "value": "geant_workshops", + "text": "We participate in GÉANT crisis workshops such as CLAW" + }, + { + "value": "national_excercises", + "text": "We participated in national crisis exercises " + }, + { + "value": "tabletop_exercises", + "text": "We run our own tabletop exercises" + }, + { + "value": "simulation_excercises", + "text": "We run our own simulation exercises" + }, + { + "value": "other_excercises", + "text": " We have done/participated in other exercises or trainings" + }, + { + "value": "real_crisis", + "text": "We had a real crisis" + }, + { + "value": "internal_security_programme", + "text": "We run an internal security awareness programme" + }, + { + "value": "none", + "text": "No, we have not done any crisis exercises or trainings" + } + ] + }, + { + "type": "checkbox", + "name": "security_controls", + "title": "Do you utilise security controls such as anti-virus, integrity checkers and systemic firewalls to protect your assets?", + "choices": [ + { + "value": "anti_virus", + "text": "Anti Virus" + }, + { + "value": "anti_spam", + "text": "Anti-Spam" + }, + { + "value": "firewall", + "text": "Firewall" + }, + { + "value": "ddos_mitigation", + "text": "DDoS mitigation" + }, + { + "value": "monitoring", + "text": "Network monitoring" + }, + { + "value": "ips_ids", + "text": "IPS/IDS" + }, + { + "value": "acl", + "text": "ACL" + }, + { + "value": "segmentation", + "text": "Network segmentation" + }, + { + "value": "integrity_checking", + "text": "Integrity checking" + } + ], + "showOtherItem": true + } + ], + "title": "Standards" + }, + { + "type": "comment", + "name": "policy_comments", + "title": "Comments regarding this section:" + } + ], + "title": "Standards & Policies" + }, + { + "name": "connected_users", + "elements": [ + { + "type": "panel", + "name": "panel2", + "elements": [ + { + "type": "matrixdynamic", + "name": "connected_sites_lists", + "title": "Please provide the URLs of the webpages listing the institutions or organisations that are connected to the NREN, if available:", + "description": "Many NRENs have one or more pages on their website listing user institutions. Please provide these links here if you have them.", + "columns": [ + { + "name": "connected_sites_url", + "title": "Url:", + "cellType": "text", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + } + ], + "rowCount": 1, + "maxRowCount": 50 + }, + { + "type": "matrixdropdown", + "name": "connectivity_proportions", + "title": "Please give an estimate of the proportion of the institutions in each category for which your NREN provides IP connectivity: ", + "description": "For the questions in this section, please use the ISCED 2011 classification system (the UNESCO scheme for International Standard Classification of Education) as follows: * Level 8 - Doctorate or equivalent level * Level 7 - Masters or equivalent level * Level 6 - Bachelors or equivalent level * Level 5 - Short-cycle tertiary education * Level 4 - Post-secondary non-tertiary education (This can include, for example, short vocational training programmes) * Levels 2 and 3: Secondary education * Level 1: Primary or basic education * Level 0: Pre-primary education", + "columns": [ + { + "name": "covered", + "title": "Does your remit cover connectivity to this institution type:", + "cellType": "dropdown", + "choices": [ + { + "value": "yes_incl_other", + "text": "Yes - including transit to other networks" + }, + { + "value": "yes_national_nren", + "text": "Yes - national NREN access" + }, + { + "value": "sometimes", + "text": "In some circumstances" + }, + { + "value": "no_policy", + "text": "No - not eligible for policy reasons" + }, + { + "value": "no_financial", + "text": "No - financial restrictions (NREN is unable to charge/cover costs)" + }, + { + "value": "no_other", + "text": "No - other reason" + }, + { + "value": "unsure", + "text": "Unsure/unclear" + } + ] + }, + { + "name": "nr_connected", + "title": "Number of institutions connected in this category (actual number):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please use a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "market_share_percentage", + "title": "% market share of institutions connected in this category:", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "nr_of_users", + "title": "Number of users served in this category (actual number):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please use a whole number", + "regex": "^[0-9]*$" + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + }, + { + "type": "matrixdropdown", + "name": "connectivity_level", + "title": "Level of IP connectivity by Institution type:", + "description": "This table explores the average level of connectivity for each type of institution to the NREN. Please enter the typical and the highest capacity at which institutions in this category are connected ( in Mbit/s). As a minimum, please provide this information for Universities and Research Institutes.", + "columns": [ + { + "name": "typical_speed", + "title": "Typical link speed (Mbit/s):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please use a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "highest_speed", + "title": "Highest speed link (Mbit/s):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please use a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "highest_speed_connection_percentage", + "title": "Proportionally how many institutions in this category are connected at the highest capacity? (%):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + }, + { + "type": "matrixdropdown", + "name": "traffic_carriers", + "title": "How is the traffic carried?", + "columns": [ + { + "name": "carry_mechanism", + "title": "Carry mechanism", + "cellType": "dropdown", + "choices": [ + { + "value": "nren_local_loops", + "text": "NREN provides the local loops" + }, + { + "value": "regional_nren_backbone", + "text": "Traffic carried to the backbone by regional NREN" + }, + { + "value": "commercial_provider_backbone", + "text": "Traffic carried to the backbone by commercial providers" + }, + { + "value": "man", + "text": "MAN" + }, + { + "value": "other", + "text": "Other" + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + }, + { + "type": "matrixdropdown", + "name": "traffic_load", + "title": "What are the traffic loads in Mbit/s?", + "columns": [ + { + "name": "average_from_institutions_to_network", + "title": "Average load from institutions to the network", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "average_to_institutions_from_network", + "title": "Average load to institutions from the network", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "peak_from_institutions_to_network", + "title": "Peak load from institutions to the network", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "name": "peak_to_institutions_from_network", + "title": "Peak load to institutions from the network", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + }, + { + "type": "matrixdropdown", + "name": "traffic_growth", + "title": "What do you expect the traffic growth to be in the next 3 years?", + "columns": [ + { + "name": "growth_rate", + "title": "% growth", + "cellType": "text", + "validators": [ + { + "type": "numeric" + } + ] + } + ], + "rows": [ + { + "value": "universities", + "text": "Universities & Other (ISCED 6-8)" + }, + { + "value": "further_education", + "text": "Further education (ISCED 4-5)" + }, + { + "value": "secondary_schools", + "text": "Secondary schools (ISCED 2-3)" + }, + { + "value": "primary_schools", + "text": "Primary schools (ISCED 1)" + }, + { + "value": "institutes", + "text": "Research Institutes" + }, + { + "value": "cultural", + "text": "Libraries, Museums, Archives, Cultural institutions" + }, + { + "value": "hospitals", + "text": "Non-university public Hospitals" + }, + { + "value": "government", + "text": "Government departments (national, regional, local)" + }, + { + "value": "iros", + "text": "International (virtual) research organisations" + }, + { + "value": "for_profit_orgs", + "text": "For-profit organisations" + } + ] + } + ], + "title": "CONNECTED USERS" + }, + { + "type": "panel", + "name": "connected_users_commercial", + "elements": [ + { + "type": "matrixdropdown", + "name": "commercial_organizations", + "title": "What types of commercial organisations do you connect?", + "columns": [ + { + "name": "connection", + "title": "Connection", + "cellType": "dropdown", + "choices": [ + { + "value": "yes_incl_other", + "text": "Yes - Including transit to other networks" + }, + { + "value": "yes_national_nren", + "text": "Yes - National NREN access only" + }, + { + "value": "yes_if_sponsored", + "text": "Yes - only if sponsored by a connected institution" + }, + { + "value": "no_but_direct_peering", + "text": "No - but we offer a direct or IX peering" + }, + { + "value": "no_policy", + "text": "No - not eligible for policy reasons" + }, + { + "value": "no_financial", + "text": "No - financial restrictions (NREN is unable to charge/recover costs)" + }, + { + "value": "no_other", + "text": "No - other reason / unsure" + } + ] + } + ], + "rows": [ + { + "value": "commercial_r_e", + "text": "Commercial R&E traffic only" + }, + { + "value": "commercial_general", + "text": "Commercial general" + }, + { + "value": "commercial_collaboration", + "text": "Commercial for collaboration only (project/time limited)" + }, + { + "value": "commercial_service_provider", + "text": "Commercial Service Provider" + }, + { + "value": "university_spin_off", + "text": "University Spin Off/Incubator" + } + ] + }, + { + "type": "matrixdropdown", + "name": "commercial_charging_levels", + "title": "What are the typical charging levels for the following types of commercial connections? Please tick all that apply:", + "columns": [ + { + "name": "charging_level", + "title": "Charging level", + "cellType": "dropdown", + "choices": [ + { + "value": "higher_than_r_e_charges", + "text": "Charges typically higher than for R&E users" + }, + { + "value": "same_as_r_e_charges", + "text": "Same charging model as for R&E users" + }, + { + "value": "no_charges_if_r_e_requested", + "text": "No charges applied if requested by R&E users" + }, + { + "value": "lower_than_r_e_charges", + "text": "Charges typically lower than for R&E users" + } + ] + } + ], + "rows": [ + { + "value": "collaboration", + "text": "Connection to your network for collaboration with R&E users" + }, + { + "value": "services", + "text": "Connection to your network for supplying services for R&E" + }, + { + "value": "peering", + "text": "Direct peering (e.g. direct peering or cloud peering)" + } + ] + }, + { + "type": "radiogroup", + "name": "remote_campuses", + "title": "Do you provide connectivity to any remote campuses in other countries?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "remote_campuses_specifics", + "visibleIf": "{remote_campuses} = 'Yes'", + "indent": 1, + "title": "Please specify:", + "columns": [ + { + "name": "country", + "title": "Which Country", + "cellType": "text", + "isRequired": true + }, + { + "name": "connected", + "title": "Connected to local R&E network", + "cellType": "radiogroup", + "choices": [ + "Yes", + "No" + ] + } + ], + "rowCount": 0, + "maxRowCount": 50 + }, + { + "type": "comment", + "name": "connected_users_comments", + "title": "Comments regarding this section:" + } + ], + "title": "CONNECTED USERS - COMMERCIAL" + } + ], + "title": "Connected Users" + }, + { + "name": "network", + "elements": [ + { + "type": "panel", + "name": "connectivity", + "elements": [ + { + "type": "radiogroup", + "name": "dark_fibre_lease", + "title": "Does your NREN have an IRU or lease of dark fibre?", + "description": "An Indefeasible Right of Use (IRU) is essentially a long-term lease of a portion of the capacity of a cable. Please do not include fibre that you have installed and own yourself, this is covered in a later question.", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "dark_fibre_lease_kilometers_inside_country", + "visibleIf": "{dark_fibre_lease} = 'Yes'", + "indent": 1, + "title": "Please state the number of kilometres of such fibre in your country:", + "description": "Please include only the fibre inside your country for this answer. The distance is the number of kilometers of your fibre pairs or if you are using bidirectional traffic on a single fibre please treat this as a fibre pair.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "text", + "name": "dark_fibre_lease_kilometers_outside_country", + "visibleIf": "{dark_fibre_lease} = 'Yes'", + "indent": 1, + "title": "Please state the number of kilometers of such fibre that is outside your country:", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "text", + "name": "dark_fibre_lease_duration", + "visibleIf": "{dark_fibre_lease} = 'Yes'", + "indent": 1, + "title": "What is the average duration, in years, of your IRU?", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "type": "radiogroup", + "name": "dark_fibre_nren", + "title": "Has your NREN physically laid any dark fibre cables in your network?", + "description": "Please include only cables that you laid yourself. If this cable was installed and is owned by a third party, this is included in a previous question.", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "dark_fibre_nren_kilometers_inside_country", + "visibleIf": "{dark_fibre_nren} = 'Yes'", + "indent": 1, + "title": "Please state the number of kilometers of such fibre in your network:", + "description": "Please include only the fibre inside your country for this answer. The distance is the number of kilometers of your fibre pairs, or If you are using bi-directional traffic on a single fibre please treat this as a fibre pair.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "radiogroup", + "name": "fibre_light", + "title": "How do you light your fibre network:", + "choices": [ + { + "value": "nren_owns_and_operates", + "text": "NREN owns and operates equipment" + }, + { + "value": "nren_owns_outsourced_operation", + "text": "NREN owns equipment and operation is outsourced" + }, + { + "value": "outsourced_ownership_and_operation", + "text": "Ownership and management are out-sourced (turn-key model)" + } + ], + "showOtherItem": true, + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "network_map_urls", + "title": "Please provide a network map for layers 1, 2 and 3 of your network:", + "columns": [ + { + "name": "network_map_url", + "title": "Url:", + "cellType": "text", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + } + ], + "rowCount": 1, + "maxRowCount": 50 + } + ], + "title": "Connectivity" + }, + { + "type": "panel", + "name": "performance", + "elements": [ + { + "type": "checkbox", + "name": "monitoring_tools", + "title": "Which tools do you offer your client institutions for monitoring or troubleshooting the network?", + "choices": [ + { + "value": "looking_glass", + "text": "Looking Glass" + }, + { + "value": "status_dashboard", + "text": "Network or Services Status Dashboard" + }, + { + "value": "historical_traffic_volumes", + "text": "Historical traffic volume information" + }, + { + "value": "netflow_analysis", + "text": "Netflow analysis tool" + } + ], + "showOtherItem": true + }, + { + "type": "text", + "name": "netflow_vendors", + "title": "If you process NetFlow, please, indicate the system name and vendor:" + }, + { + "type": "radiogroup", + "name": "passive_monitoring", + "title": "Do you passively monitor international traffic?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "radiogroup", + "name": "passive_monitoring_tech", + "visibleIf": "{passive_monitoring} = 'Yes'", + "indent": 1, + "title": "Do you use:", + "choices": [ + { + "value": "span_ports", + "text": "SPAN ports" + }, + { + "value": "taps", + "text": "Passive optical TAPS" + }, + { + "value": "both", + "text": "Both" + } + ], + "showClearButton": true + }, + { + "type": "radiogroup", + "name": "traffic_statistics", + "title": "Do you have traffic statistics on your website?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "matrixdynamic", + "name": "traffic_statistics_urls", + "visibleIf": "{traffic_statistics} = 'Yes'", + "indent": 1, + "title": "Please give the URL(s):", + "columns": [ + { + "name": "traffic_statistics_url", + "title": "Url:", + "cellType": "text", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + } + ], + "rowCount": 1, + "maxRowCount": 50, + "validators": [ + { + "type": "answercount", + "text": "Please provide at least one valid website url including http:// or https:// for traffic statistics", + "minCount": 1, + "maxCount": 50 + } + ] + }, + { + "type": "checkbox", + "name": "siem_soc_vendor", + "title": "If you use a SIEM/SOC system, please indicate the name of the vendor you use here:", + "choices": [ + "Splunk", + "IBM Qradar", + "Exabeam", + "LogRythm", + "Securonix" + ], + "showOtherItem": true + }, + { + "type": "checkbox", + "name": "certificate_service", + "title": "Which certificate service do you use?", + "choices": [ + "TCS", + "Digicert", + { + "value": "Sectigo", + "text": "Sectigo (outside of TCS)" + }, + "Let's Encrypt", + "Entrust Datacard" + ], + "showOtherItem": true + }, + { + "type": "radiogroup", + "name": "network_weather", + "title": "Do you have an online weather map of your network?", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "network_weather_url", + "visibleIf": "{network_weather} = 'Yes'", + "indent": 1, + "title": "Please give the URL:", + "validators": [ + { + "type": "expression", + "text": "Please provide a single valid website url including http:// or https://", + "expression": "validateQuestion('validateWebsiteUrl')" + } + ], + "inputType": "url" + }, + { + "type": "radiogroup", + "name": "pert_team", + "title": "Do you run a PERT team?", + "choices": [ + "Yes", + "No", + "Planned" + ], + "showClearButton": true + } + ], + "title": "PERFORMANCE MONITORING AND MANAGEMENT" + }, + { + "type": "panel", + "name": "alienwave", + "elements": [ + { + "type": "radiogroup", + "name": "alienwave_services", + "title": "Does your NREN make use of alien wavelength/lightpath services provided by third parties?", + "description": "This does not include alien waves used internally inside your network e.g. coloured optics on routers as they are covered in a later question.", + "choices": [ + "Yes", + "No", + "Planned" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "alienwave_services_number", + "visibleIf": "{alienwave_services} = 'yes'", + "indent": 1, + "title": "Please state the number of individual alien wavelength services:", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "radiogroup", + "name": "alienwave_internal", + "title": "Are you using alien waves internally in your network?", + "description": "This includes for example alien waves used between two equipment vendors e.g. coloured optics on routers carried over DWDM equipment.", + "choices": [ + "Yes", + "No" + ], + "showClearButton": true + } + ], + "title": "Alienwave", + "description": "Pure optical connectivity services provided by the NREN without a known framing, such as foreign, alien wavelength." + }, + { + "type": "panel", + "name": "capacity", + "elements": [ + { + "type": "text", + "name": "max_capacity", + "title": "What is the capacity (in Gbit/s) of the largest link in your network used for internet traffic (either shared or dedicated)?", + "description": "Please provide the sum of aggregated links, but don't include backup capacity.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "type": "text", + "name": "typical_capacity", + "title": "What is the current typical core usable backbone IP capacity of your network in Gbit/s?", + "description": "Note this refers to circuit capacity not traffic e.g. 2 x 10GE LAG aggregated links.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "type": "matrixdynamic", + "name": "external_connections", + "title": "Please provide a list of the operational external IP connections (usable links excluding backup links): ", + "description": "This should include links to your regional backbone (GÉANT, APAN, RedCLARA etc.), to other research locations, to the commercial Internet, peerings to Internet exchanges, cross-border dark fibre links and any other links you may have. Note that we are interested in the capacity for production purposes, not in any additional links that may be there for the purpose of giving resilience. Some of your capacity to your regional backbone may be used for transiting to intercontinental services; please include these too. Cross-border fibre links means those links that have been commissioned or established by the NREN from a point on the network that is near the border to another point near the border on the network of a neighbouring NREN, for example.", + "columns": [ + { + "name": "link_name", + "title": "Link name", + "cellType": "text" + }, + { + "name": "capacity", + "title": "Capacity (Gbit/s):", + "cellType": "text", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "from_organization", + "title": "The link is from (organisation):", + "cellType": "text" + }, + { + "name": "to_organization", + "title": "To (organisation):", + "cellType": "text" + }, + { + "name": "interconnection_method", + "title": "Interconnection method:", + "cellType": "dropdown", + "choices": [ + { + "value": "internet_exchange", + "text": "Internet exchange points" + }, + { + "value": "open_exchange", + "text": "Open Exchange points" + }, + { + "value": "direct", + "text": "Directly connected R&E peers" + }, + { + "value": "geant", + "text": "GEANT" + }, + { + "value": "other", + "text": "Other" + } + ] + } + ], + "rowCount": 1, + "maxRowCount": 50 + }, + { + "type": "text", + "name": "non_r_and_e_peers", + "title": "Please state how many non-R&E networks you are peering with:", + "description": "This should include all direct IP-peerings to commercial networks e.g. Google.", + "validators": [ + { + "type": "numeric", + "minValue": 0 + }, + { + "type": "regex", + "text": "Please round to a whole number", + "regex": "^[0-9]*$" + } + ] + }, + { + "type": "multipletext", + "name": "traffic_estimate", + "title": "Please supply an estimate of the total amount of traffic in Terabytes in the last 12 months for the following:", + "items": [ + { + "name": "from_customers", + "title": "Traffic from NREN customers (sources that are part of the remit of the NREN's domain)", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "to_customers", + "title": "Traffic to NREN customers (sources that are part of the remit of the NREN's domain)", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "from_external", + "title": "Traffic from external networks (sources that are outside the NREN's domain, such as, GÉANT, General/Commercial Internet, Internet exchanges, peerings, other NRENs etc)", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + }, + { + "name": "to_external", + "title": "Traffic to external networks (sources that are outside the NREN's domain, such as, GÉANT, General/Commercial Internet, Internet exchanges, peerings, other NRENs etc)", + "validators": [ + { + "type": "numeric", + "minValue": 0 + } + ] + } + ] + }, + { + "type": "multipletext", + "name": "commodity_vs_r_e", + "title": "What is the ratio of commodity vs R&E traffic in your network?", + "validators": [ + { + "type": "expression", + "text": "The percentages should add up to 100", + "expression": "(sum({commodity_vs_r_e.r_e}, {commodity_vs_r_e.commodity}) = 100) or ({commodity_vs_r_e} empty)" + } + ], + "items": [ + { + "name": "r_e", + "title": "R&E percentage", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + }, + { + "name": "commodity", + "title": "Commodity percentage", + "validators": [ + { + "type": "numeric", + "minValue": 0, + "maxValue": 100 + } + ] + } + ] + } + ], + "title": "CAPACITY" + }, + { + "type": "panel", + "name": "sdn", + "elements": [ + { + "type": "radiogroup", + "name": "operational_process_automation", + "title": "Are you automating your operational processes?", + "choices": [ + "Yes", + "No", + "Planned" + ], + "showClearButton": true + }, + { + "type": "text", + "name": "operational_process_automation_tools", + "visibleIf": "{operational_process_automation} = 'yes'", + "indent": 1, + "title": "Please specify which processes and the name/s of the automation software and tools you use for it:" + }, + { + "type": "radiogroup", + "name": "nfv", + "title": "Do you use any kind of NFV?", + "choices": [ + "Yes", + "No", + "Planned" + ], + "showClearButton": true + }, + { + "type": "checkbox", + "name": "nfv_types", + "visibleIf": "{nfv} = 'yes' or {nfv} = 'planned'", + "indent": 1, + "title": "What kind of NFV do/will you use:", + "choices": [ + { + "value": "routers", + "text": "Routers/switches" + }, + { + "value": "firewalls", + "text": "Firewalls" + }, + { + "value": "load_balancers", + "text": "Load balancers" + }, + { + "value": "vpn_concentrators", + "text": "VPN Concentrator Services" + } + ], + "showOtherItem": true + }, + { + "type": "radiogroup", + "name": "network_automation", + "title": "Do you use automation on your network?", + "choices": [ + "Yes", + "No", + "Planned" + ], + "showClearButton": true + }, + { + "type": "checkbox", + "name": "network_automation_tasks", + "visibleIf": "{network_automation} = 'yes' or {network_automation} = 'planned'", + "indent": 1, + "title": "What kind of task do you use it for?", + "choices": [ + { + "value": "provisioning", + "text": "Device Provisioning" + }, + { + "value": "data_collection", + "text": "Data Collection" + }, + { + "value": "config_management", + "text": "Configuration Management" + }, + { + "value": "compliance", + "text": "Compliance" + }, + { + "value": "reporting", + "text": "Reporting" + }, + { + "value": "troubleshooting", + "text": "Troubleshooting" + } + ] + } + ], + "title": "SOFTWARE-DEFINED NETWORKING (SDN) AND NETWORK FUNCTION VIRTUALISATION (NFV)" + }, + { + "type": "comment", + "name": "network_comments", + "title": "Comments regarding this section:" + } + ], + "title": "Network" + }, + { + "name": "services", + "elements": [ + { + "type": "html", + "name": "services_hover_explanation", + "html": "Descriptions of the individual services are shown if you hover over the service name." + }, + { + "type": "matrixdropdown", + "name": "services_network", + "state": "collapsed", + "title": "Network services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "connectivity", + "text": "IP Connectivity", + "customDescription": "Basic IP connectivity services inc R&E and commodity internet" + }, + { + "value": "home-vpn", + "text": "Remote access VPN server", + "customDescription": "Remote access VPN for end users (e.g. eduVPN) or site-to-site VPN with the possibility of encryption" + }, + { + "value": "ipv6", + "text": "IPv6", + "customDescription": "The new version of the internet protocol (IP) that will eventually replace IPv4" + }, + { + "value": "dedicated-optical-connections", + "text": "Dedicated optical connections", + "customDescription": "Provision of dedicated optical connections to users, e.g. Layer 1 optical channels or Open Lightpath exchanges." + }, + { + "value": "managed-router", + "text": "Managed router service", + "customDescription": "Remote router support for institutions" + }, + { + "value": "multicast", + "text": "Multicast", + "customDescription": "Extension to the IP protocol which allows individual packets to be sent to multiple hosts on the internet" + }, + { + "value": "netflow", + "text": "Netflow tool", + "customDescription": "Network protocol of collecting IP traffic and monitoring network traffic" + }, + { + "value": "user-monitoring", + "text": "Network troubleshooting", + "customDescription": "Providing a support service to identify, diagnose and resolve problems and issues within a computer network, using network monitoring system tools like Looking Glass." + }, + { + "value": "network-monitoring", + "text": "Network monitoring", + "customDescription": "Network Monitoring systems is software/hardware tools that can track various aspects of a network and its operation. These systems can detect devices and other network elements that comprise or touch the network, as well as provide status updates." + }, + { + "value": "pert", + "text": "PERT", + "customDescription": "Team supporting resolution of end-to-end performance problems for networked applications" + }, + { + "value": "point-to-point-circuit-vpn", + "text": "Virtual circuits/network VPNs", + "customDescription": "Virtual point to point circuits or VPNs to create virtual networks between geographically distinct sites" + }, + { + "value": "quality-of-service", + "text": "Quality of Service", + "customDescription": "Preferential service to specific applications or classes of applications" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_isp", + "state": "collapsed", + "title": "ISP services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "domain-registration", + "text": "Domain name registration", + "customDescription": "Adminstration/registration of top and second level domain names" + }, + { + "value": "ip-address-allocation", + "text": "IP address allocation", + "customDescription": "Allocating addresses for users according to the RIPE policies" + }, + { + "value": "ix-operation", + "text": "National IX operation", + "customDescription": "Operating an IX with national importance" + }, + { + "value": "nameserver", + "text": "Nameserver services", + "customDescription": "Operation of nameservers and maintenance of DNS information on behalf of users" + }, + { + "value": "timeserver-ntp", + "text": "NTP service", + "customDescription": "Allows the synchronization of computer clocks over the internet" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_security", + "state": "collapsed", + "title": "Security services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "anti-spam", + "text": "Anti-spam solution", + "customDescription": "Anti-Spam solutions for detecting and eliminating viruses and spam mails" + }, + { + "value": "csirt", + "text": "CERT/CSIRT", + "customDescription": "A single point of contact for users to deal with computer security incidents and prevention" + }, + { + "value": "ddos-prevention", + "text": "DDoS mitigation", + "customDescription": "Tools and techniques for mitigating Distributed Denial of Service attacks" + }, + { + "value": "external-security-awareness", + "text": "External security awareness programme", + "customDescription": "An external security awareness programme is a service offering to users (consisting for example in training, workshops, toolkits, campaigns, awareness games, …) aimed at helping member institutions to effectively manage human risk" + }, + { + "value": "firewall-on-demand", + "text": "Firewall-on-Demand", + "customDescription": "Provision of a dynamic firewall services to mitigate against DDoS attacks" + }, + { + "value": "intrusion", + "text": "Intrusion detection", + "customDescription": "System for detecting and preventing Intrusions (IDS/IPS)" + }, + { + "value": "pgp-key", + "text": "PGP key server", + "customDescription": "Operation of PGP key server" + }, + { + "value": "security-audit", + "text": "Security auditing", + "customDescription": "Carrying out vulnerability assessments and security reviews of user systems and resources on their behalf" + }, + { + "value": "vulnerability-testing", + "text": "Vulnerability scanning", + "customDescription": "Vulnerability service that allows users to scan their own IP networks fo security holes" + }, + { + "value": "web-filtering", + "text": "Web filtering", + "customDescription": "Centralised web content filtering service for protection against access to inappropriate content" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_identity", + "state": "collapsed", + "title": "Identity services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "aai", + "text": "Hosted campus AAI", + "customDescription": "Hosting of an Identity Provider service on behalf of connected Institutions to authenticate users" + }, + { + "value": "eduroam-wifi", + "text": "eduroam", + "customDescription": "Inter-WLAN service to facilitate easy and secure Internet access for roaming educationals users" + }, + { + "value": "interfederation", + "text": "Interfederation", + "customDescription": "Participation in an interfederation (i.e. eduGAIN, KALMAR)" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_hosting", + "state": "collapsed", + "title": "Storage & hosting services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "cloud-service-end-user", + "text": "Cloud storage (end user)", + "customDescription": "Browser-based virtual storage service for individuals" + }, + { + "value": "content-delivery-hosting", + "text": "Content delivery hosting", + "customDescription": "Hosting of contenct delivery servers, e.g. Akamai" + }, + { + "value": "disaster-recovery", + "text": "Disaster recovery", + "customDescription": "Off site backup services" + }, + { + "value": "dns-server", + "text": "DNS hosting", + "customDescription": "Hosting of primary and secondary DNS servers" + }, + { + "value": "email-services", + "text": "Email server hosting", + "customDescription": "NREN hosted email servers." + }, + { + "value": "filesender", + "text": "FileSender", + "customDescription": "Web-based application that allows authenticated userds to securely and easily send arbitrarily large files" + }, + { + "value": "saas", + "text": "SaaS", + "customDescription": "Software as a service e.g. Google Apps for Education" + }, + { + "value": "storage-co-location", + "text": "Housing/co-location", + "customDescription": "Hosting of user equipment in a managed data centre" + }, + { + "value": "virtual-machines-iaas", + "text": "Virtual machines/IaaS", + "customDescription": "Access to virtual computing resources" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_multimedia", + "state": "collapsed", + "title": "Multimedia services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "internet-radio-tv", + "text": "TV/radio streaming", + "customDescription": "Internet and radio streaming services" + }, + { + "value": "videoconferencing", + "text": "Event recording/streaming", + "customDescription": "Provision of equipment and/or software to support event streaming/recording" + }, + { + "value": "video-portal", + "text": "Provision of content portal", + "customDescription": "Multi-media content portal" + }, + { + "value": "web-conferencing", + "text": "Web/desktop conferencing", + "customDescription": "Video conferencing service to desktops and hand-held devices using software" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_collaboration", + "state": "collapsed", + "title": "Collaboration services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "e-portfolio", + "text": "E-portfolio service", + "customDescription": "Functions to create user professional and career portfolios" + }, + { + "value": "identifier-reg", + "text": "Identifier Registry", + "customDescription": "Registering of unique and automatically-processable identifiers in the form of text or numeric strings" + }, + { + "value": "journal-library-access", + "text": "Journal access", + "customDescription": "Access to academic journals" + }, + { + "value": "mailing-lists", + "text": "Mailing lists", + "customDescription": "Service for operation of electronic discussion lists" + }, + { + "value": "project-collaboration-toolkit", + "text": "Project collaboration", + "customDescription": "Packaged services or virtual project groups e.g. mailing lists, storage, web meetings, wiki." + }, + { + "value": "scheduling-tool", + "text": "Scheduling tool", + "customDescription": "Provision of tools to users for scheduling appointments or classes" + }, + { + "value": "survey-tool", + "text": "Survey/polling tool", + "customDescription": "Provision of applications for creating surveys or polls" + }, + { + "value": "virtual-learning-environment", + "text": "VLE", + "customDescription": "Online e-learning education system that provides virtual access to resources used in teaching" + }, + { + "value": "voip", + "text": "VoIP", + "customDescription": "Service to deliver voice communications and multimedia sessions over Internet Protocal (IP) networks" + }, + { + "value": "web-email-hosting", + "text": "Web hosting", + "customDescription": "Service to provide space on central web servers for users to publish their website" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "matrixdropdown", + "name": "services_professional", + "state": "collapsed", + "title": "Professional services", + "hideCheckboxLabels": true, + "columns": [ + { + "name": "offered", + "title": "Service offered", + "cellType": "checkbox", + "choices": [ + { + "value": "yes", + "text": "Yes" + } + ] + }, + { + "name": "name", + "title": "Service name", + "cellType": "text", + "width": "20%", + "visibleIf": "{row.offered} = ['yes']" + }, + { + "name": "description", + "title": "Official description", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2 + }, + { + "name": "additional_information", + "title": "Additional information", + "cellType": "comment", + "visibleIf": "{row.offered} = ['yes']", + "rows": 2, + "placeholder": "(e.g. software used, third party service, links, etc.)" + } + ], + "cellType": "checkbox", + "rows": [ + { + "value": "consultancy", + "text": "Consultancy/training", + "customDescription": "Training and consultancy services provided by the NREN" + }, + { + "value": "dissemination", + "text": "Dissemination", + "customDescription": "Dissemination of information to users e.g newsletters and magazines" + }, + { + "value": "procurement", + "text": "Procurement brokerage", + "customDescription": "Procurement support for users to purchase services (e.g. cloud services), software/software licenses (e.g. VLE or office software) or hardware either directly or via procurement frameworks." + }, + { + "value": "user-conference", + "text": "User conferences", + "customDescription": "Hosting of regular user conferences" + }, + { + "value": "user-portal", + "text": "User portals", + "customDescription": "User portals for service management and monitoring" + } + ], + "rowTitleWidth": "20%" + }, + { + "type": "comment", + "name": "service_comments", + "title": "Comments regarding this section:" + } + ], + "title": "Services" + } + ], + "showQuestionNumbers": "onPage", + "checkErrorsMode": "onValueChanged" + } \ No newline at end of file diff --git a/test/test_legacy_survey_generation.py b/test/test_legacy_survey_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..d857f9d7ba579e380f91dd03578f2ac68d7fb4eb --- /dev/null +++ b/test/test_legacy_survey_generation.py @@ -0,0 +1,272 @@ +import os +import openpyxl +import json + +from compendium_v2.db import db, presentation_models +from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db import FundingSource, \ + StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion +from compendium_v2.db.survey_models import Survey, SurveyStatus, SurveyResponse +from compendium_v2.publishers.legacy_publisher.survey_publisher_legacy import _cli, VALID_KEYS_2024_SURVEY + + +def _test_data(filename): + data_filename = os.path.join(os.path.dirname(__file__), 'data', filename) + + with open(data_filename, "r") as f: + return json.load(f) + + +def org_data(question): + """ + This function defines test data for the org questions. + + The following data is defined for the appropriate questions as modeled in Compendium: + + nren1,CYNET-CSIRT,cert team + nren1,DFN-CERT,CERT + nren2,Educampus Services,MIS shared services for third level. + nren3,VilniusTech,Technical centre + nren3,KU,Technical centre + nren3,VDU,Technical centre + nren3,VU,Technical centre + nren3,org_data +org_data +org_data +org_data +org_dataKTU,"NOC, administrative authority" + """ + + if question == OrgQuestion.PARENT_ORG_NAME: + return [ + ('nren1', 'Org1'), + ('nren3', 'Org3'), + ] + + if str(question.name).endswith('1_NAME'): + return [ + ('nren1', 'CYNET-CSIRT'), + ('nren2', 'Educampus Services'), + ('nren3', 'VilniusTech'), + ] + + if str(question.name).endswith('2_NAME'): + return [ + ('nren1', 'DFN-CERT'), + ('nren3', 'KU'), + ] + + if str(question.name).endswith('3_NAME'): + return [ + ('nren3', 'VDU'), + ] + + if str(question.name).endswith('4_NAME'): + return [ + ('nren3', 'VU'), + ] + + if str(question.name).endswith('5_NAME'): + return [ + ('nren3', 'KTU'), + ] + + if str(question.name).endswith('1_CHOICE'): + return [ + ('nren1', 'other'), + ('nren2', 'other'), + ('nren3', 'Technical centre'), + ] + + if str(question.name).endswith('2_CHOICE'): + return [ + ('nren1', 'other'), + ('nren3', 'Technical centre'), + ] + + if str(question.name).endswith('3_CHOICE'): + return [ + ('nren3', 'Technical centre'), + ] + + if str(question.name).endswith('4_CHOICE'): + return [ + ('nren3', 'Technical centre'), + ] + + if str(question.name).endswith('5_CHOICE'): + return [ + ('nren3', 'other'), + ] + + if str(question.name).endswith('1_ROLE'): + return [ + ('nren1', 'cert team'), + ('nren2', 'MIS shared services for third level.') + ] + + if str(question.name).endswith('2_ROLE'): + return [ + ('nren1', 'CERT'), + ] + + if str(question.name).endswith('3_ROLE'): + return [] + + if str(question.name).endswith('4_ROLE'): + return [] + + if str(question.name).endswith('5_ROLE'): + return [ + ('nren3', 'NOC, administrative authority') + ] + + +def setup_survey_db(app_with_survey_db, mocker): + global org_data + + def get_rows_as_tuples(*args, **kwargs): + return [ + ('nren1', '100'), + ('nren2', '200'), + ('nren3', '300'), + ('nren4', 'abcd') + ] + + def funding_source_data(): + yield FundingSource.CLIENT_INSTITUTIONS, [ + ('nren1', '10'), + ('nren2', '80'), + ('nren3', '30'), + ] + yield FundingSource.EUROPEAN_FUNDING, [ + ('nren1', '50'), + ('nren2', '20'), + ('nren3', '30'), + ] + yield FundingSource.OTHER, [ + ('nren1', '40'), + ('nren2', 'abc'), + ('nren3', '30'), + ] + + def question_data(question): + if question == StaffQuestion.NON_TECHNICAL_FTE: + return [ + ('nren1', '10'), + ('nren2', '80'), + ('nren3', '30'), + ] + + if question == StaffQuestion.TECHNICAL_FTE: + return [ + ('nren1', '50'), + ('nren2', '20'), + ('nren3', '30'), + ] + + if question == StaffQuestion.PERMANENT_FTE: + return [ + ('nren1', '60'), + ('nren2', 'abc'), + ('nren3', '30'), + ] + + if question == StaffQuestion.SUBCONTRACTED_FTE: + return [ + ('nren1', '0'), + ('nren2', '0'), + ('nren3', '0'), + ] + if question in OrgQuestion: + return org_data(question) + + if question == ChargingStructure.charging_structure: + return [ + ('nren1', 'We do not charge them directly'), + ('nren2', 'We charge a usage-based fee'), + ('nren3', 'Other'), + ] + + if question in ECQuestion: + return [ + ('nren1', '[""'), + ('nren2', '["project1", "project2"]'), + ('nren3', '["project3"]'), + ] + + def question_id_data(question_id, year): + if question_id in [ + 16469, 16064, 15720, 15305, 14910, 16471, 16066, 15722, 15307, 14912, 16473, 16378, + 16475, 16068, 15724, 15309, 14914, 16477, 16070, 15726, 15311, 14916, 16479, 16072, 15728, 15575, + 16481, 16074, 15730, 15577, 16761]: + return [ + ('nren1', f'www.nren.com/somepolicy{year}.pdf'), + ('nren2', 'policyemail@nren.com'), + ('nren3', 'n.a. online'), + ] + + def institutions_urls_data(question_id): + if question_id == 16507: + return [ + (87483, 'ANA', 2013, "http://www.rash.al/index.php/network/points-of-presence-pop"), + (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"), + ] + else: + return [] + + mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_budget', + get_rows_as_tuples) + mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_funding_sources', + funding_source_data) + mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_question', + question_data) + mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.query_question_id', + question_id_data) + mocker.patch('compendium_v2.publishers.legacy_publisher.survey_publisher_legacy_db.recursive_query', + institutions_urls_data) + + +def test_legacy_publisher(app_with_survey_db, mocker, nren_services): + setup_survey_db(app_with_survey_db, mocker) + nren_services(app_with_survey_db) + EXCEL_FILE_ORGANISATION = openpyxl.load_workbook(os.path.join(os.path.dirname( + __file__), "data", "2021_Organisation_DataSeries.xlsx"), data_only=True, read_only=True) + EXCEL_FILE_USERS = openpyxl.load_workbook(os.path.join(os.path.dirname( + __file__), "data", "2022_Connected_Users_DataSeries.xlsx"), data_only=True, read_only=True) + EXCEL_FILE_NETWORKS = openpyxl.load_workbook(os.path.join(os.path.dirname( + __file__), "data", "2022_Networks_DataSeries.xlsx"), data_only=True, read_only=True) + EXCEL_FILE_NREN_SERVICES = openpyxl.load_workbook(os.path.join(os.path.dirname( + __file__), "data", "NREN-Services-prefills_2023_Recovered.xlsx"), data_only=True, read_only=True) + mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE_ORGANISATION) + mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_USERS', EXCEL_FILE_USERS) + mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NETWORKS', EXCEL_FILE_NETWORKS) + mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NREN_SERVICES', EXCEL_FILE_NREN_SERVICES) + + survey_2024 = _test_data('survey_model_2024_testdata.json') + + nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM', + 'SIKT', 'LAT', 'RASH', 'AzScienceNet', 'GRNET', 'CSC', 'PSNC'] + with app_with_survey_db.app_context(): + db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names]) + db.session.commit() + + with app_with_survey_db.app_context(): + survey = Survey(year=2024, survey=survey_2024, status=SurveyStatus.published) + db.session.add(survey) + db.session.commit() + + _cli(app_with_survey_db) + + with app_with_survey_db.app_context(): + responses = db.session.query(SurveyResponse).all() + assert len(responses) == 78 + + nrens = set([resp.nren.name for resp in responses]) + + assert len(nrens) == len(nren_names) - 2 # Nren4 and University of Malta have no valid data + + for response in responses: + data = response.answers['data'] + assert data + for key in data: + assert key in VALID_KEYS_2024_SURVEY diff --git a/tox.ini b/tox.ini index f659a24cdcd7c95456a4ab87e0c593e4d972cfae..7b8651adec32ad31345c1925a3b0e430bde66b00 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ exclude = .tox,compendium-frontend,survey-frontend,compendium-survey-creator [coverage:run] concurrency = multiprocessing,thread +; it's legacy stuff, ignore it for coverage +omit = */survey_publisher_legacy_db* [testenv] passenv = XDG_CACHE_HOME