Skip to content
Snippets Groups Projects
Commit a7cdfbba authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

Merge branch 'feature/COMP-218_migrating_2022_data' into 'develop' 

Feature/comp 218 migrating 2022 data

See merge request !47
parents 033e8aeb 5a68a352
Branches
Tags
1 merge request!47Feature/comp 218 migrating 2022 data
Show whitespace changes
Inline Side-by-side
Showing
with 905 additions and 28 deletions
compendium_v2/conversion/__init__.py 0 → 100644
+ 0
− 0
View file @ a7cdfbba
compendium_v2/conversion/conversion.py 0 → 100644
+ 152
− 0
View file @ a7cdfbba
"""
conversion
=========================
This module loads the survey data from 2022 from the survey database
and stores the data in the json structure of the new survey, so that
it can be used to prefill the 2023 survey.
"""
import logging
import click
import json
from sqlalchemy import delete, text, select
import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
from compendium_v2.survey_db import model as survey_model
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse
from compendium_v2.conversion import mapping
setup_logging()
logger = logging.getLogger('conversion')
def query_nren(nren_id: int):
query = mapping.ANSWERS_2022_QUERY.format(nren_id)
answers = {}
for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
answers[row[0]] = row[1]
return answers
def convert_answers(answers):
data = {}
for id, question_name in mapping.ID_TO_NAME.items():
if id not in answers:
continue
answer = answers[id]
if len(answer) > 1 and answer[0] == '"' and answer[-1] == '"':
answer = answer[1:-1]
if id in mapping.VALUE_TO_CODE_MAPPING:
answer = mapping.VALUE_TO_CODE_MAPPING[id][answer]
if len(answer) > 1 and answer[0] == '[' and answer[-1] == ']':
answer = json.loads(answer)
if id in mapping.VALUE_TO_CODE_MAPPING:
mapped_answer = []
for entry in answer:
mapped_answer.append(mapping.VALUE_TO_CODE_MAPPING[id][entry])
answer = mapped_answer
# code to convert my description in the mapping to a json structure
question_names = question_name.split(":")
subdict = data
for name in question_names[0:-1]:
if name[-1] == "]":
index = name[-2]
if index == "[":
subdict = subdict.setdefault(name[:-2], [])
break # special case where json list is mapped to a list of dicts (part 1)
sublist = subdict.setdefault(name[:-3], [])
index = int(index)
while len(sublist) <= index:
sublist.append({})
subdict = sublist[index]
else:
subdict = subdict.setdefault(name, {})
if type(subdict) == list: # special case where json list is mapped to a list of dicts (part 2)
for answer_entry in answer:
subdict.append({question_names[-1]: answer_entry})
elif question_names[-1] == "available": # special case because we changed the policies questions a bit
if answer == "Yes":
subdict[question_names[-1]] = ["yes"]
else:
subdict[question_names[-1]] = answer
for id, question_name in mapping.ID_TO_NAME_SERVICES.items():
if id not in answers:
continue
answer = answers[id]
answer = json.loads(answer)
for user_type in answer:
user_type_code = mapping.SERVICE_USER_TYPE_TO_CODE[user_type]
formatted_question_name = question_name.format(user_type_code)
question_names = formatted_question_name.split(":")
subdict = data
for name in question_names[0:-2]:
subdict = subdict.setdefault(name, {})
sublist = subdict.setdefault(question_names[-2], [])
sublist.append(question_names[-1])
return {"data": data}
def _cli(app):
with app.app_context():
nren_surveys = {}
for nren in db.session.scalars(select(NREN)):
survey_db_nren_id = mapping.NREN_IDS[nren.name]
nren_surveys[nren] = query_nren(survey_db_nren_id)
db.session.execute(delete(SurveyResponse).where(
SurveyResponse.survey_year == 2022
))
db.session.execute(delete(Survey).where(
Survey.year == 2022
))
survey = Survey(year=2022, survey={})
db.session.add(survey)
for nren, answers in nren_surveys.items():
survey_dict = convert_answers(answers)
response = SurveyResponse(
nren=nren,
nren_id=nren.id,
survey_year=2022,
survey=survey,
answers=survey_dict
)
db.session.add(response)
db.session.commit()
@click.command()
@click.option('--config', type=click.STRING, default='config.json')
def cli(config):
app_config = load(open(config, 'r'))
app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
app = compendium_v2._create_app_with_db(app_config)
_cli(app)
if __name__ == "__main__":
cli()
compendium_v2/conversion/mapping.py 0 → 100644
+ 583
− 0
View file @ a7cdfbba
ANSWERS_2022_QUERY = """
SELECT question_id, value
FROM answers a
WHERE nren_id = {}
AND value NOT IN ('""', '[]', '"NA"', '"N/A"', '[""]', '["-"]', '["/"]')
AND NOT EXISTS (
SELECT 1 FROM answers a2 WHERE a.question_id = a2.question_id AND a.nren_id = a2.nren_id AND a2.id > a.id
)
AND question_id IN (16402,16405,16406,16407,16408,16409,16410,16413,16414,16416,16417,16418,16419,16420,16422,
16426,16429,16430,16432,16433,16434,16435,16438,16439,16446,16448,16449,16450,16451,16452,
16453,16455,16456,16457,16458,16459,16460,16461,16462,16463,16464,16465,16468,16469,16470,
16471,16472,16473,16474,16475,16476,16477,16478,16479,16480,16481,16482,16483,16484,16485,
16486,16488,16489,16490,16491,16492,16493,16494,16495,16496,16497,16499,16500,16501,16502,
16503,16504,16760,16761,16762,16763,16507,16509,16510,16511,16512,16513,16514,16515,16516,
16517,16518,16519,16520,16521,16522,16523,16524,16525,16526,16527,16528,16529,16530,16531,
16532,16533,16534,16535,16536,16537,16538,16539,16540,16541,16542,16543,16544,16545,16546,
16547,16548,16550,16551,16552,16553,16554,16555,16556,16557,16558,16559,16560,16561,16562,
16563,16564,16565,16566,16567,16568,16569,16570,16571,16572,16573,16574,16575,16576,16577,
16578,16579,16581,16582,16583,16584,16585,16586,16587,16588,16589,16590,16592,16593,16594,
16595,16596,16597,16598,16599,16600,16601,16602,16603,16604,16605,16606,16607,16608,16609,
16610,16611,16613,16614,16615,16616,16617,16618,16619,16620,16621,16622,16623,16624,16625,
16626,16627,16628,16629,16630,16631,16632,16634,16635,16636,16637,16638,16639,16640,16641,
16642,16643,16646,16647,16648,16649,16650,16652,16653,16654,16656,16658,16659,16662,16663,
16664,16665,16666,16667,16668,16669,16670,16672,16673,16674,16675,16676,16677,16678,16679,
16680,16681,16682,16683,16684,16685,16687,16688,16689,16691,16692,16694,16695,16696,16697,
16698,16699,16700,16701,16702,16703,16704,16705,16706,16707,16708,16709,16710,16711,16712,
16713,16714,16715,16716,16717,16718,16719,16720,16721,16722,16723,16724,16725,16726,16727,
16728,16729,16730,16731,16732,16733,16734,16735,16736,16737,16738,16739,16740,16741,16742,
16743,16744,16746,16747,16748,16749,16750,16752,16753,16754,16755,16756,16757,16758);
"""
NREN_IDS = {
"ANAS": 49,
"GRNET": 17,
"Restena": 28,
"University of Malta": 29,
"DeiC": 9,
"CSC": 12,
"Sikt": 33,
"ACOnet": 4,
"AMRES": 48,
"ARNES": 39,
"ASNET-AM": 200,
"BASNET": 54,
"Belnet": 5,
"BREN": 58,
"CARNET": 6,
"CESNET": 8,
"CYNET": 7,
"DFN": 16,
"EENet": 11,
"FCCN": 35,
"GARR": 23,
"GRENA": 15,
"HEAnet": 21,
"IUCC": 22,
"Jisc": 46,
"KIFU": 18,
"LAT": 51,
"LITNET": 27,
"MARnet": 14,
"MREN": 100,
"PIONIER": 34,
"RASH": 1,
"RedIRIS": 40,
"RENAM": 30,
"RENATER": 13,
"RHnet": 19,
"RoEduNet": 52,
"SANET": 38,
"SUNET": 41,
"SURF": 32,
"SWITCH": 42,
"ULAKBIM": 44,
"URAN": 45,
}
# query to get the data required to make the following mapping:
#
# select a.value, q.id, q.title, q.cell_column, q.cell_row, q.identifier, q.choices,
# q2.identifier, q2.column_headers, q2.row_headers
# from answers a join questions q on a.question_id = q.id
# left join questions q2 on q.parent_id = q2.id
# where a.value <> '""' and q.section_id > 225
# AND NOT EXISTS (
# SELECT 1 FROM answers a2 WHERE a.question_id = a2.question_id AND a.nren_id = a2.nren_id AND a2.id > a.id
# )
# order by q.id, a.id;
ID_TO_NAME_SERVICES = {
16488: "service_matrix:{}:service_types:identity",
16489: "service_matrix:{}:service_types:network_services",
16490: "service_matrix:{}:service_types:collaboration",
16491: "service_matrix:{}:service_types:security",
16492: "service_matrix:{}:service_types:isp_support",
16493: "service_matrix:{}:service_types:storage_and_hosting",
16494: "service_matrix:{}:service_types:multimedia",
16495: "service_matrix:{}:service_types:professional_services",
}
SERVICE_USER_TYPE_TO_CODE = {
"Universities": "universities",
"Further education": "further_education",
"Research institutes": "institutes",
"Primary Schools": "primary_schools",
"Government": "government",
"For profit organisations": "for_profit_orgs",
"International Research Organisations": "iros",
"Cultural Institutions": "cultural",
"(Non-research) hospitals": "hospitals",
"Secondary Schools": "secondary_schools"
}
ID_TO_NAME = {
16402: "budget",
16405: "income_sources:client_institutions",
16406: "income_sources:european_funding",
16407: "income_sources:commercial",
16408: "income_sources:other",
16409: "income_sources:gov_public_bodies",
16410: "charging_mechanism",
16413: "staff_employment_type:subcontracted_fte",
16414: "staff_employment_type:permanent_fte",
16416: "staff_roles:technical_fte",
16417: "staff_roles:nontechnical_fte",
16418: "parent_organization", # "Yes" "No"
16419: "parent_organization_name",
16420: "suborganizations", # "Yes" "No"
16422: "suborganization_details[0]:suborganization_name",
16426: "suborganization_details[0]:suborganization_role-Comment",
16429: "suborganization_details[1]:suborganization_name",
16430: "suborganization_details[2]:suborganization_name",
16432: "suborganization_details[3]:suborganization_name",
16433: "suborganization_details[4]:suborganization_name",
16434: "suborganization_details[1]:suborganization_role-Comment",
16435: "suborganization_details[2]:suborganization_role-Comment",
16438: "suborganization_details[3]:suborganization_role-Comment",
16439: "suborganization_details[4]:suborganization_role-Comment",
16446: "suborganization_details[2]:suborganization_role",
16448: "suborganization_details[1]:suborganization_role",
16449: "suborganization_details[0]:suborganization_role",
16450: "suborganization_details[4]:suborganization_role",
16451: "suborganization_details[3]:suborganization_role",
16452: "ec_projects", # "Yes" "No"
16453: "ec_project_names[]:ec_project_name", # json list with double quotes
16455: "full_name_english",
16456: "full_name_national_languages",
16457: "abbreviation_english",
16458: "abbreviation_national_languages",
16459: "street_name_and_number",
16460: "city",
16461: "postal_code",
16462: "country", # NB some country names are a bit different
16463: "phone_number",
16464: "email_address",
16465: "website",
# 16466: "organization_comments",
16468: "corporate_strategy", # "Yes" "No"
16469: "corporate_strategy_url",
16470: "policies:environmental_policy:available", # "Yes" "No"
16471: "policies:environmental_policy:url",
16472: "policies:equal_opportunity_policy:available", # "Yes" "No"
16473: "policies:equal_opportunity_policy:url",
16474: "policies:connectivity_policy:available", # "Yes" "No"
16475: "policies:connectivity_policy:url",
16476: "policies:acceptable_use_policy:available", # "Yes" "No"
16477: "policies:acceptable_use_policy:url",
16478: "policies:privacy_notice:available", # "Yes" "No"
16479: "policies:privacy_notice:url",
16480: "policies:data_protection_contact:available", # "Yes" "No"
16481: "policies:data_protection_contact:url",
16482: "central_software_procurement", # "Yes" "No"
16483: "central_procurement_amount",
16484: "formal_service_management_framework", # "Yes" "No"
16485: "service_level_targets", # "Yes" "No"
16486: "service_portfolio_changes", # "Yes" "No"
16496: "service_portfolio_eosc_portal", # "Yes" "No"
16497: "services_on_eosc_portal_list[]:service_name", # json list with double quotes
16499: "audits", # "Yes" "No"
16500: "audit_specifics",
16501: "business_continuity_plans", # "Yes" "No"
16502: "business_continuity_plans_specifics",
16503: "security_controls", # json list with double quotes
16504: "security_controls-Comment",
# 16505: "policy_comments",
16760: "policies:gender_equality_policy:available", # "Yes" "No"
16761: "policies:gender_equality_policy:url",
16762: "crisis_management_procedure", # "Yes" "No"
16763: "crisis_exercises", # json list with double quotes
16507: "connected_sites_lists[]:connected_sites_url", # json list with double quotes
16509: "connectivity_proportions:cultural:nr_connected",
16510: "connectivity_proportions:universities:nr_connected",
16511: "connectivity_proportions:secondary_schools:nr_connected",
16512: "connectivity_proportions:hospitals:nr_connected",
16513: "connectivity_proportions:primary_schools:nr_connected",
16514: "connectivity_proportions:for_profit_orgs:nr_connected",
16515: "connectivity_proportions:further_education:nr_connected",
16516: "connectivity_proportions:government:nr_connected",
16517: "connectivity_proportions:institutes:nr_connected",
16518: "connectivity_proportions:universities:market_share_percentage",
16519: "connectivity_proportions:iros:nr_connected",
16520: "connectivity_proportions:further_education:market_share_percentage",
16521: "connectivity_proportions:secondary_schools:market_share_percentage",
16522: "connectivity_proportions:primary_schools:market_share_percentage",
16523: "connectivity_proportions:cultural:market_share_percentage",
16524: "connectivity_proportions:institutes:market_share_percentage",
16525: "connectivity_proportions:iros:market_share_percentage",
16526: "connectivity_proportions:hospitals:market_share_percentage",
16527: "connectivity_proportions:government:market_share_percentage",
16528: "connectivity_proportions:for_profit_orgs:market_share_percentage",
16529: "connectivity_proportions:universities:nr_of_users",
16530: "connectivity_proportions:institutes:nr_of_users",
16531: "connectivity_proportions:further_education:nr_of_users",
16532: "connectivity_proportions:secondary_schools:nr_of_users",
16533: "connectivity_proportions:cultural:nr_of_users",
16534: "connectivity_proportions:primary_schools:nr_of_users",
16535: "connectivity_proportions:hospitals:nr_of_users",
16536: "connectivity_proportions:government:nr_of_users",
16537: "connectivity_proportions:for_profit_orgs:nr_of_users",
16538: "connectivity_proportions:iros:nr_of_users",
16539: "connectivity_proportions:further_education:covered",
16540: "connectivity_proportions:secondary_schools:covered",
16541: "connectivity_proportions:for_profit_orgs:covered",
16542: "connectivity_proportions:primary_schools:covered",
16543: "connectivity_proportions:institutes:covered",
16544: "connectivity_proportions:government:covered",
16545: "connectivity_proportions:iros:covered",
16546: "connectivity_proportions:cultural:covered",
16547: "connectivity_proportions:universities:covered",
16548: "connectivity_proportions:hospitals:covered",
16550: "connectivity_level:for_profit_orgs:typical_speed",
16551: "connectivity_level:further_education:highest_speed_connection_percentage",
16552: "connectivity_level:institutes:highest_speed",
16553: "connectivity_level:further_education:highest_speed",
16554: "connectivity_level:secondary_schools:highest_speed_connection_percentage",
16555: "connectivity_level:secondary_schools:typical_speed",
16556: "connectivity_level:primary_schools:highest_speed_connection_percentage",
16557: "connectivity_level:secondary_schools:highest_speed",
16558: "connectivity_level:primary_schools:highest_speed",
16559: "connectivity_level:universities:typical_speed",
16560: "connectivity_level:further_education:typical_speed",
16561: "connectivity_level:cultural:typical_speed",
16562: "connectivity_level:cultural:highest_speed_connection_percentage",
16563: "connectivity_level:primary_schools:typical_speed",
16564: "connectivity_level:institutes:highest_speed_connection_percentage",
16565: "connectivity_level:institutes:typical_speed",
16566: "connectivity_level:cultural:highest_speed",
16567: "connectivity_level:hospitals:typical_speed",
16568: "connectivity_level:hospitals:highest_speed",
16569: "connectivity_level:iros:typical_speed",
16570: "connectivity_level:hospitals:highest_speed_connection_percentage",
16571: "connectivity_level:for_profit_orgs:highest_speed",
16572: "connectivity_level:government:typical_speed",
16573: "connectivity_level:iros:highest_speed",
16574: "connectivity_level:government:highest_speed",
16575: "connectivity_level:for_profit_orgs:highest_speed_connection_percentage",
16576: "connectivity_level:government:highest_speed_connection_percentage",
16577: "connectivity_level:iros:highest_speed_connection_percentage",
16578: "connectivity_level:universities:highest_speed",
16579: "connectivity_level:universities:highest_speed_connection_percentage",
16581: "traffic_carriers:government:carry_mechanism",
16582: "traffic_carriers:universities:carry_mechanism",
16583: "traffic_carriers:secondary_schools:carry_mechanism",
16584: "traffic_carriers:institutes:carry_mechanism",
16585: "traffic_carriers:cultural:carry_mechanism",
16586: "traffic_carriers:hospitals:carry_mechanism",
16587: "traffic_carriers:iros:carry_mechanism",
16588: "traffic_carriers:for_profit_orgs:carry_mechanism",
16589: "traffic_carriers:primary_schools:carry_mechanism",
16590: "traffic_carriers:further_education:carry_mechanism",
16592: "traffic_load:government:average_from_institutions_to_network",
16593: "traffic_load:iros:average_from_institutions_to_network",
16594: "traffic_load:for_profit_orgs:average_from_institutions_to_network",
16595: "traffic_load:universities:average_from_institutions_to_network",
16596: "traffic_load:further_education:average_from_institutions_to_network",
16597: "traffic_load:secondary_schools:average_from_institutions_to_network",
16598: "traffic_load:primary_schools:average_from_institutions_to_network",
16599: "traffic_load:institutes:average_from_institutions_to_network",
16600: "traffic_load:cultural:average_from_institutions_to_network",
16601: "traffic_load:hospitals:average_from_institutions_to_network",
16602: "traffic_load:universities:average_to_institutions_from_network",
16603: "traffic_load:further_education:average_to_institutions_from_network",
16604: "traffic_load:secondary_schools:average_to_institutions_from_network",
16605: "traffic_load:primary_schools:average_to_institutions_from_network",
16606: "traffic_load:institutes:average_to_institutions_from_network",
16607: "traffic_load:cultural:average_to_institutions_from_network",
16608: "traffic_load:hospitals:average_to_institutions_from_network",
16609: "traffic_load:government:average_to_institutions_from_network",
16610: "traffic_load:iros:average_to_institutions_from_network",
16611: "traffic_load:for_profit_orgs:average_to_institutions_from_network",
16613: "traffic_load:secondary_schools:peak_to_institutions_from_network",
16614: "traffic_load:institutes:peak_to_institutions_from_network",
16615: "traffic_load:universities:peak_from_institutions_to_network",
16616: "traffic_load:for_profit_orgs:peak_from_institutions_to_network",
16617: "traffic_load:further_education:peak_from_institutions_to_network",
16618: "traffic_load:hospitals:peak_to_institutions_from_network",
16619: "traffic_load:secondary_schools:peak_from_institutions_to_network",
16620: "traffic_load:universities:peak_to_institutions_from_network",
16621: "traffic_load:primary_schools:peak_from_institutions_to_network",
16622: "traffic_load:primary_schools:peak_to_institutions_from_network",
16623: "traffic_load:institutes:peak_from_institutions_to_network",
16624: "traffic_load:cultural:peak_from_institutions_to_network",
16625: "traffic_load:hospitals:peak_from_institutions_to_network",
16626: "traffic_load:further_education:peak_to_institutions_from_network",
16627: "traffic_load:government:peak_from_institutions_to_network",
16628: "traffic_load:government:peak_to_institutions_from_network",
16629: "traffic_load:iros:peak_from_institutions_to_network",
16630: "traffic_load:for_profit_orgs:peak_to_institutions_from_network",
16631: "traffic_load:cultural:peak_to_institutions_from_network",
16632: "traffic_load:iros:peak_to_institutions_from_network",
16634: "traffic_growth:iros:growth_rate",
16635: "traffic_growth:for_profit_orgs:growth_rate",
16636: "traffic_growth:institutes:growth_rate",
16637: "traffic_growth:primary_schools:growth_rate",
16638: "traffic_growth:further_education:growth_rate",
16639: "traffic_growth:universities:growth_rate",
16640: "traffic_growth:hospitals:growth_rate",
16641: "traffic_growth:secondary_schools:growth_rate",
16642: "traffic_growth:government:growth_rate",
16643: "traffic_growth:cultural:growth_rate",
16646: "commercial_organizations:commercial_service_provider:connection",
16647: "commercial_organizations:commercial_collaboration:connection",
16648: "commercial_organizations:commercial_r_e:connection",
16649: "commercial_organizations:commercial_general:connection",
16650: "commercial_organizations:university_spin_off:connection",
16652: "commercial_charging_levels:collaboration:charging_level",
16653: "commercial_charging_levels:services:charging_level",
16654: "commercial_charging_levels:peering:charging_level",
16656: "remote_campuses", # "Yes" "No"
16658: "remote_campuses_specifics[0]:connected", # NB free text, doesnt map properly
16659: "remote_campuses_specifics[0]:country", # NB free text, doesnt map properly
# 16660: "connected_users_comments",
16662: "dark_fibre_lease", # "Yes" "No"
16663: "dark_fibre_lease_kilometers_inside_country",
16664: "dark_fibre_lease_kilometers_outside_country",
16665: "dark_fibre_lease_duration",
16666: "dark_fibre_nren", # "Yes" "No"
16667: "dark_fibre_nren_kilometers_inside_country",
16668: "fibre_light",
16669: "fibre_light-Comment",
16670: "network_map_urls[]:network_map_url", # json list with double quotes
16672: "monitoring_tools", # json list with double quotes
16673: "monitoring_tools-Comment",
16674: "netflow_vendors",
16675: "passive_monitoring", # "Yes" "No"
16676: "passive_monitoring_tech",
16677: "traffic_statistics", # "Yes" "No"
16678: "traffic_statistics_urls[0]:traffic_statistics_url",
16679: "siem_soc_vendor", # json list with double quotes
16680: "siem_soc_vendor-Comment",
16681: "certificate_service", # json list with double quotes
16682: "certificate_service-Comment",
16683: "network_weather", # "Yes" "No"
16684: "network_weather_url",
16685: "pert_team", # "Yes" "No" "Planned"
16687: "alienwave_services", # "Yes" "No" "Planned"
16688: "alienwave_services_number",
16689: "alienwave_internal", # "Yes" "No"
16691: "max_capacity",
16692: "typical_capacity",
16694: "external_connections[5]:capacity",
16695: "external_connections[7]:capacity",
16696: "external_connections[6]:capacity",
16697: "external_connections[7]:from_organization",
16698: "external_connections[1]:to_organization",
16699: "external_connections[8]:to_organization",
16700: "external_connections[9]:to_organization",
16701: "external_connections[1]:from_organization",
16702: "external_connections[8]:capacity",
16703: "external_connections[5]:to_organization",
16704: "external_connections[0]:link_name",
16705: "external_connections[1]:link_name",
16706: "external_connections[9]:capacity",
16707: "external_connections[2]:link_name",
16708: "external_connections[0]:from_organization",
16709: "external_connections[4]:link_name",
16710: "external_connections[3]:link_name",
16711: "external_connections[9]:link_name",
16712: "external_connections[7]:link_name",
16713: "external_connections[8]:link_name",
16714: "external_connections[6]:link_name",
16715: "external_connections[5]:link_name",
16716: "external_connections[4]:from_organization",
16717: "external_connections[5]:from_organization",
16718: "external_connections[6]:from_organization",
16719: "external_connections[2]:to_organization",
16720: "external_connections[3]:to_organization",
16721: "external_connections[4]:to_organization",
16722: "external_connections[6]:to_organization",
16723: "external_connections[7]:to_organization",
16724: "external_connections[2]:interconnection_method",
16725: "external_connections[3]:interconnection_method",
16726: "external_connections[4]:interconnection_method",
16727: "external_connections[5]:interconnection_method",
16728: "external_connections[8]:from_organization",
16729: "external_connections[9]:from_organization",
16730: "external_connections[0]:to_organization",
16731: "external_connections[0]:capacity",
16732: "external_connections[1]:capacity",
16733: "external_connections[2]:capacity",
16734: "external_connections[3]:capacity",
16735: "external_connections[4]:capacity",
16736: "external_connections[3]:from_organization",
16737: "external_connections[2]:from_organization",
16738: "external_connections[1]:interconnection_method",
16739: "external_connections[7]:interconnection_method",
16740: "external_connections[8]:interconnection_method",
16741: "external_connections[0]:interconnection_method",
16742: "external_connections[9]:interconnection_method",
16743: "external_connections[6]:interconnection_method",
16744: "non_r_and_e_peers",
16746: "traffic_estimate:from_customers",
16747: "traffic_estimate:to_customers",
16748: "traffic_estimate:from_external",
16749: "traffic_estimate:to_external",
16750: "commodity_vs_r_e:r_e", # note that this is free text, so it doesnt map properly
16752: "operational_process_automation", # "Yes" "No" "Planned"
16753: "operational_process_automation_tools",
16754: "nfv", # "Yes" "No" "Planned"
16755: "nfv_types", # json list with double quotes
16756: "nfv_types-Comment",
16757: "network_automation", # "Yes" "No" "Planned"
16758: "network_automation_tasks" # json list with double quotes
# 16759: "network_comments"
}
SUBORGANIZATION_ROLES = {
"Other": "other",
"HPC centre": "hpc",
"IDC federation": "idc"
}
CARRIERS = {
"NREN provides the local loops": "nren_local_loops",
"MAN": "man",
"Traffic carried to the backbone by commercial providers": "commercial_provider_backbone",
"Traffic carried to the backbone by regional NREN": "regional_nren_backbone",
"Other": "other"
}
CONNECTION = {
"Yes - including transit to other networks": "yes_incl_other",
"Yes - national NREN access only": "yes_national_nren",
"Yes - only if sponsored by a connected institution": "yes_if_sponsored",
"No - not eligible for policy reasons": "no_policy",
"No - but we offer a direct or IX peering": "no_but_direct_peering",
"No - financial restrictions (NREN is unable to charge/recover costs)": "no_financial",
"No - other reason / unsure": "no_other"
}
CHARGING_LEVELS = {
"Charges typically higher than for R+E users": "higher_than_r_e_charges",
"Same charging model as for R+E users": "same_as_r_e_charges",
"Charges typically lower than for R+E users": "lower_than_r_e_charges",
"No charges applied if requested by R+E users": "no_charges_if_r_e_requested"
}
INTERCONNECTION = {
"Internet exchange points": "internet_exchange",
"Peering at IXP": "internet_exchange",
"Open Exchange points": "open_exchange",
"Open Exchange points ": "open_exchange",
"Directly connected R&E peers": "direct",
"Direct peering": "direct",
"GEANT": "geant",
"Other": "other",
"CBF": "other"
}
VALUE_TO_CODE_MAPPING = {
16410: {
"Other": "other",
"We use a combination of flat fee and usage-based fee": "combination",
"We charge a usage-based fee": "usage_based_fee",
"We charge a flat fee, based on bandwidth": "flat_fee",
"We do not charge them directly": "no_charge"
},
16446: SUBORGANIZATION_ROLES,
16448: SUBORGANIZATION_ROLES,
16449: SUBORGANIZATION_ROLES,
16450: SUBORGANIZATION_ROLES,
16451: SUBORGANIZATION_ROLES,
16503: {
"Anti Virus": "anti_virus",
"Anti-Spam": "anti_spam",
"Firewall": "firewall",
"DDoS mitigation": "ddos_mitigation",
"Network monitoring": "monitoring",
"IPS/IDS": "ips_ids",
"ACL": "acl",
"Network segmentation": "segmentation",
"Integrity checking": "integrity_checking",
"Other": "other"
},
16763: {
"We participate in GEANT Crisis workshops such as CLAW": "geant_workshops",
"No, we have not done any crisis exercises or trainings": "none",
"We participated in National crisis exercises ": "national_excercises",
"We had a real crisis": "real_crisis",
"We run our own simulation exercises": "simulation_excercises",
"We run our own tabletop exercises": "tabletop_exercises",
"We have done/participated in other exercises or trainings": "other_excercises"
},
16581: CARRIERS,
16582: CARRIERS,
16583: CARRIERS,
16584: CARRIERS,
16585: CARRIERS,
16586: CARRIERS,
16587: CARRIERS,
16588: CARRIERS,
16589: CARRIERS,
16590: CARRIERS,
16646: CONNECTION,
16647: CONNECTION,
16648: CONNECTION,
16649: CONNECTION,
16650: CONNECTION,
16652: CHARGING_LEVELS,
16653: CHARGING_LEVELS,
16654: CHARGING_LEVELS,
16668: {
"NREN owns and operates equipment": "nren_owns_and_operates",
"NREN owns equipment and operation is outsourced": "nren_owns_outsourced_operation",
"Ownership and management are out-sourced (turn-key model)": "outsourced_ownership_and_operation",
"Other": "other"
},
16672: {
"Looking Glass": "looking_glass",
"Network or services Status Dashboard": "status_dashboard",
"Historical traffic volume information": "historical_traffic_volumes",
"Netflow analysis tool": "netflow_analysis",
"Other": "other"
},
16676: {
"SPAN ports": "span_ports",
"Passive optical TAPS": "taps",
"Both": "both"
},
16679: {
"LogRythm": "LogRythm",
"Securonix": "Securonix",
"Splunk": "Splunk",
"Other": "other"
},
16681: {
"Sectigo (outside of TCS)": "Sectigo",
"GlobalSign": "GlobalSign",
"TCS": "TCS",
"GeoDaddy": "GeoDaddy",
"GeoTrust": "GeoTrust",
"Digicert": "Digicert",
"Other": "other"
},
16724: INTERCONNECTION,
16725: INTERCONNECTION,
16726: INTERCONNECTION,
16727: INTERCONNECTION,
16738: INTERCONNECTION,
16739: INTERCONNECTION,
16740: INTERCONNECTION,
16741: INTERCONNECTION,
16742: INTERCONNECTION,
16743: INTERCONNECTION,
16755: {
"Routers/switches": "routers",
"Firewalls": "firewalls",
"Load balancers": "load_balancers",
"VPN Concentrator Services": "vpn_concentrators",
"Other": "other"
},
16758: {
"Device Provisioning": "provisioning",
"Data Collection": "data_collection",
"Configuration Management": "config_management",
"Compliance": "compliance",
"Reporting": "reporting",
"Troubleshooting": "troubleshooting"
}
}
compendium_v2/routes/survey.py
+ 18
− 6
View file @ a7cdfbba
...@@ -27,13 +27,24 @@ class VerificationStatus(str, Enum): ...@@ -27,13 +27,24 @@ class VerificationStatus(str, Enum):
Edited = "edited" # a question for which last years answer was edited Edited = "edited" # a question for which last years answer was edited
@routes.route('/open', methods=['GET']) @routes.route('/nrens', methods=['GET'])
@common.require_accepts_json @common.require_accepts_json
def open_survey() -> Any: def get_nrens() -> Any:
entries = [
{"id": entry.id, "name": entry.name}
for entry in db.session.scalars(select(NREN).order_by(NREN.name))
]
return jsonify(entries)
@routes.route('/open/<string:nren_name>', methods=['GET'])
@common.require_accepts_json
def open_survey(nren_name) -> Any:
# just a hardcoded year and nren for development for now # just a hardcoded year and nren for development for now
nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one() nren = db.session.execute(select(NREN).filter(NREN.name == nren_name)).scalar_one()
year = 1989 year = 1989
last_year = 2022
survey = db.session.scalar(select(Survey).where(Survey.year == year)) survey = db.session.scalar(select(Survey).where(Survey.year == year))
if survey is None or survey.survey == {}: if survey is None or survey.survey == {}:
...@@ -43,17 +54,18 @@ def open_survey() -> Any: ...@@ -43,17 +54,18 @@ def open_survey() -> Any:
with p.open('r') as f: with p.open('r') as f:
survey = json.load(f) survey = json.load(f)
# TODO add some magic strings in the json (like the year, url validation regex, maybe countries list) and interpolate them here # TODO add some magic strings in the json (like the year, url validation regex, maybe countries list)
# and interpolate them here
data: Optional[dict] = None data: Optional[dict] = None
page = 0 page = 0
verification_status: dict[str, str] = {"budget": VerificationStatus.Unverified, "TODO": "remove or set everything to new"} verification_status = {"budget": VerificationStatus.Unverified, "TODO": "remove or set all to new"}
response = db.session.scalar( response = db.session.scalar(
select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id) select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
) )
previous_response = db.session.scalar( previous_response = db.session.scalar(
select(SurveyResponse).where(SurveyResponse.survey_year == year - 1).where(SurveyResponse.nren_id == nren.id) select(SurveyResponse).where(SurveyResponse.survey_year == last_year).where(SurveyResponse.nren_id == nren.id)
) )
if response: if response:
... ...
......
compendium_v2/routes/survey_model.json
+ 23
− 14
View file @ a7cdfbba
...@@ -293,7 +293,7 @@ ...@@ -293,7 +293,7 @@
}, },
{ {
"type": "text", "type": "text",
"name": "abbreviation_local_languages", "name": "abbreviation_national_languages",
"title": "Abbreviation in the national language(s) (if applicable):", "title": "Abbreviation in the national language(s) (if applicable):",
"description": "Please always provide this information. Please use the Latin alphabet." "description": "Please always provide this information. Please use the Latin alphabet."
}, },
...@@ -506,7 +506,7 @@ ...@@ -506,7 +506,7 @@
"type": "radiogroup", "type": "radiogroup",
"name": "service_portfolio_changes", "name": "service_portfolio_changes",
"title": "Has your NREN made any changes to its service portfolio for its users?", "title": "Has your NREN made any changes to its service portfolio for its users?",
"description": "Please note any changes here and amend the Service Matrix directly at https://compendiumdatabase.geant.org/reports/nrens_services", "description": "Please note any changes here and amend the Service Matrix at the last page of this survey",
"choices": [ "choices": [
"Yes", "Yes",
"No" "No"
...@@ -519,8 +519,8 @@ ...@@ -519,8 +519,8 @@
"title": "Which service types are available for which user types?", "title": "Which service types are available for which user types?",
"columns": [ "columns": [
{ {
"name": "user_types", "name": "service_types",
"title": "User types", "title": "Service types",
"cellType": "checkbox", "cellType": "checkbox",
"showInMultipleColumns": true, "showInMultipleColumns": true,
"choices": [ "choices": [
...@@ -778,9 +778,14 @@ ...@@ -778,9 +778,14 @@
"name": "panel2", "name": "panel2",
"elements": [ "elements": [
{ {
"type": "text", "type": "matrixdynamic",
"name": "connected_sites_list", "name": "connected_sites_lists",
"title": "Please provide a URL that lists the sites that are connected to the NREN, if available:", "title": "Please provide the URLs that lists the sites that are connected to the NREN, if available:",
"columns": [
{
"name": "connected_sites_url",
"title": "Url:",
"cellType": "text",
"inputType": "url", "inputType": "url",
"validators": [ "validators": [
{ {
...@@ -789,6 +794,10 @@ ...@@ -789,6 +794,10 @@
"regex": "^(https?:\\/\\/)?([\\da-zA-Z\\.-]+\\.[a-zA-Z\\.]{2,6}|[\\d\\.]+)([\\/:?=&#%]{1}[\\d_a-zA-Z\\.-]+)*[\\/\\?]?$" "regex": "^(https?:\\/\\/)?([\\da-zA-Z\\.-]+\\.[a-zA-Z\\.]{2,6}|[\\d\\.]+)([\\/:?=&#%]{1}[\\d_a-zA-Z\\.-]+)*[\\/\\?]?$"
} }
] ]
}
],
"rowCount": 1,
"maxRowCount": 50
}, },
{ {
"type": "matrixdropdown", "type": "matrixdropdown",
...@@ -826,7 +835,7 @@ ...@@ -826,7 +835,7 @@
"text": "No - other reason" "text": "No - other reason"
}, },
{ {
"value": "item7", "value": "unsure",
"text": "Unsure/unclear" "text": "Unsure/unclear"
} }
] ]
... ...
......
compendium_v2/static/survey-bundle.js
+ 1
− 1
View file @ a7cdfbba
Source diff could not be displayed: it is too large. Options to address this: view the blob.
setup.py
+ 1
− 0
View file @ a7cdfbba
...@@ -28,6 +28,7 @@ setup( ...@@ -28,6 +28,7 @@ setup(
'console_scripts': [ 'console_scripts': [
'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa 'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa
'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa 'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa
'conversion=compendium_v2.conversion.conversion:cli', # noqa
] ]
} }
) )
survey-frontend/src/SurveyComponent.tsx
+ 5
− 5
View file @ a7cdfbba
...@@ -23,7 +23,7 @@ enum VerificationStatus { ...@@ -23,7 +23,7 @@ enum VerificationStatus {
} }
function SurveyComponent() { function SurveyComponent({ nrenName }) {
const [surveyModel, setSurveyModel] = useState<Model>(); const [surveyModel, setSurveyModel] = useState<Model>();
const [progress, setProgress] = useState<Progress[]>([]); const [progress, setProgress] = useState<Progress[]>([]);
const verificationStatus = useRef<Map<string, VerificationStatus>>(new Map()); const verificationStatus = useRef<Map<string, VerificationStatus>>(new Map());
...@@ -64,7 +64,7 @@ function SurveyComponent() { ...@@ -64,7 +64,7 @@ function SurveyComponent() {
async function getModel() async function getModel()
{ {
const response = await fetch('/api/survey/open'); const response = await fetch('/api/survey/open/' + nrenName);
const json = await response.json(); const json = await response.json();
for (const questionName in json["verification_status"]) { for (const questionName in json["verification_status"]) {
... ...
......
survey-frontend/src/SurveySelectionComponent.tsx 0 → 100644
+ 52
− 0
View file @ a7cdfbba
import React, { useState, useEffect } from "react";
import SurveyComponent from "./SurveyComponent";
interface Nren {
id: number
name: string
}
function SurveySelectionComponent() {
const [nrens, setNrens] = useState<Nren[]>([]);
const [selectedNren, setSelectedNren] = useState<Nren | null>(null);
useEffect(() => {
// Fetch organizations from the API
fetchNrens();
}, []);
const fetchNrens = async () => {
try {
const response = await fetch('/api/survey/nrens');
const data = await response.json();
setNrens(data);
} catch (error) {
console.error('Error fetching organizations:', error);
}
};
const handleNrenSelect = (nren) => {
setSelectedNren(nren);
};
if (!selectedNren) {
return (
<div>
<h2>Select an organization:</h2>
<ul>
{nrens.map((nren) => (
<li key={nren.id} onClick={() => handleNrenSelect(nren)}>
{nren.name}
</li>
))}
</ul>
</div>
);
}
return <SurveyComponent nrenName={selectedNren.name} />;
}
export default SurveySelectionComponent;
\ No newline at end of file
survey-frontend/src/index.tsx
+ 2
− 2
View file @ a7cdfbba
import React from 'react'; import React from 'react';
import { createRoot } from 'react-dom/client'; import { createRoot } from 'react-dom/client';
import SurveyComponent from './SurveyComponent'; import SurveySelectionComponent from './SurveySelectionComponent';
const container = document.getElementById('root') as HTMLElement; const container = document.getElementById('root') as HTMLElement;
...@@ -10,6 +10,6 @@ const root = createRoot(container); ...@@ -10,6 +10,6 @@ const root = createRoot(container);
root.render( root.render(
<React.StrictMode> <React.StrictMode>
<SurveyComponent /> <SurveySelectionComponent />
</React.StrictMode> </React.StrictMode>
) )
\ No newline at end of file
test/test_conversion.py 0 → 100644
+ 68
− 0
View file @ a7cdfbba
from sqlalchemy import select
from compendium_v2.db import db
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse
from compendium_v2.conversion.conversion import _cli, convert_answers
def mock_convert_answers(_):
return {"data": {}}
def mock_query_nren(_):
return {16455: "answer1"}
def test_queries(app_with_survey_db, mocker):
with app_with_survey_db.app_context():
db.session.add(NREN(name='Restena', country='country'))
db.session.commit()
mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
_cli(app_with_survey_db)
with app_with_survey_db.app_context():
surveys = db.session.scalars(select(Survey)).all()
assert len(surveys) == 1
assert surveys[0].year == 2022
responses = db.session.scalars(select(SurveyResponse).order_by(SurveyResponse.nren_id)).all()
assert len(responses) == 1
assert responses[0].answers == {"data": {}}
def test_conversion():
answers = {
16455: '"full nren name"',
16453: '["ec project1", "ec project2"]',
16632: '"3434"',
16432: '"suborg name 3"',
16410: '"We use a combination of flat fee and usage-based fee"',
16474: '"Yes"',
16476: '"No"',
16491: '["Universities", "Further education"]',
16492: '["Research institutes", "Universities"]'
}
converted_answers = convert_answers(answers)
assert converted_answers == {
'data': {
'charging_mechanism': 'combination',
'suborganization_details': [{}, {}, {}, {'suborganization_name': 'suborg name 3'}],
'ec_project_names': [{'ec_project_name': 'ec project1'}, {'ec_project_name': 'ec project2'}],
'full_name_english': 'full nren name',
'policies': {'connectivity_policy': {'available': ['yes']}, 'acceptable_use_policy': {}},
'traffic_load': {'iros': {'peak_to_institutions_from_network': '3434'}},
'service_matrix': {
'universities': {'service_types': ['security', 'isp_support']},
'further_education': {'service_types': ['security']},
'institutes': {'service_types': ['isp_support']}
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment