Skip to content
Snippets Groups Projects
Commit 271d3d4d authored by Remco Tukker's avatar Remco Tukker
Browse files

complete mapping

parent 15fdfd42
No related branches found
No related tags found
1 merge request!47Feature/comp 218 migrating 2022 data
Expand all Hide whitespace changes
Inline Side-by-side
compendium_v2/conversion/conversion.py
+ 13
− 86
View file @ 271d3d4d
...@@ -21,93 +21,15 @@ from compendium_v2.survey_db import model as survey_model ...@@ -21,93 +21,15 @@ from compendium_v2.survey_db import model as survey_model
from compendium_v2.db.model import NREN from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse from compendium_v2.db.survey_model import Survey, SurveyResponse
from compendium_v2.conversion.mapping import id_to_name, id_to_name_services, service_user_types_to_code from compendium_v2.conversion import mapping
setup_logging() setup_logging()
logger = logging.getLogger('conversion') logger = logging.getLogger('conversion')
ANSWERS_2022_QUERY = """
SELECT question_id, value
FROM answers a
WHERE nren_id = {}
AND value NOT IN ('""', '[]', '"NA"', '"N/A"', '[""]', '["-"]', '["/"]')
AND NOT EXISTS (
SELECT 1 FROM answers a2 WHERE a.question_id = a2.question_id AND a.nren_id = a2.nren_id AND a2.id > a.id
)
AND question_id IN (16402,16405,16406,16407,16408,16409,16410,16413,16414,16416,16417,16418,16419,16420,16422,
16426,16429,16430,16432,16433,16434,16435,16438,16439,16446,16448,16449,16450,16451,16452,
16453,16455,16456,16457,16458,16459,16460,16461,16462,16463,16464,16465,16468,16469,16470,
16471,16472,16473,16474,16475,16476,16477,16478,16479,16480,16481,16482,16483,16484,16485,
16486,16488,16489,16490,16491,16492,16493,16494,16495,16496,16497,16499,16500,16501,16502,
16503,16504,16760,16761,16762,16763,16507,16509,16510,16511,16512,16513,16514,16515,16516,
16517,16518,16519,16520,16521,16522,16523,16524,16525,16526,16527,16528,16529,16530,16531,
16532,16533,16534,16535,16536,16537,16538,16539,16540,16541,16542,16543,16544,16545,16546,
16547,16548,16550,16551,16552,16553,16554,16555,16556,16557,16558,16559,16560,16561,16562,
16563,16564,16565,16566,16567,16568,16569,16570,16571,16572,16573,16574,16575,16576,16577,
16578,16579,16581,16582,16583,16584,16585,16586,16587,16588,16589,16590,16592,16593,16594,
16595,16596,16597,16598,16599,16600,16601,16602,16603,16604,16605,16606,16607,16608,16609,
16610,16611,16613,16614,16615,16616,16617,16618,16619,16620,16621,16622,16623,16624,16625,
16626,16627,16628,16629,16630,16631,16632,16634,16635,16636,16637,16638,16639,16640,16641,
16642,16643,16646,16647,16648,16649,16650,16652,16653,16654,16656,16658,16659,16662,16663,
16664,16665,16666,16667,16668,16669,16670,16672,16673,16674,16675,16676,16677,16678,16679,
16680,16681,16682,16683,16684,16685,16687,16688,16689,16691,16692,16694,16695,16696,16697,
16698,16699,16700,16701,16702,16703,16704,16705,16706,16707,16708,16709,16710,16711,16712,
16713,16714,16715,16716,16717,16718,16719,16720,16721,16722,16723,16724,16725,16726,16727,
16728,16729,16730,16731,16732,16733,16734,16735,16736,16737,16738,16739,16740,16741,16742,
16743,16744,16746,16747,16748,16749,16750,16752,16753,16754,16755,16756,16757,16758);
"""
NREN_IDS = {
"ANAS": 49,
"GRNET": 17,
"Restena": 28,
"University of Malta": 29,
"DeiC": 9,
"CSC": 12,
"Sikt": 33,
"ACOnet": 4,
"AMRES": 48,
"ARNES": 39,
"ASNET-AM": 200,
"BASNET": 54,
"Belnet": 5,
"BREN": 58,
"CARNET": 6,
"CESNET": 8,
"CYNET": 7,
"DFN": 16,
"EENet": 11,
"FCCN": 35,
"GARR": 23,
"GRENA": 15,
"HEAnet": 21,
"IUCC": 22,
"Jisc": 46,
"KIFU": 18,
"LAT": 51,
"LITNET": 27,
"MARnet": 14,
"MREN": 100,
"PIONIER": 34,
"RASH": 1,
"RedIRIS": 40,
"RENAM": 30,
"RENATER": 13,
"RHnet": 19,
"RoEduNet": 52,
"SANET": 38,
"SUNET": 41,
"SURF": 32,
"SWITCH": 42,
"ULAKBIM": 44,
"URAN": 45,
}
def query_nren(nren_id: int): def query_nren(nren_id: int):
query = ANSWERS_2022_QUERY.format(nren_id) query = mapping.ANSWERS_2022_QUERY.format(nren_id)
answers = {} answers = {}
for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}): for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
answers[row[0]] = row[1] answers[row[0]] = row[1]
...@@ -117,17 +39,22 @@ def query_nren(nren_id: int): ...@@ -117,17 +39,22 @@ def query_nren(nren_id: int):
def convert_answers(answers): def convert_answers(answers):
data = {} data = {}
for id, question_name in id_to_name.items(): for id, question_name in mapping.ID_TO_NAME.items():
if not id in answers: if not id in answers:
continue continue
answer = answers[id] answer = answers[id]
if len(answer) > 1 and answer[0] == '"' and answer[-1] == '"': if len(answer) > 1 and answer[0] == '"' and answer[-1] == '"':
answer = answer[1:-1] answer = answer[1:-1]
if id in mapping.VALUE_TO_CODE_MAPPING:
answer = mapping.VALUE_TO_CODE_MAPPING[id][answer]
if len(answer) > 1 and answer[0] == '[' and answer[-1] == ']': if len(answer) > 1 and answer[0] == '[' and answer[-1] == ']':
answer = json.loads(answer) answer = json.loads(answer)
if id in mapping.VALUE_TO_CODE_MAPPING:
# TODO mapping of the answers to the codes for a lot of questions mapped_answer = []
for entry in answer:
mapped_answer.append(mapping.VALUE_TO_CODE_MAPPING[id][entry])
answer = mapped_answer
# code to convert my description in the mapping to a json structure # code to convert my description in the mapping to a json structure
question_names = question_name.split(":") question_names = question_name.split(":")
...@@ -156,14 +83,14 @@ def convert_answers(answers): ...@@ -156,14 +83,14 @@ def convert_answers(answers):
subdict[question_names[-1]] = answer subdict[question_names[-1]] = answer
for id, question_name in id_to_name_services.items(): for id, question_name in mapping.ID_TO_NAME_SERVICES.items():
if not id in answers: if not id in answers:
continue continue
answer = answers[id] answer = answers[id]
answer = json.loads(answer) answer = json.loads(answer)
for user_type in answer: for user_type in answer:
user_type_code = service_user_types_to_code[user_type] user_type_code = mapping.SERVICE_USER_TYPE_TO_CODE[user_type]
formatted_question_name = question_name.format(user_type_code) formatted_question_name = question_name.format(user_type_code)
question_names = formatted_question_name.split(":") question_names = formatted_question_name.split(":")
...@@ -183,7 +110,7 @@ def _cli(app): ...@@ -183,7 +110,7 @@ def _cli(app):
nren_surveys = {} nren_surveys = {}
for nren in db.session.scalars(select(NREN)): for nren in db.session.scalars(select(NREN)):
survey_db_nren_id = NREN_IDS[nren.name] survey_db_nren_id = mapping.NREN_IDS[nren.name]
nren_surveys[nren] = query_nren(survey_db_nren_id) nren_surveys[nren] = query_nren(survey_db_nren_id)
db.session.execute(delete(SurveyResponse).where( db.session.execute(delete(SurveyResponse).where(
......
compendium_v2/conversion/mapping.py
+ 294
− 63
View file @ 271d3d4d
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment