From 3b8d694cac9e5ef9dd0728d338b4a97cd18cd5e3 Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Thu, 29 Jun 2023 15:35:32 +0200
Subject: [PATCH] start of conversion script
---
compendium_v2/conversion/__init__.py | 0
compendium_v2/conversion/conversion.py | 149 +++++++++++++++++++++++++
setup.py | 1 +
3 files changed, 150 insertions(+)
create mode 100644 compendium_v2/conversion/__init__.py
create mode 100644 compendium_v2/conversion/conversion.py
diff --git a/compendium_v2/conversion/__init__.py b/compendium_v2/conversion/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
new file mode 100644
index 00000000..bdd8d96a
--- /dev/null
+++ b/compendium_v2/conversion/conversion.py
@@ -0,0 +1,149 @@
+"""
+conversion
+=========================
+
+This module loads the survey data from 2022 from the survey database
+and stores the data in the json structure of the new survey, so that
+it can be used to prefill the 2023 survey.
+
+"""
+import logging
+import click
+
+from sqlalchemy import delete, text, select
+
+import compendium_v2
+from compendium_v2.environment import setup_logging
+from compendium_v2.db import db
+from compendium_v2.config import load
+from compendium_v2.survey_db import model as survey_model
+from compendium_v2.db.model import NREN
+from compendium_v2.db.survey_model import Survey, SurveyResponse
+
+setup_logging()
+
+logger = logging.getLogger('conversion')
+
+
+ANSWERS_2022_QUERY = """
+SELECT question_id, value
+FROM answers a
+WHERE nren_id = {}
+ AND value NOT IN ('""', '[]', '"NA"', '"N/A"', '[""]', '["-"]', '["/"]')
+ AND NOT EXISTS (SELECT 1 FROM answers a2 WHERE a.question_id = a2.question_id AND a.nren_id = a2.nren_id AND a2.id > a.id)
+ AND question_id IN (16402,16405,16406,16407,16408,16409,16410,16413,16414,16416,16417,16418,16419,16420,16422,16426,16429,16430,16432,16433,16434,16435,16438,16439,16446,16448,16449,
+ 16450,16451,16452,16453,16455,16456,16457,16458,16459,16460,16461,16462,16463,16464,16465,16468,16469,16470,16471,16472,16473,16474,16475,16476,16477,16478,16479,
+ 16480,16481,16482,16483,16484,16485,16486,16488,16489,16490,16491,16492,16493,16494,16495,16496,16497,16499,16500,16501,16502,16503,16504,16760,16761,16762,16763,
+ 16507,16509,16510,16511,16512,16513,16514,16515,16516,16517,16518,16519,16520,16521,16522,16523,16524,16525,16526,16527,16528,16529,16530,16531,16532,16533,16534,
+ 16535,16536,16537,16538,16539,16540,16541,16542,16543,16544,16545,16546,16547,16548,16550,16551,16552,16553,16554,16555,16556,16557,16558,16559,16560,16561,16562,
+ 16563,16564,16565,16566,16567,16568,16569,16570,16571,16572,16573,16574,16575,16576,16577,16578,16579,16581,16582,16583,16584,16585,16586,16587,16588,16589,16590,
+ 16592,16593,16594,16595,16596,16597,16598,16599,16600,16601,16602,16603,16604,16605,16606,16607,16608,16609,16610,16611,16613,16614,16615,16616,16617,16618,16619,
+ 16620,16621,16622,16623,16624,16625,16626,16627,16628,16629,16630,16631,16632,16634,16635,16636,16637,16638,16639,16640,16641,16642,16643,16646,16647,16648,16649,
+ 16650,16652,16653,16654,16656,16658,16659,16662,16663,16664,16665,16666,16667,16668,16669,16670,16672,16673,16674,16675,16676,16677,16678,16679,16680,16681,16682,
+ 16683,16684,16685,16687,16688,16689,16691,16692,16694,16695,16696,16697,16698,16699,16700,16701,16702,16703,16704,16705,16706,16707,16708,16709,16710,16711,16712,
+ 16713,16714,16715,16716,16717,16718,16719,16720,16721,16722,16723,16724,16725,16726,16727,16728,16729,16730,16731,16732,16733,16734,16735,16736,16737,16738,16739,
+ 16740,16741,16742,16743,16744,16746,16747,16748,16749,16750,16752,16753,16754,16755,16756,16757,16758);
+"""
+
+NREN_IDS = {
+ "ANAS": 49,
+ "GRNET": 17,
+ "Restena": 28,
+ "University of Malta": 29,
+ "DeiC": 9,
+ "CSC": 12,
+ "Sikt": 33,
+ "ACOnet": 4,
+ "AMRES": 48,
+ "ARNES": 39,
+ "ASNET-AM": 200,
+ "BASNET": 54,
+ "Belnet": 5,
+ "BREN": 58,
+ "CARNET": 6,
+ "CESNET": 8,
+ "CYNET": 7,
+ "DFN": 16,
+ "EENet": 11,
+ "FCCN": 35,
+ "GARR": 23,
+ "GRENA": 15,
+ "HEAnet": 21,
+ "IUCC": 22,
+ "Jisc": 46,
+ "KIFU": 18,
+ "LAT": 51,
+ "LITNET": 27,
+ "MARnet": 14,
+ "MREN": 100,
+ "PIONIER": 34,
+ "RASH": 1,
+ "RedIRIS": 40,
+ "RENAM": 30,
+ "RENATER": 13,
+ "RHnet": 19,
+ "RoEduNet": 52,
+ "SANET": 38,
+ "SUNET": 41,
+ "SURF": 32,
+ "SWITCH": 42,
+ "ULAKBIM": 44,
+ "URAN": 45,
+}
+
+
+
+def query_nren(nren_id: int):
+ query = ANSWERS_2022_QUERY.format(nren_id)
+ answers = {}
+ for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
+ answers[row[0]] = row[1]
+ return answers
+
+
+def _cli(app):
+ with app.app_context():
+
+ nren_surveys = {}
+
+ for nren in db.session.scalars(select(NREN)):
+ survey_db_nren_id = NREN_IDS[nren.name]
+ nren_surveys[nren] = query_nren(survey_db_nren_id)
+
+ db.session.execute(delete(SurveyResponse).where(
+ SurveyResponse.survey_year == 2022
+ ))
+
+ db.session.execute(delete(Survey).where(
+ Survey.year == 2022
+ ))
+
+ survey = Survey(year=2022, survey={})
+ db.session.add(survey)
+
+ for nren, answers in nren_surveys.items():
+ response = SurveyResponse(
+ nren=nren,
+ nren_id=nren.id,
+ survey_year=2022,
+ survey=survey,
+ answers=answers # TODO structure should be different
+ )
+ db.session.add(response)
+
+ db.session.commit()
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+ app_config = load(open(config, 'r'))
+
+ app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
+
+ app = compendium_v2._create_app_with_db(app_config)
+ _cli(app)
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/setup.py b/setup.py
index 87daee04..6d982c18 100644
--- a/setup.py
+++ b/setup.py
@@ -28,6 +28,7 @@ setup(
'console_scripts': [
'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa
'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa
+ 'conversion=compendium_v2.conversion.conversion:cli', # noqa
]
}
)
--
GitLab