start of conversion script

compendium_v2/conversion/conversion.py

+"""
+conversion
+=========================
+
+This module loads the survey data from 2022 from the survey database
+and stores the data in the json structure of the new survey, so that
+it can be used to prefill the 2023 survey.
+
+"""
+import logging
+import click
+
+from sqlalchemy import delete, text, select
+
+import compendium_v2
+from compendium_v2.environment import setup_logging
+from compendium_v2.db import db
+from compendium_v2.config import load
+from compendium_v2.survey_db import model as survey_model
+from compendium_v2.db.model import NREN
+from compendium_v2.db.survey_model import Survey, SurveyResponse
+
+setup_logging()
+
+logger = logging.getLogger('conversion')
+
+
+ANSWERS_2022_QUERY = """
+SELECT question_id, value
+FROM answers a
+WHERE nren_id = {}
+      AND value NOT IN ('""', '[]', '"NA"', '"N/A"', '[""]', '["-"]', '["/"]')
+      AND NOT EXISTS (SELECT 1 FROM answers a2 WHERE a.question_id = a2.question_id AND a.nren_id = a2.nren_id AND a2.id > a.id)
+      AND question_id IN (16402,16405,16406,16407,16408,16409,16410,16413,16414,16416,16417,16418,16419,16420,16422,16426,16429,16430,16432,16433,16434,16435,16438,16439,16446,16448,16449,
+                          16450,16451,16452,16453,16455,16456,16457,16458,16459,16460,16461,16462,16463,16464,16465,16468,16469,16470,16471,16472,16473,16474,16475,16476,16477,16478,16479,
+                          16480,16481,16482,16483,16484,16485,16486,16488,16489,16490,16491,16492,16493,16494,16495,16496,16497,16499,16500,16501,16502,16503,16504,16760,16761,16762,16763,
+                          16507,16509,16510,16511,16512,16513,16514,16515,16516,16517,16518,16519,16520,16521,16522,16523,16524,16525,16526,16527,16528,16529,16530,16531,16532,16533,16534,
+                          16535,16536,16537,16538,16539,16540,16541,16542,16543,16544,16545,16546,16547,16548,16550,16551,16552,16553,16554,16555,16556,16557,16558,16559,16560,16561,16562,
+                          16563,16564,16565,16566,16567,16568,16569,16570,16571,16572,16573,16574,16575,16576,16577,16578,16579,16581,16582,16583,16584,16585,16586,16587,16588,16589,16590,
+                          16592,16593,16594,16595,16596,16597,16598,16599,16600,16601,16602,16603,16604,16605,16606,16607,16608,16609,16610,16611,16613,16614,16615,16616,16617,16618,16619,
+                          16620,16621,16622,16623,16624,16625,16626,16627,16628,16629,16630,16631,16632,16634,16635,16636,16637,16638,16639,16640,16641,16642,16643,16646,16647,16648,16649,
+                          16650,16652,16653,16654,16656,16658,16659,16662,16663,16664,16665,16666,16667,16668,16669,16670,16672,16673,16674,16675,16676,16677,16678,16679,16680,16681,16682,
+                          16683,16684,16685,16687,16688,16689,16691,16692,16694,16695,16696,16697,16698,16699,16700,16701,16702,16703,16704,16705,16706,16707,16708,16709,16710,16711,16712,
+                          16713,16714,16715,16716,16717,16718,16719,16720,16721,16722,16723,16724,16725,16726,16727,16728,16729,16730,16731,16732,16733,16734,16735,16736,16737,16738,16739,
+                          16740,16741,16742,16743,16744,16746,16747,16748,16749,16750,16752,16753,16754,16755,16756,16757,16758);
+"""
+
+NREN_IDS = {
+    "ANAS": 49,
+    "GRNET": 17,
+    "Restena": 28,
+    "University of Malta": 29,
+    "DeiC": 9,
+    "CSC": 12,
+    "Sikt": 33,
+    "ACOnet": 4,
+    "AMRES": 48,
+    "ARNES": 39,
+    "ASNET-AM": 200,
+    "BASNET": 54,
+    "Belnet": 5,
+    "BREN": 58,
+    "CARNET": 6,
+    "CESNET": 8,
+    "CYNET": 7,
+    "DFN": 16,
+    "EENet": 11,
+    "FCCN": 35,
+    "GARR": 23,
+    "GRENA": 15,
+    "HEAnet": 21,
+    "IUCC": 22,
+    "Jisc": 46,
+    "KIFU": 18,
+    "LAT": 51,
+    "LITNET": 27,
+    "MARnet": 14,
+    "MREN": 100,
+    "PIONIER": 34,
+    "RASH": 1,
+    "RedIRIS": 40,
+    "RENAM": 30,
+    "RENATER": 13,
+    "RHnet": 19,
+    "RoEduNet": 52,
+    "SANET": 38,
+    "SUNET": 41,
+    "SURF": 32,
+    "SWITCH": 42,
+    "ULAKBIM": 44,
+    "URAN": 45,
+}
+
+
+
+def query_nren(nren_id: int):
+    query = ANSWERS_2022_QUERY.format(nren_id)
+    answers = {}
+    for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
+        answers[row[0]] = row[1]
+    return answers
+
+
+def _cli(app):
+    with app.app_context():
+
+        nren_surveys = {}
+
+        for nren in db.session.scalars(select(NREN)):
+            survey_db_nren_id = NREN_IDS[nren.name]
+            nren_surveys[nren] = query_nren(survey_db_nren_id)
+
+        db.session.execute(delete(SurveyResponse).where(
+            SurveyResponse.survey_year == 2022
+        ))
+
+        db.session.execute(delete(Survey).where(
+            Survey.year == 2022
+        ))
+
+        survey = Survey(year=2022, survey={})
+        db.session.add(survey)
+
+        for nren, answers in nren_surveys.items():
+            response = SurveyResponse(
+                nren=nren,
+                nren_id=nren.id,
+                survey_year=2022,
+                survey=survey,
+                answers=answers  # TODO structure should be different
+            )
+            db.session.add(response)
+
+        db.session.commit()
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+    app_config = load(open(config, 'r'))
+
+    app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
+
+    app = compendium_v2._create_app_with_db(app_config)
+    _cli(app)
+
+
+if __name__ == "__main__":
+    cli()

setup.py

@@ -28,6 +28,7 @@ setup(
         'console_scripts': [
             'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli',  # noqa
             'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli',  # noqa
+            'conversion=compendium_v2.conversion.conversion:cli',  # noqa
         ]
     }
 )