make service conversion work

compendium_v2/conversion/conversion.py

@@ -110,8 +110,7 @@ def convert_answers(answers):
     return {"data": data}
 
 
-def _cli(app):
-
+def load_service_data():
     wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
     ws = wb["Sheet1"]
     rows = list(ws.rows)
@@ -128,26 +127,30 @@ def _cli(app):
     nren_service_data = {}
 
     for nren_name, start_column in nren_service_data_columns.items():
+        nren_service_data[nren_name] = {}
         for row_index in range(2, 61):
             row = rows[row_index]
             service_name = row[0].value
-            question_key_base = mapping.SERVICES_MAPPING[service_name]
-            nren_service_data[nren_name] = {
-                'question_key_base': question_key_base,
-                'offered': row[start_column].value,
-                'service_name': row[start_column + 1].value,
-                'additional': row[start_column + 2].value
-            }
+            if row[start_column].value and row[start_column].value.upper() == 'YES':
+                question_data = {"offered": ["yes"]}
+                if row[start_column + 1].value:
+                    question_data["name"] = row[start_column + 1].value
+                if row[start_column + 2].value:
+                    question_data["additional_information"] = row[start_column + 2].value
 
-    with app.app_context():
+                question_key_base = mapping.SERVICES_MAPPING[service_name]
+                question_name, subquestion_name = question_key_base.split(':')
+                question_dict = nren_service_data[nren_name].setdefault(question_name, {})
+                question_dict[subquestion_name] = question_data
+
+    return nren_service_data
 
-        nren_names = set()
-        for nren in db.session.scalars(select(NREN)):
-            nren_names.add(nren.name.upper())
-        for nren_name in nren_service_data.keys():
-            if nren_name not in nren_names:
-                raise Exception('NREN in excel not found in db!')
-                # TODO also check if its right that we dont have ANAS data..
+
+def _cli(app):
+
+    nren_service_data = load_service_data()
+
+    with app.app_context():
 
         nren_surveys = {}
 
@@ -155,6 +158,10 @@ def _cli(app):
             survey_db_nren_id = mapping.NREN_IDS[nren.name]
             nren_surveys[nren] = query_nren(survey_db_nren_id)
 
+        for nren_name in nren_service_data.keys():
+            if nren_name not in [n.name.upper() for n in nren_surveys.keys()]:
+                raise Exception('NREN in excel not found in source dataset!')
+
         db.session.execute(delete(SurveyResponse).where(
             SurveyResponse.survey_year == 2022
         ))
@@ -163,9 +170,9 @@ def _cli(app):
 
         for nren, answers in nren_surveys.items():
             survey_dict = convert_answers(answers)
+            survey_dict["data"].update(nren_service_data.get(nren.name.upper(), {}))
             survey_dict["page"] = 0
             survey_dict["verification_status"] = {}
-            # TODO add service data to the dict
             response = SurveyResponse(
                 nren=nren,
                 nren_id=nren.id,

test/test_conversion.py

@@ -15,6 +15,10 @@ def mock_query_nren(_):
     return {16455: "answer1"}
 
 
+def mock_load_service_data():
+    return {}
+
+
 def test_queries(app_with_survey_db, mocker):
 
     with app_with_survey_db.app_context():
@@ -24,6 +28,7 @@ def test_queries(app_with_survey_db, mocker):
 
     mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
     mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
+    mocker.patch('compendium_v2.conversion.conversion.load_service_data', mock_load_service_data)
 
     _cli(app_with_survey_db)