Merge branch 'develop' into feature/COMP-208-Google-OAuth-POC

MANIFEST.in

 recursive-include compendium_v2/static *
-include compendium_v2/templates/index.html *
-include compendium_v2/templates/survey-index.html *
+include compendium_v2/templates/index.html
+include compendium_v2/templates/survey-index.html
 recursive-include compendium_v2/migrations/versions *
-include compendium_v2/migrations/alembic.ini *
+include compendium_v2/migrations/alembic.ini
 recursive-include compendium_v2/background_task/xlsx *
+include compendium_v2/routes/survey_model.json

compendium_v2/conversion/conversion.py

+"""
+conversion
+=========================
+
+This module loads the survey data from 2022 from the survey database
+and stores the data in the json structure of the new survey, so that
+it can be used to prefill the 2023 survey.
+
+"""
+import logging
+import click
+import json
+
+from sqlalchemy import delete, text, select
+
+import compendium_v2
+from compendium_v2.environment import setup_logging
+from compendium_v2.db import db
+from compendium_v2.config import load
+from compendium_v2.survey_db import model as survey_model
+from compendium_v2.db.model import NREN
+from compendium_v2.db.survey_model import Survey, SurveyResponse
+
+from compendium_v2.conversion import mapping
+
+setup_logging()
+
+logger = logging.getLogger('conversion')
+
+
+def query_nren(nren_id: int):
+    query = mapping.ANSWERS_2022_QUERY.format(nren_id)
+    answers = {}
+    for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
+        answers[row[0]] = row[1]
+    return answers
+
+
+def convert_answers(answers):
+    data = {}
+
+    for id, question_name in mapping.ID_TO_NAME.items():
+        if id not in answers:
+            continue
+
+        answer = answers[id]
+        if len(answer) > 1 and answer[0] == '"' and answer[-1] == '"':
+            answer = answer[1:-1]
+            if id in mapping.VALUE_TO_CODE_MAPPING:
+                answer = mapping.VALUE_TO_CODE_MAPPING[id][answer]
+        if len(answer) > 1 and answer[0] == '[' and answer[-1] == ']':
+            answer = json.loads(answer)
+            if id in mapping.VALUE_TO_CODE_MAPPING:
+                mapped_answer = []
+                for entry in answer:
+                    mapped_answer.append(mapping.VALUE_TO_CODE_MAPPING[id][entry])
+                answer = mapped_answer
+
+        # code to convert my description in the mapping to a json structure
+        question_names = question_name.split(":")
+        subdict = data
+        for name in question_names[0:-1]:
+            if name[-1] == "]":
+                index = name[-2]
+                if index == "[":
+                    subdict = subdict.setdefault(name[:-2], [])
+                    break  # special case where json list is mapped to a list of dicts (part 1)
+                sublist = subdict.setdefault(name[:-3], [])
+                index = int(index)
+                while len(sublist) <= index:
+                    sublist.append({})
+                subdict = sublist[index]
+            else:
+                subdict = subdict.setdefault(name, {})
+
+        if type(subdict) == list:  # special case where json list is mapped to a list of dicts (part 2)
+            for answer_entry in answer:
+                subdict.append({question_names[-1]: answer_entry})
+        elif question_names[-1] == "available":  # special case because we changed the policies questions a bit
+            if answer == "Yes":
+                subdict[question_names[-1]] = ["yes"]
+        else:
+            subdict[question_names[-1]] = answer
+
+    for id, question_name in mapping.ID_TO_NAME_SERVICES.items():
+        if id not in answers:
+            continue
+
+        answer = answers[id]
+        answer = json.loads(answer)
+        for user_type in answer:
+            user_type_code = mapping.SERVICE_USER_TYPE_TO_CODE[user_type]
+            formatted_question_name = question_name.format(user_type_code)
+
+            question_names = formatted_question_name.split(":")
+            subdict = data
+            for name in question_names[0:-2]:
+                subdict = subdict.setdefault(name, {})
+
+            sublist = subdict.setdefault(question_names[-2], [])
+            sublist.append(question_names[-1])
+
+    return {"data": data}
+
+
+def _cli(app):
+    with app.app_context():
+
+        nren_surveys = {}
+
+        for nren in db.session.scalars(select(NREN)):
+            survey_db_nren_id = mapping.NREN_IDS[nren.name]
+            nren_surveys[nren] = query_nren(survey_db_nren_id)
+
+        db.session.execute(delete(SurveyResponse).where(
+            SurveyResponse.survey_year == 2022
+        ))
+
+        db.session.execute(delete(Survey).where(
+            Survey.year == 2022
+        ))
+
+        survey = Survey(year=2022, survey={})
+        db.session.add(survey)
+
+        for nren, answers in nren_surveys.items():
+            survey_dict = convert_answers(answers)
+            response = SurveyResponse(
+                nren=nren,
+                nren_id=nren.id,
+                survey_year=2022,
+                survey=survey,
+                answers=survey_dict
+            )
+            db.session.add(response)
+
+        db.session.commit()
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+    app_config = load(open(config, 'r'))
+
+    app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
+
+    app = compendium_v2._create_app_with_db(app_config)
+    _cli(app)
+
+
+if __name__ == "__main__":
+    cli()

compendium_v2/routes/survey.py

 import json
 import logging
+from enum import Enum
 from pathlib import Path
-from typing import Any, List, Optional
+from typing import Any, Optional
 
 from flask import Blueprint, jsonify, request
 from sqlalchemy import select
@@ -18,48 +19,71 @@ logger = logging.getLogger(__name__)
 
 # TODO (partial) schemas
 
+class VerificationStatus(str, Enum):
+    New = "new"                   # a question that was not answered last year
+    Answered = "answered"         # a question that was not answered last year but has an answer now
+    Unverified = "unverified"     # a question that has its answered copied from last year
+    Verified = "verified"         # a question for which last years answer was verified
+    Edited = "edited"             # a question for which last years answer was edited
 
-@routes.route('/open', methods=['GET'])
+
+@routes.route('/nrens', methods=['GET'])
 @common.require_accepts_json
-def open_survey() -> Any:
+def get_nrens() -> Any:
+    entries = [
+        {"id": entry.id, "name": entry.name}
+        for entry in db.session.scalars(select(NREN).order_by(NREN.name))
+    ]
+    return jsonify(entries)
+
+
+@routes.route('/open/<string:nren_name>', methods=['GET'])
+@common.require_accepts_json
+def open_survey(nren_name) -> Any:
 
     # just a hardcoded year and nren for development for now
-    nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one()
-    year = 1988
+    nren = db.session.execute(select(NREN).filter(NREN.name == nren_name)).scalar_one()
+    year = 1989
+    last_year = 2022
 
     survey = db.session.scalar(select(Survey).where(Survey.year == year))
     if survey is None or survey.survey == {}:
         # TODO remove this at some point, its just convenient for now while we are changing the survey model a lot
+        # or is it really?? we can also keep the surveys on disk, which makes it really easy to diff, etc
         p = Path(__file__).with_name('survey_model.json')
         with p.open('r') as f:
             survey = json.load(f)
 
-    # TODO add some magic strings in the json (like the year) and interpolate them here
+    # TODO add some magic strings in the json (like the year, url validation regex, maybe countries list)
+    # and interpolate them here
 
     data: Optional[dict] = None
     page = 0
-    unvalidated: List[str] = []  # or should we keep track of what _was_ validated?
+    verification_status = {"budget": VerificationStatus.Unverified, "TODO": "remove or set all to new"}
 
     response = db.session.scalar(
         select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
     )
     previous_response = db.session.scalar(
-        select(SurveyResponse).where(SurveyResponse.survey_year == year - 1).where(SurveyResponse.nren_id == nren.id)
+        select(SurveyResponse).where(SurveyResponse.survey_year == last_year).where(SurveyResponse.nren_id == nren.id)
     )
 
     if response:
         data = response.answers["data"]
         page = response.answers["page"]
-        unvalidated = response.answers["unvalidated"]
+        verification_status = response.answers["verification_status"]
     elif previous_response:
+        # TODO add a 'migration' hook here for updating data per year
+        # TODO i suppose we also need to remove the data that isnt asked anymore because
+        # i dont think the frontend will remove it
         data = previous_response.answers["data"]
-        unvalidated = ["TODO everything?"]
+        verification_status = {"budget": VerificationStatus.Unverified, "TODO": "all other questions"}
 
     open_survey: dict = {
         "model": survey,
         "data": data,
         "page": page,
-        "unvalidated": unvalidated
+        "verification_status": verification_status
     }
 
     return jsonify(open_survey)
@@ -71,7 +95,7 @@ def save_survey() -> Any:
 
     # just a hardcoded year and nren for development for now
     nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one()
-    year = 1988
+    year = 1989
     survey = db.session.scalar(select(Survey).where(Survey.year == year))
     if survey is None:
         survey = Survey(year=year, survey={})
@@ -91,7 +115,7 @@ def save_survey() -> Any:
     response.answers = {
         "data": save_survey["data"],
         "page": save_survey["page"],
-        "unvalidated": save_survey["unvalidated"]
+        "verification_status": save_survey["verification_status"]
     }
 
     db.session.commit()

setup.py

@@ -29,6 +29,7 @@ setup(
         'console_scripts': [
             'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli',  # noqa
             'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli',  # noqa
+            'conversion=compendium_v2.conversion.conversion:cli',  # noqa
         ]
     },
     license='MIT',

survey-frontend/src/ProgressBar.tsx

+import React from "react";
+
+function ProgressBar({
+  completionPercentage,
+  unansweredPercentage,
+  pages,
+  pageTitle,
+}) {
+  const progressBarContainerStyle: React.CSSProperties = {
+    display: "flex",
+    flexWrap: "wrap",
+    height: "10px",
+    margin: "5px",
+    width: `${100 / pages}%`,
+  };
+
+  const progressBarFillStyle: React.CSSProperties = {
+    height: "100%",
+    transition: "width 0.3s ease",
+  };
+
+  const progressBarFillStyleCopy: React.CSSProperties = {
+    ...progressBarFillStyle,
+    width: `${completionPercentage}%`,
+    backgroundColor: "#1ab394",
+  };
+  const unansweredProgressBarFillStyle: React.CSSProperties = {
+    ...progressBarFillStyle,
+    width: `${unansweredPercentage}%`,
+    backgroundColor: "#9d9d9d",
+  };
+
+  const pageTitleStyle: React.CSSProperties ={
+    width: "100%",
+    textAlign: "center"
+  }
+  
+
+  return (
+    <div style={progressBarContainerStyle}>
+      <div style={progressBarFillStyleCopy} />
+      <div style={unansweredProgressBarFillStyle} />
+      <div style={pageTitleStyle}>{pageTitle}</div>
+    </div>
+  );
+}
+
+export default ProgressBar;

survey-frontend/src/SurveyComponent.tsx

-import React, { useState, useEffect } from "react";
-import { Model } from "survey-core";
+import React, { useState, useEffect, useRef } from "react";
+import { Model, Serializer, ComputedUpdater, Question } from "survey-core";
 import { Survey } from "survey-react-ui";
 import "survey-core/modern.min.css";
+import './survey.scss';
+import ProgressBar from "./ProgressBar";
 
+Serializer.addProperty("itemvalue", "customDescription:text");
+Serializer.addProperty("question", "hideCheckboxLabels:boolean");
 
-function SurveyComponent() {
+interface Progress {
+  completionPercentage: number;
+  unansweredPercentage: number;
+  totalPages: number;
+  pageTitle: string;
+}
+enum VerificationStatus {
+  New = "new",                   // a question that was not answered last year
+  Answered = "answered",         // a question that was not answered last year but has an answer now
+  Unverified = "unverified",     // a question that has its answered copied from last year
+  Verified = "verified",         // a question for which last years answer was verified
+  Edited = "edited"              // a question for which last years answer was edited
+}
 
+
+function SurveyComponent({ nrenName }) {
     const [surveyModel, setSurveyModel] = useState<Model>();
+    const [progress, setProgress] = useState<Progress[]>([]);
+    const verificationStatus = useRef<Map<string, VerificationStatus>>(new Map());
+
+    function setVerifyButton(question: Question, state: VerificationStatus) {
+
+      verificationStatus.current.set(question.name, state);
+
+      const btn = document.createElement("button");
+      btn.type = "button";
+      btn.className =  "sv-action-bar-item verification";
+      btn.innerHTML = state;
+
+      if (state == VerificationStatus.Unverified) {
+        btn.innerHTML = "Verify last years data";
+        btn.className += " verification-required";
+        btn.onclick = function() {
+            question.validate();
+            setVerifyButton(question, VerificationStatus.Verified)
+        }
+      } else {
+        btn.className += " verification-ok";
+      }
+
+      const selector = '[data-name="' + question.name + '"]';
+      const header = document.querySelector(selector)!.querySelector('h5')!;
+      const old = header.querySelector(".verification");
+      if (old) {
+        old.replaceWith(btn);
+      } else {
+        header.appendChild(btn);
+      }
+    }
 
-    // const surveyComplete = useCallback((sender) => {
-    //   console.log(sender.data);
-    // }, []);
+  // const surveyComplete = useCallback((sender) => {
+  //   console.log(sender.data);
+  // }, []);
 
     async function getModel()
     {
-        const response = await fetch('/api/survey/open');
+        const response = await fetch('/api/survey/open/' + nrenName);
         const json = await response.json();
-        const survey = new Model(json['model']);
 
-        if (json['data'] !== null)
-        {
-            survey.data = json['data'];
+        for (const questionName in json["verification_status"]) {
+          verificationStatus.current.set(questionName, json["verification_status"][questionName]);
         }
 
-        // survey.onComplete.add((sender, options) => {
-        //     console.log(JSON.stringify(sender.data, null, 3));
-        // });
-
-        // TODO also use data and page info
-        // TODO also manage the list of (un)validated answers from last year
+        const survey = new Model(json['model']);
 
-        survey.onComplete.add((sender, options) => {
+    if (json['data'] !== null) {
+      survey.data = json['data'];
+    }
 
-            console.log(sender.data)
+        // TODO also use data and page info
 
-            options.showSaveInProgress();
-            const xhr = new XMLHttpRequest();
-            xhr.open("POST", "/api/survey/save");
-            xhr.setRequestHeader("Content-Type", "application/json; charset=utf-8");
-            xhr.onload = xhr.onerror = function () {
-              if (xhr.status == 200) {
-                // Display the "Success" message (pass a string value to display a custom message)
-                options.showSaveSuccess();
-                // Alternatively, you can clear all messages:
-                // options.clearSaveMessages();
-              } else {
-                // Display the "Error" message (pass a string value to display a custom message)
-                options.showSaveError();
+        survey.addNavigationItem({
+          id: "sv-nav-compendium-complete",
+          title: "Complete",
+          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+          // @ts-expect-error: visible may be a ComputedUpdater but the types are not (yet?) aware of this
+          visible: new ComputedUpdater(() => survey.isLastPage),
+          action: () => {
+            let firstValidationError = '';
+            const verificationValidator = (survey, options) => {
+              const status = verificationStatus.current.get(options.name);
+              if (status == VerificationStatus.Unverified) {
+                if (firstValidationError == '') {
+                  firstValidationError = options.name;
+                }
+                options.error = 'Please verify that last years data is correct by editing the value or pressing the verification button!';
               }
             };
+            survey.onValidateQuestion.add(verificationValidator);
+            const validSurvey = survey.validate();
+            survey.onValidateQuestion.remove(verificationValidator);
+            if (validSurvey) {
+              survey.completeLastPage();  // continue with usual completion process
+            } else {
+              survey.focusQuestion(firstValidationError);
+            }
+          },
+          innerCss: "sv-btn sv-btn--navigation sv-footer__complete-btn"
+        });
+
+    survey.onComplete.add((sender, options) => {
+      console.log(sender.data);
+
+      options.showSaveInProgress();
+      const xhr = new XMLHttpRequest();
+      xhr.open("POST", "/api/survey/save");
+      xhr.setRequestHeader("Content-Type", "application/json; charset=utf-8");
+      xhr.onload = xhr.onerror = function () {
+        if (xhr.status == 200) {
+          // Display the "Success" message (pass a string value to display a custom message)
+          options.showSaveSuccess();
+          // Alternatively, you can clear all messages:
+          // options.clearSaveMessages();
+        } else {
+          // Display the "Error" message (pass a string value to display a custom message)
+          options.showSaveError();
+        }
+      };
 
             const saveData = {
                 data: sender.data,
                 page: 2,
-                unvalidated: []
+                verification_status: Object.fromEntries(verificationStatus.current)
             }
             xhr.send(JSON.stringify(saveData));
         });
         survey.onPartialSend.add((sender, options) => {
             console.log(sender.data)
 
-            // TODO same as above
+      // TODO same as above
+    });
 
+        survey.onAfterRenderQuestion.add(function(survey, options){
+            const status = verificationStatus.current.get(options.question.name);
+            if (status) {
+              setVerifyButton(options.question, status);
+            }
         });
 
-        survey.onGetQuestionTitleActions.add((_, opt) => {
-          // opt.question TODO check what we can do with this..
-          console.log(opt.question.title, opt.question.value, opt.question.validators);
-          opt.titleActions = [
-            {
-              title: 'Validate pre-filled value',
-              innerCss: 'validate-pre-filled-value',
-              action: () => {
-                console.log('verified!')
-              },
-            },
-          ];
+        survey.onValueChanged.add(function(survey, options) {
+            const currentStatus = verificationStatus.current.get(options.question.name);
+            if (currentStatus == VerificationStatus.New) {
+                setVerifyButton(options.question, VerificationStatus.Answered);
+            } else if (currentStatus == VerificationStatus.Unverified) {
+                setVerifyButton(options.question, VerificationStatus.Edited);
+            }
         });
 
-        setSurveyModel(survey);
+        survey.onUpdateQuestionCssClasses.add(function(_, options) {
+            if (options.question.hideCheckboxLabels) {
+                const classes = options.cssClasses;
+                classes.root += " hidden-checkbox-labels";
+            }
+        });
 
+    survey.onMatrixAfterCellRender.add((survey, options) => {
+      // get the customDescription for matrix rows and set it in the title
+      // attribute so that it shows up as a hover popup
+      // NB I would have preferred using onAfterRenderQuestion, but unfortunately that is
+      // not always triggered on re-renders (specifically when extra column become visble or invisible)
+
+      if (options.column['indexValue'] == 0 && 'item' in options.row) {
+        const item = options.row['item'] as object;
+        if (item['customDescription'] !== undefined) {
+          options.htmlElement.parentElement?.children[0].setAttribute(
+            "title",
+            item['customDescription']
+          );
+        }
+      }
+    });
+
+    survey.onCurrentPageChanged.add((sender) => {
+      console.log("sender--> " + sender);
+      calculateProgress(sender);
+    });
+
+    setSurveyModel(survey);
+  }
+  const filterCallback = (question) => {
+    return question.value !== null && question.value !== undefined;
+  };
+
+  const calculateProgress = (survey) => {
+    // console.log("survey--> "+ survey);
+    if (survey && survey.pages) {
+      console.log("survey.page--> " + survey.pages);
+      const progressArray: Progress[] = [];
+      survey.pages.forEach((page) => {
+        const sectionQuestions = page.questions.filter(
+          (question) => question.startWithNewLine
+        );
+        const questionCount = sectionQuestions.length;
+        const answeredCount = sectionQuestions.filter(filterCallback).length;
+        const unansweredCount = questionCount - answeredCount;
+        const completionPercentage = answeredCount / questionCount;
+
+
+        progressArray.push({
+          completionPercentage: completionPercentage * 100,
+          unansweredPercentage: (unansweredCount / questionCount) * 100,
+          totalPages: survey.pages.length,
+          pageTitle: page.title,
+        });
+      });
+      setProgress(progressArray);
     }
+  };
 
-    useEffect(() => {
-        getModel();
-    }, []);
+  useEffect(() => {
+    getModel();
+  }, []);
 
-
-    if (surveyModel)
-    {
-        return (<Survey model={surveyModel} />);
+  useEffect(() => {
+    if (surveyModel) {
+      calculateProgress(surveyModel);
     }
-    else
-    {
-        return (<span>loading...</span>);
-    }
-
+  }, [surveyModel]);
+
+  if (surveyModel) {
+    return (
+      <div className="survey-container">
+        <div className="survey-progress">
+          {progress.map((sectionProgress, index) => (
+            <ProgressBar
+              key={index}
+              completionPercentage={sectionProgress.completionPercentage}
+              unansweredPercentage={sectionProgress.unansweredPercentage}
+              pages={sectionProgress.totalPages}
+              pageTitle={sectionProgress.pageTitle}
+            />
+          ))}
+        </div>
+        <Survey model={surveyModel} />
+      </div>
+    );
+  } else {
+    return <span>loading...</span>;
+  }
 }
 
-export default SurveyComponent;
\ No newline at end of file
+export default SurveyComponent;

survey-frontend/src/SurveySelectionComponent.tsx

+import React, { useState, useEffect } from "react";
+import SurveyComponent from "./SurveyComponent";
+
+
+interface Nren {
+    id: number
+    name: string
+}
+
+function SurveySelectionComponent() {
+    const [nrens, setNrens] = useState<Nren[]>([]);
+    const [selectedNren, setSelectedNren] = useState<Nren | null>(null);
+
+    useEffect(() => {
+      // Fetch organizations from the API
+      fetchNrens();
+    }, []);
+
+    const fetchNrens = async () => {
+      try {
+        const response = await fetch('/api/survey/nrens');
+        const data = await response.json();
+        setNrens(data);
+      } catch (error) {
+        console.error('Error fetching organizations:', error);
+      }
+    };
+
+    const handleNrenSelect = (nren) => {
+      setSelectedNren(nren);
+    };
+
+    if (!selectedNren) {
+      return (
+        <div>
+          <h2>Select an organization:</h2>
+          <ul>
+            {nrens.map((nren) => (
+              <li key={nren.id} onClick={() => handleNrenSelect(nren)}>
+                {nren.name}
+              </li>
+            ))}
+          </ul>
+        </div>
+      );
+    }
+
+    return <SurveyComponent nrenName={selectedNren.name} />;
+
+}
+
+export default SurveySelectionComponent;
\ No newline at end of file

survey-frontend/src/index.tsx

 
 import React from 'react';
 import { createRoot } from 'react-dom/client';
-import SurveyComponent from './SurveyComponent';
+import SurveySelectionComponent from './SurveySelectionComponent';
 
 
 const container = document.getElementById('root') as HTMLElement;
@@ -10,6 +10,6 @@ const root = createRoot(container);
 
 root.render(
   <React.StrictMode>
-   <SurveyComponent />
+   <SurveySelectionComponent />
   </React.StrictMode>
 )
\ No newline at end of file

survey-frontend/src/survey.scss

+
+.hidden-checkbox-labels .sv-checkbox .sv-item__control-label {
+	visibility: hidden;
+}
+
+#sv-nav-complete {
+	width: 0px;
+	height: 0px;
+	overflow: hidden;
+	visibility: hidden;
+}
+
+.verification {
+	display: inline-block;
+	margin-left: 20px;
+}
+
+.verification-required {
+	color: red;
+	border-style: solid;
+	border-width: 1px;
+}
+
+.sv-action-bar-item.verification.verification-ok:hover {
+	cursor: auto;
+	background-color: transparent;
+}
+
+.survey-progress {
+	display: flex;
+}
\ No newline at end of file

test/test_conversion.py

+from sqlalchemy import select
+
+from compendium_v2.db import db
+from compendium_v2.db.model import NREN
+from compendium_v2.db.survey_model import Survey, SurveyResponse
+from compendium_v2.conversion.conversion import _cli, convert_answers
+
+
+def mock_convert_answers(_):
+    return {"data": {}}
+
+
+def mock_query_nren(_):
+    return {16455: "answer1"}
+
+
+def test_queries(app_with_survey_db, mocker):
+
+    with app_with_survey_db.app_context():
+        db.session.add(NREN(name='Restena', country='country'))
+        db.session.commit()
+
+    mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
+    mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
+
+    _cli(app_with_survey_db)
+
+    with app_with_survey_db.app_context():
+        surveys = db.session.scalars(select(Survey)).all()
+        assert len(surveys) == 1
+        assert surveys[0].year == 2022
+
+        responses = db.session.scalars(select(SurveyResponse).order_by(SurveyResponse.nren_id)).all()
+        assert len(responses) == 1
+        assert responses[0].answers == {"data": {}}
+
+
+def test_conversion():
+
+    answers = {
+        16455: '"full nren name"',
+        16453: '["ec project1", "ec project2"]',
+        16632: '"3434"',
+        16432: '"suborg name 3"',
+        16410: '"We use a combination of flat fee and usage-based fee"',
+        16474: '"Yes"',
+        16476: '"No"',
+        16491: '["Universities", "Further education"]',
+        16492: '["Research institutes", "Universities"]'
+    }
+
+    converted_answers = convert_answers(answers)
+
+    assert converted_answers == {
+        'data': {
+            'charging_mechanism': 'combination',
+            'suborganization_details': [{}, {}, {}, {'suborganization_name': 'suborg name 3'}],
+            'ec_project_names': [{'ec_project_name': 'ec project1'}, {'ec_project_name': 'ec project2'}],
+            'full_name_english': 'full nren name',
+            'policies': {'connectivity_policy': {'available': ['yes']}, 'acceptable_use_policy': {}},
+            'traffic_load': {'iros': {'peak_to_institutions_from_network': '3434'}},
+            'service_matrix': {
+                'universities': {'service_types': ['security', 'isp_support']},
+                'further_education': {'service_types': ['security']},
+                'institutes': {'service_types': ['isp_support']}
+            }
+        }
+    }