Skip to content
Snippets Groups Projects
Commit a7cdfbba authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

Merge branch 'feature/COMP-218_migrating_2022_data' into 'develop'

Feature/comp 218 migrating 2022 data

See merge request !47
parents 033e8aeb 5a68a352
No related branches found
No related tags found
1 merge request!47Feature/comp 218 migrating 2022 data
"""
conversion
=========================
This module loads the survey data from 2022 from the survey database
and stores the data in the json structure of the new survey, so that
it can be used to prefill the 2023 survey.
"""
import logging
import click
import json
from sqlalchemy import delete, text, select
import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
from compendium_v2.survey_db import model as survey_model
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse
from compendium_v2.conversion import mapping
setup_logging()
logger = logging.getLogger('conversion')
def query_nren(nren_id: int):
query = mapping.ANSWERS_2022_QUERY.format(nren_id)
answers = {}
for row in db.session.execute(text(query), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]}):
answers[row[0]] = row[1]
return answers
def convert_answers(answers):
data = {}
for id, question_name in mapping.ID_TO_NAME.items():
if id not in answers:
continue
answer = answers[id]
if len(answer) > 1 and answer[0] == '"' and answer[-1] == '"':
answer = answer[1:-1]
if id in mapping.VALUE_TO_CODE_MAPPING:
answer = mapping.VALUE_TO_CODE_MAPPING[id][answer]
if len(answer) > 1 and answer[0] == '[' and answer[-1] == ']':
answer = json.loads(answer)
if id in mapping.VALUE_TO_CODE_MAPPING:
mapped_answer = []
for entry in answer:
mapped_answer.append(mapping.VALUE_TO_CODE_MAPPING[id][entry])
answer = mapped_answer
# code to convert my description in the mapping to a json structure
question_names = question_name.split(":")
subdict = data
for name in question_names[0:-1]:
if name[-1] == "]":
index = name[-2]
if index == "[":
subdict = subdict.setdefault(name[:-2], [])
break # special case where json list is mapped to a list of dicts (part 1)
sublist = subdict.setdefault(name[:-3], [])
index = int(index)
while len(sublist) <= index:
sublist.append({})
subdict = sublist[index]
else:
subdict = subdict.setdefault(name, {})
if type(subdict) == list: # special case where json list is mapped to a list of dicts (part 2)
for answer_entry in answer:
subdict.append({question_names[-1]: answer_entry})
elif question_names[-1] == "available": # special case because we changed the policies questions a bit
if answer == "Yes":
subdict[question_names[-1]] = ["yes"]
else:
subdict[question_names[-1]] = answer
for id, question_name in mapping.ID_TO_NAME_SERVICES.items():
if id not in answers:
continue
answer = answers[id]
answer = json.loads(answer)
for user_type in answer:
user_type_code = mapping.SERVICE_USER_TYPE_TO_CODE[user_type]
formatted_question_name = question_name.format(user_type_code)
question_names = formatted_question_name.split(":")
subdict = data
for name in question_names[0:-2]:
subdict = subdict.setdefault(name, {})
sublist = subdict.setdefault(question_names[-2], [])
sublist.append(question_names[-1])
return {"data": data}
def _cli(app):
with app.app_context():
nren_surveys = {}
for nren in db.session.scalars(select(NREN)):
survey_db_nren_id = mapping.NREN_IDS[nren.name]
nren_surveys[nren] = query_nren(survey_db_nren_id)
db.session.execute(delete(SurveyResponse).where(
SurveyResponse.survey_year == 2022
))
db.session.execute(delete(Survey).where(
Survey.year == 2022
))
survey = Survey(year=2022, survey={})
db.session.add(survey)
for nren, answers in nren_surveys.items():
survey_dict = convert_answers(answers)
response = SurveyResponse(
nren=nren,
nren_id=nren.id,
survey_year=2022,
survey=survey,
answers=survey_dict
)
db.session.add(response)
db.session.commit()
@click.command()
@click.option('--config', type=click.STRING, default='config.json')
def cli(config):
app_config = load(open(config, 'r'))
app_config['SQLALCHEMY_BINDS'] = {survey_model.SURVEY_DB_BIND: app_config['SURVEY_DATABASE_URI']}
app = compendium_v2._create_app_with_db(app_config)
_cli(app)
if __name__ == "__main__":
cli()
This diff is collapsed.
...@@ -27,13 +27,24 @@ class VerificationStatus(str, Enum): ...@@ -27,13 +27,24 @@ class VerificationStatus(str, Enum):
Edited = "edited" # a question for which last years answer was edited Edited = "edited" # a question for which last years answer was edited
@routes.route('/open', methods=['GET']) @routes.route('/nrens', methods=['GET'])
@common.require_accepts_json @common.require_accepts_json
def open_survey() -> Any: def get_nrens() -> Any:
entries = [
{"id": entry.id, "name": entry.name}
for entry in db.session.scalars(select(NREN).order_by(NREN.name))
]
return jsonify(entries)
@routes.route('/open/<string:nren_name>', methods=['GET'])
@common.require_accepts_json
def open_survey(nren_name) -> Any:
# just a hardcoded year and nren for development for now # just a hardcoded year and nren for development for now
nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one() nren = db.session.execute(select(NREN).filter(NREN.name == nren_name)).scalar_one()
year = 1989 year = 1989
last_year = 2022
survey = db.session.scalar(select(Survey).where(Survey.year == year)) survey = db.session.scalar(select(Survey).where(Survey.year == year))
if survey is None or survey.survey == {}: if survey is None or survey.survey == {}:
...@@ -43,17 +54,18 @@ def open_survey() -> Any: ...@@ -43,17 +54,18 @@ def open_survey() -> Any:
with p.open('r') as f: with p.open('r') as f:
survey = json.load(f) survey = json.load(f)
# TODO add some magic strings in the json (like the year, url validation regex, maybe countries list) and interpolate them here # TODO add some magic strings in the json (like the year, url validation regex, maybe countries list)
# and interpolate them here
data: Optional[dict] = None data: Optional[dict] = None
page = 0 page = 0
verification_status: dict[str, str] = {"budget": VerificationStatus.Unverified, "TODO": "remove or set everything to new"} verification_status = {"budget": VerificationStatus.Unverified, "TODO": "remove or set all to new"}
response = db.session.scalar( response = db.session.scalar(
select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id) select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
) )
previous_response = db.session.scalar( previous_response = db.session.scalar(
select(SurveyResponse).where(SurveyResponse.survey_year == year - 1).where(SurveyResponse.nren_id == nren.id) select(SurveyResponse).where(SurveyResponse.survey_year == last_year).where(SurveyResponse.nren_id == nren.id)
) )
if response: if response:
......
...@@ -293,7 +293,7 @@ ...@@ -293,7 +293,7 @@
}, },
{ {
"type": "text", "type": "text",
"name": "abbreviation_local_languages", "name": "abbreviation_national_languages",
"title": "Abbreviation in the national language(s) (if applicable):", "title": "Abbreviation in the national language(s) (if applicable):",
"description": "Please always provide this information. Please use the Latin alphabet." "description": "Please always provide this information. Please use the Latin alphabet."
}, },
...@@ -506,7 +506,7 @@ ...@@ -506,7 +506,7 @@
"type": "radiogroup", "type": "radiogroup",
"name": "service_portfolio_changes", "name": "service_portfolio_changes",
"title": "Has your NREN made any changes to its service portfolio for its users?", "title": "Has your NREN made any changes to its service portfolio for its users?",
"description": "Please note any changes here and amend the Service Matrix directly at https://compendiumdatabase.geant.org/reports/nrens_services", "description": "Please note any changes here and amend the Service Matrix at the last page of this survey",
"choices": [ "choices": [
"Yes", "Yes",
"No" "No"
...@@ -519,8 +519,8 @@ ...@@ -519,8 +519,8 @@
"title": "Which service types are available for which user types?", "title": "Which service types are available for which user types?",
"columns": [ "columns": [
{ {
"name": "user_types", "name": "service_types",
"title": "User types", "title": "Service types",
"cellType": "checkbox", "cellType": "checkbox",
"showInMultipleColumns": true, "showInMultipleColumns": true,
"choices": [ "choices": [
...@@ -778,17 +778,26 @@ ...@@ -778,17 +778,26 @@
"name": "panel2", "name": "panel2",
"elements": [ "elements": [
{ {
"type": "text", "type": "matrixdynamic",
"name": "connected_sites_list", "name": "connected_sites_lists",
"title": "Please provide a URL that lists the sites that are connected to the NREN, if available:", "title": "Please provide the URLs that lists the sites that are connected to the NREN, if available:",
"inputType": "url", "columns": [
"validators": [
{ {
"type": "regex", "name": "connected_sites_url",
"text": "Please provide a single valid url", "title": "Url:",
"regex": "^(https?:\\/\\/)?([\\da-zA-Z\\.-]+\\.[a-zA-Z\\.]{2,6}|[\\d\\.]+)([\\/:?=&#%]{1}[\\d_a-zA-Z\\.-]+)*[\\/\\?]?$" "cellType": "text",
"inputType": "url",
"validators": [
{
"type": "regex",
"text": "Please provide a single valid url",
"regex": "^(https?:\\/\\/)?([\\da-zA-Z\\.-]+\\.[a-zA-Z\\.]{2,6}|[\\d\\.]+)([\\/:?=&#%]{1}[\\d_a-zA-Z\\.-]+)*[\\/\\?]?$"
}
]
} }
] ],
"rowCount": 1,
"maxRowCount": 50
}, },
{ {
"type": "matrixdropdown", "type": "matrixdropdown",
...@@ -826,7 +835,7 @@ ...@@ -826,7 +835,7 @@
"text": "No - other reason" "text": "No - other reason"
}, },
{ {
"value": "item7", "value": "unsure",
"text": "Unsure/unclear" "text": "Unsure/unclear"
} }
] ]
......
This diff is collapsed.
...@@ -28,6 +28,7 @@ setup( ...@@ -28,6 +28,7 @@ setup(
'console_scripts': [ 'console_scripts': [
'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa 'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa
'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa 'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa
'conversion=compendium_v2.conversion.conversion:cli', # noqa
] ]
} }
) )
...@@ -23,10 +23,10 @@ enum VerificationStatus { ...@@ -23,10 +23,10 @@ enum VerificationStatus {
} }
function SurveyComponent() { function SurveyComponent({ nrenName }) {
const [surveyModel, setSurveyModel] = useState<Model>(); const [surveyModel, setSurveyModel] = useState<Model>();
const [progress, setProgress] = useState<Progress[]>([]); const [progress, setProgress] = useState<Progress[]>([]);
const verificationStatus = useRef<Map<string, VerificationStatus>>(new Map()); const verificationStatus = useRef<Map<string, VerificationStatus>>(new Map());
function setVerifyButton(question: Question, state: VerificationStatus) { function setVerifyButton(question: Question, state: VerificationStatus) {
...@@ -64,7 +64,7 @@ function SurveyComponent() { ...@@ -64,7 +64,7 @@ function SurveyComponent() {
async function getModel() async function getModel()
{ {
const response = await fetch('/api/survey/open'); const response = await fetch('/api/survey/open/' + nrenName);
const json = await response.json(); const json = await response.json();
for (const questionName in json["verification_status"]) { for (const questionName in json["verification_status"]) {
......
import React, { useState, useEffect } from "react";
import SurveyComponent from "./SurveyComponent";
interface Nren {
id: number
name: string
}
function SurveySelectionComponent() {
const [nrens, setNrens] = useState<Nren[]>([]);
const [selectedNren, setSelectedNren] = useState<Nren | null>(null);
useEffect(() => {
// Fetch organizations from the API
fetchNrens();
}, []);
const fetchNrens = async () => {
try {
const response = await fetch('/api/survey/nrens');
const data = await response.json();
setNrens(data);
} catch (error) {
console.error('Error fetching organizations:', error);
}
};
const handleNrenSelect = (nren) => {
setSelectedNren(nren);
};
if (!selectedNren) {
return (
<div>
<h2>Select an organization:</h2>
<ul>
{nrens.map((nren) => (
<li key={nren.id} onClick={() => handleNrenSelect(nren)}>
{nren.name}
</li>
))}
</ul>
</div>
);
}
return <SurveyComponent nrenName={selectedNren.name} />;
}
export default SurveySelectionComponent;
\ No newline at end of file
import React from 'react'; import React from 'react';
import { createRoot } from 'react-dom/client'; import { createRoot } from 'react-dom/client';
import SurveyComponent from './SurveyComponent'; import SurveySelectionComponent from './SurveySelectionComponent';
const container = document.getElementById('root') as HTMLElement; const container = document.getElementById('root') as HTMLElement;
...@@ -10,6 +10,6 @@ const root = createRoot(container); ...@@ -10,6 +10,6 @@ const root = createRoot(container);
root.render( root.render(
<React.StrictMode> <React.StrictMode>
<SurveyComponent /> <SurveySelectionComponent />
</React.StrictMode> </React.StrictMode>
) )
\ No newline at end of file
from sqlalchemy import select
from compendium_v2.db import db
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse
from compendium_v2.conversion.conversion import _cli, convert_answers
def mock_convert_answers(_):
return {"data": {}}
def mock_query_nren(_):
return {16455: "answer1"}
def test_queries(app_with_survey_db, mocker):
with app_with_survey_db.app_context():
db.session.add(NREN(name='Restena', country='country'))
db.session.commit()
mocker.patch('compendium_v2.conversion.conversion.convert_answers', mock_convert_answers)
mocker.patch('compendium_v2.conversion.conversion.query_nren', mock_query_nren)
_cli(app_with_survey_db)
with app_with_survey_db.app_context():
surveys = db.session.scalars(select(Survey)).all()
assert len(surveys) == 1
assert surveys[0].year == 2022
responses = db.session.scalars(select(SurveyResponse).order_by(SurveyResponse.nren_id)).all()
assert len(responses) == 1
assert responses[0].answers == {"data": {}}
def test_conversion():
answers = {
16455: '"full nren name"',
16453: '["ec project1", "ec project2"]',
16632: '"3434"',
16432: '"suborg name 3"',
16410: '"We use a combination of flat fee and usage-based fee"',
16474: '"Yes"',
16476: '"No"',
16491: '["Universities", "Further education"]',
16492: '["Research institutes", "Universities"]'
}
converted_answers = convert_answers(answers)
assert converted_answers == {
'data': {
'charging_mechanism': 'combination',
'suborganization_details': [{}, {}, {}, {'suborganization_name': 'suborg name 3'}],
'ec_project_names': [{'ec_project_name': 'ec project1'}, {'ec_project_name': 'ec project2'}],
'full_name_english': 'full nren name',
'policies': {'connectivity_policy': {'available': ['yes']}, 'acceptable_use_policy': {}},
'traffic_load': {'iros': {'peak_to_institutions_from_network': '3434'}},
'service_matrix': {
'universities': {'service_types': ['security', 'isp_support']},
'further_education': {'service_types': ['security']},
'institutes': {'service_types': ['isp_support']}
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment