From 0494ec7f868b10f485afdfdbe7a3872433cbf8f3 Mon Sep 17 00:00:00 2001 From: Mohammad Torkashvand <mohammad.torkashvand@geant.org> Date: Fri, 15 Sep 2023 08:59:16 +0000 Subject: [PATCH] refactor publisher and make Excel files consistent --- MANIFEST.in | 3 +- compendium_v2/background_task/__init__.py | 0 compendium_v2/conversion/conversion.py | 8 +-- compendium_v2/db/auth_model.py | 2 +- .../db/{model.py => presentation_models.py} | 0 .../db/{survey_model.py => survey_models.py} | 2 +- compendium_v2/migrations/dump_survey_model.py | 2 +- .../excel_parser.py} | 6 +- compendium_v2/publishers/helpers.py | 4 +- ...ey_publisher_v2.py => survey_publisher.py} | 19 ++++-- ...v1.py => survey_publisher_legacy_excel.py} | 43 +++++++------ ...022.py => survey_publisher_old_db_2022.py} | 43 +++++++------ .../2021_Organisation_DataSeries.xlsx | Bin .../2022_Connected_Users_DataSeries.xlsx | Bin .../2022_Networks_DataSeries.xlsx | Bin ...NREN-Services-prefills_2023_Recovered.xlsx | Bin compendium_v2/routes/budget.py | 2 +- compendium_v2/routes/charging.py | 2 +- compendium_v2/routes/common.py | 2 +- compendium_v2/routes/ec_projects.py | 2 +- compendium_v2/routes/funding.py | 2 +- compendium_v2/routes/institutions_urls.py | 2 +- compendium_v2/routes/organization.py | 2 +- compendium_v2/routes/policy.py | 2 +- compendium_v2/routes/response.py | 4 +- compendium_v2/routes/staff.py | 2 +- compendium_v2/routes/survey.py | 6 +- compendium_v2/routes/traffic.py | 2 +- compendium_v2/routes/user.py | 2 +- setup.py | 4 +- test/conftest.py | 58 +++++++++--------- test/test_budget.py | 2 +- test/test_conversion.py | 4 +- ...22.py => test_db_survey_publisher_2022.py} | 40 ++++++------ test/test_dump_survey_model.py | 2 +- test/test_response.py | 2 +- test/test_survey.py | 2 +- ...blisher_v2.py => test_survey_publisher.py} | 40 ++++++------ ... => test_survey_publisher_legacy_excel.py} | 22 ++++--- 39 files changed, 180 insertions(+), 160 deletions(-) delete mode 100644 compendium_v2/background_task/__init__.py rename compendium_v2/db/{model.py => presentation_models.py} (100%) rename compendium_v2/db/{survey_model.py => survey_models.py} (97%) rename compendium_v2/{background_task/parse_excel_data.py => publishers/excel_parser.py} (98%) rename compendium_v2/publishers/{survey_publisher_v2.py => survey_publisher.py} (88%) rename compendium_v2/publishers/{survey_publisher_v1.py => survey_publisher_legacy_excel.py} (83%) rename compendium_v2/publishers/{survey_publisher_2022.py => survey_publisher_old_db_2022.py} (93%) rename compendium_v2/{background_task/xlsx => resources}/2021_Organisation_DataSeries.xlsx (100%) rename compendium_v2/{background_task/xlsx => resources}/2022_Connected_Users_DataSeries.xlsx (100%) rename compendium_v2/{background_task/xlsx => resources}/2022_Networks_DataSeries.xlsx (100%) rename compendium_v2/{conversion => resources}/NREN-Services-prefills_2023_Recovered.xlsx (100%) rename test/{test_survey_publisher_2022.py => test_db_survey_publisher_2022.py} (82%) rename test/{test_survey_publisher_v2.py => test_survey_publisher.py} (65%) rename test/{test_survey_publisher_v1.py => test_survey_publisher_legacy_excel.py} (80%) diff --git a/MANIFEST.in b/MANIFEST.in index edff315a..58874ea5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,5 @@ include compendium_v2/templates/survey-index.html include compendium_v2/migrations/alembic.ini recursive-include compendium_v2/migrations/versions * recursive-include compendium_v2/migrations/surveymodels * -recursive-include compendium_v2/background_task/xlsx * -include compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx +recursive-include compendium_v2/background_task/resources * recursive-exclude test * diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py index a963b446..6da4572a 100644 --- a/compendium_v2/conversion/conversion.py +++ b/compendium_v2/conversion/conversion.py @@ -2,7 +2,7 @@ conversion ========================= -This module loads the survey data from 2022 from the survey database +This module loads the survey data from 2022 from the old survey database and stores the data in the json structure of the new survey, so that it can be used to prefill the 2023 survey. It also loads an xlsx file with the data for the services questions. @@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging from compendium_v2.db import db from compendium_v2.config import load from compendium_v2.survey_db import model as survey_model -from compendium_v2.db.model import NREN -from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus +from compendium_v2.db.presentation_models import NREN +from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus from compendium_v2.conversion import mapping @@ -30,7 +30,7 @@ setup_logging() logger = logging.getLogger('conversion') -EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx") +EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx") def query_nren(nren_id: int): diff --git a/compendium_v2/db/auth_model.py b/compendium_v2/db/auth_model.py index 50be7e3e..8605ab39 100644 --- a/compendium_v2/db/auth_model.py +++ b/compendium_v2/db/auth_model.py @@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey from flask_login import UserMixin # type: ignore from compendium_v2.db import db -from compendium_v2.db.model import NREN +from compendium_v2.db.presentation_models import NREN logger = logging.getLogger(__name__) diff --git a/compendium_v2/db/model.py b/compendium_v2/db/presentation_models.py similarity index 100% rename from compendium_v2/db/model.py rename to compendium_v2/db/presentation_models.py diff --git a/compendium_v2/db/survey_model.py b/compendium_v2/db/survey_models.py similarity index 97% rename from compendium_v2/db/survey_model.py rename to compendium_v2/db/survey_models.py index 866df323..fd688d38 100644 --- a/compendium_v2/db/survey_model.py +++ b/compendium_v2/db/survey_models.py @@ -15,7 +15,7 @@ from sqlalchemy.types import JSON from compendium_v2.db import db from compendium_v2.db.auth_model import User -from compendium_v2.db.model import NREN +from compendium_v2.db.presentation_models import NREN logger = logging.getLogger(__name__) diff --git a/compendium_v2/migrations/dump_survey_model.py b/compendium_v2/migrations/dump_survey_model.py index 314fda6d..3869a4a6 100644 --- a/compendium_v2/migrations/dump_survey_model.py +++ b/compendium_v2/migrations/dump_survey_model.py @@ -24,7 +24,7 @@ import compendium_v2 from compendium_v2.environment import setup_logging from compendium_v2.db import db from compendium_v2.config import load -from compendium_v2.db.survey_model import Survey +from compendium_v2.db.survey_models import Survey setup_logging() diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/publishers/excel_parser.py similarity index 98% rename from compendium_v2/background_task/parse_excel_data.py rename to compendium_v2/publishers/excel_parser.py index aba3ef6f..594031b9 100644 --- a/compendium_v2/background_task/parse_excel_data.py +++ b/compendium_v2/publishers/excel_parser.py @@ -2,15 +2,15 @@ import logging import openpyxl import os -from compendium_v2.db.model import FeeType +from compendium_v2.db.presentation_models import FeeType from compendium_v2.environment import setup_logging setup_logging() logger = logging.getLogger(__name__) -EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx") -NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx") +EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx") +NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx") def fetch_budget_excel_data(): diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py index f6492d2d..aeb4ecd9 100644 --- a/compendium_v2/publishers/helpers.py +++ b/compendium_v2/publishers/helpers.py @@ -3,7 +3,7 @@ from typing import List from sqlalchemy import select -from compendium_v2.db import db, model +from compendium_v2.db import db, presentation_models URL_PATTERN = re.compile( (r'\b(https?://[^\s<>";,(){}\[\]!\\]+' @@ -17,7 +17,7 @@ def get_uppercase_nren_dict(): """ :return: a dictionary of all known NRENs db entities keyed on the uppercased name """ - current_nrens = db.session.scalars(select(model.NREN)) + current_nrens = db.session.scalars(select(presentation_models.NREN)) nren_dict = {nren.name.upper(): nren for nren in current_nrens} # add aliases that are used in the source data: nren_dict['ASNET'] = nren_dict['ASNET-AM'] diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher.py similarity index 88% rename from compendium_v2/publishers/survey_publisher_v2.py rename to compendium_v2/publishers/survey_publisher.py index ebede055..3b88d5b9 100644 --- a/compendium_v2/publishers/survey_publisher_v2.py +++ b/compendium_v2/publishers/survey_publisher.py @@ -1,14 +1,25 @@ +""" +This module handles adding responses from the latest 2023 survey into the 2023 presentation_models. +The process is straightforward as the data is new and can be directly inserted into the presentation_models. + +For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions +will need to be addressed. + +Usage: + Used in publish_survey API in compendium_v2/routes/survey.py +""" + from decimal import Decimal from sqlalchemy import delete, select from compendium_v2.db import db -from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \ +from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \ InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume -from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse +from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse -def map_2023(nren, answers): +def _map_2023(nren, answers): year = 2023 for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs, @@ -112,7 +123,7 @@ def publish(year): ).unique() question_mapping = { - 2023: map_2023 + 2023: _map_2023 } mapping_function = question_mapping[year] diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py similarity index 83% rename from compendium_v2/publishers/survey_publisher_v1.py rename to compendium_v2/publishers/survey_publisher_legacy_excel.py index a6e2376f..6503b79f 100644 --- a/compendium_v2/publishers/survey_publisher_v1.py +++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py @@ -2,7 +2,7 @@ survey_publisher_v1 ========================= -This module loads the survey data from before 2022 from an excel file. +This module loads the survey data from before 2022 from a legacy Excel files. Missing info is filled in from the survey db for some questions. Registered as click cli command when installing compendium-v2. @@ -16,15 +16,14 @@ from sqlalchemy import select import compendium_v2 from compendium_v2.environment import setup_logging -from compendium_v2.background_task import parse_excel_data from compendium_v2.config import load -from compendium_v2.db import db, model +from compendium_v2.db import db, presentation_models from compendium_v2.survey_db import model as survey_model -from compendium_v2.publishers import helpers +from compendium_v2.publishers import helpers, excel_parser setup_logging() -logger = logging.getLogger('survey-publisher-v1') +logger = logging.getLogger('survey-publisher-legacy-excel') def db_budget_migration(nren_dict): @@ -46,7 +45,7 @@ def db_budget_migration(nren_dict): logger.warning(f'{abbrev} unknown. Skipping.') continue - budget_entry = model.BudgetEntry( + budget_entry = presentation_models.BudgetEntry( nren=nren_dict[abbrev], nren_id=nren_dict[abbrev].id, budget=float(budget.budget), @@ -55,7 +54,7 @@ def db_budget_migration(nren_dict): db.session.merge(budget_entry) # Import the data from excel sheet to database - exceldata = parse_excel_data.fetch_budget_excel_data() + exceldata = excel_parser.fetch_budget_excel_data() for abbrev, budget, year in exceldata: if abbrev not in nren_dict: @@ -65,7 +64,7 @@ def db_budget_migration(nren_dict): if budget > 200: logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})') - budget_entry = model.BudgetEntry( + budget_entry = presentation_models.BudgetEntry( nren=nren_dict[abbrev], nren_id=nren_dict[abbrev].id, budget=budget, @@ -77,7 +76,7 @@ def db_budget_migration(nren_dict): def db_funding_migration(nren_dict): # Import the data to database - data = parse_excel_data.fetch_funding_excel_data() + data = excel_parser.fetch_funding_excel_data() for (abbrev, year, client_institution, european_funding, @@ -93,7 +92,7 @@ def db_funding_migration(nren_dict): logger.warning(f'{abbrev} unknown. Skipping.') continue - budget_entry = model.FundingSource( + budget_entry = presentation_models.FundingSource( nren=nren_dict[abbrev], nren_id=nren_dict[abbrev].id, year=year, @@ -108,14 +107,14 @@ def db_funding_migration(nren_dict): def db_charging_structure_migration(nren_dict): # Import the data to database - data = parse_excel_data.fetch_charging_structure_excel_data() + data = excel_parser.fetch_charging_structure_excel_data() for (abbrev, year, charging_structure) in data: if abbrev not in nren_dict: logger.warning(f'{abbrev} unknown. Skipping.') continue - charging_structure_entry = model.ChargingStructure( + charging_structure_entry = presentation_models.ChargingStructure( nren=nren_dict[abbrev], nren_id=nren_dict[abbrev].id, year=year, @@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict): def db_staffing_migration(nren_dict): - staff_data = parse_excel_data.fetch_staffing_excel_data() + staff_data = excel_parser.fetch_staffing_excel_data() nren_staff_map = {} for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data: @@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict): continue nren = nren_dict[abbrev] - nren_staff_map[(nren.id, year)] = model.NrenStaff( + nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff( nren=nren, nren_id=nren.id, year=year, @@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict): non_technical_fte=0 ) - function_data = parse_excel_data.fetch_staff_function_excel_data() + function_data = excel_parser.fetch_staff_function_excel_data() for (abbrev, year, technical_fte, non_technical_fte) in function_data: if abbrev not in nren_dict: logger.warning(f'{abbrev} unknown. Skipping staff function data.') @@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict): nren_staff_map[(nren.id, year)].technical_fte = technical_fte nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte else: - nren_staff_map[(nren.id, year)] = model.NrenStaff( + nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff( nren=nren, nren_id=nren.id, year=year, @@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict): def db_ecprojects_migration(nren_dict): - ecproject_data = parse_excel_data.fetch_ecproject_excel_data() + ecproject_data = excel_parser.fetch_ecproject_excel_data() for (abbrev, year, project) in ecproject_data: if abbrev not in nren_dict: logger.warning(f'{abbrev} unknown. Skipping.') continue nren = nren_dict[abbrev] - ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project) + ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project) db.session.merge(ecproject_entry) db.session.commit() def db_organizations_migration(nren_dict): - organization_data = parse_excel_data.fetch_organization_excel_data() + organization_data = excel_parser.fetch_organization_excel_data() for (abbrev, year, org) in organization_data: if abbrev not in nren_dict: logger.warning(f'{abbrev} unknown. Skipping.') continue nren = nren_dict[abbrev] - org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org) + org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org) db.session.merge(org_entry) db.session.commit() def db_traffic_volume_migration(nren_dict): - traffic_data = parse_excel_data.fetch_traffic_excel_data() + traffic_data = excel_parser.fetch_traffic_excel_data() for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data: if abbrev not in nren_dict: logger.warning(f'{abbrev} unknown. Skipping.') continue nren = nren_dict[abbrev] - traffic_entry = model.TrafficVolume( + traffic_entry = presentation_models.TrafficVolume( nren=nren, nren_id=nren.id, year=year, diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py similarity index 93% rename from compendium_v2/publishers/survey_publisher_2022.py rename to compendium_v2/publishers/survey_publisher_old_db_2022.py index 39038d1e..ca704690 100644 --- a/compendium_v2/publishers/survey_publisher_2022.py +++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py @@ -2,7 +2,7 @@ survey_publisher_2022 ========================= -This module loads the survey data from 2022 from the survey database. +This module loads the survey data from 2022 from the old survey database into presentation_models. Registered as click cli command when installing compendium-v2. """ @@ -17,17 +17,17 @@ from sqlalchemy import delete, text from collections import defaultdict import compendium_v2 -from compendium_v2.db.model import FeeType +from compendium_v2.db.presentation_models import FeeType from compendium_v2.environment import setup_logging from compendium_v2.config import load from compendium_v2.publishers.helpers import extract_urls from compendium_v2.survey_db import model as survey_model -from compendium_v2.db import db, model +from compendium_v2.db import db, presentation_models from compendium_v2.publishers import helpers setup_logging() -logger = logging.getLogger('survey-publisher-2022') +logger = logging.getLogger('survey-publisher-old-db-2022') BUDGET_QUERY = """ SELECT DISTINCT ON (n.id, a.question_id) @@ -189,7 +189,7 @@ def transfer_budget(nren_dict): logger.info(f'{nren_name} unknown. Skipping.') continue - budget_entry = model.BudgetEntry( + budget_entry = presentation_models.BudgetEntry( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, budget=budget, @@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict): logger.info(f'{nren_name} has no urls for {year}. Skipping.') continue - institution_urls = model.InstitutionURLs( + institution_urls = presentation_models.InstitutionURLs( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, urls=urls, @@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict): logger.info(f'{nren_name} unknown. Skipping.') continue - funding_source = model.FundingSource( + funding_source = presentation_models.FundingSource( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict): for nren_name, nren_info in data.items(): if sum([nren_info[question] for question in StaffQuestion]) == 0: logger.info(f'{nren_name} has no staff data. Deleting if exists.') - db.session.execute(delete(model.NrenStaff).where( - model.NrenStaff.nren_id == nren_dict[nren_name].id, - model.NrenStaff.year == 2022 + db.session.execute(delete(presentation_models.NrenStaff).where( + presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id, + presentation_models.NrenStaff.year == 2022 )) continue @@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict): logger.info(f'{nren_name} FTE do not equal across employed/technical categories.' f' ({employed} != {technical})') - staff_data = model.NrenStaff( + staff_data = presentation_models.NrenStaff( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict): 'We are affiliated to ' ] - db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022)) + db.session.execute(delete(presentation_models.ParentOrganization).where( + presentation_models.ParentOrganization.year == 2022) + ) rows = query_question(OrgQuestion.PARENT_ORG_NAME) for row in rows: @@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict): logger.info(f'{nren_name} unknown. Skipping.') continue - parent_org = model.ParentOrganization( + parent_org = presentation_models.ParentOrganization( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict): (OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE), (OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE) ] - lookup = defaultdict(list) - db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022)) + db.session.execute(delete(presentation_models.SubOrganization).where( + presentation_models.SubOrganization.year == 2022) + ) for name, choice, role in suborg_questions: _name_rows = query_question(name) @@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict): for nren_name, suborgs in lookup.items(): for suborg_name, role in suborgs: - suborg = model.SubOrganization( + suborg = presentation_models.SubOrganization( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict): else: charging_structure = None - charging_structure = model.ChargingStructure( + charging_structure = presentation_models.ChargingStructure( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict): def transfer_ec_projects(nren_dict): # delete all existing EC projects, in case something changed db.session.execute( - delete(model.ECProject).where(model.ECProject.year == 2022) + delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022) ) rows = query_question(ECQuestion.EC_PROJECT) @@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict): # some answers include contract numbers, which we don't want here val = val.split('(contract n')[0] - ec_project = model.ECProject( + ec_project = presentation_models.ECProject( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=2022, @@ -536,7 +539,7 @@ def transfer_policies(nren_dict): data[(nren_name, year)][question_key] = value for (nren_name, year), nren_info in data.items(): - policy_data = model.Policy( + policy_data = presentation_models.Policy( nren=nren_dict[nren_name], nren_id=nren_dict[nren_name].id, year=year, diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/resources/2021_Organisation_DataSeries.xlsx similarity index 100% rename from compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx rename to compendium_v2/resources/2021_Organisation_DataSeries.xlsx diff --git a/compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx b/compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx similarity index 100% rename from compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx rename to compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx diff --git a/compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx b/compendium_v2/resources/2022_Networks_DataSeries.xlsx similarity index 100% rename from compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx rename to compendium_v2/resources/2022_Networks_DataSeries.xlsx diff --git a/compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx b/compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx similarity index 100% rename from compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx rename to compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py index 2f11be47..b0cff93d 100644 --- a/compendium_v2/routes/budget.py +++ b/compendium_v2/routes/budget.py @@ -3,7 +3,7 @@ from typing import Any from flask import Blueprint, jsonify -from compendium_v2.db.model import BudgetEntry +from compendium_v2.db.presentation_models import BudgetEntry from compendium_v2.routes import common diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py index d61df556..8c40679b 100644 --- a/compendium_v2/routes/charging.py +++ b/compendium_v2/routes/charging.py @@ -3,7 +3,7 @@ from typing import Any from flask import Blueprint, jsonify -from compendium_v2.db.model import ChargingStructure +from compendium_v2.db.presentation_models import ChargingStructure from compendium_v2.routes import common diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py index cc7b9fd1..be84fbfd 100644 --- a/compendium_v2/routes/common.py +++ b/compendium_v2/routes/common.py @@ -4,7 +4,7 @@ Utilities used by multiple route blueprints. import functools import logging from compendium_v2 import db -from compendium_v2.db.model import NREN, PreviewYear +from compendium_v2.db.presentation_models import NREN, PreviewYear from flask import Response, request from sqlalchemy import select diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py index aaf1c74f..501d2206 100644 --- a/compendium_v2/routes/ec_projects.py +++ b/compendium_v2/routes/ec_projects.py @@ -3,7 +3,7 @@ from typing import Any from flask import Blueprint, jsonify -from compendium_v2.db.model import ECProject +from compendium_v2.db.presentation_models import ECProject from compendium_v2.routes import common diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py index 2ca91eb3..0c110190 100644 --- a/compendium_v2/routes/funding.py +++ b/compendium_v2/routes/funding.py @@ -3,7 +3,7 @@ import logging from flask import Blueprint, jsonify from compendium_v2.routes import common -from compendium_v2.db.model import FundingSource +from compendium_v2.db.presentation_models import FundingSource from typing import Any diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py index 7153a8fc..8af1bbb6 100644 --- a/compendium_v2/routes/institutions_urls.py +++ b/compendium_v2/routes/institutions_urls.py @@ -4,7 +4,7 @@ from flask import Blueprint, jsonify from sqlalchemy import select from compendium_v2.db import db -from compendium_v2.db.model import NREN, InstitutionURLs +from compendium_v2.db.presentation_models import NREN, InstitutionURLs from compendium_v2.routes import common routes = Blueprint('institutions-urls', __name__) diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py index 473994b9..b39f7fd8 100644 --- a/compendium_v2/routes/organization.py +++ b/compendium_v2/routes/organization.py @@ -3,7 +3,7 @@ from typing import Any from flask import Blueprint, jsonify -from compendium_v2.db.model import ParentOrganization, SubOrganization +from compendium_v2.db.presentation_models import ParentOrganization, SubOrganization from compendium_v2.routes import common diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py index 68c6add0..a97dcf22 100644 --- a/compendium_v2/routes/policy.py +++ b/compendium_v2/routes/policy.py @@ -2,7 +2,7 @@ import logging from typing import Any from flask import Blueprint, jsonify -from compendium_v2.db.model import Policy +from compendium_v2.db.presentation_models import Policy from compendium_v2.routes import common routes = Blueprint('policy', __name__) diff --git a/compendium_v2/routes/response.py b/compendium_v2/routes/response.py index 9ba078e5..1c691431 100644 --- a/compendium_v2/routes/response.py +++ b/compendium_v2/routes/response.py @@ -9,8 +9,8 @@ from sqlalchemy import select from sqlalchemy.orm import lazyload from compendium_v2.db import db -from compendium_v2.db.model import NREN -from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED +from compendium_v2.db.presentation_models import NREN +from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED from compendium_v2.routes import common from compendium_v2.auth.session_management import admin_required, User diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py index 3c049385..3a23cd65 100644 --- a/compendium_v2/routes/staff.py +++ b/compendium_v2/routes/staff.py @@ -2,7 +2,7 @@ import logging from flask import Blueprint, jsonify -from compendium_v2.db.model import NrenStaff +from compendium_v2.db.presentation_models import NrenStaff from compendium_v2.routes import common from typing import Any diff --git a/compendium_v2/routes/survey.py b/compendium_v2/routes/survey.py index 5d685e7f..35766d19 100644 --- a/compendium_v2/routes/survey.py +++ b/compendium_v2/routes/survey.py @@ -7,9 +7,9 @@ from sqlalchemy import delete, select from sqlalchemy.orm import joinedload, load_only from compendium_v2.db import db -from compendium_v2.db.model import NREN, PreviewYear -from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED -from compendium_v2.publishers.survey_publisher_v2 import publish +from compendium_v2.db.presentation_models import NREN, PreviewYear +from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED +from compendium_v2.publishers.survey_publisher import publish from compendium_v2.routes import common from compendium_v2.auth.session_management import admin_required diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py index 70255d0f..56668fc4 100644 --- a/compendium_v2/routes/traffic.py +++ b/compendium_v2/routes/traffic.py @@ -3,7 +3,7 @@ import logging from flask import Blueprint, jsonify from compendium_v2.routes import common -from compendium_v2.db.model import TrafficVolume +from compendium_v2.db.presentation_models import TrafficVolume from typing import Any diff --git a/compendium_v2/routes/user.py b/compendium_v2/routes/user.py index f526e33a..ab9cce2e 100644 --- a/compendium_v2/routes/user.py +++ b/compendium_v2/routes/user.py @@ -8,7 +8,7 @@ from sqlalchemy import select from compendium_v2.auth.session_management import admin_required from compendium_v2.db import db from compendium_v2.db.auth_model import User, ROLES -from compendium_v2.db.model import NREN +from compendium_v2.db.presentation_models import NREN from compendium_v2.routes import common routes = Blueprint('user', __name__) diff --git a/setup.py b/setup.py index e4b9e4e3..f8258a33 100644 --- a/setup.py +++ b/setup.py @@ -29,8 +29,8 @@ setup( include_package_data=True, entry_points={ 'console_scripts': [ - 'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa - 'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa + 'excel-survey-publisher=compendium_v2.publishers.survey_publisher_legacy_excel:cli', # noqa + 'db-publisher-2022=compendium_v2.publishers.survey_publisher_old_db_2022:cli', # noqa 'conversion=compendium_v2.conversion.conversion:cli', # noqa 'dump_survey_model=compendium_v2.migrations.dump_survey_model:cli', # noqa ] diff --git a/test/conftest.py b/test/conftest.py index 2d8b1abb..591fe482 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,7 @@ import random from sqlalchemy import select from flask_login import LoginManager # type: ignore import compendium_v2 -from compendium_v2.db import db, model, survey_model +from compendium_v2.db import db, presentation_models, survey_models from compendium_v2.survey_db import model as survey_db_model from compendium_v2.auth.session_management import setup_login_manager, User, ROLES @@ -30,7 +30,7 @@ def mocked_admin_user(app, test_survey_data, mocker): with app.app_context(): user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub', roles=ROLES.admin) - nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2')) + nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2')) user.nrens.append(nren2) db.session.add(user) db.session.commit() @@ -46,7 +46,7 @@ def mocked_user(app, test_survey_data, mocker): with app.app_context(): user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub') - nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2')) + nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2')) user.nrens.append(nren2) db.session.add(user) db.session.commit() @@ -75,7 +75,7 @@ def test_budget_data(app): with app.app_context(): data = [row for row in _test_data_csv("BudgetTestData.csv")] nren_names = set([row["nren"] for row in data]) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for row in data: @@ -83,7 +83,7 @@ def test_budget_data(app): budget = row["budget"] year = row["year"] - db.session.add(model.BudgetEntry(nren=nren, budget=float(budget), year=int(year))) + db.session.add(presentation_models.BudgetEntry(nren=nren, budget=float(budget), year=int(year))) db.session.commit() @@ -92,7 +92,7 @@ def test_funding_source_data(app): with app.app_context(): data = [row for row in _test_data_csv("FundingSourceTestData.csv")] nren_names = set([row["nren"] for row in data]) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for row in data: @@ -105,7 +105,7 @@ def test_funding_source_data(app): other = row["other"] db.session.add( - model.FundingSource( + presentation_models.FundingSource( nren=nren, year=year, client_institutions=client, european_funding=european, @@ -137,7 +137,7 @@ def test_staff_data(app): data = list(_generate_rows()) nren_names = set(d['nren'] for d in data) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for row in data: @@ -149,7 +149,7 @@ def test_staff_data(app): non_technical_fte = row["non_technical_fte"] db.session.add( - model.NrenStaff( + presentation_models.NrenStaff( nren=nren, year=year, permanent_fte=permanent_fte, @@ -165,23 +165,23 @@ def test_staff_data(app): def test_survey_data(app): with app.app_context(): nren_names = ['nren1', 'nren2', 'nren3', 'nren4'] - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) - survey2021 = survey_model.Survey(year=2021, survey={}, status=survey_model.SurveyStatus.published) - survey2022 = survey_model.Survey(year=2022, survey={}, status=survey_model.SurveyStatus.published) - survey2023 = survey_model.Survey( + survey2021 = survey_models.Survey(year=2021, survey={}, status=survey_models.SurveyStatus.published) + survey2022 = survey_models.Survey(year=2022, survey={}, status=survey_models.SurveyStatus.published) + survey2023 = survey_models.Survey( year=2023, survey={'part1': [{'title': 'ha', 'visibleIf': 'false'}]}, - status=survey_model.SurveyStatus.open + status=survey_models.SurveyStatus.open ) db.session.add_all([survey2021, survey2022, survey2023]) - db.session.add(survey_model.SurveyResponse( + db.session.add(survey_models.SurveyResponse( nren=nren_dict['nren1'], survey=survey2023, answers={}, - status=survey_model.ResponseStatus.completed + status=survey_models.ResponseStatus.completed )) db.session.commit() @@ -219,7 +219,7 @@ def test_charging_structure_data(app): with app.app_context(): data = [row for row in _test_data_csv("ChargingStructureTestData.csv")] nren_names = set([row["nren"] for row in data]) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for row in data: @@ -230,7 +230,7 @@ def test_charging_structure_data(app): fee_type = None db.session.add( - model.ChargingStructure( + presentation_models.ChargingStructure( nren=nren, year=year, fee_type=fee_type) ) @@ -264,7 +264,7 @@ def test_organization_data(app): sub_org_data = list(_generate_sub_org_data()) nren_names = set(d['nren'] for d in [*org_data, *sub_org_data]) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for org in org_data: @@ -272,7 +272,7 @@ def test_organization_data(app): year = org["year"] name = org["name"] - db.session.add(model.ParentOrganization(nren=nren, year=year, organization=name)) + db.session.add(presentation_models.ParentOrganization(nren=nren, year=year, organization=name)) for sub_org in sub_org_data: nren = nren_dict[sub_org["nren"]] @@ -280,7 +280,7 @@ def test_organization_data(app): name = sub_org["name"] role = sub_org["role"] - db.session.add(model.SubOrganization(nren=nren, year=year, organization=name, role=role)) + db.session.add(presentation_models.SubOrganization(nren=nren, year=year, organization=name, role=role)) db.session.commit() @@ -307,7 +307,7 @@ def test_ec_project_data(app): ec_project_data = list(_generate_ec_project_data()) nren_names = set(d['nren'] for d in ec_project_data) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for ec_project in ec_project_data: @@ -315,7 +315,7 @@ def test_ec_project_data(app): year = ec_project["year"] project = ec_project["project"] - db.session.add(model.ECProject(nren=nren, year=year, project=project)) + db.session.add(presentation_models.ECProject(nren=nren, year=year, project=project)) db.session.commit() @@ -326,13 +326,13 @@ def test_policy_data(app): nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)] nren_names = set(ny[0] for ny in nrens_and_years) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for (nren_name, year) in nrens_and_years: nren = nren_dict[nren_name] - db.session.add(model.Policy( + db.session.add(presentation_models.Policy( nren=nren, year=year, strategic_plan='a strategy', @@ -353,14 +353,14 @@ def test_traffic_data(app): with app.app_context(): nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)] nren_names = set(ny[0] for ny in nrens_and_years) - nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names} db.session.add_all(nren_dict.values()) for (nren_name, year) in nrens_and_years: nren = nren_dict[nren_name] db.session.add( - model.TrafficVolume( + presentation_models.TrafficVolume( nren=nren, year=year, from_customers=2.23, @@ -377,7 +377,7 @@ def test_institution_urls_data(app): def _create_and_save_nrens(nren_names): nrens = {} for nren_name in nren_names: - nren_instance = model.NREN(name=nren_name, country='country') + nren_instance = presentation_models.NREN(name=nren_name, country='country') nrens[nren_name] = nren_instance db.session.add(nren_instance) return nrens @@ -387,7 +387,7 @@ def test_institution_urls_data(app): nren_instance = nrens[nren_name] urls = ['https://example.com', 'http://example.org'] - institution_urls_model = model.InstitutionURLs( + institution_urls_model = presentation_models.InstitutionURLs( nren=nren_instance, year=year, urls=urls diff --git a/test/test_budget.py b/test/test_budget.py index 77ea3452..de0206bc 100644 --- a/test/test_budget.py +++ b/test/test_budget.py @@ -1,7 +1,7 @@ import json import jsonschema from compendium_v2 import db -from compendium_v2.db.model import PreviewYear +from compendium_v2.db.presentation_models import PreviewYear from compendium_v2.routes.budget import BUDGET_RESPONSE_SCHEMA diff --git a/test/test_conversion.py b/test/test_conversion.py index 0f480505..d0e2282f 100644 --- a/test/test_conversion.py +++ b/test/test_conversion.py @@ -2,8 +2,8 @@ from sqlalchemy import select from sqlalchemy.orm import lazyload from compendium_v2.db import db -from compendium_v2.db.model import NREN -from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus +from compendium_v2.db.presentation_models import NREN +from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus from compendium_v2.conversion.conversion import _cli, convert_answers, load_service_data diff --git a/test/test_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py similarity index 82% rename from test/test_survey_publisher_2022.py rename to test/test_db_survey_publisher_2022.py index e8eaa8b4..ab4154de 100644 --- a/test/test_survey_publisher_2022.py +++ b/test/test_db_survey_publisher_2022.py @@ -1,7 +1,7 @@ from sqlalchemy import select -from compendium_v2.db import db, model -from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \ +from compendium_v2.db import db, presentation_models +from compendium_v2.publishers.survey_publisher_old_db_2022 import _cli, FundingSource, \ StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion @@ -200,30 +200,31 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"), ] - mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget', get_rows_as_tuples) - mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_funding_sources', funding_source_data) - mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question', question_data) - mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question_id', question_id_data) - mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_institutions_urls', institutions_urls_data) + mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_budget', get_rows_as_tuples) + mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_funding_sources', funding_source_data) + mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question', question_data) + mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question_id', question_id_data) + mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_institutions_urls', + institutions_urls_data) nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC'] with app_with_survey_db.app_context(): - db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names]) + db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names]) db.session.commit() _cli(dummy_config, app_with_survey_db) with app_with_survey_db.app_context(): budgets = db.session.scalars( - select(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc()) + select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc()) ).all() assert len(budgets) == 3 assert budgets[0].nren.name.lower() == 'nren1' assert budgets[0].budget == 100 funding_sources = db.session.scalars( - select(model.FundingSource).order_by(model.FundingSource.nren_id.asc()) + select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc()) ).all() assert len(funding_sources) == 3 assert funding_sources[0].nren.name.lower() == 'nren1' @@ -242,7 +243,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert funding_sources[2].other == 30 staff_data = db.session.scalars( - select(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc()) + select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc()) ).all() assert len(staff_data) == 3 @@ -265,7 +266,8 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert staff_data[2].subcontracted_fte == 0 _org_data = db.session.scalars( - select(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc()) + select(presentation_models.ParentOrganization).order_by( + presentation_models.ParentOrganization.nren_id.asc()) ).all() assert len(_org_data) == 2 @@ -276,18 +278,18 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert _org_data[1].organization == 'Org3' charging_structures = db.session.scalars( - select(model.ChargingStructure).order_by(model.ChargingStructure.nren_id.asc()) + select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc()) ).all() assert len(charging_structures) == 3 assert charging_structures[0].nren.name.lower() == 'nren1' - assert charging_structures[0].fee_type == model.FeeType.no_charge + assert charging_structures[0].fee_type == presentation_models.FeeType.no_charge assert charging_structures[1].nren.name.lower() == 'nren2' - assert charging_structures[1].fee_type == model.FeeType.usage_based_fee + assert charging_structures[1].fee_type == presentation_models.FeeType.usage_based_fee assert charging_structures[2].nren.name.lower() == 'nren3' - assert charging_structures[2].fee_type == model.FeeType.other + assert charging_structures[2].fee_type == presentation_models.FeeType.other _ec_data = db.session.scalars( - select(model.ECProject).order_by(model.ECProject.nren_id.asc()) + select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc()) ).all() assert len(_ec_data) == 3 @@ -301,7 +303,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert _ec_data[2].project == 'project3' policy_data = db.session.scalars( - select(model.Policy).order_by(model.Policy.nren_id.asc()) + select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc()) ).all() policy_data_2020 = [p for p in policy_data if p.year == 2020] policy_data_2022 = [p for p in policy_data if p.year == 2022] @@ -311,7 +313,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com' _institution_urls_data = db.session.scalars( - select(model.InstitutionURLs).order_by(model.InstitutionURLs.nren_id.asc()) + select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc()) ).all() assert len(_institution_urls_data) == 2 assert _institution_urls_data[0].nren.name.lower() == 'rash' diff --git a/test/test_dump_survey_model.py b/test/test_dump_survey_model.py index 722f21dc..6c49c6fb 100644 --- a/test/test_dump_survey_model.py +++ b/test/test_dump_survey_model.py @@ -1,5 +1,5 @@ from compendium_v2.db import db -from compendium_v2.db.survey_model import Survey, SurveyStatus +from compendium_v2.db.survey_models import Survey, SurveyStatus from compendium_v2.migrations.dump_survey_model import _cli diff --git a/test/test_response.py b/test/test_response.py index 8d870bb2..917ec051 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -1,7 +1,7 @@ import json import jsonschema from compendium_v2.db.auth_model import User -from compendium_v2.db.survey_model import ResponseStatus +from compendium_v2.db.survey_models import ResponseStatus from compendium_v2.routes.response import SURVEY_RESPONSE_SCHEMA, VerificationStatus, SURVEY_LOCK_SCHEMA, \ SURVEY_STATUS_RESPONSE_SCHEMA diff --git a/test/test_survey.py b/test/test_survey.py index dabf95cb..09144127 100644 --- a/test/test_survey.py +++ b/test/test_survey.py @@ -1,7 +1,7 @@ import json import jsonschema from compendium_v2.db import db -from compendium_v2.db.survey_model import Survey, SurveyStatus +from compendium_v2.db.survey_models import Survey, SurveyStatus from compendium_v2.routes.survey import LIST_SURVEYS_RESPONSE_SCHEMA diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher.py similarity index 65% rename from test/test_survey_publisher_v2.py rename to test/test_survey_publisher.py index 68b4b2d0..91f3d70d 100644 --- a/test/test_survey_publisher_v2.py +++ b/test/test_survey_publisher.py @@ -5,8 +5,8 @@ import os from sqlalchemy import func, select from compendium_v2 import db -from compendium_v2.db import model -from compendium_v2.publishers.survey_publisher_v2 import map_2023 +from compendium_v2.db import presentation_models +from compendium_v2.publishers.survey_publisher import _map_2023 JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json") @@ -16,15 +16,15 @@ def test_v2_publisher_empty(app): data = {} with app.app_context(): - nren = model.NREN(name='name', country='country') + nren = presentation_models.NREN(name='name', country='country') db.session.commit() with app.app_context(): - map_2023(nren, {"data": data}) + _map_2023(nren, {"data": data}) db.session.commit() with app.app_context(): - budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year))) + budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year))) assert budget_count == 0 # the main thing is actually that it doesnt crash @@ -34,37 +34,39 @@ def test_v2_publisher_full(app): data = json.load(json_data) with app.app_context(): - nren = model.NREN(name='name', country='country') + nren = presentation_models.NREN(name='name', country='country') db.session.commit() with app.app_context(): - map_2023(nren, {"data": data}) + _map_2023(nren, {"data": data}) db.session.commit() with app.app_context(): - budget = db.session.scalar(select(model.BudgetEntry.budget)) + budget = db.session.scalar(select(presentation_models.BudgetEntry.budget)) assert budget == Decimal("124.76") - funding_source = db.session.scalar(select(model.FundingSource)) + funding_source = db.session.scalar(select(presentation_models.FundingSource)) assert funding_source.client_institutions == Decimal("0") assert funding_source.european_funding == Decimal("20") assert funding_source.gov_public_bodies == Decimal("70") assert funding_source.commercial == Decimal("0") assert funding_source.other == Decimal("10") - charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type)) - assert charging_structure == model.FeeType.usage_based_fee + charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type)) + assert charging_structure == presentation_models.FeeType.usage_based_fee - staff = db.session.scalar(select(model.NrenStaff)) + staff = db.session.scalar(select(presentation_models.NrenStaff)) assert staff.permanent_fte == Decimal("5.6") assert staff.subcontracted_fte == Decimal("56") assert staff.technical_fte == Decimal("2") assert staff.non_technical_fte == Decimal("1") - parent = db.session.scalar(select(model.ParentOrganization.organization)) + parent = db.session.scalar(select(presentation_models.ParentOrganization.organization)) assert parent == "sdtfgd" - subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization)) + subs = db.session.scalars(select(presentation_models.SubOrganization).order_by( + presentation_models.SubOrganization.organization) + ) subs = [s for s in subs] assert subs[0].organization == "Aaerer" assert subs[0].role == "Treer" @@ -73,13 +75,15 @@ def test_v2_publisher_full(app): assert subs[2].organization == "werser" assert subs[2].role == "" - projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project)) + projects = db.session.scalars(select(presentation_models.ECProject.project).order_by( + presentation_models.ECProject.project) + ) projects = [p for p in projects] assert projects[0] == "dgdg" assert projects[1] == "rrrrr" assert projects[2] == "st" - policy = db.session.scalar(select(model.Policy)) + policy = db.session.scalar(select(presentation_models.Policy)) assert policy.strategic_plan == "https://serere.com" assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf" assert policy.equal_opportunity == "" @@ -89,11 +93,11 @@ def test_v2_publisher_full(app): assert policy.data_protection == "" assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore" # noqa: E501 - traffic = db.session.scalar(select(model.TrafficVolume)) + traffic = db.session.scalar(select(presentation_models.TrafficVolume)) assert traffic.to_customers == Decimal("3") assert traffic.from_customers == Decimal("34") assert traffic.to_external == Decimal("22") assert traffic.from_external == Decimal("3") - client_urls = db.session.scalar(select(model.InstitutionURLs)) + client_urls = db.session.scalar(select(presentation_models.InstitutionURLs)) assert client_urls.urls == ["http://erse.com", "https://wwe.com"] diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_legacy_excel.py similarity index 80% rename from test/test_survey_publisher_v1.py rename to test/test_survey_publisher_legacy_excel.py index d3bd42b6..5a20fc06 100644 --- a/test/test_survey_publisher_v1.py +++ b/test/test_survey_publisher_legacy_excel.py @@ -3,30 +3,32 @@ import os from sqlalchemy import select, func from compendium_v2 import db -from compendium_v2.db import model -from compendium_v2.publishers.survey_publisher_v1 import _cli +from compendium_v2.db import presentation_models +from compendium_v2.publishers.survey_publisher_legacy_excel import _cli EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx") def test_publisher(app_with_survey_db, mocker, dummy_config): - mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE) + mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE) with app_with_survey_db.app_context(): nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC'] - db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names]) + db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names]) db.session.commit() _cli(dummy_config, app_with_survey_db) with app_with_survey_db.app_context(): - budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year))) + budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year))) assert budget_count - funding_source_count = db.session.scalar(select(func.count(model.FundingSource.year))) + funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year))) assert funding_source_count - charging_structure_count = db.session.scalar(select(func.count(model.ChargingStructure.year))) + charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year))) assert charging_structure_count - staff_data = db.session.scalars(select(model.NrenStaff).order_by(model.NrenStaff.year.asc())).all() + staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by( + presentation_models.NrenStaff.year.asc()) + ).all() # data should only be saved for the NRENs we have saved in the database staff_data_nrens = set([staff.nren.name for staff in staff_data]) @@ -72,7 +74,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert kifu_data[5].technical_fte == 133 assert kifu_data[5].non_technical_fte == 45 - ecproject_data = db.session.scalars(select(model.ECProject)).all() + ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all() # test a couple of random entries surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017] assert len(surf2017) == 1 @@ -86,7 +88,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config): assert len(kifu2019) == 4 assert kifu2019[3].project == 'SuperHeroes for Science' - parent_data = db.session.scalars(select(model.ParentOrganization)).all() + parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all() # test a random entry asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021] assert len(asnet2021) == 1 -- GitLab