diff --git a/MANIFEST.in b/MANIFEST.in
index edff315a66901ecf2ff0533fa800de7c5fd7bd52..58874ea53d89bcd754171ef8f9c65be2a37b38ad 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,6 +4,5 @@ include compendium_v2/templates/survey-index.html
include compendium_v2/migrations/alembic.ini
recursive-include compendium_v2/migrations/versions *
recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/xlsx *
-include compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
+recursive-include compendium_v2/background_task/resources *
recursive-exclude test *
diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
index a963b4466b1347467ab28cb01f9df3322b26c65f..6da4572a93b3b07456d4acbf734df82cf81ebf23 100644
--- a/compendium_v2/conversion/conversion.py
+++ b/compendium_v2/conversion/conversion.py
@@ -2,7 +2,7 @@
conversion
=========================
-This module loads the survey data from 2022 from the survey database
+This module loads the survey data from 2022 from the old survey database
and stores the data in the json structure of the new survey, so that
it can be used to prefill the 2023 survey.
It also loads an xlsx file with the data for the services questions.
@@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus
from compendium_v2.conversion import mapping
@@ -30,7 +30,7 @@ setup_logging()
logger = logging.getLogger('conversion')
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx")
def query_nren(nren_id: int):
diff --git a/compendium_v2/db/auth_model.py b/compendium_v2/db/auth_model.py
index 50be7e3e009ef40c6762738a72b9049cf8923496..8605ab39c0173d99411e1b231c46d5371bd92450 100644
--- a/compendium_v2/db/auth_model.py
+++ b/compendium_v2/db/auth_model.py
@@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey
from flask_login import UserMixin # type: ignore
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/presentation_models.py
similarity index 100%
rename from compendium_v2/db/model.py
rename to compendium_v2/db/presentation_models.py
diff --git a/compendium_v2/db/survey_model.py b/compendium_v2/db/survey_models.py
similarity index 97%
rename from compendium_v2/db/survey_model.py
rename to compendium_v2/db/survey_models.py
index 866df323a0a1bf74925246eda08a813aeca458e3..fd688d38bc4a7b8dcc32a251d93f0038ebe44dc5 100644
--- a/compendium_v2/db/survey_model.py
+++ b/compendium_v2/db/survey_models.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import JSON
from compendium_v2.db import db
from compendium_v2.db.auth_model import User
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/migrations/dump_survey_model.py b/compendium_v2/migrations/dump_survey_model.py
index 314fda6dd27301fab9f57e34b1a285821b96c7d5..3869a4a6e37059c45e5b4138822008cbeed6a5b9 100644
--- a/compendium_v2/migrations/dump_survey_model.py
+++ b/compendium_v2/migrations/dump_survey_model.py
@@ -24,7 +24,7 @@ import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
-from compendium_v2.db.survey_model import Survey
+from compendium_v2.db.survey_models import Survey
setup_logging()
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/publishers/excel_parser.py
similarity index 98%
rename from compendium_v2/background_task/parse_excel_data.py
rename to compendium_v2/publishers/excel_parser.py
index aba3ef6fb09ce2ec5f7b523788dfe22c2a7ff00c..594031b9bc404b31b1634084e5c9b08768c6ce4c 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -2,15 +2,15 @@ import logging
import openpyxl
import os
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
def fetch_budget_excel_data():
diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py
index f6492d2d93e97db681a6a93b768ef7a70c87f0d1..aeb4ecd94aa3d8b80743617fcad8d5e742abaa29 100644
--- a/compendium_v2/publishers/helpers.py
+++ b/compendium_v2/publishers/helpers.py
@@ -3,7 +3,7 @@ from typing import List
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
URL_PATTERN = re.compile(
(r'\b(https?://[^\s<>";,(){}\[\]!\\]+'
@@ -17,7 +17,7 @@ def get_uppercase_nren_dict():
"""
:return: a dictionary of all known NRENs db entities keyed on the uppercased name
"""
- current_nrens = db.session.scalars(select(model.NREN))
+ current_nrens = db.session.scalars(select(presentation_models.NREN))
nren_dict = {nren.name.upper(): nren for nren in current_nrens}
# add aliases that are used in the source data:
nren_dict['ASNET'] = nren_dict['ASNET-AM']
diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher.py
similarity index 88%
rename from compendium_v2/publishers/survey_publisher_v2.py
rename to compendium_v2/publishers/survey_publisher.py
index ebede0556d6ea28c3db6df1c1ef267be59aee346..3b88d5b97cabd99bd27cb0232c49269e5d4e3cc8 100644
--- a/compendium_v2/publishers/survey_publisher_v2.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -1,14 +1,25 @@
+"""
+This module handles adding responses from the latest 2023 survey into the 2023 presentation_models.
+The process is straightforward as the data is new and can be directly inserted into the presentation_models.
+
+For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions
+will need to be addressed.
+
+Usage:
+ Used in publish_survey API in compendium_v2/routes/survey.py
+"""
+
from decimal import Decimal
from sqlalchemy import delete, select
from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
-from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
-def map_2023(nren, answers):
+def _map_2023(nren, answers):
year = 2023
for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
@@ -112,7 +123,7 @@ def publish(year):
).unique()
question_mapping = {
- 2023: map_2023
+ 2023: _map_2023
}
mapping_function = question_mapping[year]
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
similarity index 83%
rename from compendium_v2/publishers/survey_publisher_v1.py
rename to compendium_v2/publishers/survey_publisher_legacy_excel.py
index a6e2376f5f1f12571e6f8fefa8b89c04fb405a3e..6503b79faeda6e296c9fe2bf043658e95f4b8c5b 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -2,7 +2,7 @@
survey_publisher_v1
=========================
-This module loads the survey data from before 2022 from an excel file.
+This module loads the survey data from before 2022 from a legacy Excel files.
Missing info is filled in from the survey db for some questions.
Registered as click cli command when installing compendium-v2.
@@ -16,15 +16,14 @@ from sqlalchemy import select
import compendium_v2
from compendium_v2.environment import setup_logging
-from compendium_v2.background_task import parse_excel_data
from compendium_v2.config import load
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.publishers import helpers
+from compendium_v2.publishers import helpers, excel_parser
setup_logging()
-logger = logging.getLogger('survey-publisher-v1')
+logger = logging.getLogger('survey-publisher-legacy-excel')
def db_budget_migration(nren_dict):
@@ -46,7 +45,7 @@ def db_budget_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=float(budget.budget),
@@ -55,7 +54,7 @@ def db_budget_migration(nren_dict):
db.session.merge(budget_entry)
# Import the data from excel sheet to database
- exceldata = parse_excel_data.fetch_budget_excel_data()
+ exceldata = excel_parser.fetch_budget_excel_data()
for abbrev, budget, year in exceldata:
if abbrev not in nren_dict:
@@ -65,7 +64,7 @@ def db_budget_migration(nren_dict):
if budget > 200:
logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=budget,
@@ -77,7 +76,7 @@ def db_budget_migration(nren_dict):
def db_funding_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_funding_excel_data()
+ data = excel_parser.fetch_funding_excel_data()
for (abbrev, year, client_institution,
european_funding,
@@ -93,7 +92,7 @@ def db_funding_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.FundingSource(
+ budget_entry = presentation_models.FundingSource(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -108,14 +107,14 @@ def db_funding_migration(nren_dict):
def db_charging_structure_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_charging_structure_excel_data()
+ data = excel_parser.fetch_charging_structure_excel_data()
for (abbrev, year, charging_structure) in data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- charging_structure_entry = model.ChargingStructure(
+ charging_structure_entry = presentation_models.ChargingStructure(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict):
def db_staffing_migration(nren_dict):
- staff_data = parse_excel_data.fetch_staffing_excel_data()
+ staff_data = excel_parser.fetch_staffing_excel_data()
nren_staff_map = {}
for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
@@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict):
continue
nren = nren_dict[abbrev]
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict):
non_technical_fte=0
)
- function_data = parse_excel_data.fetch_staff_function_excel_data()
+ function_data = excel_parser.fetch_staff_function_excel_data()
for (abbrev, year, technical_fte, non_technical_fte) in function_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping staff function data.')
@@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict):
nren_staff_map[(nren.id, year)].technical_fte = technical_fte
nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
else:
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict):
def db_ecprojects_migration(nren_dict):
- ecproject_data = parse_excel_data.fetch_ecproject_excel_data()
+ ecproject_data = excel_parser.fetch_ecproject_excel_data()
for (abbrev, year, project) in ecproject_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
+ ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
db.session.merge(ecproject_entry)
db.session.commit()
def db_organizations_migration(nren_dict):
- organization_data = parse_excel_data.fetch_organization_excel_data()
+ organization_data = excel_parser.fetch_organization_excel_data()
for (abbrev, year, org) in organization_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
+ org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
db.session.merge(org_entry)
db.session.commit()
def db_traffic_volume_migration(nren_dict):
- traffic_data = parse_excel_data.fetch_traffic_excel_data()
+ traffic_data = excel_parser.fetch_traffic_excel_data()
for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- traffic_entry = model.TrafficVolume(
+ traffic_entry = presentation_models.TrafficVolume(
nren=nren,
nren_id=nren.id,
year=year,
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
similarity index 93%
rename from compendium_v2/publishers/survey_publisher_2022.py
rename to compendium_v2/publishers/survey_publisher_old_db_2022.py
index 39038d1ef878af99256d97e09b8931d58858f278..ca704690f10db37c956bde87795d47b0c7b4d8ca 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -2,7 +2,7 @@
survey_publisher_2022
=========================
-This module loads the survey data from 2022 from the survey database.
+This module loads the survey data from 2022 from the old survey database into presentation_models.
Registered as click cli command when installing compendium-v2.
"""
@@ -17,17 +17,17 @@ from sqlalchemy import delete, text
from collections import defaultdict
import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.publishers import helpers
setup_logging()
-logger = logging.getLogger('survey-publisher-2022')
+logger = logging.getLogger('survey-publisher-old-db-2022')
BUDGET_QUERY = """
SELECT DISTINCT ON (n.id, a.question_id)
@@ -189,7 +189,7 @@ def transfer_budget(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
budget=budget,
@@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict):
logger.info(f'{nren_name} has no urls for {year}. Skipping.')
continue
- institution_urls = model.InstitutionURLs(
+ institution_urls = presentation_models.InstitutionURLs(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
urls=urls,
@@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- funding_source = model.FundingSource(
+ funding_source = presentation_models.FundingSource(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict):
for nren_name, nren_info in data.items():
if sum([nren_info[question] for question in StaffQuestion]) == 0:
logger.info(f'{nren_name} has no staff data. Deleting if exists.')
- db.session.execute(delete(model.NrenStaff).where(
- model.NrenStaff.nren_id == nren_dict[nren_name].id,
- model.NrenStaff.year == 2022
+ db.session.execute(delete(presentation_models.NrenStaff).where(
+ presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id,
+ presentation_models.NrenStaff.year == 2022
))
continue
@@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict):
logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
f' ({employed} != {technical})')
- staff_data = model.NrenStaff(
+ staff_data = presentation_models.NrenStaff(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict):
'We are affiliated to '
]
- db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.ParentOrganization).where(
+ presentation_models.ParentOrganization.year == 2022)
+ )
rows = query_question(OrgQuestion.PARENT_ORG_NAME)
for row in rows:
@@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- parent_org = model.ParentOrganization(
+ parent_org = presentation_models.ParentOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict):
(OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
(OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
]
-
lookup = defaultdict(list)
- db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.SubOrganization).where(
+ presentation_models.SubOrganization.year == 2022)
+ )
for name, choice, role in suborg_questions:
_name_rows = query_question(name)
@@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict):
for nren_name, suborgs in lookup.items():
for suborg_name, role in suborgs:
- suborg = model.SubOrganization(
+ suborg = presentation_models.SubOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict):
else:
charging_structure = None
- charging_structure = model.ChargingStructure(
+ charging_structure = presentation_models.ChargingStructure(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict):
def transfer_ec_projects(nren_dict):
# delete all existing EC projects, in case something changed
db.session.execute(
- delete(model.ECProject).where(model.ECProject.year == 2022)
+ delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022)
)
rows = query_question(ECQuestion.EC_PROJECT)
@@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict):
# some answers include contract numbers, which we don't want here
val = val.split('(contract n')[0]
- ec_project = model.ECProject(
+ ec_project = presentation_models.ECProject(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -536,7 +539,7 @@ def transfer_policies(nren_dict):
data[(nren_name, year)][question_key] = value
for (nren_name, year), nren_info in data.items():
- policy_data = model.Policy(
+ policy_data = presentation_models.Policy(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=year,
diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/resources/2021_Organisation_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx
rename to compendium_v2/resources/2021_Organisation_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx b/compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx
rename to compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx b/compendium_v2/resources/2022_Networks_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx
rename to compendium_v2/resources/2022_Networks_DataSeries.xlsx
diff --git a/compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx b/compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
similarity index 100%
rename from compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
rename to compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index 2f11be47b5661f7bf3d43dfb12722f68959926dd..b0cff93d8f4bbffcd7a9754f61392d94055c8820 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import BudgetEntry
+from compendium_v2.db.presentation_models import BudgetEntry
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index d61df556bf337dc17c3a41ae378565ec2c4a6308..8c40679b4521c945ccdc19932ad60cb6642b356b 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ChargingStructure
+from compendium_v2.db.presentation_models import ChargingStructure
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index cc7b9fd1e605518f5ce60df14423fc0c28288367..be84fbfdd5dc40c4c913838be3d13a149617d9dd 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -4,7 +4,7 @@ Utilities used by multiple route blueprints.
import functools
import logging
from compendium_v2 import db
-from compendium_v2.db.model import NREN, PreviewYear
+from compendium_v2.db.presentation_models import NREN, PreviewYear
from flask import Response, request
from sqlalchemy import select
diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py
index aaf1c74fdbea718bf50adfe73115eee34156b225..501d2206bdf3238a738b5a71af14bea75aa3f68a 100644
--- a/compendium_v2/routes/ec_projects.py
+++ b/compendium_v2/routes/ec_projects.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ECProject
+from compendium_v2.db.presentation_models import ECProject
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 2ca91eb33ef9bb649c93ba5dc0d326982c692a18..0c1101900f038bc198e689d21e10962921022e9a 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import FundingSource
+from compendium_v2.db.presentation_models import FundingSource
from typing import Any
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 7153a8fc2da1f6c6fb23b42c53a008b4a6ca9898..8af1bbb67cd51df9a922ca0be240ba4278856713 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -4,7 +4,7 @@ from flask import Blueprint, jsonify
from sqlalchemy import select
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import NREN, InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py
index 473994b9a4d703769ecc71828288faf72320cc93..b39f7fd859b3a3f3261cf26f3381b03bb641fe40 100644
--- a/compendium_v2/routes/organization.py
+++ b/compendium_v2/routes/organization.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ParentOrganization, SubOrganization
+from compendium_v2.db.presentation_models import ParentOrganization, SubOrganization
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py
index 68c6add03d2726792e5ef287a6a1520ad7efc27c..a97dcf224517acf82519f9f77f932d476a353735 100644
--- a/compendium_v2/routes/policy.py
+++ b/compendium_v2/routes/policy.py
@@ -2,7 +2,7 @@ import logging
from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import Policy
+from compendium_v2.db.presentation_models import Policy
from compendium_v2.routes import common
routes = Blueprint('policy', __name__)
diff --git a/compendium_v2/routes/response.py b/compendium_v2/routes/response.py
index 9ba078e5db8c00cf4a7bf7fc0588508b7fc603e3..1c691431e22b47634c32d9ec3095612bb16b6ba2 100644
--- a/compendium_v2/routes/response.py
+++ b/compendium_v2/routes/response.py
@@ -9,8 +9,8 @@ from sqlalchemy import select
from sqlalchemy.orm import lazyload
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required, User
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 3c049385c9a568bde26bd37cc93dc97b4e03df2a..3a23cd6503dd12fc91ad6411cb9023372f191e70 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -2,7 +2,7 @@ import logging
from flask import Blueprint, jsonify
-from compendium_v2.db.model import NrenStaff
+from compendium_v2.db.presentation_models import NrenStaff
from compendium_v2.routes import common
from typing import Any
diff --git a/compendium_v2/routes/survey.py b/compendium_v2/routes/survey.py
index 5d685e7f29789734997ff9d94e93b13d7aca0e43..35766d19748433c9c1e82a547cf6bb25011f2d7b 100644
--- a/compendium_v2/routes/survey.py
+++ b/compendium_v2/routes/survey.py
@@ -7,9 +7,9 @@ from sqlalchemy import delete, select
from sqlalchemy.orm import joinedload, load_only
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, PreviewYear
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
-from compendium_v2.publishers.survey_publisher_v2 import publish
+from compendium_v2.db.presentation_models import NREN, PreviewYear
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
+from compendium_v2.publishers.survey_publisher import publish
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required
diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py
index 70255d0f847be12d54b62308cdc4763e6b263dde..56668fc41cf347be32c08afece9e773244cd7417 100644
--- a/compendium_v2/routes/traffic.py
+++ b/compendium_v2/routes/traffic.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import TrafficVolume
+from compendium_v2.db.presentation_models import TrafficVolume
from typing import Any
diff --git a/compendium_v2/routes/user.py b/compendium_v2/routes/user.py
index f526e33a3ee0a5ef44adf815045ae0c0dc5a42b8..ab9cce2e4e39ee9c0e84f019d9b460314c93f730 100644
--- a/compendium_v2/routes/user.py
+++ b/compendium_v2/routes/user.py
@@ -8,7 +8,7 @@ from sqlalchemy import select
from compendium_v2.auth.session_management import admin_required
from compendium_v2.db import db
from compendium_v2.db.auth_model import User, ROLES
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
from compendium_v2.routes import common
routes = Blueprint('user', __name__)
diff --git a/setup.py b/setup.py
index e4b9e4e357d2df31b4af06380ef961eab9cd72a4..f8258a33a0f4a3a9a4cefdd1001fb1e7cf9ee34a 100644
--- a/setup.py
+++ b/setup.py
@@ -29,8 +29,8 @@ setup(
include_package_data=True,
entry_points={
'console_scripts': [
- 'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli', # noqa
- 'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli', # noqa
+ 'excel-survey-publisher=compendium_v2.publishers.survey_publisher_legacy_excel:cli', # noqa
+ 'db-publisher-2022=compendium_v2.publishers.survey_publisher_old_db_2022:cli', # noqa
'conversion=compendium_v2.conversion.conversion:cli', # noqa
'dump_survey_model=compendium_v2.migrations.dump_survey_model:cli', # noqa
]
diff --git a/test/conftest.py b/test/conftest.py
index 2d8b1abb9c2695a390b0720920c6a3627f1aa925..591fe482257bc5bf6e96244309c9f7ed484dc57f 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -6,7 +6,7 @@ import random
from sqlalchemy import select
from flask_login import LoginManager # type: ignore
import compendium_v2
-from compendium_v2.db import db, model, survey_model
+from compendium_v2.db import db, presentation_models, survey_models
from compendium_v2.survey_db import model as survey_db_model
from compendium_v2.auth.session_management import setup_login_manager, User, ROLES
@@ -30,7 +30,7 @@ def mocked_admin_user(app, test_survey_data, mocker):
with app.app_context():
user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub', roles=ROLES.admin)
- nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2'))
+ nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2'))
user.nrens.append(nren2)
db.session.add(user)
db.session.commit()
@@ -46,7 +46,7 @@ def mocked_user(app, test_survey_data, mocker):
with app.app_context():
user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub')
- nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2'))
+ nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2'))
user.nrens.append(nren2)
db.session.add(user)
db.session.commit()
@@ -75,7 +75,7 @@ def test_budget_data(app):
with app.app_context():
data = [row for row in _test_data_csv("BudgetTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for row in data:
@@ -83,7 +83,7 @@ def test_budget_data(app):
budget = row["budget"]
year = row["year"]
- db.session.add(model.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
+ db.session.add(presentation_models.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
db.session.commit()
@@ -92,7 +92,7 @@ def test_funding_source_data(app):
with app.app_context():
data = [row for row in _test_data_csv("FundingSourceTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for row in data:
@@ -105,7 +105,7 @@ def test_funding_source_data(app):
other = row["other"]
db.session.add(
- model.FundingSource(
+ presentation_models.FundingSource(
nren=nren, year=year,
client_institutions=client,
european_funding=european,
@@ -137,7 +137,7 @@ def test_staff_data(app):
data = list(_generate_rows())
nren_names = set(d['nren'] for d in data)
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for row in data:
@@ -149,7 +149,7 @@ def test_staff_data(app):
non_technical_fte = row["non_technical_fte"]
db.session.add(
- model.NrenStaff(
+ presentation_models.NrenStaff(
nren=nren,
year=year,
permanent_fte=permanent_fte,
@@ -165,23 +165,23 @@ def test_staff_data(app):
def test_survey_data(app):
with app.app_context():
nren_names = ['nren1', 'nren2', 'nren3', 'nren4']
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
- survey2021 = survey_model.Survey(year=2021, survey={}, status=survey_model.SurveyStatus.published)
- survey2022 = survey_model.Survey(year=2022, survey={}, status=survey_model.SurveyStatus.published)
- survey2023 = survey_model.Survey(
+ survey2021 = survey_models.Survey(year=2021, survey={}, status=survey_models.SurveyStatus.published)
+ survey2022 = survey_models.Survey(year=2022, survey={}, status=survey_models.SurveyStatus.published)
+ survey2023 = survey_models.Survey(
year=2023,
survey={'part1': [{'title': 'ha', 'visibleIf': 'false'}]},
- status=survey_model.SurveyStatus.open
+ status=survey_models.SurveyStatus.open
)
db.session.add_all([survey2021, survey2022, survey2023])
- db.session.add(survey_model.SurveyResponse(
+ db.session.add(survey_models.SurveyResponse(
nren=nren_dict['nren1'],
survey=survey2023,
answers={},
- status=survey_model.ResponseStatus.completed
+ status=survey_models.ResponseStatus.completed
))
db.session.commit()
@@ -219,7 +219,7 @@ def test_charging_structure_data(app):
with app.app_context():
data = [row for row in _test_data_csv("ChargingStructureTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for row in data:
@@ -230,7 +230,7 @@ def test_charging_structure_data(app):
fee_type = None
db.session.add(
- model.ChargingStructure(
+ presentation_models.ChargingStructure(
nren=nren, year=year,
fee_type=fee_type)
)
@@ -264,7 +264,7 @@ def test_organization_data(app):
sub_org_data = list(_generate_sub_org_data())
nren_names = set(d['nren'] for d in [*org_data, *sub_org_data])
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for org in org_data:
@@ -272,7 +272,7 @@ def test_organization_data(app):
year = org["year"]
name = org["name"]
- db.session.add(model.ParentOrganization(nren=nren, year=year, organization=name))
+ db.session.add(presentation_models.ParentOrganization(nren=nren, year=year, organization=name))
for sub_org in sub_org_data:
nren = nren_dict[sub_org["nren"]]
@@ -280,7 +280,7 @@ def test_organization_data(app):
name = sub_org["name"]
role = sub_org["role"]
- db.session.add(model.SubOrganization(nren=nren, year=year, organization=name, role=role))
+ db.session.add(presentation_models.SubOrganization(nren=nren, year=year, organization=name, role=role))
db.session.commit()
@@ -307,7 +307,7 @@ def test_ec_project_data(app):
ec_project_data = list(_generate_ec_project_data())
nren_names = set(d['nren'] for d in ec_project_data)
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for ec_project in ec_project_data:
@@ -315,7 +315,7 @@ def test_ec_project_data(app):
year = ec_project["year"]
project = ec_project["project"]
- db.session.add(model.ECProject(nren=nren, year=year, project=project))
+ db.session.add(presentation_models.ECProject(nren=nren, year=year, project=project))
db.session.commit()
@@ -326,13 +326,13 @@ def test_policy_data(app):
nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
nren_names = set(ny[0] for ny in nrens_and_years)
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for (nren_name, year) in nrens_and_years:
nren = nren_dict[nren_name]
- db.session.add(model.Policy(
+ db.session.add(presentation_models.Policy(
nren=nren,
year=year,
strategic_plan='a strategy',
@@ -353,14 +353,14 @@ def test_traffic_data(app):
with app.app_context():
nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
nren_names = set(ny[0] for ny in nrens_and_years)
- nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+ nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for (nren_name, year) in nrens_and_years:
nren = nren_dict[nren_name]
db.session.add(
- model.TrafficVolume(
+ presentation_models.TrafficVolume(
nren=nren,
year=year,
from_customers=2.23,
@@ -377,7 +377,7 @@ def test_institution_urls_data(app):
def _create_and_save_nrens(nren_names):
nrens = {}
for nren_name in nren_names:
- nren_instance = model.NREN(name=nren_name, country='country')
+ nren_instance = presentation_models.NREN(name=nren_name, country='country')
nrens[nren_name] = nren_instance
db.session.add(nren_instance)
return nrens
@@ -387,7 +387,7 @@ def test_institution_urls_data(app):
nren_instance = nrens[nren_name]
urls = ['https://example.com', 'http://example.org']
- institution_urls_model = model.InstitutionURLs(
+ institution_urls_model = presentation_models.InstitutionURLs(
nren=nren_instance,
year=year,
urls=urls
diff --git a/test/test_budget.py b/test/test_budget.py
index 77ea3452c64da2bdaa766de0cd89465630eb3d7b..de0206bcfa0504ce1ba07ca2dba7e389ef9b49c9 100644
--- a/test/test_budget.py
+++ b/test/test_budget.py
@@ -1,7 +1,7 @@
import json
import jsonschema
from compendium_v2 import db
-from compendium_v2.db.model import PreviewYear
+from compendium_v2.db.presentation_models import PreviewYear
from compendium_v2.routes.budget import BUDGET_RESPONSE_SCHEMA
diff --git a/test/test_conversion.py b/test/test_conversion.py
index 0f48050554fadf9b9df037cee1d5e76dfc809d89..d0e2282f4eba63dc1b06274a6cb535522c5a541b 100644
--- a/test/test_conversion.py
+++ b/test/test_conversion.py
@@ -2,8 +2,8 @@ from sqlalchemy import select
from sqlalchemy.orm import lazyload
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus
from compendium_v2.conversion.conversion import _cli, convert_answers, load_service_data
diff --git a/test/test_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py
similarity index 82%
rename from test/test_survey_publisher_2022.py
rename to test/test_db_survey_publisher_2022.py
index e8eaa8b42c488aec35bd99c7fd044a3234cb1f56..ab4154de07ab5102a5e6534a82ba895a455ae09d 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_db_survey_publisher_2022.py
@@ -1,7 +1,7 @@
from sqlalchemy import select
-from compendium_v2.db import db, model
-from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
+from compendium_v2.db import db, presentation_models
+from compendium_v2.publishers.survey_publisher_old_db_2022 import _cli, FundingSource, \
StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
@@ -200,30 +200,31 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
(163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"),
]
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget', get_rows_as_tuples)
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_funding_sources', funding_source_data)
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question', question_data)
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question_id', question_id_data)
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_institutions_urls', institutions_urls_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_budget', get_rows_as_tuples)
+ mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_funding_sources', funding_source_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question', question_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question_id', question_id_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_institutions_urls',
+ institutions_urls_data)
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
with app_with_survey_db.app_context():
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
budgets = db.session.scalars(
- select(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc())
+ select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc())
).all()
assert len(budgets) == 3
assert budgets[0].nren.name.lower() == 'nren1'
assert budgets[0].budget == 100
funding_sources = db.session.scalars(
- select(model.FundingSource).order_by(model.FundingSource.nren_id.asc())
+ select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc())
).all()
assert len(funding_sources) == 3
assert funding_sources[0].nren.name.lower() == 'nren1'
@@ -242,7 +243,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert funding_sources[2].other == 30
staff_data = db.session.scalars(
- select(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc())
+ select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc())
).all()
assert len(staff_data) == 3
@@ -265,7 +266,8 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert staff_data[2].subcontracted_fte == 0
_org_data = db.session.scalars(
- select(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc())
+ select(presentation_models.ParentOrganization).order_by(
+ presentation_models.ParentOrganization.nren_id.asc())
).all()
assert len(_org_data) == 2
@@ -276,18 +278,18 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _org_data[1].organization == 'Org3'
charging_structures = db.session.scalars(
- select(model.ChargingStructure).order_by(model.ChargingStructure.nren_id.asc())
+ select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc())
).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == model.FeeType.no_charge
+ assert charging_structures[0].fee_type == presentation_models.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+ assert charging_structures[1].fee_type == presentation_models.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == model.FeeType.other
+ assert charging_structures[2].fee_type == presentation_models.FeeType.other
_ec_data = db.session.scalars(
- select(model.ECProject).order_by(model.ECProject.nren_id.asc())
+ select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc())
).all()
assert len(_ec_data) == 3
@@ -301,7 +303,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _ec_data[2].project == 'project3'
policy_data = db.session.scalars(
- select(model.Policy).order_by(model.Policy.nren_id.asc())
+ select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc())
).all()
policy_data_2020 = [p for p in policy_data if p.year == 2020]
policy_data_2022 = [p for p in policy_data if p.year == 2022]
@@ -311,7 +313,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com'
_institution_urls_data = db.session.scalars(
- select(model.InstitutionURLs).order_by(model.InstitutionURLs.nren_id.asc())
+ select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc())
).all()
assert len(_institution_urls_data) == 2
assert _institution_urls_data[0].nren.name.lower() == 'rash'
diff --git a/test/test_dump_survey_model.py b/test/test_dump_survey_model.py
index 722f21dc178f9396f8931130f03a3c6ab71df719..6c49c6fbcff0073aad3000ceaa30ebe9f94113f1 100644
--- a/test/test_dump_survey_model.py
+++ b/test/test_dump_survey_model.py
@@ -1,5 +1,5 @@
from compendium_v2.db import db
-from compendium_v2.db.survey_model import Survey, SurveyStatus
+from compendium_v2.db.survey_models import Survey, SurveyStatus
from compendium_v2.migrations.dump_survey_model import _cli
diff --git a/test/test_response.py b/test/test_response.py
index 8d870bb2b0cef3036799497ce9e4ee7128901abe..917ec05187bb3675c9bf4be58324441ccb404b96 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -1,7 +1,7 @@
import json
import jsonschema
from compendium_v2.db.auth_model import User
-from compendium_v2.db.survey_model import ResponseStatus
+from compendium_v2.db.survey_models import ResponseStatus
from compendium_v2.routes.response import SURVEY_RESPONSE_SCHEMA, VerificationStatus, SURVEY_LOCK_SCHEMA, \
SURVEY_STATUS_RESPONSE_SCHEMA
diff --git a/test/test_survey.py b/test/test_survey.py
index dabf95cbb050ec52e7ab5e374f35902b0e93872d..091441279eee0839ddd8fdfab2cb785a5e43f081 100644
--- a/test/test_survey.py
+++ b/test/test_survey.py
@@ -1,7 +1,7 @@
import json
import jsonschema
from compendium_v2.db import db
-from compendium_v2.db.survey_model import Survey, SurveyStatus
+from compendium_v2.db.survey_models import Survey, SurveyStatus
from compendium_v2.routes.survey import LIST_SURVEYS_RESPONSE_SCHEMA
diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher.py
similarity index 65%
rename from test/test_survey_publisher_v2.py
rename to test/test_survey_publisher.py
index 68b4b2d090765a5d36a0681a137197bf0b7f5ca8..91f3d70d36d46eddfa07c49b93ef18eb124841b3 100644
--- a/test/test_survey_publisher_v2.py
+++ b/test/test_survey_publisher.py
@@ -5,8 +5,8 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v2 import map_2023
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher import _map_2023
JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
@@ -16,15 +16,15 @@ def test_v2_publisher_empty(app):
data = {}
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count == 0
# the main thing is actually that it doesnt crash
@@ -34,37 +34,39 @@ def test_v2_publisher_full(app):
data = json.load(json_data)
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget = db.session.scalar(select(model.BudgetEntry.budget))
+ budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
assert budget == Decimal("124.76")
- funding_source = db.session.scalar(select(model.FundingSource))
+ funding_source = db.session.scalar(select(presentation_models.FundingSource))
assert funding_source.client_institutions == Decimal("0")
assert funding_source.european_funding == Decimal("20")
assert funding_source.gov_public_bodies == Decimal("70")
assert funding_source.commercial == Decimal("0")
assert funding_source.other == Decimal("10")
- charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
- assert charging_structure == model.FeeType.usage_based_fee
+ charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
+ assert charging_structure == presentation_models.FeeType.usage_based_fee
- staff = db.session.scalar(select(model.NrenStaff))
+ staff = db.session.scalar(select(presentation_models.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
assert staff.subcontracted_fte == Decimal("56")
assert staff.technical_fte == Decimal("2")
assert staff.non_technical_fte == Decimal("1")
- parent = db.session.scalar(select(model.ParentOrganization.organization))
+ parent = db.session.scalar(select(presentation_models.ParentOrganization.organization))
assert parent == "sdtfgd"
- subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization))
+ subs = db.session.scalars(select(presentation_models.SubOrganization).order_by(
+ presentation_models.SubOrganization.organization)
+ )
subs = [s for s in subs]
assert subs[0].organization == "Aaerer"
assert subs[0].role == "Treer"
@@ -73,13 +75,15 @@ def test_v2_publisher_full(app):
assert subs[2].organization == "werser"
assert subs[2].role == ""
- projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project))
+ projects = db.session.scalars(select(presentation_models.ECProject.project).order_by(
+ presentation_models.ECProject.project)
+ )
projects = [p for p in projects]
assert projects[0] == "dgdg"
assert projects[1] == "rrrrr"
assert projects[2] == "st"
- policy = db.session.scalar(select(model.Policy))
+ policy = db.session.scalar(select(presentation_models.Policy))
assert policy.strategic_plan == "https://serere.com"
assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf"
assert policy.equal_opportunity == ""
@@ -89,11 +93,11 @@ def test_v2_publisher_full(app):
assert policy.data_protection == ""
assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore" # noqa: E501
- traffic = db.session.scalar(select(model.TrafficVolume))
+ traffic = db.session.scalar(select(presentation_models.TrafficVolume))
assert traffic.to_customers == Decimal("3")
assert traffic.from_customers == Decimal("34")
assert traffic.to_external == Decimal("22")
assert traffic.from_external == Decimal("3")
- client_urls = db.session.scalar(select(model.InstitutionURLs))
+ client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_legacy_excel.py
similarity index 80%
rename from test/test_survey_publisher_v1.py
rename to test/test_survey_publisher_legacy_excel.py
index d3bd42b6631950951a99df6e545d8a94030c91dc..5a20fc063e843f3d930ce463034f5f910762bf5b 100644
--- a/test/test_survey_publisher_v1.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -3,30 +3,32 @@ import os
from sqlalchemy import select, func
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v1 import _cli
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
def test_publisher(app_with_survey_db, mocker, dummy_config):
- mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
+ mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
with app_with_survey_db.app_context():
nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count
- funding_source_count = db.session.scalar(select(func.count(model.FundingSource.year)))
+ funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year)))
assert funding_source_count
- charging_structure_count = db.session.scalar(select(func.count(model.ChargingStructure.year)))
+ charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year)))
assert charging_structure_count
- staff_data = db.session.scalars(select(model.NrenStaff).order_by(model.NrenStaff.year.asc())).all()
+ staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by(
+ presentation_models.NrenStaff.year.asc())
+ ).all()
# data should only be saved for the NRENs we have saved in the database
staff_data_nrens = set([staff.nren.name for staff in staff_data])
@@ -72,7 +74,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert kifu_data[5].technical_fte == 133
assert kifu_data[5].non_technical_fte == 45
- ecproject_data = db.session.scalars(select(model.ECProject)).all()
+ ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all()
# test a couple of random entries
surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017]
assert len(surf2017) == 1
@@ -86,7 +88,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert len(kifu2019) == 4
assert kifu2019[3].project == 'SuperHeroes for Science'
- parent_data = db.session.scalars(select(model.ParentOrganization)).all()
+ parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all()
# test a random entry
asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021]
assert len(asnet2021) == 1