diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
index a963b4466b1347467ab28cb01f9df3322b26c65f..0da96978dbb06d7ef4f7191a6e25c6e850e2bd75 100644
--- a/compendium_v2/conversion/conversion.py
+++ b/compendium_v2/conversion/conversion.py
@@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus
from compendium_v2.conversion import mapping
@@ -30,7 +30,7 @@ setup_logging()
logger = logging.getLogger('conversion')
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx")
def query_nren(nren_id: int):
diff --git a/compendium_v2/db/auth_model.py b/compendium_v2/db/auth_model.py
index 50be7e3e009ef40c6762738a72b9049cf8923496..8605ab39c0173d99411e1b231c46d5371bd92450 100644
--- a/compendium_v2/db/auth_model.py
+++ b/compendium_v2/db/auth_model.py
@@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey
from flask_login import UserMixin # type: ignore
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/presentation_models.py
similarity index 100%
rename from compendium_v2/db/model.py
rename to compendium_v2/db/presentation_models.py
diff --git a/compendium_v2/db/survey_model.py b/compendium_v2/db/survey_models.py
similarity index 97%
rename from compendium_v2/db/survey_model.py
rename to compendium_v2/db/survey_models.py
index 866df323a0a1bf74925246eda08a813aeca458e3..fd688d38bc4a7b8dcc32a251d93f0038ebe44dc5 100644
--- a/compendium_v2/db/survey_model.py
+++ b/compendium_v2/db/survey_models.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import JSON
from compendium_v2.db import db
from compendium_v2.db.auth_model import User
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/migrations/dump_survey_model.py b/compendium_v2/migrations/dump_survey_model.py
index 314fda6dd27301fab9f57e34b1a285821b96c7d5..3869a4a6e37059c45e5b4138822008cbeed6a5b9 100644
--- a/compendium_v2/migrations/dump_survey_model.py
+++ b/compendium_v2/migrations/dump_survey_model.py
@@ -24,7 +24,7 @@ import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
-from compendium_v2.db.survey_model import Survey
+from compendium_v2.db.survey_models import Survey
setup_logging()
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/publishers/excel_parser.py
similarity index 98%
rename from compendium_v2/background_task/parse_excel_data.py
rename to compendium_v2/publishers/excel_parser.py
index aba3ef6fb09ce2ec5f7b523788dfe22c2a7ff00c..594031b9bc404b31b1634084e5c9b08768c6ce4c 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -2,15 +2,15 @@ import logging
import openpyxl
import os
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
def fetch_budget_excel_data():
diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py
index f6492d2d93e97db681a6a93b768ef7a70c87f0d1..aeb4ecd94aa3d8b80743617fcad8d5e742abaa29 100644
--- a/compendium_v2/publishers/helpers.py
+++ b/compendium_v2/publishers/helpers.py
@@ -3,7 +3,7 @@ from typing import List
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
URL_PATTERN = re.compile(
(r'\b(https?://[^\s<>";,(){}\[\]!\\]+'
@@ -17,7 +17,7 @@ def get_uppercase_nren_dict():
"""
:return: a dictionary of all known NRENs db entities keyed on the uppercased name
"""
- current_nrens = db.session.scalars(select(model.NREN))
+ current_nrens = db.session.scalars(select(presentation_models.NREN))
nren_dict = {nren.name.upper(): nren for nren in current_nrens}
# add aliases that are used in the source data:
nren_dict['ASNET'] = nren_dict['ASNET-AM']
diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher.py
similarity index 88%
rename from compendium_v2/publishers/survey_publisher_v2.py
rename to compendium_v2/publishers/survey_publisher.py
index ebede0556d6ea28c3db6df1c1ef267be59aee346..3b88d5b97cabd99bd27cb0232c49269e5d4e3cc8 100644
--- a/compendium_v2/publishers/survey_publisher_v2.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -1,14 +1,25 @@
+"""
+This module handles adding responses from the latest 2023 survey into the 2023 presentation_models.
+The process is straightforward as the data is new and can be directly inserted into the presentation_models.
+
+For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions
+will need to be addressed.
+
+Usage:
+ Used in publish_survey API in compendium_v2/routes/survey.py
+"""
+
from decimal import Decimal
from sqlalchemy import delete, select
from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
-from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
-def map_2023(nren, answers):
+def _map_2023(nren, answers):
year = 2023
for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
@@ -112,7 +123,7 @@ def publish(year):
).unique()
question_mapping = {
- 2023: map_2023
+ 2023: _map_2023
}
mapping_function = question_mapping[year]
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
similarity index 84%
rename from compendium_v2/publishers/survey_publisher_v1.py
rename to compendium_v2/publishers/survey_publisher_legacy_excel.py
index a6e2376f5f1f12571e6f8fefa8b89c04fb405a3e..f265231aabb52bd9fb970dc29e87a527ae76ddf3 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -2,7 +2,7 @@
survey_publisher_v1
=========================
-This module loads the survey data from before 2022 from an excel file.
+This module loads the survey data from before 2022 from a legacy Excel files.
Missing info is filled in from the survey db for some questions.
Registered as click cli command when installing compendium-v2.
@@ -16,15 +16,14 @@ from sqlalchemy import select
import compendium_v2
from compendium_v2.environment import setup_logging
-from compendium_v2.background_task import parse_excel_data
from compendium_v2.config import load
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.publishers import helpers
+from compendium_v2.publishers import helpers, excel_parser
setup_logging()
-logger = logging.getLogger('survey-publisher-v1')
+logger = logging.getLogger(__file__)
def db_budget_migration(nren_dict):
@@ -46,7 +45,7 @@ def db_budget_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=float(budget.budget),
@@ -55,7 +54,7 @@ def db_budget_migration(nren_dict):
db.session.merge(budget_entry)
# Import the data from excel sheet to database
- exceldata = parse_excel_data.fetch_budget_excel_data()
+ exceldata = excel_parser.fetch_budget_excel_data()
for abbrev, budget, year in exceldata:
if abbrev not in nren_dict:
@@ -65,7 +64,7 @@ def db_budget_migration(nren_dict):
if budget > 200:
logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=budget,
@@ -77,7 +76,7 @@ def db_budget_migration(nren_dict):
def db_funding_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_funding_excel_data()
+ data = excel_parser.fetch_funding_excel_data()
for (abbrev, year, client_institution,
european_funding,
@@ -93,7 +92,7 @@ def db_funding_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.FundingSource(
+ budget_entry = presentation_models.FundingSource(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -108,14 +107,14 @@ def db_funding_migration(nren_dict):
def db_charging_structure_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_charging_structure_excel_data()
+ data = excel_parser.fetch_charging_structure_excel_data()
for (abbrev, year, charging_structure) in data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- charging_structure_entry = model.ChargingStructure(
+ charging_structure_entry = presentation_models.ChargingStructure(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict):
def db_staffing_migration(nren_dict):
- staff_data = parse_excel_data.fetch_staffing_excel_data()
+ staff_data = excel_parser.fetch_staffing_excel_data()
nren_staff_map = {}
for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
@@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict):
continue
nren = nren_dict[abbrev]
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict):
non_technical_fte=0
)
- function_data = parse_excel_data.fetch_staff_function_excel_data()
+ function_data = excel_parser.fetch_staff_function_excel_data()
for (abbrev, year, technical_fte, non_technical_fte) in function_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping staff function data.')
@@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict):
nren_staff_map[(nren.id, year)].technical_fte = technical_fte
nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
else:
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict):
def db_ecprojects_migration(nren_dict):
- ecproject_data = parse_excel_data.fetch_ecproject_excel_data()
+ ecproject_data = excel_parser.fetch_ecproject_excel_data()
for (abbrev, year, project) in ecproject_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
+ ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
db.session.merge(ecproject_entry)
db.session.commit()
def db_organizations_migration(nren_dict):
- organization_data = parse_excel_data.fetch_organization_excel_data()
+ organization_data = excel_parser.fetch_organization_excel_data()
for (abbrev, year, org) in organization_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
+ org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
db.session.merge(org_entry)
db.session.commit()
def db_traffic_volume_migration(nren_dict):
- traffic_data = parse_excel_data.fetch_traffic_excel_data()
+ traffic_data = excel_parser.fetch_traffic_excel_data()
for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- traffic_entry = model.TrafficVolume(
+ traffic_entry = presentation_models.TrafficVolume(
nren=nren,
nren_id=nren.id,
year=year,
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
similarity index 93%
rename from compendium_v2/publishers/survey_publisher_2022.py
rename to compendium_v2/publishers/survey_publisher_old_db_2022.py
index 39038d1ef878af99256d97e09b8931d58858f278..fc028fc6a8f8bb12079b57a5912fa0a70481613d 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -2,7 +2,7 @@
survey_publisher_2022
=========================
-This module loads the survey data from 2022 from the survey database.
+This module loads the survey data from 2022 from the old survey database into presentation_models.
Registered as click cli command when installing compendium-v2.
"""
@@ -17,17 +17,17 @@ from sqlalchemy import delete, text
from collections import defaultdict
import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.publishers import helpers
setup_logging()
-logger = logging.getLogger('survey-publisher-2022')
+logger = logging.getLogger(__name__)
BUDGET_QUERY = """
SELECT DISTINCT ON (n.id, a.question_id)
@@ -189,7 +189,7 @@ def transfer_budget(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
budget=budget,
@@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict):
logger.info(f'{nren_name} has no urls for {year}. Skipping.')
continue
- institution_urls = model.InstitutionURLs(
+ institution_urls = presentation_models.InstitutionURLs(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
urls=urls,
@@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- funding_source = model.FundingSource(
+ funding_source = presentation_models.FundingSource(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict):
for nren_name, nren_info in data.items():
if sum([nren_info[question] for question in StaffQuestion]) == 0:
logger.info(f'{nren_name} has no staff data. Deleting if exists.')
- db.session.execute(delete(model.NrenStaff).where(
- model.NrenStaff.nren_id == nren_dict[nren_name].id,
- model.NrenStaff.year == 2022
+ db.session.execute(delete(presentation_models.NrenStaff).where(
+ presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id,
+ presentation_models.NrenStaff.year == 2022
))
continue
@@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict):
logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
f' ({employed} != {technical})')
- staff_data = model.NrenStaff(
+ staff_data = presentation_models.NrenStaff(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict):
'We are affiliated to '
]
- db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.ParentOrganization).where(
+ presentation_models.ParentOrganization.year == 2022)
+ )
rows = query_question(OrgQuestion.PARENT_ORG_NAME)
for row in rows:
@@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- parent_org = model.ParentOrganization(
+ parent_org = presentation_models.ParentOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict):
(OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
(OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
]
-
lookup = defaultdict(list)
- db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.SubOrganization).where(
+ presentation_models.SubOrganization.year == 2022)
+ )
for name, choice, role in suborg_questions:
_name_rows = query_question(name)
@@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict):
for nren_name, suborgs in lookup.items():
for suborg_name, role in suborgs:
- suborg = model.SubOrganization(
+ suborg = presentation_models.SubOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict):
else:
charging_structure = None
- charging_structure = model.ChargingStructure(
+ charging_structure = presentation_models.ChargingStructure(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict):
def transfer_ec_projects(nren_dict):
# delete all existing EC projects, in case something changed
db.session.execute(
- delete(model.ECProject).where(model.ECProject.year == 2022)
+ delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022)
)
rows = query_question(ECQuestion.EC_PROJECT)
@@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict):
# some answers include contract numbers, which we don't want here
val = val.split('(contract n')[0]
- ec_project = model.ECProject(
+ ec_project = presentation_models.ECProject(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -536,7 +539,7 @@ def transfer_policies(nren_dict):
data[(nren_name, year)][question_key] = value
for (nren_name, year), nren_info in data.items():
- policy_data = model.Policy(
+ policy_data = presentation_models.Policy(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=year,
diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/resources/2021_Organisation_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx
rename to compendium_v2/resources/2021_Organisation_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx b/compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx
rename to compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx b/compendium_v2/resources/2022_Networks_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx
rename to compendium_v2/resources/2022_Networks_DataSeries.xlsx
diff --git a/compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx b/compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
similarity index 100%
rename from compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
rename to compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index 2f11be47b5661f7bf3d43dfb12722f68959926dd..b0cff93d8f4bbffcd7a9754f61392d94055c8820 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import BudgetEntry
+from compendium_v2.db.presentation_models import BudgetEntry
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index d61df556bf337dc17c3a41ae378565ec2c4a6308..8c40679b4521c945ccdc19932ad60cb6642b356b 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ChargingStructure
+from compendium_v2.db.presentation_models import ChargingStructure
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index cc7b9fd1e605518f5ce60df14423fc0c28288367..be84fbfdd5dc40c4c913838be3d13a149617d9dd 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -4,7 +4,7 @@ Utilities used by multiple route blueprints.
import functools
import logging
from compendium_v2 import db
-from compendium_v2.db.model import NREN, PreviewYear
+from compendium_v2.db.presentation_models import NREN, PreviewYear
from flask import Response, request
from sqlalchemy import select
diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py
index aaf1c74fdbea718bf50adfe73115eee34156b225..501d2206bdf3238a738b5a71af14bea75aa3f68a 100644
--- a/compendium_v2/routes/ec_projects.py
+++ b/compendium_v2/routes/ec_projects.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ECProject
+from compendium_v2.db.presentation_models import ECProject
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 2ca91eb33ef9bb649c93ba5dc0d326982c692a18..0c1101900f038bc198e689d21e10962921022e9a 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import FundingSource
+from compendium_v2.db.presentation_models import FundingSource
from typing import Any
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 7153a8fc2da1f6c6fb23b42c53a008b4a6ca9898..8af1bbb67cd51df9a922ca0be240ba4278856713 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -4,7 +4,7 @@ from flask import Blueprint, jsonify
from sqlalchemy import select
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import NREN, InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py
index 473994b9a4d703769ecc71828288faf72320cc93..b39f7fd859b3a3f3261cf26f3381b03bb641fe40 100644
--- a/compendium_v2/routes/organization.py
+++ b/compendium_v2/routes/organization.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ParentOrganization, SubOrganization
+from compendium_v2.db.presentation_models import ParentOrganization, SubOrganization
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py
index 68c6add03d2726792e5ef287a6a1520ad7efc27c..a97dcf224517acf82519f9f77f932d476a353735 100644
--- a/compendium_v2/routes/policy.py
+++ b/compendium_v2/routes/policy.py
@@ -2,7 +2,7 @@ import logging
from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import Policy
+from compendium_v2.db.presentation_models import Policy
from compendium_v2.routes import common
routes = Blueprint('policy', __name__)
diff --git a/compendium_v2/routes/response.py b/compendium_v2/routes/response.py
index 9ba078e5db8c00cf4a7bf7fc0588508b7fc603e3..1c691431e22b47634c32d9ec3095612bb16b6ba2 100644
--- a/compendium_v2/routes/response.py
+++ b/compendium_v2/routes/response.py
@@ -9,8 +9,8 @@ from sqlalchemy import select
from sqlalchemy.orm import lazyload
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required, User
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 3c049385c9a568bde26bd37cc93dc97b4e03df2a..3a23cd6503dd12fc91ad6411cb9023372f191e70 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -2,7 +2,7 @@ import logging
from flask import Blueprint, jsonify
-from compendium_v2.db.model import NrenStaff
+from compendium_v2.db.presentation_models import NrenStaff
from compendium_v2.routes import common
from typing import Any
diff --git a/compendium_v2/routes/survey.py b/compendium_v2/routes/survey.py
index 5d685e7f29789734997ff9d94e93b13d7aca0e43..35766d19748433c9c1e82a547cf6bb25011f2d7b 100644
--- a/compendium_v2/routes/survey.py
+++ b/compendium_v2/routes/survey.py
@@ -7,9 +7,9 @@ from sqlalchemy import delete, select
from sqlalchemy.orm import joinedload, load_only
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, PreviewYear
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
-from compendium_v2.publishers.survey_publisher_v2 import publish
+from compendium_v2.db.presentation_models import NREN, PreviewYear
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
+from compendium_v2.publishers.survey_publisher import publish
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required
diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py
index 70255d0f847be12d54b62308cdc4763e6b263dde..56668fc41cf347be32c08afece9e773244cd7417 100644
--- a/compendium_v2/routes/traffic.py
+++ b/compendium_v2/routes/traffic.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import TrafficVolume
+from compendium_v2.db.presentation_models import TrafficVolume
from typing import Any
diff --git a/compendium_v2/routes/user.py b/compendium_v2/routes/user.py
index f526e33a3ee0a5ef44adf815045ae0c0dc5a42b8..ab9cce2e4e39ee9c0e84f019d9b460314c93f730 100644
--- a/compendium_v2/routes/user.py
+++ b/compendium_v2/routes/user.py
@@ -8,7 +8,7 @@ from sqlalchemy import select
from compendium_v2.auth.session_management import admin_required
from compendium_v2.db import db
from compendium_v2.db.auth_model import User, ROLES
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
from compendium_v2.routes import common
routes = Blueprint('user', __name__)
diff --git a/test/test_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py
similarity index 88%
rename from test/test_survey_publisher_2022.py
rename to test/test_db_survey_publisher_2022.py
index e8eaa8b42c488aec35bd99c7fd044a3234cb1f56..6e9c90968cf475ae009fc3eeb5b5af79cd0bc12a 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_db_survey_publisher_2022.py
@@ -1,6 +1,6 @@
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
@@ -209,21 +209,21 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
with app_with_survey_db.app_context():
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
budgets = db.session.scalars(
- select(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc())
+ select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc())
).all()
assert len(budgets) == 3
assert budgets[0].nren.name.lower() == 'nren1'
assert budgets[0].budget == 100
funding_sources = db.session.scalars(
- select(model.FundingSource).order_by(model.FundingSource.nren_id.asc())
+ select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc())
).all()
assert len(funding_sources) == 3
assert funding_sources[0].nren.name.lower() == 'nren1'
@@ -242,7 +242,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert funding_sources[2].other == 30
staff_data = db.session.scalars(
- select(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc())
+ select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc())
).all()
assert len(staff_data) == 3
@@ -265,7 +265,8 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert staff_data[2].subcontracted_fte == 0
_org_data = db.session.scalars(
- select(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc())
+ select(presentation_models.ParentOrganization).order_by(
+ presentation_models.ParentOrganization.nren_id.asc())
).all()
assert len(_org_data) == 2
@@ -276,18 +277,18 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _org_data[1].organization == 'Org3'
charging_structures = db.session.scalars(
- select(model.ChargingStructure).order_by(model.ChargingStructure.nren_id.asc())
+ select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc())
).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == model.FeeType.no_charge
+ assert charging_structures[0].fee_type == presentation_models.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+ assert charging_structures[1].fee_type == presentation_models.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == model.FeeType.other
+ assert charging_structures[2].fee_type == presentation_models.FeeType.other
_ec_data = db.session.scalars(
- select(model.ECProject).order_by(model.ECProject.nren_id.asc())
+ select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc())
).all()
assert len(_ec_data) == 3
@@ -301,7 +302,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _ec_data[2].project == 'project3'
policy_data = db.session.scalars(
- select(model.Policy).order_by(model.Policy.nren_id.asc())
+ select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc())
).all()
policy_data_2020 = [p for p in policy_data if p.year == 2020]
policy_data_2022 = [p for p in policy_data if p.year == 2022]
@@ -311,7 +312,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com'
_institution_urls_data = db.session.scalars(
- select(model.InstitutionURLs).order_by(model.InstitutionURLs.nren_id.asc())
+ select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc())
).all()
assert len(_institution_urls_data) == 2
assert _institution_urls_data[0].nren.name.lower() == 'rash'
diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher.py
similarity index 65%
rename from test/test_survey_publisher_v2.py
rename to test/test_survey_publisher.py
index 68b4b2d090765a5d36a0681a137197bf0b7f5ca8..91f3d70d36d46eddfa07c49b93ef18eb124841b3 100644
--- a/test/test_survey_publisher_v2.py
+++ b/test/test_survey_publisher.py
@@ -5,8 +5,8 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v2 import map_2023
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher import _map_2023
JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
@@ -16,15 +16,15 @@ def test_v2_publisher_empty(app):
data = {}
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count == 0
# the main thing is actually that it doesnt crash
@@ -34,37 +34,39 @@ def test_v2_publisher_full(app):
data = json.load(json_data)
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget = db.session.scalar(select(model.BudgetEntry.budget))
+ budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
assert budget == Decimal("124.76")
- funding_source = db.session.scalar(select(model.FundingSource))
+ funding_source = db.session.scalar(select(presentation_models.FundingSource))
assert funding_source.client_institutions == Decimal("0")
assert funding_source.european_funding == Decimal("20")
assert funding_source.gov_public_bodies == Decimal("70")
assert funding_source.commercial == Decimal("0")
assert funding_source.other == Decimal("10")
- charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
- assert charging_structure == model.FeeType.usage_based_fee
+ charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
+ assert charging_structure == presentation_models.FeeType.usage_based_fee
- staff = db.session.scalar(select(model.NrenStaff))
+ staff = db.session.scalar(select(presentation_models.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
assert staff.subcontracted_fte == Decimal("56")
assert staff.technical_fte == Decimal("2")
assert staff.non_technical_fte == Decimal("1")
- parent = db.session.scalar(select(model.ParentOrganization.organization))
+ parent = db.session.scalar(select(presentation_models.ParentOrganization.organization))
assert parent == "sdtfgd"
- subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization))
+ subs = db.session.scalars(select(presentation_models.SubOrganization).order_by(
+ presentation_models.SubOrganization.organization)
+ )
subs = [s for s in subs]
assert subs[0].organization == "Aaerer"
assert subs[0].role == "Treer"
@@ -73,13 +75,15 @@ def test_v2_publisher_full(app):
assert subs[2].organization == "werser"
assert subs[2].role == ""
- projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project))
+ projects = db.session.scalars(select(presentation_models.ECProject.project).order_by(
+ presentation_models.ECProject.project)
+ )
projects = [p for p in projects]
assert projects[0] == "dgdg"
assert projects[1] == "rrrrr"
assert projects[2] == "st"
- policy = db.session.scalar(select(model.Policy))
+ policy = db.session.scalar(select(presentation_models.Policy))
assert policy.strategic_plan == "https://serere.com"
assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf"
assert policy.equal_opportunity == ""
@@ -89,11 +93,11 @@ def test_v2_publisher_full(app):
assert policy.data_protection == ""
assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore" # noqa: E501
- traffic = db.session.scalar(select(model.TrafficVolume))
+ traffic = db.session.scalar(select(presentation_models.TrafficVolume))
assert traffic.to_customers == Decimal("3")
assert traffic.from_customers == Decimal("34")
assert traffic.to_external == Decimal("22")
assert traffic.from_external == Decimal("3")
- client_urls = db.session.scalar(select(model.InstitutionURLs))
+ client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_legacy_excel.py
similarity index 80%
rename from test/test_survey_publisher_v1.py
rename to test/test_survey_publisher_legacy_excel.py
index d3bd42b6631950951a99df6e545d8a94030c91dc..5a20fc063e843f3d930ce463034f5f910762bf5b 100644
--- a/test/test_survey_publisher_v1.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -3,30 +3,32 @@ import os
from sqlalchemy import select, func
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v1 import _cli
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
def test_publisher(app_with_survey_db, mocker, dummy_config):
- mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
+ mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
with app_with_survey_db.app_context():
nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count
- funding_source_count = db.session.scalar(select(func.count(model.FundingSource.year)))
+ funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year)))
assert funding_source_count
- charging_structure_count = db.session.scalar(select(func.count(model.ChargingStructure.year)))
+ charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year)))
assert charging_structure_count
- staff_data = db.session.scalars(select(model.NrenStaff).order_by(model.NrenStaff.year.asc())).all()
+ staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by(
+ presentation_models.NrenStaff.year.asc())
+ ).all()
# data should only be saved for the NRENs we have saved in the database
staff_data_nrens = set([staff.nren.name for staff in staff_data])
@@ -72,7 +74,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert kifu_data[5].technical_fte == 133
assert kifu_data[5].non_technical_fte == 45
- ecproject_data = db.session.scalars(select(model.ECProject)).all()
+ ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all()
# test a couple of random entries
surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017]
assert len(surf2017) == 1
@@ -86,7 +88,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert len(kifu2019) == 4
assert kifu2019[3].project == 'SuperHeroes for Science'
- parent_data = db.session.scalars(select(model.ParentOrganization)).all()
+ parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all()
# test a random entry
asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021]
assert len(asnet2021) == 1