From c094c536b0b5d56292cdec75e537ad867d183c10 Mon Sep 17 00:00:00 2001
From: Mohammad Torkashvand <mohammad.torkashvand@geant.org>
Date: Thu, 14 Sep 2023 16:23:39 +0200
Subject: [PATCH] refacgtor modules name and structures
---
compendium_v2/background_task/__init__.py | 0
compendium_v2/conversion/conversion.py | 6 +--
compendium_v2/db/auth_model.py | 2 +-
.../db/{model.py => presentation_models.py} | 0
.../db/{survey_model.py => survey_models.py} | 2 +-
compendium_v2/migrations/dump_survey_model.py | 2 +-
.../excel_parser.py} | 6 +--
compendium_v2/publishers/helpers.py | 4 +-
...ey_publisher_v2.py => survey_publisher.py} | 19 ++++++--
...v1.py => survey_publisher_legacy_excel.py} | 43 +++++++++---------
...022.py => survey_publisher_old_db_2022.py} | 43 ++++++++++--------
.../2021_Organisation_DataSeries.xlsx | Bin
.../2022_Connected_Users_DataSeries.xlsx | Bin
.../2022_Networks_DataSeries.xlsx | Bin
...NREN-Services-prefills_2023_Recovered.xlsx | Bin
compendium_v2/routes/budget.py | 2 +-
compendium_v2/routes/charging.py | 2 +-
compendium_v2/routes/common.py | 2 +-
compendium_v2/routes/ec_projects.py | 2 +-
compendium_v2/routes/funding.py | 2 +-
compendium_v2/routes/institutions_urls.py | 2 +-
compendium_v2/routes/organization.py | 2 +-
compendium_v2/routes/policy.py | 2 +-
compendium_v2/routes/response.py | 4 +-
compendium_v2/routes/staff.py | 2 +-
compendium_v2/routes/survey.py | 6 +--
compendium_v2/routes/traffic.py | 2 +-
compendium_v2/routes/user.py | 2 +-
...22.py => test_db_survey_publisher_2022.py} | 27 +++++------
...blisher_v2.py => test_survey_publisher.py} | 40 ++++++++--------
... => test_survey_publisher_legacy_excel.py} | 22 +++++----
31 files changed, 134 insertions(+), 114 deletions(-)
delete mode 100644 compendium_v2/background_task/__init__.py
rename compendium_v2/db/{model.py => presentation_models.py} (100%)
rename compendium_v2/db/{survey_model.py => survey_models.py} (97%)
rename compendium_v2/{background_task/parse_excel_data.py => publishers/excel_parser.py} (98%)
rename compendium_v2/publishers/{survey_publisher_v2.py => survey_publisher.py} (88%)
rename compendium_v2/publishers/{survey_publisher_v1.py => survey_publisher_legacy_excel.py} (84%)
rename compendium_v2/publishers/{survey_publisher_2022.py => survey_publisher_old_db_2022.py} (93%)
rename compendium_v2/{background_task/xlsx => resources}/2021_Organisation_DataSeries.xlsx (100%)
rename compendium_v2/{background_task/xlsx => resources}/2022_Connected_Users_DataSeries.xlsx (100%)
rename compendium_v2/{background_task/xlsx => resources}/2022_Networks_DataSeries.xlsx (100%)
rename compendium_v2/{conversion => resources}/NREN-Services-prefills_2023_Recovered.xlsx (100%)
rename test/{test_survey_publisher_2022.py => test_db_survey_publisher_2022.py} (88%)
rename test/{test_survey_publisher_v2.py => test_survey_publisher.py} (65%)
rename test/{test_survey_publisher_v1.py => test_survey_publisher_legacy_excel.py} (80%)
diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
index a963b446..0da96978 100644
--- a/compendium_v2/conversion/conversion.py
+++ b/compendium_v2/conversion/conversion.py
@@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus
from compendium_v2.conversion import mapping
@@ -30,7 +30,7 @@ setup_logging()
logger = logging.getLogger('conversion')
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx")
def query_nren(nren_id: int):
diff --git a/compendium_v2/db/auth_model.py b/compendium_v2/db/auth_model.py
index 50be7e3e..8605ab39 100644
--- a/compendium_v2/db/auth_model.py
+++ b/compendium_v2/db/auth_model.py
@@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey
from flask_login import UserMixin # type: ignore
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/presentation_models.py
similarity index 100%
rename from compendium_v2/db/model.py
rename to compendium_v2/db/presentation_models.py
diff --git a/compendium_v2/db/survey_model.py b/compendium_v2/db/survey_models.py
similarity index 97%
rename from compendium_v2/db/survey_model.py
rename to compendium_v2/db/survey_models.py
index 866df323..fd688d38 100644
--- a/compendium_v2/db/survey_model.py
+++ b/compendium_v2/db/survey_models.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import JSON
from compendium_v2.db import db
from compendium_v2.db.auth_model import User
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
logger = logging.getLogger(__name__)
diff --git a/compendium_v2/migrations/dump_survey_model.py b/compendium_v2/migrations/dump_survey_model.py
index 314fda6d..3869a4a6 100644
--- a/compendium_v2/migrations/dump_survey_model.py
+++ b/compendium_v2/migrations/dump_survey_model.py
@@ -24,7 +24,7 @@ import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2.db import db
from compendium_v2.config import load
-from compendium_v2.db.survey_model import Survey
+from compendium_v2.db.survey_models import Survey
setup_logging()
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/publishers/excel_parser.py
similarity index 98%
rename from compendium_v2/background_task/parse_excel_data.py
rename to compendium_v2/publishers/excel_parser.py
index aba3ef6f..594031b9 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -2,15 +2,15 @@ import logging
import openpyxl
import os
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
def fetch_budget_excel_data():
diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py
index f6492d2d..aeb4ecd9 100644
--- a/compendium_v2/publishers/helpers.py
+++ b/compendium_v2/publishers/helpers.py
@@ -3,7 +3,7 @@ from typing import List
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
URL_PATTERN = re.compile(
(r'\b(https?://[^\s<>";,(){}\[\]!\\]+'
@@ -17,7 +17,7 @@ def get_uppercase_nren_dict():
"""
:return: a dictionary of all known NRENs db entities keyed on the uppercased name
"""
- current_nrens = db.session.scalars(select(model.NREN))
+ current_nrens = db.session.scalars(select(presentation_models.NREN))
nren_dict = {nren.name.upper(): nren for nren in current_nrens}
# add aliases that are used in the source data:
nren_dict['ASNET'] = nren_dict['ASNET-AM']
diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher.py
similarity index 88%
rename from compendium_v2/publishers/survey_publisher_v2.py
rename to compendium_v2/publishers/survey_publisher.py
index ebede055..3b88d5b9 100644
--- a/compendium_v2/publishers/survey_publisher_v2.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -1,14 +1,25 @@
+"""
+This module handles adding responses from the latest 2023 survey into the 2023 presentation_models.
+The process is straightforward as the data is new and can be directly inserted into the presentation_models.
+
+For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions
+will need to be addressed.
+
+Usage:
+ Used in publish_survey API in compendium_v2/routes/survey.py
+"""
+
from decimal import Decimal
from sqlalchemy import delete, select
from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
-from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
-def map_2023(nren, answers):
+def _map_2023(nren, answers):
year = 2023
for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
@@ -112,7 +123,7 @@ def publish(year):
).unique()
question_mapping = {
- 2023: map_2023
+ 2023: _map_2023
}
mapping_function = question_mapping[year]
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
similarity index 84%
rename from compendium_v2/publishers/survey_publisher_v1.py
rename to compendium_v2/publishers/survey_publisher_legacy_excel.py
index a6e2376f..f265231a 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -2,7 +2,7 @@
survey_publisher_v1
=========================
-This module loads the survey data from before 2022 from an excel file.
+This module loads the survey data from before 2022 from a legacy Excel files.
Missing info is filled in from the survey db for some questions.
Registered as click cli command when installing compendium-v2.
@@ -16,15 +16,14 @@ from sqlalchemy import select
import compendium_v2
from compendium_v2.environment import setup_logging
-from compendium_v2.background_task import parse_excel_data
from compendium_v2.config import load
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.publishers import helpers
+from compendium_v2.publishers import helpers, excel_parser
setup_logging()
-logger = logging.getLogger('survey-publisher-v1')
+logger = logging.getLogger(__file__)
def db_budget_migration(nren_dict):
@@ -46,7 +45,7 @@ def db_budget_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=float(budget.budget),
@@ -55,7 +54,7 @@ def db_budget_migration(nren_dict):
db.session.merge(budget_entry)
# Import the data from excel sheet to database
- exceldata = parse_excel_data.fetch_budget_excel_data()
+ exceldata = excel_parser.fetch_budget_excel_data()
for abbrev, budget, year in exceldata:
if abbrev not in nren_dict:
@@ -65,7 +64,7 @@ def db_budget_migration(nren_dict):
if budget > 200:
logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
budget=budget,
@@ -77,7 +76,7 @@ def db_budget_migration(nren_dict):
def db_funding_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_funding_excel_data()
+ data = excel_parser.fetch_funding_excel_data()
for (abbrev, year, client_institution,
european_funding,
@@ -93,7 +92,7 @@ def db_funding_migration(nren_dict):
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.FundingSource(
+ budget_entry = presentation_models.FundingSource(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -108,14 +107,14 @@ def db_funding_migration(nren_dict):
def db_charging_structure_migration(nren_dict):
# Import the data to database
- data = parse_excel_data.fetch_charging_structure_excel_data()
+ data = excel_parser.fetch_charging_structure_excel_data()
for (abbrev, year, charging_structure) in data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
- charging_structure_entry = model.ChargingStructure(
+ charging_structure_entry = presentation_models.ChargingStructure(
nren=nren_dict[abbrev],
nren_id=nren_dict[abbrev].id,
year=year,
@@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict):
def db_staffing_migration(nren_dict):
- staff_data = parse_excel_data.fetch_staffing_excel_data()
+ staff_data = excel_parser.fetch_staffing_excel_data()
nren_staff_map = {}
for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
@@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict):
continue
nren = nren_dict[abbrev]
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict):
non_technical_fte=0
)
- function_data = parse_excel_data.fetch_staff_function_excel_data()
+ function_data = excel_parser.fetch_staff_function_excel_data()
for (abbrev, year, technical_fte, non_technical_fte) in function_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping staff function data.')
@@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict):
nren_staff_map[(nren.id, year)].technical_fte = technical_fte
nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
else:
- nren_staff_map[(nren.id, year)] = model.NrenStaff(
+ nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
nren=nren,
nren_id=nren.id,
year=year,
@@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict):
def db_ecprojects_migration(nren_dict):
- ecproject_data = parse_excel_data.fetch_ecproject_excel_data()
+ ecproject_data = excel_parser.fetch_ecproject_excel_data()
for (abbrev, year, project) in ecproject_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
+ ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
db.session.merge(ecproject_entry)
db.session.commit()
def db_organizations_migration(nren_dict):
- organization_data = parse_excel_data.fetch_organization_excel_data()
+ organization_data = excel_parser.fetch_organization_excel_data()
for (abbrev, year, org) in organization_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
+ org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
db.session.merge(org_entry)
db.session.commit()
def db_traffic_volume_migration(nren_dict):
- traffic_data = parse_excel_data.fetch_traffic_excel_data()
+ traffic_data = excel_parser.fetch_traffic_excel_data()
for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
- traffic_entry = model.TrafficVolume(
+ traffic_entry = presentation_models.TrafficVolume(
nren=nren,
nren_id=nren.id,
year=year,
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
similarity index 93%
rename from compendium_v2/publishers/survey_publisher_2022.py
rename to compendium_v2/publishers/survey_publisher_old_db_2022.py
index 39038d1e..fc028fc6 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -2,7 +2,7 @@
survey_publisher_2022
=========================
-This module loads the survey data from 2022 from the survey database.
+This module loads the survey data from 2022 from the old survey database into presentation_models.
Registered as click cli command when installing compendium-v2.
"""
@@ -17,17 +17,17 @@ from sqlalchemy import delete, text
from collections import defaultdict
import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.publishers import helpers
setup_logging()
-logger = logging.getLogger('survey-publisher-2022')
+logger = logging.getLogger(__name__)
BUDGET_QUERY = """
SELECT DISTINCT ON (n.id, a.question_id)
@@ -189,7 +189,7 @@ def transfer_budget(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
+ budget_entry = presentation_models.BudgetEntry(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
budget=budget,
@@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict):
logger.info(f'{nren_name} has no urls for {year}. Skipping.')
continue
- institution_urls = model.InstitutionURLs(
+ institution_urls = presentation_models.InstitutionURLs(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
urls=urls,
@@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- funding_source = model.FundingSource(
+ funding_source = presentation_models.FundingSource(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict):
for nren_name, nren_info in data.items():
if sum([nren_info[question] for question in StaffQuestion]) == 0:
logger.info(f'{nren_name} has no staff data. Deleting if exists.')
- db.session.execute(delete(model.NrenStaff).where(
- model.NrenStaff.nren_id == nren_dict[nren_name].id,
- model.NrenStaff.year == 2022
+ db.session.execute(delete(presentation_models.NrenStaff).where(
+ presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id,
+ presentation_models.NrenStaff.year == 2022
))
continue
@@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict):
logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
f' ({employed} != {technical})')
- staff_data = model.NrenStaff(
+ staff_data = presentation_models.NrenStaff(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict):
'We are affiliated to '
]
- db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.ParentOrganization).where(
+ presentation_models.ParentOrganization.year == 2022)
+ )
rows = query_question(OrgQuestion.PARENT_ORG_NAME)
for row in rows:
@@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict):
logger.info(f'{nren_name} unknown. Skipping.')
continue
- parent_org = model.ParentOrganization(
+ parent_org = presentation_models.ParentOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict):
(OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
(OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
]
-
lookup = defaultdict(list)
- db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022))
+ db.session.execute(delete(presentation_models.SubOrganization).where(
+ presentation_models.SubOrganization.year == 2022)
+ )
for name, choice, role in suborg_questions:
_name_rows = query_question(name)
@@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict):
for nren_name, suborgs in lookup.items():
for suborg_name, role in suborgs:
- suborg = model.SubOrganization(
+ suborg = presentation_models.SubOrganization(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict):
else:
charging_structure = None
- charging_structure = model.ChargingStructure(
+ charging_structure = presentation_models.ChargingStructure(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict):
def transfer_ec_projects(nren_dict):
# delete all existing EC projects, in case something changed
db.session.execute(
- delete(model.ECProject).where(model.ECProject.year == 2022)
+ delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022)
)
rows = query_question(ECQuestion.EC_PROJECT)
@@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict):
# some answers include contract numbers, which we don't want here
val = val.split('(contract n')[0]
- ec_project = model.ECProject(
+ ec_project = presentation_models.ECProject(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=2022,
@@ -536,7 +539,7 @@ def transfer_policies(nren_dict):
data[(nren_name, year)][question_key] = value
for (nren_name, year), nren_info in data.items():
- policy_data = model.Policy(
+ policy_data = presentation_models.Policy(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
year=year,
diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/resources/2021_Organisation_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx
rename to compendium_v2/resources/2021_Organisation_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx b/compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx
rename to compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx b/compendium_v2/resources/2022_Networks_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx
rename to compendium_v2/resources/2022_Networks_DataSeries.xlsx
diff --git a/compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx b/compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
similarity index 100%
rename from compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
rename to compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index 2f11be47..b0cff93d 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import BudgetEntry
+from compendium_v2.db.presentation_models import BudgetEntry
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index d61df556..8c40679b 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ChargingStructure
+from compendium_v2.db.presentation_models import ChargingStructure
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index cc7b9fd1..be84fbfd 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -4,7 +4,7 @@ Utilities used by multiple route blueprints.
import functools
import logging
from compendium_v2 import db
-from compendium_v2.db.model import NREN, PreviewYear
+from compendium_v2.db.presentation_models import NREN, PreviewYear
from flask import Response, request
from sqlalchemy import select
diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py
index aaf1c74f..501d2206 100644
--- a/compendium_v2/routes/ec_projects.py
+++ b/compendium_v2/routes/ec_projects.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ECProject
+from compendium_v2.db.presentation_models import ECProject
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 2ca91eb3..0c110190 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import FundingSource
+from compendium_v2.db.presentation_models import FundingSource
from typing import Any
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 7153a8fc..8af1bbb6 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -4,7 +4,7 @@ from flask import Blueprint, jsonify
from sqlalchemy import select
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import NREN, InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py
index 473994b9..b39f7fd8 100644
--- a/compendium_v2/routes/organization.py
+++ b/compendium_v2/routes/organization.py
@@ -3,7 +3,7 @@ from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import ParentOrganization, SubOrganization
+from compendium_v2.db.presentation_models import ParentOrganization, SubOrganization
from compendium_v2.routes import common
diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py
index 68c6add0..a97dcf22 100644
--- a/compendium_v2/routes/policy.py
+++ b/compendium_v2/routes/policy.py
@@ -2,7 +2,7 @@ import logging
from typing import Any
from flask import Blueprint, jsonify
-from compendium_v2.db.model import Policy
+from compendium_v2.db.presentation_models import Policy
from compendium_v2.routes import common
routes = Blueprint('policy', __name__)
diff --git a/compendium_v2/routes/response.py b/compendium_v2/routes/response.py
index 9ba078e5..1c691431 100644
--- a/compendium_v2/routes/response.py
+++ b/compendium_v2/routes/response.py
@@ -9,8 +9,8 @@ from sqlalchemy import select
from sqlalchemy.orm import lazyload
from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required, User
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 3c049385..3a23cd65 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -2,7 +2,7 @@ import logging
from flask import Blueprint, jsonify
-from compendium_v2.db.model import NrenStaff
+from compendium_v2.db.presentation_models import NrenStaff
from compendium_v2.routes import common
from typing import Any
diff --git a/compendium_v2/routes/survey.py b/compendium_v2/routes/survey.py
index 5d685e7f..35766d19 100644
--- a/compendium_v2/routes/survey.py
+++ b/compendium_v2/routes/survey.py
@@ -7,9 +7,9 @@ from sqlalchemy import delete, select
from sqlalchemy.orm import joinedload, load_only
from compendium_v2.db import db
-from compendium_v2.db.model import NREN, PreviewYear
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
-from compendium_v2.publishers.survey_publisher_v2 import publish
+from compendium_v2.db.presentation_models import NREN, PreviewYear
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
+from compendium_v2.publishers.survey_publisher import publish
from compendium_v2.routes import common
from compendium_v2.auth.session_management import admin_required
diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py
index 70255d0f..56668fc4 100644
--- a/compendium_v2/routes/traffic.py
+++ b/compendium_v2/routes/traffic.py
@@ -3,7 +3,7 @@ import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
-from compendium_v2.db.model import TrafficVolume
+from compendium_v2.db.presentation_models import TrafficVolume
from typing import Any
diff --git a/compendium_v2/routes/user.py b/compendium_v2/routes/user.py
index f526e33a..ab9cce2e 100644
--- a/compendium_v2/routes/user.py
+++ b/compendium_v2/routes/user.py
@@ -8,7 +8,7 @@ from sqlalchemy import select
from compendium_v2.auth.session_management import admin_required
from compendium_v2.db import db
from compendium_v2.db.auth_model import User, ROLES
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
from compendium_v2.routes import common
routes = Blueprint('user', __name__)
diff --git a/test/test_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py
similarity index 88%
rename from test/test_survey_publisher_2022.py
rename to test/test_db_survey_publisher_2022.py
index e8eaa8b4..6e9c9096 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_db_survey_publisher_2022.py
@@ -1,6 +1,6 @@
from sqlalchemy import select
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
@@ -209,21 +209,21 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
with app_with_survey_db.app_context():
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
budgets = db.session.scalars(
- select(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc())
+ select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc())
).all()
assert len(budgets) == 3
assert budgets[0].nren.name.lower() == 'nren1'
assert budgets[0].budget == 100
funding_sources = db.session.scalars(
- select(model.FundingSource).order_by(model.FundingSource.nren_id.asc())
+ select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc())
).all()
assert len(funding_sources) == 3
assert funding_sources[0].nren.name.lower() == 'nren1'
@@ -242,7 +242,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert funding_sources[2].other == 30
staff_data = db.session.scalars(
- select(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc())
+ select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc())
).all()
assert len(staff_data) == 3
@@ -265,7 +265,8 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert staff_data[2].subcontracted_fte == 0
_org_data = db.session.scalars(
- select(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc())
+ select(presentation_models.ParentOrganization).order_by(
+ presentation_models.ParentOrganization.nren_id.asc())
).all()
assert len(_org_data) == 2
@@ -276,18 +277,18 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _org_data[1].organization == 'Org3'
charging_structures = db.session.scalars(
- select(model.ChargingStructure).order_by(model.ChargingStructure.nren_id.asc())
+ select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc())
).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == model.FeeType.no_charge
+ assert charging_structures[0].fee_type == presentation_models.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+ assert charging_structures[1].fee_type == presentation_models.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == model.FeeType.other
+ assert charging_structures[2].fee_type == presentation_models.FeeType.other
_ec_data = db.session.scalars(
- select(model.ECProject).order_by(model.ECProject.nren_id.asc())
+ select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc())
).all()
assert len(_ec_data) == 3
@@ -301,7 +302,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert _ec_data[2].project == 'project3'
policy_data = db.session.scalars(
- select(model.Policy).order_by(model.Policy.nren_id.asc())
+ select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc())
).all()
policy_data_2020 = [p for p in policy_data if p.year == 2020]
policy_data_2022 = [p for p in policy_data if p.year == 2022]
@@ -311,7 +312,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com'
_institution_urls_data = db.session.scalars(
- select(model.InstitutionURLs).order_by(model.InstitutionURLs.nren_id.asc())
+ select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc())
).all()
assert len(_institution_urls_data) == 2
assert _institution_urls_data[0].nren.name.lower() == 'rash'
diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher.py
similarity index 65%
rename from test/test_survey_publisher_v2.py
rename to test/test_survey_publisher.py
index 68b4b2d0..91f3d70d 100644
--- a/test/test_survey_publisher_v2.py
+++ b/test/test_survey_publisher.py
@@ -5,8 +5,8 @@ import os
from sqlalchemy import func, select
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v2 import map_2023
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher import _map_2023
JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
@@ -16,15 +16,15 @@ def test_v2_publisher_empty(app):
data = {}
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count == 0
# the main thing is actually that it doesnt crash
@@ -34,37 +34,39 @@ def test_v2_publisher_full(app):
data = json.load(json_data)
with app.app_context():
- nren = model.NREN(name='name', country='country')
+ nren = presentation_models.NREN(name='name', country='country')
db.session.commit()
with app.app_context():
- map_2023(nren, {"data": data})
+ _map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
- budget = db.session.scalar(select(model.BudgetEntry.budget))
+ budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
assert budget == Decimal("124.76")
- funding_source = db.session.scalar(select(model.FundingSource))
+ funding_source = db.session.scalar(select(presentation_models.FundingSource))
assert funding_source.client_institutions == Decimal("0")
assert funding_source.european_funding == Decimal("20")
assert funding_source.gov_public_bodies == Decimal("70")
assert funding_source.commercial == Decimal("0")
assert funding_source.other == Decimal("10")
- charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
- assert charging_structure == model.FeeType.usage_based_fee
+ charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
+ assert charging_structure == presentation_models.FeeType.usage_based_fee
- staff = db.session.scalar(select(model.NrenStaff))
+ staff = db.session.scalar(select(presentation_models.NrenStaff))
assert staff.permanent_fte == Decimal("5.6")
assert staff.subcontracted_fte == Decimal("56")
assert staff.technical_fte == Decimal("2")
assert staff.non_technical_fte == Decimal("1")
- parent = db.session.scalar(select(model.ParentOrganization.organization))
+ parent = db.session.scalar(select(presentation_models.ParentOrganization.organization))
assert parent == "sdtfgd"
- subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization))
+ subs = db.session.scalars(select(presentation_models.SubOrganization).order_by(
+ presentation_models.SubOrganization.organization)
+ )
subs = [s for s in subs]
assert subs[0].organization == "Aaerer"
assert subs[0].role == "Treer"
@@ -73,13 +75,15 @@ def test_v2_publisher_full(app):
assert subs[2].organization == "werser"
assert subs[2].role == ""
- projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project))
+ projects = db.session.scalars(select(presentation_models.ECProject.project).order_by(
+ presentation_models.ECProject.project)
+ )
projects = [p for p in projects]
assert projects[0] == "dgdg"
assert projects[1] == "rrrrr"
assert projects[2] == "st"
- policy = db.session.scalar(select(model.Policy))
+ policy = db.session.scalar(select(presentation_models.Policy))
assert policy.strategic_plan == "https://serere.com"
assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf"
assert policy.equal_opportunity == ""
@@ -89,11 +93,11 @@ def test_v2_publisher_full(app):
assert policy.data_protection == ""
assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore" # noqa: E501
- traffic = db.session.scalar(select(model.TrafficVolume))
+ traffic = db.session.scalar(select(presentation_models.TrafficVolume))
assert traffic.to_customers == Decimal("3")
assert traffic.from_customers == Decimal("34")
assert traffic.to_external == Decimal("22")
assert traffic.from_external == Decimal("3")
- client_urls = db.session.scalar(select(model.InstitutionURLs))
+ client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_legacy_excel.py
similarity index 80%
rename from test/test_survey_publisher_v1.py
rename to test/test_survey_publisher_legacy_excel.py
index d3bd42b6..5a20fc06 100644
--- a/test/test_survey_publisher_v1.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -3,30 +3,32 @@ import os
from sqlalchemy import select, func
from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v1 import _cli
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
def test_publisher(app_with_survey_db, mocker, dummy_config):
- mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
+ mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
with app_with_survey_db.app_context():
nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
- db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+ db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
db.session.commit()
_cli(dummy_config, app_with_survey_db)
with app_with_survey_db.app_context():
- budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+ budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count
- funding_source_count = db.session.scalar(select(func.count(model.FundingSource.year)))
+ funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year)))
assert funding_source_count
- charging_structure_count = db.session.scalar(select(func.count(model.ChargingStructure.year)))
+ charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year)))
assert charging_structure_count
- staff_data = db.session.scalars(select(model.NrenStaff).order_by(model.NrenStaff.year.asc())).all()
+ staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by(
+ presentation_models.NrenStaff.year.asc())
+ ).all()
# data should only be saved for the NRENs we have saved in the database
staff_data_nrens = set([staff.nren.name for staff in staff_data])
@@ -72,7 +74,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert kifu_data[5].technical_fte == 133
assert kifu_data[5].non_technical_fte == 45
- ecproject_data = db.session.scalars(select(model.ECProject)).all()
+ ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all()
# test a couple of random entries
surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017]
assert len(surf2017) == 1
@@ -86,7 +88,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
assert len(kifu2019) == 4
assert kifu2019[3].project == 'SuperHeroes for Science'
- parent_data = db.session.scalars(select(model.ParentOrganization)).all()
+ parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all()
# test a random entry
asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021]
assert len(asnet2021) == 1
--
GitLab