From 0494ec7f868b10f485afdfdbe7a3872433cbf8f3 Mon Sep 17 00:00:00 2001
From: Mohammad Torkashvand <mohammad.torkashvand@geant.org>
Date: Fri, 15 Sep 2023 08:59:16 +0000
Subject: [PATCH] refactor publisher and make Excel files consistent

---
 MANIFEST.in                                   |   3 +-
 compendium_v2/background_task/__init__.py     |   0
 compendium_v2/conversion/conversion.py        |   8 +--
 compendium_v2/db/auth_model.py                |   2 +-
 .../db/{model.py => presentation_models.py}   |   0
 .../db/{survey_model.py => survey_models.py}  |   2 +-
 compendium_v2/migrations/dump_survey_model.py |   2 +-
 .../excel_parser.py}                          |   6 +-
 compendium_v2/publishers/helpers.py           |   4 +-
 ...ey_publisher_v2.py => survey_publisher.py} |  19 ++++--
 ...v1.py => survey_publisher_legacy_excel.py} |  43 +++++++------
 ...022.py => survey_publisher_old_db_2022.py} |  43 +++++++------
 .../2021_Organisation_DataSeries.xlsx         | Bin
 .../2022_Connected_Users_DataSeries.xlsx      | Bin
 .../2022_Networks_DataSeries.xlsx             | Bin
 ...NREN-Services-prefills_2023_Recovered.xlsx | Bin
 compendium_v2/routes/budget.py                |   2 +-
 compendium_v2/routes/charging.py              |   2 +-
 compendium_v2/routes/common.py                |   2 +-
 compendium_v2/routes/ec_projects.py           |   2 +-
 compendium_v2/routes/funding.py               |   2 +-
 compendium_v2/routes/institutions_urls.py     |   2 +-
 compendium_v2/routes/organization.py          |   2 +-
 compendium_v2/routes/policy.py                |   2 +-
 compendium_v2/routes/response.py              |   4 +-
 compendium_v2/routes/staff.py                 |   2 +-
 compendium_v2/routes/survey.py                |   6 +-
 compendium_v2/routes/traffic.py               |   2 +-
 compendium_v2/routes/user.py                  |   2 +-
 setup.py                                      |   4 +-
 test/conftest.py                              |  58 +++++++++---------
 test/test_budget.py                           |   2 +-
 test/test_conversion.py                       |   4 +-
 ...22.py => test_db_survey_publisher_2022.py} |  40 ++++++------
 test/test_dump_survey_model.py                |   2 +-
 test/test_response.py                         |   2 +-
 test/test_survey.py                           |   2 +-
 ...blisher_v2.py => test_survey_publisher.py} |  40 ++++++------
 ... => test_survey_publisher_legacy_excel.py} |  22 ++++---
 39 files changed, 180 insertions(+), 160 deletions(-)
 delete mode 100644 compendium_v2/background_task/__init__.py
 rename compendium_v2/db/{model.py => presentation_models.py} (100%)
 rename compendium_v2/db/{survey_model.py => survey_models.py} (97%)
 rename compendium_v2/{background_task/parse_excel_data.py => publishers/excel_parser.py} (98%)
 rename compendium_v2/publishers/{survey_publisher_v2.py => survey_publisher.py} (88%)
 rename compendium_v2/publishers/{survey_publisher_v1.py => survey_publisher_legacy_excel.py} (83%)
 rename compendium_v2/publishers/{survey_publisher_2022.py => survey_publisher_old_db_2022.py} (93%)
 rename compendium_v2/{background_task/xlsx => resources}/2021_Organisation_DataSeries.xlsx (100%)
 rename compendium_v2/{background_task/xlsx => resources}/2022_Connected_Users_DataSeries.xlsx (100%)
 rename compendium_v2/{background_task/xlsx => resources}/2022_Networks_DataSeries.xlsx (100%)
 rename compendium_v2/{conversion => resources}/NREN-Services-prefills_2023_Recovered.xlsx (100%)
 rename test/{test_survey_publisher_2022.py => test_db_survey_publisher_2022.py} (82%)
 rename test/{test_survey_publisher_v2.py => test_survey_publisher.py} (65%)
 rename test/{test_survey_publisher_v1.py => test_survey_publisher_legacy_excel.py} (80%)

diff --git a/MANIFEST.in b/MANIFEST.in
index edff315a..58874ea5 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,6 +4,5 @@ include compendium_v2/templates/survey-index.html
 include compendium_v2/migrations/alembic.ini
 recursive-include compendium_v2/migrations/versions *
 recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/xlsx *
-include compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
+recursive-include compendium_v2/background_task/resources *
 recursive-exclude test *
diff --git a/compendium_v2/background_task/__init__.py b/compendium_v2/background_task/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/compendium_v2/conversion/conversion.py b/compendium_v2/conversion/conversion.py
index a963b446..6da4572a 100644
--- a/compendium_v2/conversion/conversion.py
+++ b/compendium_v2/conversion/conversion.py
@@ -2,7 +2,7 @@
 conversion
 =========================
 
-This module loads the survey data from 2022 from the survey database
+This module loads the survey data from 2022 from the old survey database
 and stores the data in the json structure of the new survey, so that
 it can be used to prefill the 2023 survey.
 It also loads an xlsx file with the data for the services questions.
@@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging
 from compendium_v2.db import db
 from compendium_v2.config import load
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus
 
 from compendium_v2.conversion import mapping
 
@@ -30,7 +30,7 @@ setup_logging()
 
 logger = logging.getLogger('conversion')
 
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx")
 
 
 def query_nren(nren_id: int):
diff --git a/compendium_v2/db/auth_model.py b/compendium_v2/db/auth_model.py
index 50be7e3e..8605ab39 100644
--- a/compendium_v2/db/auth_model.py
+++ b/compendium_v2/db/auth_model.py
@@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey
 from flask_login import UserMixin  # type: ignore
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
 
 logger = logging.getLogger(__name__)
 
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/presentation_models.py
similarity index 100%
rename from compendium_v2/db/model.py
rename to compendium_v2/db/presentation_models.py
diff --git a/compendium_v2/db/survey_model.py b/compendium_v2/db/survey_models.py
similarity index 97%
rename from compendium_v2/db/survey_model.py
rename to compendium_v2/db/survey_models.py
index 866df323..fd688d38 100644
--- a/compendium_v2/db/survey_model.py
+++ b/compendium_v2/db/survey_models.py
@@ -15,7 +15,7 @@ from sqlalchemy.types import JSON
 
 from compendium_v2.db import db
 from compendium_v2.db.auth_model import User
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
 
 
 logger = logging.getLogger(__name__)
diff --git a/compendium_v2/migrations/dump_survey_model.py b/compendium_v2/migrations/dump_survey_model.py
index 314fda6d..3869a4a6 100644
--- a/compendium_v2/migrations/dump_survey_model.py
+++ b/compendium_v2/migrations/dump_survey_model.py
@@ -24,7 +24,7 @@ import compendium_v2
 from compendium_v2.environment import setup_logging
 from compendium_v2.db import db
 from compendium_v2.config import load
-from compendium_v2.db.survey_model import Survey
+from compendium_v2.db.survey_models import Survey
 
 setup_logging()
 
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/publishers/excel_parser.py
similarity index 98%
rename from compendium_v2/background_task/parse_excel_data.py
rename to compendium_v2/publishers/excel_parser.py
index aba3ef6f..594031b9 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/publishers/excel_parser.py
@@ -2,15 +2,15 @@ import logging
 import openpyxl
 import os
 
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
 from compendium_v2.environment import setup_logging
 
 setup_logging()
 
 logger = logging.getLogger(__name__)
 
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
 
 
 def fetch_budget_excel_data():
diff --git a/compendium_v2/publishers/helpers.py b/compendium_v2/publishers/helpers.py
index f6492d2d..aeb4ecd9 100644
--- a/compendium_v2/publishers/helpers.py
+++ b/compendium_v2/publishers/helpers.py
@@ -3,7 +3,7 @@ from typing import List
 
 from sqlalchemy import select
 
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 
 URL_PATTERN = re.compile(
     (r'\b(https?://[^\s<>";,(){}\[\]!\\]+'
@@ -17,7 +17,7 @@ def get_uppercase_nren_dict():
     """
     :return: a dictionary of all known NRENs db entities keyed on the uppercased name
     """
-    current_nrens = db.session.scalars(select(model.NREN))
+    current_nrens = db.session.scalars(select(presentation_models.NREN))
     nren_dict = {nren.name.upper(): nren for nren in current_nrens}
     # add aliases that are used in the source data:
     nren_dict['ASNET'] = nren_dict['ASNET-AM']
diff --git a/compendium_v2/publishers/survey_publisher_v2.py b/compendium_v2/publishers/survey_publisher.py
similarity index 88%
rename from compendium_v2/publishers/survey_publisher_v2.py
rename to compendium_v2/publishers/survey_publisher.py
index ebede055..3b88d5b9 100644
--- a/compendium_v2/publishers/survey_publisher_v2.py
+++ b/compendium_v2/publishers/survey_publisher.py
@@ -1,14 +1,25 @@
+"""
+This module handles adding responses from the latest 2023 survey into the 2023 presentation_models.
+The process is straightforward as the data is new and can be directly inserted into the presentation_models.
+
+For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions
+will need to be addressed.
+
+Usage:
+    Used in publish_survey API in compendium_v2/routes/survey.py
+"""
+
 from decimal import Decimal
 
 from sqlalchemy import delete, select
 
 from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
     InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
-from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
 
 
-def map_2023(nren, answers):
+def _map_2023(nren, answers):
     year = 2023
 
     for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
@@ -112,7 +123,7 @@ def publish(year):
     ).unique()
 
     question_mapping = {
-        2023: map_2023
+        2023: _map_2023
     }
 
     mapping_function = question_mapping[year]
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_legacy_excel.py
similarity index 83%
rename from compendium_v2/publishers/survey_publisher_v1.py
rename to compendium_v2/publishers/survey_publisher_legacy_excel.py
index a6e2376f..6503b79f 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_legacy_excel.py
@@ -2,7 +2,7 @@
 survey_publisher_v1
 =========================
 
-This module loads the survey data from before 2022 from an excel file.
+This module loads the survey data from before 2022 from a legacy Excel files.
 Missing info is filled in from the survey db for some questions.
 Registered as click cli command when installing compendium-v2.
 
@@ -16,15 +16,14 @@ from sqlalchemy import select
 
 import compendium_v2
 from compendium_v2.environment import setup_logging
-from compendium_v2.background_task import parse_excel_data
 from compendium_v2.config import load
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.publishers import helpers
+from compendium_v2.publishers import helpers, excel_parser
 
 setup_logging()
 
-logger = logging.getLogger('survey-publisher-v1')
+logger = logging.getLogger('survey-publisher-legacy-excel')
 
 
 def db_budget_migration(nren_dict):
@@ -46,7 +45,7 @@ def db_budget_migration(nren_dict):
                 logger.warning(f'{abbrev} unknown. Skipping.')
                 continue
 
-            budget_entry = model.BudgetEntry(
+            budget_entry = presentation_models.BudgetEntry(
                 nren=nren_dict[abbrev],
                 nren_id=nren_dict[abbrev].id,
                 budget=float(budget.budget),
@@ -55,7 +54,7 @@ def db_budget_migration(nren_dict):
             db.session.merge(budget_entry)
 
     # Import the data from excel sheet to database
-    exceldata = parse_excel_data.fetch_budget_excel_data()
+    exceldata = excel_parser.fetch_budget_excel_data()
 
     for abbrev, budget, year in exceldata:
         if abbrev not in nren_dict:
@@ -65,7 +64,7 @@ def db_budget_migration(nren_dict):
         if budget > 200:
             logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
 
-        budget_entry = model.BudgetEntry(
+        budget_entry = presentation_models.BudgetEntry(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             budget=budget,
@@ -77,7 +76,7 @@ def db_budget_migration(nren_dict):
 
 def db_funding_migration(nren_dict):
     # Import the data to database
-    data = parse_excel_data.fetch_funding_excel_data()
+    data = excel_parser.fetch_funding_excel_data()
 
     for (abbrev, year, client_institution,
             european_funding,
@@ -93,7 +92,7 @@ def db_funding_migration(nren_dict):
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
-        budget_entry = model.FundingSource(
+        budget_entry = presentation_models.FundingSource(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             year=year,
@@ -108,14 +107,14 @@ def db_funding_migration(nren_dict):
 
 def db_charging_structure_migration(nren_dict):
     # Import the data to database
-    data = parse_excel_data.fetch_charging_structure_excel_data()
+    data = excel_parser.fetch_charging_structure_excel_data()
 
     for (abbrev, year, charging_structure) in data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
-        charging_structure_entry = model.ChargingStructure(
+        charging_structure_entry = presentation_models.ChargingStructure(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             year=year,
@@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict):
 
 
 def db_staffing_migration(nren_dict):
-    staff_data = parse_excel_data.fetch_staffing_excel_data()
+    staff_data = excel_parser.fetch_staffing_excel_data()
 
     nren_staff_map = {}
     for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
@@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict):
             continue
 
         nren = nren_dict[abbrev]
-        nren_staff_map[(nren.id, year)] = model.NrenStaff(
+        nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
             nren=nren,
             nren_id=nren.id,
             year=year,
@@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict):
             non_technical_fte=0
         )
 
-    function_data = parse_excel_data.fetch_staff_function_excel_data()
+    function_data = excel_parser.fetch_staff_function_excel_data()
     for (abbrev, year, technical_fte, non_technical_fte) in function_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping staff function data.')
@@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict):
             nren_staff_map[(nren.id, year)].technical_fte = technical_fte
             nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
         else:
-            nren_staff_map[(nren.id, year)] = model.NrenStaff(
+            nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
                 nren=nren,
                 nren_id=nren.id,
                 year=year,
@@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict):
 
 
 def db_ecprojects_migration(nren_dict):
-    ecproject_data = parse_excel_data.fetch_ecproject_excel_data()
+    ecproject_data = excel_parser.fetch_ecproject_excel_data()
     for (abbrev, year, project) in ecproject_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
+        ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
         db.session.merge(ecproject_entry)
     db.session.commit()
 
 
 def db_organizations_migration(nren_dict):
-    organization_data = parse_excel_data.fetch_organization_excel_data()
+    organization_data = excel_parser.fetch_organization_excel_data()
     for (abbrev, year, org) in organization_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
+        org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
         db.session.merge(org_entry)
     db.session.commit()
 
 
 def db_traffic_volume_migration(nren_dict):
-    traffic_data = parse_excel_data.fetch_traffic_excel_data()
+    traffic_data = excel_parser.fetch_traffic_excel_data()
     for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        traffic_entry = model.TrafficVolume(
+        traffic_entry = presentation_models.TrafficVolume(
             nren=nren,
             nren_id=nren.id,
             year=year,
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_old_db_2022.py
similarity index 93%
rename from compendium_v2/publishers/survey_publisher_2022.py
rename to compendium_v2/publishers/survey_publisher_old_db_2022.py
index 39038d1e..ca704690 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_old_db_2022.py
@@ -2,7 +2,7 @@
 survey_publisher_2022
 =========================
 
-This module loads the survey data from 2022 from the survey database.
+This module loads the survey data from 2022 from the old survey database into presentation_models.
 Registered as click cli command when installing compendium-v2.
 
 """
@@ -17,17 +17,17 @@ from sqlalchemy import delete, text
 from collections import defaultdict
 
 import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
 from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
 from compendium_v2.publishers.helpers import extract_urls
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 from compendium_v2.publishers import helpers
 
 setup_logging()
 
-logger = logging.getLogger('survey-publisher-2022')
+logger = logging.getLogger('survey-publisher-old-db-2022')
 
 BUDGET_QUERY = """
 SELECT DISTINCT ON (n.id, a.question_id)
@@ -189,7 +189,7 @@ def transfer_budget(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        budget_entry = model.BudgetEntry(
+        budget_entry = presentation_models.BudgetEntry(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             budget=budget,
@@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict):
             logger.info(f'{nren_name} has no urls for {year}. Skipping.')
             continue
 
-        institution_urls = model.InstitutionURLs(
+        institution_urls = presentation_models.InstitutionURLs(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             urls=urls,
@@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        funding_source = model.FundingSource(
+        funding_source = presentation_models.FundingSource(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict):
     for nren_name, nren_info in data.items():
         if sum([nren_info[question] for question in StaffQuestion]) == 0:
             logger.info(f'{nren_name} has no staff data. Deleting if exists.')
-            db.session.execute(delete(model.NrenStaff).where(
-                model.NrenStaff.nren_id == nren_dict[nren_name].id,
-                model.NrenStaff.year == 2022
+            db.session.execute(delete(presentation_models.NrenStaff).where(
+                presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id,
+                presentation_models.NrenStaff.year == 2022
             ))
             continue
 
@@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict):
             logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
                         f' ({employed} != {technical})')
 
-        staff_data = model.NrenStaff(
+        staff_data = presentation_models.NrenStaff(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict):
         'We are affiliated to '
     ]
 
-    db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022))
+    db.session.execute(delete(presentation_models.ParentOrganization).where(
+        presentation_models.ParentOrganization.year == 2022)
+    )
 
     rows = query_question(OrgQuestion.PARENT_ORG_NAME)
     for row in rows:
@@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        parent_org = model.ParentOrganization(
+        parent_org = presentation_models.ParentOrganization(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict):
         (OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
         (OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
     ]
-
     lookup = defaultdict(list)
 
-    db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022))
+    db.session.execute(delete(presentation_models.SubOrganization).where(
+        presentation_models.SubOrganization.year == 2022)
+    )
 
     for name, choice, role in suborg_questions:
         _name_rows = query_question(name)
@@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict):
 
     for nren_name, suborgs in lookup.items():
         for suborg_name, role in suborgs:
-            suborg = model.SubOrganization(
+            suborg = presentation_models.SubOrganization(
                 nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
@@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict):
         else:
             charging_structure = None
 
-        charging_structure = model.ChargingStructure(
+        charging_structure = presentation_models.ChargingStructure(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict):
 def transfer_ec_projects(nren_dict):
     # delete all existing EC projects, in case something changed
     db.session.execute(
-        delete(model.ECProject).where(model.ECProject.year == 2022)
+        delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022)
     )
 
     rows = query_question(ECQuestion.EC_PROJECT)
@@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict):
             # some answers include contract numbers, which we don't want here
             val = val.split('(contract n')[0]
 
-            ec_project = model.ECProject(
+            ec_project = presentation_models.ECProject(
                 nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
@@ -536,7 +539,7 @@ def transfer_policies(nren_dict):
                 data[(nren_name, year)][question_key] = value
 
     for (nren_name, year), nren_info in data.items():
-        policy_data = model.Policy(
+        policy_data = presentation_models.Policy(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=year,
diff --git a/compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx b/compendium_v2/resources/2021_Organisation_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx
rename to compendium_v2/resources/2021_Organisation_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx b/compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Connected_Users_DataSeries.xlsx
rename to compendium_v2/resources/2022_Connected_Users_DataSeries.xlsx
diff --git a/compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx b/compendium_v2/resources/2022_Networks_DataSeries.xlsx
similarity index 100%
rename from compendium_v2/background_task/xlsx/2022_Networks_DataSeries.xlsx
rename to compendium_v2/resources/2022_Networks_DataSeries.xlsx
diff --git a/compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx b/compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
similarity index 100%
rename from compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
rename to compendium_v2/resources/NREN-Services-prefills_2023_Recovered.xlsx
diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index 2f11be47..b0cff93d 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import BudgetEntry
+from compendium_v2.db.presentation_models import BudgetEntry
 from compendium_v2.routes import common
 
 
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index d61df556..8c40679b 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import ChargingStructure
+from compendium_v2.db.presentation_models import ChargingStructure
 from compendium_v2.routes import common
 
 
diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py
index cc7b9fd1..be84fbfd 100644
--- a/compendium_v2/routes/common.py
+++ b/compendium_v2/routes/common.py
@@ -4,7 +4,7 @@ Utilities used by multiple route blueprints.
 import functools
 import logging
 from compendium_v2 import db
-from compendium_v2.db.model import NREN, PreviewYear
+from compendium_v2.db.presentation_models import NREN, PreviewYear
 
 from flask import Response, request
 from sqlalchemy import select
diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py
index aaf1c74f..501d2206 100644
--- a/compendium_v2/routes/ec_projects.py
+++ b/compendium_v2/routes/ec_projects.py
@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import ECProject
+from compendium_v2.db.presentation_models import ECProject
 from compendium_v2.routes import common
 
 
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 2ca91eb3..0c110190 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -3,7 +3,7 @@ import logging
 from flask import Blueprint, jsonify
 
 from compendium_v2.routes import common
-from compendium_v2.db.model import FundingSource
+from compendium_v2.db.presentation_models import FundingSource
 from typing import Any
 
 
diff --git a/compendium_v2/routes/institutions_urls.py b/compendium_v2/routes/institutions_urls.py
index 7153a8fc..8af1bbb6 100644
--- a/compendium_v2/routes/institutions_urls.py
+++ b/compendium_v2/routes/institutions_urls.py
@@ -4,7 +4,7 @@ from flask import Blueprint, jsonify
 from sqlalchemy import select
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN, InstitutionURLs
+from compendium_v2.db.presentation_models import NREN, InstitutionURLs
 from compendium_v2.routes import common
 
 routes = Blueprint('institutions-urls', __name__)
diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py
index 473994b9..b39f7fd8 100644
--- a/compendium_v2/routes/organization.py
+++ b/compendium_v2/routes/organization.py
@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import ParentOrganization, SubOrganization
+from compendium_v2.db.presentation_models import ParentOrganization, SubOrganization
 from compendium_v2.routes import common
 
 
diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py
index 68c6add0..a97dcf22 100644
--- a/compendium_v2/routes/policy.py
+++ b/compendium_v2/routes/policy.py
@@ -2,7 +2,7 @@ import logging
 from typing import Any
 
 from flask import Blueprint, jsonify
-from compendium_v2.db.model import Policy
+from compendium_v2.db.presentation_models import Policy
 from compendium_v2.routes import common
 
 routes = Blueprint('policy', __name__)
diff --git a/compendium_v2/routes/response.py b/compendium_v2/routes/response.py
index 9ba078e5..1c691431 100644
--- a/compendium_v2/routes/response.py
+++ b/compendium_v2/routes/response.py
@@ -9,8 +9,8 @@ from sqlalchemy import select
 from sqlalchemy.orm import lazyload
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, ResponseStatus, RESPONSE_NOT_STARTED
 from compendium_v2.routes import common
 from compendium_v2.auth.session_management import admin_required, User
 
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 3c049385..3a23cd65 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -2,7 +2,7 @@ import logging
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import NrenStaff
+from compendium_v2.db.presentation_models import NrenStaff
 from compendium_v2.routes import common
 from typing import Any
 
diff --git a/compendium_v2/routes/survey.py b/compendium_v2/routes/survey.py
index 5d685e7f..35766d19 100644
--- a/compendium_v2/routes/survey.py
+++ b/compendium_v2/routes/survey.py
@@ -7,9 +7,9 @@ from sqlalchemy import delete, select
 from sqlalchemy.orm import joinedload, load_only
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN, PreviewYear
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
-from compendium_v2.publishers.survey_publisher_v2 import publish
+from compendium_v2.db.presentation_models import NREN, PreviewYear
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus, RESPONSE_NOT_STARTED
+from compendium_v2.publishers.survey_publisher import publish
 from compendium_v2.routes import common
 from compendium_v2.auth.session_management import admin_required
 
diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py
index 70255d0f..56668fc4 100644
--- a/compendium_v2/routes/traffic.py
+++ b/compendium_v2/routes/traffic.py
@@ -3,7 +3,7 @@ import logging
 from flask import Blueprint, jsonify
 
 from compendium_v2.routes import common
-from compendium_v2.db.model import TrafficVolume
+from compendium_v2.db.presentation_models import TrafficVolume
 from typing import Any
 
 
diff --git a/compendium_v2/routes/user.py b/compendium_v2/routes/user.py
index f526e33a..ab9cce2e 100644
--- a/compendium_v2/routes/user.py
+++ b/compendium_v2/routes/user.py
@@ -8,7 +8,7 @@ from sqlalchemy import select
 from compendium_v2.auth.session_management import admin_required
 from compendium_v2.db import db
 from compendium_v2.db.auth_model import User, ROLES
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
 from compendium_v2.routes import common
 
 routes = Blueprint('user', __name__)
diff --git a/setup.py b/setup.py
index e4b9e4e3..f8258a33 100644
--- a/setup.py
+++ b/setup.py
@@ -29,8 +29,8 @@ setup(
     include_package_data=True,
     entry_points={
         'console_scripts': [
-            'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli',  # noqa
-            'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli',  # noqa
+            'excel-survey-publisher=compendium_v2.publishers.survey_publisher_legacy_excel:cli',  # noqa
+            'db-publisher-2022=compendium_v2.publishers.survey_publisher_old_db_2022:cli',  # noqa
             'conversion=compendium_v2.conversion.conversion:cli',  # noqa
             'dump_survey_model=compendium_v2.migrations.dump_survey_model:cli',  # noqa
         ]
diff --git a/test/conftest.py b/test/conftest.py
index 2d8b1abb..591fe482 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -6,7 +6,7 @@ import random
 from sqlalchemy import select
 from flask_login import LoginManager  # type: ignore
 import compendium_v2
-from compendium_v2.db import db, model, survey_model
+from compendium_v2.db import db, presentation_models, survey_models
 from compendium_v2.survey_db import model as survey_db_model
 from compendium_v2.auth.session_management import setup_login_manager, User, ROLES
 
@@ -30,7 +30,7 @@ def mocked_admin_user(app, test_survey_data, mocker):
     with app.app_context():
         user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub', roles=ROLES.admin)
 
-        nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2'))
+        nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2'))
         user.nrens.append(nren2)
         db.session.add(user)
         db.session.commit()
@@ -46,7 +46,7 @@ def mocked_user(app, test_survey_data, mocker):
     with app.app_context():
         user = User(email='testemail123@email.local', fullname='testfullname', oidc_sub='fakesub')
 
-        nren2 = db.session.scalar(select(model.NREN).filter(model.NREN.name == 'nren2'))
+        nren2 = db.session.scalar(select(presentation_models.NREN).filter(presentation_models.NREN.name == 'nren2'))
         user.nrens.append(nren2)
         db.session.add(user)
         db.session.commit()
@@ -75,7 +75,7 @@ def test_budget_data(app):
     with app.app_context():
         data = [row for row in _test_data_csv("BudgetTestData.csv")]
         nren_names = set([row["nren"] for row in data])
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for row in data:
@@ -83,7 +83,7 @@ def test_budget_data(app):
             budget = row["budget"]
             year = row["year"]
 
-            db.session.add(model.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
+            db.session.add(presentation_models.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
         db.session.commit()
 
 
@@ -92,7 +92,7 @@ def test_funding_source_data(app):
     with app.app_context():
         data = [row for row in _test_data_csv("FundingSourceTestData.csv")]
         nren_names = set([row["nren"] for row in data])
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for row in data:
@@ -105,7 +105,7 @@ def test_funding_source_data(app):
             other = row["other"]
 
             db.session.add(
-                model.FundingSource(
+                presentation_models.FundingSource(
                     nren=nren, year=year,
                     client_institutions=client,
                     european_funding=european,
@@ -137,7 +137,7 @@ def test_staff_data(app):
         data = list(_generate_rows())
 
         nren_names = set(d['nren'] for d in data)
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for row in data:
@@ -149,7 +149,7 @@ def test_staff_data(app):
             non_technical_fte = row["non_technical_fte"]
 
             db.session.add(
-                model.NrenStaff(
+                presentation_models.NrenStaff(
                     nren=nren,
                     year=year,
                     permanent_fte=permanent_fte,
@@ -165,23 +165,23 @@ def test_staff_data(app):
 def test_survey_data(app):
     with app.app_context():
         nren_names = ['nren1', 'nren2', 'nren3', 'nren4']
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
-        survey2021 = survey_model.Survey(year=2021, survey={}, status=survey_model.SurveyStatus.published)
-        survey2022 = survey_model.Survey(year=2022, survey={}, status=survey_model.SurveyStatus.published)
-        survey2023 = survey_model.Survey(
+        survey2021 = survey_models.Survey(year=2021, survey={}, status=survey_models.SurveyStatus.published)
+        survey2022 = survey_models.Survey(year=2022, survey={}, status=survey_models.SurveyStatus.published)
+        survey2023 = survey_models.Survey(
             year=2023,
             survey={'part1': [{'title': 'ha', 'visibleIf': 'false'}]},
-            status=survey_model.SurveyStatus.open
+            status=survey_models.SurveyStatus.open
         )
         db.session.add_all([survey2021, survey2022, survey2023])
 
-        db.session.add(survey_model.SurveyResponse(
+        db.session.add(survey_models.SurveyResponse(
             nren=nren_dict['nren1'],
             survey=survey2023,
             answers={},
-            status=survey_model.ResponseStatus.completed
+            status=survey_models.ResponseStatus.completed
         ))
 
         db.session.commit()
@@ -219,7 +219,7 @@ def test_charging_structure_data(app):
     with app.app_context():
         data = [row for row in _test_data_csv("ChargingStructureTestData.csv")]
         nren_names = set([row["nren"] for row in data])
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for row in data:
@@ -230,7 +230,7 @@ def test_charging_structure_data(app):
                 fee_type = None
 
             db.session.add(
-                model.ChargingStructure(
+                presentation_models.ChargingStructure(
                     nren=nren, year=year,
                     fee_type=fee_type)
             )
@@ -264,7 +264,7 @@ def test_organization_data(app):
         sub_org_data = list(_generate_sub_org_data())
 
         nren_names = set(d['nren'] for d in [*org_data, *sub_org_data])
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for org in org_data:
@@ -272,7 +272,7 @@ def test_organization_data(app):
             year = org["year"]
             name = org["name"]
 
-            db.session.add(model.ParentOrganization(nren=nren, year=year, organization=name))
+            db.session.add(presentation_models.ParentOrganization(nren=nren, year=year, organization=name))
 
         for sub_org in sub_org_data:
             nren = nren_dict[sub_org["nren"]]
@@ -280,7 +280,7 @@ def test_organization_data(app):
             name = sub_org["name"]
             role = sub_org["role"]
 
-            db.session.add(model.SubOrganization(nren=nren, year=year, organization=name, role=role))
+            db.session.add(presentation_models.SubOrganization(nren=nren, year=year, organization=name, role=role))
 
         db.session.commit()
 
@@ -307,7 +307,7 @@ def test_ec_project_data(app):
         ec_project_data = list(_generate_ec_project_data())
 
         nren_names = set(d['nren'] for d in ec_project_data)
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for ec_project in ec_project_data:
@@ -315,7 +315,7 @@ def test_ec_project_data(app):
             year = ec_project["year"]
             project = ec_project["project"]
 
-            db.session.add(model.ECProject(nren=nren, year=year, project=project))
+            db.session.add(presentation_models.ECProject(nren=nren, year=year, project=project))
 
         db.session.commit()
 
@@ -326,13 +326,13 @@ def test_policy_data(app):
         nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
         nren_names = set(ny[0] for ny in nrens_and_years)
 
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for (nren_name, year) in nrens_and_years:
             nren = nren_dict[nren_name]
 
-            db.session.add(model.Policy(
+            db.session.add(presentation_models.Policy(
                 nren=nren,
                 year=year,
                 strategic_plan='a strategy',
@@ -353,14 +353,14 @@ def test_traffic_data(app):
     with app.app_context():
         nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
         nren_names = set(ny[0] for ny in nrens_and_years)
-        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        nren_dict = {nren_name: presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names}
         db.session.add_all(nren_dict.values())
 
         for (nren_name, year) in nrens_and_years:
             nren = nren_dict[nren_name]
 
             db.session.add(
-                model.TrafficVolume(
+                presentation_models.TrafficVolume(
                     nren=nren,
                     year=year,
                     from_customers=2.23,
@@ -377,7 +377,7 @@ def test_institution_urls_data(app):
     def _create_and_save_nrens(nren_names):
         nrens = {}
         for nren_name in nren_names:
-            nren_instance = model.NREN(name=nren_name, country='country')
+            nren_instance = presentation_models.NREN(name=nren_name, country='country')
             nrens[nren_name] = nren_instance
             db.session.add(nren_instance)
         return nrens
@@ -387,7 +387,7 @@ def test_institution_urls_data(app):
             nren_instance = nrens[nren_name]
             urls = ['https://example.com', 'http://example.org']
 
-            institution_urls_model = model.InstitutionURLs(
+            institution_urls_model = presentation_models.InstitutionURLs(
                 nren=nren_instance,
                 year=year,
                 urls=urls
diff --git a/test/test_budget.py b/test/test_budget.py
index 77ea3452..de0206bc 100644
--- a/test/test_budget.py
+++ b/test/test_budget.py
@@ -1,7 +1,7 @@
 import json
 import jsonschema
 from compendium_v2 import db
-from compendium_v2.db.model import PreviewYear
+from compendium_v2.db.presentation_models import PreviewYear
 from compendium_v2.routes.budget import BUDGET_RESPONSE_SCHEMA
 
 
diff --git a/test/test_conversion.py b/test/test_conversion.py
index 0f480505..d0e2282f 100644
--- a/test/test_conversion.py
+++ b/test/test_conversion.py
@@ -2,8 +2,8 @@ from sqlalchemy import select
 from sqlalchemy.orm import lazyload
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, SurveyStatus
 from compendium_v2.conversion.conversion import _cli, convert_answers, load_service_data
 
 
diff --git a/test/test_survey_publisher_2022.py b/test/test_db_survey_publisher_2022.py
similarity index 82%
rename from test/test_survey_publisher_2022.py
rename to test/test_db_survey_publisher_2022.py
index e8eaa8b4..ab4154de 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_db_survey_publisher_2022.py
@@ -1,7 +1,7 @@
 from sqlalchemy import select
 
-from compendium_v2.db import db, model
-from compendium_v2.publishers.survey_publisher_2022 import _cli, FundingSource, \
+from compendium_v2.db import db, presentation_models
+from compendium_v2.publishers.survey_publisher_old_db_2022 import _cli, FundingSource, \
     StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion
 
 
@@ -200,30 +200,31 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
             (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"),
         ]
 
-    mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget', get_rows_as_tuples)
-    mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_funding_sources', funding_source_data)
-    mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question', question_data)
-    mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question_id', question_id_data)
-    mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_institutions_urls', institutions_urls_data)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_budget', get_rows_as_tuples)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_funding_sources', funding_source_data)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question', question_data)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question_id', question_id_data)
+    mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_institutions_urls',
+                 institutions_urls_data)
 
     nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM',
                   'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
     with app_with_survey_db.app_context():
-        db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+        db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
         db.session.commit()
 
     _cli(dummy_config, app_with_survey_db)
 
     with app_with_survey_db.app_context():
         budgets = db.session.scalars(
-            select(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc())
+            select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc())
         ).all()
         assert len(budgets) == 3
         assert budgets[0].nren.name.lower() == 'nren1'
         assert budgets[0].budget == 100
 
         funding_sources = db.session.scalars(
-            select(model.FundingSource).order_by(model.FundingSource.nren_id.asc())
+            select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc())
         ).all()
         assert len(funding_sources) == 3
         assert funding_sources[0].nren.name.lower() == 'nren1'
@@ -242,7 +243,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert funding_sources[2].other == 30
 
         staff_data = db.session.scalars(
-            select(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc())
+            select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc())
         ).all()
 
         assert len(staff_data) == 3
@@ -265,7 +266,8 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert staff_data[2].subcontracted_fte == 0
 
         _org_data = db.session.scalars(
-            select(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc())
+            select(presentation_models.ParentOrganization).order_by(
+                presentation_models.ParentOrganization.nren_id.asc())
         ).all()
 
         assert len(_org_data) == 2
@@ -276,18 +278,18 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert _org_data[1].organization == 'Org3'
 
         charging_structures = db.session.scalars(
-            select(model.ChargingStructure).order_by(model.ChargingStructure.nren_id.asc())
+            select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc())
         ).all()
         assert len(charging_structures) == 3
         assert charging_structures[0].nren.name.lower() == 'nren1'
-        assert charging_structures[0].fee_type == model.FeeType.no_charge
+        assert charging_structures[0].fee_type == presentation_models.FeeType.no_charge
         assert charging_structures[1].nren.name.lower() == 'nren2'
-        assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
+        assert charging_structures[1].fee_type == presentation_models.FeeType.usage_based_fee
         assert charging_structures[2].nren.name.lower() == 'nren3'
-        assert charging_structures[2].fee_type == model.FeeType.other
+        assert charging_structures[2].fee_type == presentation_models.FeeType.other
 
         _ec_data = db.session.scalars(
-            select(model.ECProject).order_by(model.ECProject.nren_id.asc())
+            select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc())
         ).all()
 
         assert len(_ec_data) == 3
@@ -301,7 +303,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert _ec_data[2].project == 'project3'
 
         policy_data = db.session.scalars(
-            select(model.Policy).order_by(model.Policy.nren_id.asc())
+            select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc())
         ).all()
         policy_data_2020 = [p for p in policy_data if p.year == 2020]
         policy_data_2022 = [p for p in policy_data if p.year == 2022]
@@ -311,7 +313,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com'
 
         _institution_urls_data = db.session.scalars(
-            select(model.InstitutionURLs).order_by(model.InstitutionURLs.nren_id.asc())
+            select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc())
         ).all()
         assert len(_institution_urls_data) == 2
         assert _institution_urls_data[0].nren.name.lower() == 'rash'
diff --git a/test/test_dump_survey_model.py b/test/test_dump_survey_model.py
index 722f21dc..6c49c6fb 100644
--- a/test/test_dump_survey_model.py
+++ b/test/test_dump_survey_model.py
@@ -1,5 +1,5 @@
 from compendium_v2.db import db
-from compendium_v2.db.survey_model import Survey, SurveyStatus
+from compendium_v2.db.survey_models import Survey, SurveyStatus
 from compendium_v2.migrations.dump_survey_model import _cli
 
 
diff --git a/test/test_response.py b/test/test_response.py
index 8d870bb2..917ec051 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -1,7 +1,7 @@
 import json
 import jsonschema
 from compendium_v2.db.auth_model import User
-from compendium_v2.db.survey_model import ResponseStatus
+from compendium_v2.db.survey_models import ResponseStatus
 from compendium_v2.routes.response import SURVEY_RESPONSE_SCHEMA, VerificationStatus, SURVEY_LOCK_SCHEMA, \
     SURVEY_STATUS_RESPONSE_SCHEMA
 
diff --git a/test/test_survey.py b/test/test_survey.py
index dabf95cb..09144127 100644
--- a/test/test_survey.py
+++ b/test/test_survey.py
@@ -1,7 +1,7 @@
 import json
 import jsonschema
 from compendium_v2.db import db
-from compendium_v2.db.survey_model import Survey, SurveyStatus
+from compendium_v2.db.survey_models import Survey, SurveyStatus
 from compendium_v2.routes.survey import LIST_SURVEYS_RESPONSE_SCHEMA
 
 
diff --git a/test/test_survey_publisher_v2.py b/test/test_survey_publisher.py
similarity index 65%
rename from test/test_survey_publisher_v2.py
rename to test/test_survey_publisher.py
index 68b4b2d0..91f3d70d 100644
--- a/test/test_survey_publisher_v2.py
+++ b/test/test_survey_publisher.py
@@ -5,8 +5,8 @@ import os
 from sqlalchemy import func, select
 
 from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v2 import map_2023
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher import _map_2023
 
 
 JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
@@ -16,15 +16,15 @@ def test_v2_publisher_empty(app):
     data = {}
 
     with app.app_context():
-        nren = model.NREN(name='name', country='country')
+        nren = presentation_models.NREN(name='name', country='country')
         db.session.commit()
 
     with app.app_context():
-        map_2023(nren, {"data": data})
+        _map_2023(nren, {"data": data})
         db.session.commit()
 
     with app.app_context():
-        budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+        budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
         assert budget_count == 0
         # the main thing is actually that it doesnt crash
 
@@ -34,37 +34,39 @@ def test_v2_publisher_full(app):
         data = json.load(json_data)
 
     with app.app_context():
-        nren = model.NREN(name='name', country='country')
+        nren = presentation_models.NREN(name='name', country='country')
         db.session.commit()
 
     with app.app_context():
-        map_2023(nren, {"data": data})
+        _map_2023(nren, {"data": data})
         db.session.commit()
 
     with app.app_context():
-        budget = db.session.scalar(select(model.BudgetEntry.budget))
+        budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
         assert budget == Decimal("124.76")
 
-        funding_source = db.session.scalar(select(model.FundingSource))
+        funding_source = db.session.scalar(select(presentation_models.FundingSource))
         assert funding_source.client_institutions == Decimal("0")
         assert funding_source.european_funding == Decimal("20")
         assert funding_source.gov_public_bodies == Decimal("70")
         assert funding_source.commercial == Decimal("0")
         assert funding_source.other == Decimal("10")
 
-        charging_structure = db.session.scalar(select(model.ChargingStructure.fee_type))
-        assert charging_structure == model.FeeType.usage_based_fee
+        charging_structure = db.session.scalar(select(presentation_models.ChargingStructure.fee_type))
+        assert charging_structure == presentation_models.FeeType.usage_based_fee
 
-        staff = db.session.scalar(select(model.NrenStaff))
+        staff = db.session.scalar(select(presentation_models.NrenStaff))
         assert staff.permanent_fte == Decimal("5.6")
         assert staff.subcontracted_fte == Decimal("56")
         assert staff.technical_fte == Decimal("2")
         assert staff.non_technical_fte == Decimal("1")
 
-        parent = db.session.scalar(select(model.ParentOrganization.organization))
+        parent = db.session.scalar(select(presentation_models.ParentOrganization.organization))
         assert parent == "sdtfgd"
 
-        subs = db.session.scalars(select(model.SubOrganization).order_by(model.SubOrganization.organization))
+        subs = db.session.scalars(select(presentation_models.SubOrganization).order_by(
+            presentation_models.SubOrganization.organization)
+        )
         subs = [s for s in subs]
         assert subs[0].organization == "Aaerer"
         assert subs[0].role == "Treer"
@@ -73,13 +75,15 @@ def test_v2_publisher_full(app):
         assert subs[2].organization == "werser"
         assert subs[2].role == ""
 
-        projects = db.session.scalars(select(model.ECProject.project).order_by(model.ECProject.project))
+        projects = db.session.scalars(select(presentation_models.ECProject.project).order_by(
+            presentation_models.ECProject.project)
+        )
         projects = [p for p in projects]
         assert projects[0] == "dgdg"
         assert projects[1] == "rrrrr"
         assert projects[2] == "st"
 
-        policy = db.session.scalar(select(model.Policy))
+        policy = db.session.scalar(select(presentation_models.Policy))
         assert policy.strategic_plan == "https://serere.com"
         assert policy.environmental == "http://mren.ac.me/documents/Environmental%20policy%20for%20MREN.pdf"
         assert policy.equal_opportunity == ""
@@ -89,11 +93,11 @@ def test_v2_publisher_full(app):
         assert policy.data_protection == ""
         assert policy.gender_equality == "https://www.ucg.ac.me/objava/blog/616808/objava/148423-plan-rodne-ravnopravnosti-univerziteta-crne-gore"  # noqa: E501
 
-        traffic = db.session.scalar(select(model.TrafficVolume))
+        traffic = db.session.scalar(select(presentation_models.TrafficVolume))
         assert traffic.to_customers == Decimal("3")
         assert traffic.from_customers == Decimal("34")
         assert traffic.to_external == Decimal("22")
         assert traffic.from_external == Decimal("3")
 
-        client_urls = db.session.scalar(select(model.InstitutionURLs))
+        client_urls = db.session.scalar(select(presentation_models.InstitutionURLs))
         assert client_urls.urls == ["http://erse.com", "https://wwe.com"]
diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_legacy_excel.py
similarity index 80%
rename from test/test_survey_publisher_v1.py
rename to test/test_survey_publisher_legacy_excel.py
index d3bd42b6..5a20fc06 100644
--- a/test/test_survey_publisher_v1.py
+++ b/test/test_survey_publisher_legacy_excel.py
@@ -3,30 +3,32 @@ import os
 from sqlalchemy import select, func
 
 from compendium_v2 import db
-from compendium_v2.db import model
-from compendium_v2.publishers.survey_publisher_v1 import _cli
+from compendium_v2.db import presentation_models
+from compendium_v2.publishers.survey_publisher_legacy_excel import _cli
 
 EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
 
 
 def test_publisher(app_with_survey_db, mocker, dummy_config):
-    mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
+    mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE', EXCEL_FILE)
 
     with app_with_survey_db.app_context():
         nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'ANAS', 'GRNET', 'CSC']
-        db.session.add_all([model.NREN(name=nren_name, country='country') for nren_name in nren_names])
+        db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names])
         db.session.commit()
 
     _cli(dummy_config, app_with_survey_db)
 
     with app_with_survey_db.app_context():
-        budget_count = db.session.scalar(select(func.count(model.BudgetEntry.year)))
+        budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
         assert budget_count
-        funding_source_count = db.session.scalar(select(func.count(model.FundingSource.year)))
+        funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year)))
         assert funding_source_count
-        charging_structure_count = db.session.scalar(select(func.count(model.ChargingStructure.year)))
+        charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year)))
         assert charging_structure_count
-        staff_data = db.session.scalars(select(model.NrenStaff).order_by(model.NrenStaff.year.asc())).all()
+        staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by(
+            presentation_models.NrenStaff.year.asc())
+        ).all()
 
         # data should only be saved for the NRENs we have saved in the database
         staff_data_nrens = set([staff.nren.name for staff in staff_data])
@@ -72,7 +74,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert kifu_data[5].technical_fte == 133
         assert kifu_data[5].non_technical_fte == 45
 
-        ecproject_data = db.session.scalars(select(model.ECProject)).all()
+        ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all()
         # test a couple of random entries
         surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017]
         assert len(surf2017) == 1
@@ -86,7 +88,7 @@ def test_publisher(app_with_survey_db, mocker, dummy_config):
         assert len(kifu2019) == 4
         assert kifu2019[3].project == 'SuperHeroes for Science'
 
-        parent_data = db.session.scalars(select(model.ParentOrganization)).all()
+        parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all()
         # test a random entry
         asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021]
         assert len(asnet2021) == 1
-- 
GitLab