Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.36] - 2023-09-14
+- Fixed bug with sorting in user management page
+
 ## [0.35] - 2023-09-12
 - Added a publisher for the 2023 survey data
 - Added report pages for the traffic volume and institution urls

MANIFEST.in

@@ -4,6 +4,5 @@ include compendium_v2/templates/survey-index.html
 include compendium_v2/migrations/alembic.ini
 recursive-include compendium_v2/migrations/versions *
 recursive-include compendium_v2/migrations/surveymodels *
-recursive-include compendium_v2/background_task/xlsx *
-include compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
+recursive-include compendium_v2/background_task/resources *
 recursive-exclude test *

compendium_v2/conversion/conversion.py

@@ -2,7 +2,7 @@
 conversion
 =========================
 
-This module loads the survey data from 2022 from the survey database
+This module loads the survey data from 2022 from the old survey database
 and stores the data in the json structure of the new survey, so that
 it can be used to prefill the 2023 survey.
 It also loads an xlsx file with the data for the services questions.
@@ -21,8 +21,8 @@ from compendium_v2.environment import setup_logging
 from compendium_v2.db import db
 from compendium_v2.config import load
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
+from compendium_v2.db.presentation_models import NREN
+from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus
 
 from compendium_v2.conversion import mapping
 
@@ -30,7 +30,7 @@ setup_logging()
 
 logger = logging.getLogger('conversion')
 
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "NREN-Services-prefills_2023_Recovered.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources/NREN-Services-prefills_2023_Recovered.xlsx")
 
 
 def query_nren(nren_id: int):

compendium_v2/db/auth_model.py

@@ -15,7 +15,7 @@ from sqlalchemy.schema import ForeignKey
 from flask_login import UserMixin  # type: ignore
 
 from compendium_v2.db import db
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
 
 logger = logging.getLogger(__name__)
 

compendium_v2/db/survey_model.py → compendium_v2/db/survey_models.py

@@ -15,7 +15,7 @@ from sqlalchemy.types import JSON
 
 from compendium_v2.db import db
 from compendium_v2.db.auth_model import User
-from compendium_v2.db.model import NREN
+from compendium_v2.db.presentation_models import NREN
 
 
 logger = logging.getLogger(__name__)

compendium_v2/migrations/dump_survey_model.py

@@ -24,7 +24,7 @@ import compendium_v2
 from compendium_v2.environment import setup_logging
 from compendium_v2.db import db
 from compendium_v2.config import load
-from compendium_v2.db.survey_model import Survey
+from compendium_v2.db.survey_models import Survey
 
 setup_logging()
 

compendium_v2/background_task/parse_excel_data.py → compendium_v2/publishers/excel_parser.py

@@ -2,15 +2,15 @@ import logging
 import openpyxl
 import os
 
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
 from compendium_v2.environment import setup_logging
 
 setup_logging()
 
 logger = logging.getLogger(__name__)
 
-EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
-NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "../resources", "2022_Networks_DataSeries.xlsx")
 
 
 def fetch_budget_excel_data():

compendium_v2/publishers/helpers.py

@@ -3,7 +3,7 @@ from typing import List
 
 from sqlalchemy import select
 
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 
 URL_PATTERN = re.compile(
     (r'\b(https?://[^\s<>";,(){}\[\]!\\]+'
@@ -17,7 +17,7 @@ def get_uppercase_nren_dict():
     """
     :return: a dictionary of all known NRENs db entities keyed on the uppercased name
     """
-    current_nrens = db.session.scalars(select(model.NREN))
+    current_nrens = db.session.scalars(select(presentation_models.NREN))
     nren_dict = {nren.name.upper(): nren for nren in current_nrens}
     # add aliases that are used in the source data:
     nren_dict['ASNET'] = nren_dict['ASNET-AM']

compendium_v2/publishers/survey_publisher_v2.py → compendium_v2/publishers/survey_publisher.py

+"""
+This module handles adding responses from the latest 2023 survey into the 2023 presentation_models.
+The process is straightforward as the data is new and can be directly inserted into the presentation_models.
+
+For subsequent years like 2024, handling should be almost the same as 2023, only changes in questions
+will need to be addressed.
+
+Usage:
+    Used in publish_survey API in compendium_v2/routes/survey.py
+"""
+
 from decimal import Decimal
 
 from sqlalchemy import delete, select
 
 from compendium_v2.db import db
-from compendium_v2.db.model import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
+from compendium_v2.db.presentation_models import BudgetEntry, ChargingStructure, ECProject, FeeType, FundingSource, \
     InstitutionURLs, NrenStaff, ParentOrganization, Policy, SubOrganization, TrafficVolume
-from compendium_v2.db.survey_model import ResponseStatus, SurveyResponse
+from compendium_v2.db.survey_models import ResponseStatus, SurveyResponse
 
 
-def map_2023(nren, answers):
+def _map_2023(nren, answers):
     year = 2023
 
     for table_class in [BudgetEntry, ChargingStructure, ECProject, FundingSource, InstitutionURLs,
@@ -112,7 +123,7 @@ def publish(year):
     ).unique()
 
     question_mapping = {
-        2023: map_2023
+        2023: _map_2023
     }
 
     mapping_function = question_mapping[year]

compendium_v2/publishers/survey_publisher_v1.py → compendium_v2/publishers/survey_publisher_legacy_excel.py

@@ -2,7 +2,7 @@
 survey_publisher_v1
 =========================
 
-This module loads the survey data from before 2022 from an excel file.
+This module loads the survey data from before 2022 from a legacy Excel files.
 Missing info is filled in from the survey db for some questions.
 Registered as click cli command when installing compendium-v2.
 
@@ -16,15 +16,14 @@ from sqlalchemy import select
 
 import compendium_v2
 from compendium_v2.environment import setup_logging
-from compendium_v2.background_task import parse_excel_data
 from compendium_v2.config import load
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.publishers import helpers
+from compendium_v2.publishers import helpers, excel_parser
 
 setup_logging()
 
-logger = logging.getLogger('survey-publisher-v1')
+logger = logging.getLogger('survey-publisher-legacy-excel')
 
 
 def db_budget_migration(nren_dict):
@@ -46,7 +45,7 @@ def db_budget_migration(nren_dict):
                 logger.warning(f'{abbrev} unknown. Skipping.')
                 continue
 
-            budget_entry = model.BudgetEntry(
+            budget_entry = presentation_models.BudgetEntry(
                 nren=nren_dict[abbrev],
                 nren_id=nren_dict[abbrev].id,
                 budget=float(budget.budget),
@@ -55,7 +54,7 @@ def db_budget_migration(nren_dict):
             db.session.merge(budget_entry)
 
     # Import the data from excel sheet to database
-    exceldata = parse_excel_data.fetch_budget_excel_data()
+    exceldata = excel_parser.fetch_budget_excel_data()
 
     for abbrev, budget, year in exceldata:
         if abbrev not in nren_dict:
@@ -65,7 +64,7 @@ def db_budget_migration(nren_dict):
         if budget > 200:
             logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
 
-        budget_entry = model.BudgetEntry(
+        budget_entry = presentation_models.BudgetEntry(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             budget=budget,
@@ -77,7 +76,7 @@ def db_budget_migration(nren_dict):
 
 def db_funding_migration(nren_dict):
     # Import the data to database
-    data = parse_excel_data.fetch_funding_excel_data()
+    data = excel_parser.fetch_funding_excel_data()
 
     for (abbrev, year, client_institution,
             european_funding,
@@ -93,7 +92,7 @@ def db_funding_migration(nren_dict):
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
-        budget_entry = model.FundingSource(
+        budget_entry = presentation_models.FundingSource(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             year=year,
@@ -108,14 +107,14 @@ def db_funding_migration(nren_dict):
 
 def db_charging_structure_migration(nren_dict):
     # Import the data to database
-    data = parse_excel_data.fetch_charging_structure_excel_data()
+    data = excel_parser.fetch_charging_structure_excel_data()
 
     for (abbrev, year, charging_structure) in data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
-        charging_structure_entry = model.ChargingStructure(
+        charging_structure_entry = presentation_models.ChargingStructure(
             nren=nren_dict[abbrev],
             nren_id=nren_dict[abbrev].id,
             year=year,
@@ -126,7 +125,7 @@ def db_charging_structure_migration(nren_dict):
 
 
 def db_staffing_migration(nren_dict):
-    staff_data = parse_excel_data.fetch_staffing_excel_data()
+    staff_data = excel_parser.fetch_staffing_excel_data()
 
     nren_staff_map = {}
     for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
@@ -135,7 +134,7 @@ def db_staffing_migration(nren_dict):
             continue
 
         nren = nren_dict[abbrev]
-        nren_staff_map[(nren.id, year)] = model.NrenStaff(
+        nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
             nren=nren,
             nren_id=nren.id,
             year=year,
@@ -145,7 +144,7 @@ def db_staffing_migration(nren_dict):
             non_technical_fte=0
         )
 
-    function_data = parse_excel_data.fetch_staff_function_excel_data()
+    function_data = excel_parser.fetch_staff_function_excel_data()
     for (abbrev, year, technical_fte, non_technical_fte) in function_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping staff function data.')
@@ -156,7 +155,7 @@ def db_staffing_migration(nren_dict):
             nren_staff_map[(nren.id, year)].technical_fte = technical_fte
             nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
         else:
-            nren_staff_map[(nren.id, year)] = model.NrenStaff(
+            nren_staff_map[(nren.id, year)] = presentation_models.NrenStaff(
                 nren=nren,
                 nren_id=nren.id,
                 year=year,
@@ -179,40 +178,40 @@ def db_staffing_migration(nren_dict):
 
 
 def db_ecprojects_migration(nren_dict):
-    ecproject_data = parse_excel_data.fetch_ecproject_excel_data()
+    ecproject_data = excel_parser.fetch_ecproject_excel_data()
     for (abbrev, year, project) in ecproject_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
+        ecproject_entry = presentation_models.ECProject(nren=nren, nren_id=nren.id, year=year, project=project)
         db.session.merge(ecproject_entry)
     db.session.commit()
 
 
 def db_organizations_migration(nren_dict):
-    organization_data = parse_excel_data.fetch_organization_excel_data()
+    organization_data = excel_parser.fetch_organization_excel_data()
     for (abbrev, year, org) in organization_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
+        org_entry = presentation_models.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org)
         db.session.merge(org_entry)
     db.session.commit()
 
 
 def db_traffic_volume_migration(nren_dict):
-    traffic_data = parse_excel_data.fetch_traffic_excel_data()
+    traffic_data = excel_parser.fetch_traffic_excel_data()
     for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
 
         nren = nren_dict[abbrev]
-        traffic_entry = model.TrafficVolume(
+        traffic_entry = presentation_models.TrafficVolume(
             nren=nren,
             nren_id=nren.id,
             year=year,

compendium_v2/publishers/survey_publisher_2022.py → compendium_v2/publishers/survey_publisher_old_db_2022.py

@@ -2,7 +2,7 @@
 survey_publisher_2022
 =========================
 
-This module loads the survey data from 2022 from the survey database.
+This module loads the survey data from 2022 from the old survey database into presentation_models.
 Registered as click cli command when installing compendium-v2.
 
 """
@@ -17,17 +17,17 @@ from sqlalchemy import delete, text
 from collections import defaultdict
 
 import compendium_v2
-from compendium_v2.db.model import FeeType
+from compendium_v2.db.presentation_models import FeeType
 from compendium_v2.environment import setup_logging
 from compendium_v2.config import load
 from compendium_v2.publishers.helpers import extract_urls
 from compendium_v2.survey_db import model as survey_model
-from compendium_v2.db import db, model
+from compendium_v2.db import db, presentation_models
 from compendium_v2.publishers import helpers
 
 setup_logging()
 
-logger = logging.getLogger('survey-publisher-2022')
+logger = logging.getLogger('survey-publisher-old-db-2022')
 
 BUDGET_QUERY = """
 SELECT DISTINCT ON (n.id, a.question_id)
@@ -189,7 +189,7 @@ def transfer_budget(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        budget_entry = model.BudgetEntry(
+        budget_entry = presentation_models.BudgetEntry(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             budget=budget,
@@ -227,7 +227,7 @@ def transfer_institutions_urls(nren_dict):
             logger.info(f'{nren_name} has no urls for {year}. Skipping.')
             continue
 
-        institution_urls = model.InstitutionURLs(
+        institution_urls = presentation_models.InstitutionURLs(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             urls=urls,
@@ -267,7 +267,7 @@ def transfer_funding_sources(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        funding_source = model.FundingSource(
+        funding_source = presentation_models.FundingSource(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -303,9 +303,9 @@ def transfer_staff_data(nren_dict):
     for nren_name, nren_info in data.items():
         if sum([nren_info[question] for question in StaffQuestion]) == 0:
             logger.info(f'{nren_name} has no staff data. Deleting if exists.')
-            db.session.execute(delete(model.NrenStaff).where(
-                model.NrenStaff.nren_id == nren_dict[nren_name].id,
-                model.NrenStaff.year == 2022
+            db.session.execute(delete(presentation_models.NrenStaff).where(
+                presentation_models.NrenStaff.nren_id == nren_dict[nren_name].id,
+                presentation_models.NrenStaff.year == 2022
             ))
             continue
 
@@ -316,7 +316,7 @@ def transfer_staff_data(nren_dict):
             logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
                         f' ({employed} != {technical})')
 
-        staff_data = model.NrenStaff(
+        staff_data = presentation_models.NrenStaff(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -335,7 +335,9 @@ def transfer_nren_parent_org(nren_dict):
         'We are affiliated to '
     ]
 
-    db.session.execute(delete(model.ParentOrganization).where(model.ParentOrganization.year == 2022))
+    db.session.execute(delete(presentation_models.ParentOrganization).where(
+        presentation_models.ParentOrganization.year == 2022)
+    )
 
     rows = query_question(OrgQuestion.PARENT_ORG_NAME)
     for row in rows:
@@ -352,7 +354,7 @@ def transfer_nren_parent_org(nren_dict):
             logger.info(f'{nren_name} unknown. Skipping.')
             continue
 
-        parent_org = model.ParentOrganization(
+        parent_org = presentation_models.ParentOrganization(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -370,10 +372,11 @@ def transfer_nren_sub_org(nren_dict):
         (OrgQuestion.SUB_ORGS_4_NAME, OrgQuestion.SUB_ORGS_4_CHOICE, OrgQuestion.SUB_ORGS_4_ROLE),
         (OrgQuestion.SUB_ORGS_5_NAME, OrgQuestion.SUB_ORGS_5_CHOICE, OrgQuestion.SUB_ORGS_5_ROLE)
     ]
-
     lookup = defaultdict(list)
 
-    db.session.execute(delete(model.SubOrganization).where(model.SubOrganization.year == 2022))
+    db.session.execute(delete(presentation_models.SubOrganization).where(
+        presentation_models.SubOrganization.year == 2022)
+    )
 
     for name, choice, role in suborg_questions:
         _name_rows = query_question(name)
@@ -403,7 +406,7 @@ def transfer_nren_sub_org(nren_dict):
 
     for nren_name, suborgs in lookup.items():
         for suborg_name, role in suborgs:
-            suborg = model.SubOrganization(
+            suborg = presentation_models.SubOrganization(
                 nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
@@ -437,7 +440,7 @@ def transfer_charging_structure(nren_dict):
         else:
             charging_structure = None
 
-        charging_structure = model.ChargingStructure(
+        charging_structure = presentation_models.ChargingStructure(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=2022,
@@ -450,7 +453,7 @@ def transfer_charging_structure(nren_dict):
 def transfer_ec_projects(nren_dict):
     # delete all existing EC projects, in case something changed
     db.session.execute(
-        delete(model.ECProject).where(model.ECProject.year == 2022)
+        delete(presentation_models.ECProject).where(presentation_models.ECProject.year == 2022)
     )
 
     rows = query_question(ECQuestion.EC_PROJECT)
@@ -478,7 +481,7 @@ def transfer_ec_projects(nren_dict):
             # some answers include contract numbers, which we don't want here
             val = val.split('(contract n')[0]
 
-            ec_project = model.ECProject(
+            ec_project = presentation_models.ECProject(
                 nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
@@ -536,7 +539,7 @@ def transfer_policies(nren_dict):
                 data[(nren_name, year)][question_key] = value
 
     for (nren_name, year), nren_info in data.items():
-        policy_data = model.Policy(
+        policy_data = presentation_models.Policy(
             nren=nren_dict[nren_name],
             nren_id=nren_dict[nren_name].id,
             year=year,

compendium_v2/routes/budget.py

@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import BudgetEntry
+from compendium_v2.db.presentation_models import BudgetEntry
 from compendium_v2.routes import common
 
 

compendium_v2/routes/charging.py

@@ -3,7 +3,7 @@ from typing import Any
 
 from flask import Blueprint, jsonify
 
-from compendium_v2.db.model import ChargingStructure
+from compendium_v2.db.presentation_models import ChargingStructure
 from compendium_v2.routes import common
 
 

compendium_v2/routes/common.py

@@ -4,7 +4,7 @@ Utilities used by multiple route blueprints.
 import functools
 import logging
 from compendium_v2 import db
-from compendium_v2.db.model import NREN, PreviewYear
+from compendium_v2.db.presentation_models import NREN, PreviewYear
 
 from flask import Response, request
 from sqlalchemy import select