Skip to content
Snippets Groups Projects
Commit 13ce81ec authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

Refactor publisher

parent 1aebf8a0
No related branches found
No related tags found
No related merge requests found
......@@ -18,7 +18,6 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial
logger = logging.getLogger(__name__)
str128 = Annotated[str, 128]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
......@@ -48,13 +47,16 @@ RemoteCampus = TypedDict(
# See https://github.com/pallets-eco/flask-sqlalchemy/issues/1140
# mypy: disable-error-code="name-defined"
class PresentationModel(db.Model):
__abstract__ = True
class PreviewYear(db.Model):
class PreviewYear(PresentationModel):
__tablename__ = 'preview_year'
year: Mapped[int_pk]
class NREN(db.Model):
class NREN(PresentationModel):
__tablename__ = 'nren'
id: Mapped[int_pk]
name: Mapped[str128]
......@@ -64,7 +66,7 @@ class NREN(db.Model):
return f'<NREN {self.id} | {self.name}>'
class BudgetEntry(db.Model):
class BudgetEntry(PresentationModel):
__tablename__ = 'budgets'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -72,7 +74,7 @@ class BudgetEntry(db.Model):
budget: Mapped[Decimal]
class FundingSource(db.Model):
class FundingSource(PresentationModel):
__tablename__ = 'funding_source'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -84,7 +86,7 @@ class FundingSource(db.Model):
other: Mapped[Decimal]
class ChargingStructure(db.Model):
class ChargingStructure(PresentationModel):
__tablename__ = 'charging_structure'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -92,7 +94,7 @@ class ChargingStructure(db.Model):
fee_type: Mapped[Optional[FeeType]]
class NrenStaff(db.Model):
class NrenStaff(PresentationModel):
__tablename__ = 'nren_staff'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -103,7 +105,7 @@ class NrenStaff(db.Model):
non_technical_fte: Mapped[Decimal]
class ParentOrganization(db.Model):
class ParentOrganization(PresentationModel):
__tablename__ = 'parent_organization'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -111,7 +113,7 @@ class ParentOrganization(db.Model):
organization: Mapped[str128]
class SubOrganization(db.Model):
class SubOrganization(PresentationModel):
__tablename__ = 'sub_organization'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -120,7 +122,7 @@ class SubOrganization(db.Model):
role: Mapped[str128_pk]
class ECProject(db.Model):
class ECProject(PresentationModel):
__tablename__ = 'ec_project'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -128,7 +130,7 @@ class ECProject(db.Model):
project: Mapped[str256_pk]
class Policy(db.Model):
class Policy(PresentationModel):
__tablename__ = 'policy'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -143,7 +145,7 @@ class Policy(db.Model):
gender_equality: Mapped[str]
class TrafficVolume(db.Model):
class TrafficVolume(PresentationModel):
__tablename__ = 'traffic_volume'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -154,7 +156,7 @@ class TrafficVolume(db.Model):
from_external: Mapped[Decimal]
class InstitutionURLs(db.Model):
class InstitutionURLs(PresentationModel):
__tablename__ = 'institution_urls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -162,7 +164,7 @@ class InstitutionURLs(db.Model):
urls: Mapped[json_str_list]
class CentralProcurement(db.Model):
class CentralProcurement(PresentationModel):
__tablename__ = 'central_procurement'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -171,7 +173,7 @@ class CentralProcurement(db.Model):
amount: Mapped[Optional[Decimal]]
class ServiceManagement(db.Model):
class ServiceManagement(PresentationModel):
__tablename__ = 'service_management'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -180,7 +182,7 @@ class ServiceManagement(db.Model):
service_level_targets: Mapped[Optional[bool]]
class ServiceUserTypes(db.Model):
class ServiceUserTypes(PresentationModel):
__tablename__ = 'service_user_types'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -189,7 +191,7 @@ class ServiceUserTypes(db.Model):
service_category: Mapped[ServiceCategory] = mapped_column(primary_key=True)
class EOSCListings(db.Model):
class EOSCListings(PresentationModel):
__tablename__ = 'eosc_listings'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -197,7 +199,7 @@ class EOSCListings(db.Model):
service_names: Mapped[json_str_list]
class Standards(db.Model):
class Standards(PresentationModel):
__tablename__ = 'standards'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -209,7 +211,7 @@ class Standards(db.Model):
crisis_management_procedure: Mapped[Optional[bool]]
class CrisisExercises(db.Model):
class CrisisExercises(PresentationModel):
__tablename__ = 'crisis_exercises'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -217,7 +219,7 @@ class CrisisExercises(db.Model):
exercise_descriptions: Mapped[json_str_list]
class SecurityControls(db.Model):
class SecurityControls(PresentationModel):
__tablename__ = 'security_controls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -225,7 +227,7 @@ class SecurityControls(db.Model):
security_control_descriptions: Mapped[json_str_list]
class ConnectedProportion(db.Model):
class ConnectedProportion(PresentationModel):
__tablename__ = 'connected_proportion'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -237,7 +239,7 @@ class ConnectedProportion(db.Model):
users_served: Mapped[Optional[int]]
class ConnectivityLevel(db.Model):
class ConnectivityLevel(PresentationModel):
__tablename__ = 'connectivity_level'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -248,7 +250,7 @@ class ConnectivityLevel(db.Model):
highest_speed_proportion: Mapped[Optional[Decimal]]
class ConnectionCarrier(db.Model):
class ConnectionCarrier(PresentationModel):
__tablename__ = 'connection_carrier'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -257,7 +259,7 @@ class ConnectionCarrier(db.Model):
carry_mechanism: Mapped[CarryMechanism]
class ConnectivityLoad(db.Model):
class ConnectivityLoad(PresentationModel):
__tablename__ = 'connectivity_load'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -269,7 +271,7 @@ class ConnectivityLoad(db.Model):
peak_load_to_institutions: Mapped[Optional[int]]
class ConnectivityGrowth(db.Model):
class ConnectivityGrowth(PresentationModel):
__tablename__ = 'connectivity_growth'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -278,7 +280,7 @@ class ConnectivityGrowth(db.Model):
growth: Mapped[Decimal]
class CommercialConnectivity(db.Model):
class CommercialConnectivity(PresentationModel):
__tablename__ = 'commercial_connectivity'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -290,7 +292,7 @@ class CommercialConnectivity(db.Model):
university_spin_off: Mapped[Optional[CommercialConnectivityCoverage]]
class CommercialChargingLevel(db.Model):
class CommercialChargingLevel(PresentationModel):
__tablename__ = 'commercial_charging_level'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -300,7 +302,7 @@ class CommercialChargingLevel(db.Model):
direct_peering: Mapped[Optional[CommercialCharges]]
class RemoteCampuses(db.Model):
class RemoteCampuses(PresentationModel):
__tablename__ = 'remote_campuses'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -309,7 +311,7 @@ class RemoteCampuses(db.Model):
connections: Mapped[List[RemoteCampus]] = mapped_column(JSON)
class DarkFibreLease(db.Model):
class DarkFibreLease(PresentationModel):
__tablename__ = 'dark_fibre_lease'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -320,7 +322,7 @@ class DarkFibreLease(db.Model):
iru_duration: Mapped[Optional[Decimal]]
class DarkFibreInstalled(db.Model):
class DarkFibreInstalled(PresentationModel):
__tablename__ = 'dark_fibre_installed'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -329,7 +331,7 @@ class DarkFibreInstalled(db.Model):
fibre_length_in_country: Mapped[Optional[int]]
class FibreLight(db.Model):
class FibreLight(PresentationModel):
__tablename__ = 'fibre_light'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -337,7 +339,7 @@ class FibreLight(db.Model):
light_description: Mapped[str]
class NetworkMapUrls(db.Model):
class NetworkMapUrls(PresentationModel):
__tablename__ = 'network_map_urls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -345,7 +347,7 @@ class NetworkMapUrls(db.Model):
urls: Mapped[json_str_list]
class MonitoringTools(db.Model):
class MonitoringTools(PresentationModel):
__tablename__ = 'monitoring_tools'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -354,7 +356,7 @@ class MonitoringTools(db.Model):
netflow_processing_description: Mapped[str]
class PassiveMonitoring(db.Model):
class PassiveMonitoring(PresentationModel):
__tablename__ = 'passive_monitoring'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -363,7 +365,7 @@ class PassiveMonitoring(db.Model):
method: Mapped[Optional[MonitoringMethod]]
class TrafficStatistics(db.Model):
class TrafficStatistics(PresentationModel):
__tablename__ = 'traffic_statistics'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -372,7 +374,7 @@ class TrafficStatistics(db.Model):
urls: Mapped[json_str_list]
class SiemVendors(db.Model):
class SiemVendors(PresentationModel):
__tablename__ = 'siem_vendors'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -380,7 +382,7 @@ class SiemVendors(db.Model):
vendor_names: Mapped[json_str_list]
class CertificateProviders(db.Model):
class CertificateProviders(PresentationModel):
__tablename__ = 'certificate_providers'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -388,7 +390,7 @@ class CertificateProviders(db.Model):
provider_names: Mapped[json_str_list]
class WeatherMap(db.Model):
class WeatherMap(PresentationModel):
__tablename__ = 'weather_map'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -397,7 +399,7 @@ class WeatherMap(db.Model):
url: Mapped[str]
class PertTeam(db.Model):
class PertTeam(PresentationModel):
__tablename__ = 'pert_team'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -405,7 +407,7 @@ class PertTeam(db.Model):
pert_team: Mapped[YesNoPlanned]
class AlienWave(db.Model):
class AlienWave(PresentationModel):
__tablename__ = 'alien_wave'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -415,7 +417,7 @@ class AlienWave(db.Model):
alien_wave_internal: Mapped[Optional[bool]]
class Capacity(db.Model):
class Capacity(PresentationModel):
__tablename__ = 'capacity'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -424,7 +426,7 @@ class Capacity(db.Model):
typical_backbone_capacity: Mapped[Optional[Decimal]]
class ExternalConnections(db.Model):
class ExternalConnections(PresentationModel):
__tablename__ = 'external_connections'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -432,7 +434,7 @@ class ExternalConnections(db.Model):
connections: Mapped[List[ExternalConnection]] = mapped_column(JSON)
class NonREPeers(db.Model):
class NonREPeers(PresentationModel):
__tablename__ = 'non_r_and_e_peers'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -440,7 +442,7 @@ class NonREPeers(db.Model):
nr_of_non_r_and_e_peers: Mapped[int]
class TrafficRatio(db.Model):
class TrafficRatio(PresentationModel):
__tablename__ = 'traffic_ratio'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -449,7 +451,7 @@ class TrafficRatio(db.Model):
commodity_percentage: Mapped[Decimal]
class OpsAutomation(db.Model):
class OpsAutomation(PresentationModel):
__tablename__ = 'ops_automation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -458,7 +460,7 @@ class OpsAutomation(db.Model):
ops_automation_specifics: Mapped[str]
class NetworkFunctionVirtualisation(db.Model):
class NetworkFunctionVirtualisation(PresentationModel):
__tablename__ = 'network_function_virtualisation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -467,7 +469,7 @@ class NetworkFunctionVirtualisation(db.Model):
nfv_specifics: Mapped[json_str_list]
class NetworkAutomation(db.Model):
class NetworkAutomation(PresentationModel):
__tablename__ = 'network_automation'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......@@ -476,7 +478,7 @@ class NetworkAutomation(db.Model):
network_automation_specifics: Mapped[json_str_list]
class Service(db.Model):
class Service(PresentationModel):
__tablename__ = 'service'
name_key: Mapped[str128_pk]
name: Mapped[str128]
......@@ -484,7 +486,7 @@ class Service(db.Model):
description: Mapped[str]
class NRENService(db.Model):
class NRENService(PresentationModel):
__tablename__ = 'nren_service'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
......
This diff is collapsed.
from decimal import Decimal
def int_or_none(answers_dict, key):
if key in answers_dict:
value = answers_dict[key]
if isinstance(value, str):
value = value.replace(",", ".")
return int(answers_dict[key])
return None
def decimal_or_none(answers_dict, key):
if key in answers_dict:
value = answers_dict[key]
if isinstance(value, str):
value = value.replace(",", ".")
return Decimal(value)
return None
def decimal_or_zero(answers_dict, key):
value = answers_dict.get(key, 0)
if isinstance(value, str):
value = value.replace(",", ".")
return Decimal(value)
def bool_or_none(answer, key=None):
if key:
answer = answer.get(key)
if answer:
return answer == "Yes"
return None
This diff is collapsed.
......@@ -6,7 +6,8 @@ from sqlalchemy import func, select
from compendium_v2 import db
from compendium_v2.db import presentation_model_enums as model_enums, presentation_models
from compendium_v2.publishers.survey_publisher import _map_2023
from compendium_v2.db.survey_models import Survey, SurveyResponse, ResponseStatus, SurveyStatus
from compendium_v2.publishers.survey_publisher import publish
JSON_FILE = os.path.join(os.path.dirname(__file__), "data", "2023_all_questions_answered.json")
......@@ -17,13 +18,13 @@ def test_v2_publisher_empty(app):
with app.app_context():
nren = presentation_models.NREN(name='name', country='country')
db.session.add(nren)
survey = Survey(year=2023, survey={}, status=SurveyStatus.open)
response = SurveyResponse(survey=survey, nren=nren, answers={"data": data}, status=ResponseStatus.completed)
db.session.add(survey)
db.session.add(response)
db.session.commit()
with app.app_context():
_map_2023(nren, {"data": data})
db.session.commit()
with app.app_context():
budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year)))
assert budget_count == 0
# the main thing is actually that it doesnt crash
......@@ -35,13 +36,15 @@ def test_v2_publisher_full(app):
with app.app_context():
nren = presentation_models.NREN(name='name', country='country')
db.session.add(nren)
survey = Survey(year=2023, survey={}, status=SurveyStatus.open)
response = SurveyResponse(survey=survey, nren=nren, answers={"data": data}, status=ResponseStatus.completed)
db.session.add(survey)
db.session.add(response)
db.session.commit()
with app.app_context():
_map_2023(nren, {"data": data})
db.session.commit()
publish(2023)
with app.app_context():
budget = db.session.scalar(select(presentation_models.BudgetEntry.budget))
assert budget == Decimal("124.76")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment