Skip to content
Snippets Groups Projects
Commit 2130df71 authored by Bjarke Madsen's avatar Bjarke Madsen
Browse files

Use minimum 256 length strings

parent 3d42d465
No related branches found
No related tags found
No related merge requests found
...@@ -18,8 +18,7 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial ...@@ -18,8 +18,7 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
str128 = Annotated[str, 128] str256 = Annotated[str, mapped_column(String(256))]
str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)] str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
int_pk = Annotated[int, mapped_column(primary_key=True)] int_pk = Annotated[int, mapped_column(primary_key=True)]
int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)] int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
...@@ -65,8 +64,8 @@ class NREN(PresentationModel): ...@@ -65,8 +64,8 @@ class NREN(PresentationModel):
""" """
__tablename__ = 'nren' __tablename__ = 'nren'
id: Mapped[int_pk] id: Mapped[int_pk]
name: Mapped[str128] name: Mapped[str256]
country: Mapped[str128] country: Mapped[str256]
def to_dict(self, download=False): def to_dict(self, download=False):
return { return {
...@@ -222,7 +221,7 @@ class ParentOrganization(PresentationModel): ...@@ -222,7 +221,7 @@ class ParentOrganization(PresentationModel):
nren_id: Mapped[int_pk_fkNREN] nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined') nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk] year: Mapped[int_pk]
organization: Mapped[str128] organization: Mapped[str256]
def compare_dict(self): def compare_dict(self):
return { return {
...@@ -252,8 +251,8 @@ class SubOrganization(PresentationModel): ...@@ -252,8 +251,8 @@ class SubOrganization(PresentationModel):
nren_id: Mapped[int_pk_fkNREN] nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined') nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk] year: Mapped[int_pk]
organization: Mapped[str128_pk] organization: Mapped[str256_pk]
role: Mapped[str128_pk] role: Mapped[str256_pk]
def compare_dict(self): def compare_dict(self):
return { return {
...@@ -1280,8 +1279,8 @@ class Service(PresentationModel): ...@@ -1280,8 +1279,8 @@ class Service(PresentationModel):
.. autoenum:: compendium_v2.db.presentation_model_enums.ServiceCategory .. autoenum:: compendium_v2.db.presentation_model_enums.ServiceCategory
""" """
__tablename__ = 'service' __tablename__ = 'service'
name_key: Mapped[str128_pk] name_key: Mapped[str256_pk]
name: Mapped[str128] name: Mapped[str256]
category: Mapped[ServiceCategory] category: Mapped[ServiceCategory]
description: Mapped[str] description: Mapped[str]
...@@ -1297,9 +1296,9 @@ class NRENService(PresentationModel): ...@@ -1297,9 +1296,9 @@ class NRENService(PresentationModel):
nren_id: Mapped[int_pk_fkNREN] nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined') nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk] year: Mapped[int_pk]
service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True) service_key: Mapped[str256] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
service: Mapped[Service] = relationship(lazy='joined') service: Mapped[Service] = relationship(lazy='joined')
product_name: Mapped[str128] product_name: Mapped[str256]
additional_information: Mapped[str] additional_information: Mapped[str]
official_description: Mapped[str] official_description: Mapped[str]
......
"""Use 256 strings
Revision ID: 4b531785f8d4
Revises: 8298b45f8e3d
Create Date: 2025-01-22 14:54:42.695483
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '4b531785f8d4'
down_revision = '8298b45f8e3d'
branch_labels = None
depends_on = None
def upgrade():
with op.batch_alter_table('nren', schema=None) as batch_op:
batch_op.alter_column(
'name',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
with op.batch_alter_table('nren_service', schema=None) as batch_op:
batch_op.alter_column(
'service_key',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
with op.batch_alter_table('parent_organization', schema=None) as batch_op:
batch_op.alter_column(
'organization',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
with op.batch_alter_table('service', schema=None) as batch_op:
batch_op.alter_column(
'name_key',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
with op.batch_alter_table('sub_organization', schema=None) as batch_op:
batch_op.alter_column(
'organization',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
batch_op.alter_column(
'role',
existing_type=sa.VARCHAR(length=128),
type_=sa.String(length=256),
existing_nullable=False)
def downgrade():
with op.batch_alter_table('sub_organization', schema=None) as batch_op:
batch_op.alter_column(
'role',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
batch_op.alter_column(
'organization',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
with op.batch_alter_table('service', schema=None) as batch_op:
batch_op.alter_column(
'name_key',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
with op.batch_alter_table('parent_organization', schema=None) as batch_op:
batch_op.alter_column(
'organization',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
with op.batch_alter_table('nren_service', schema=None) as batch_op:
batch_op.alter_column(
'service_key',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
with op.batch_alter_table('nren', schema=None) as batch_op:
batch_op.alter_column(
'name',
existing_type=sa.String(length=256),
type_=sa.VARCHAR(length=128),
existing_nullable=False)
import logging
from decimal import Decimal from decimal import Decimal
from typing import List, Dict, Any, Type from typing import List, Dict, Any, Type
from collections import defaultdict from collections import defaultdict
...@@ -16,6 +17,8 @@ from compendium_v2.db.survey_models import SurveyResponse ...@@ -16,6 +17,8 @@ from compendium_v2.db.survey_models import SurveyResponse
from compendium_v2.publishers.utils import float_or_zero, float_or_none, bool_or_none, int_or_none, sanitize from compendium_v2.publishers.utils import float_or_zero, float_or_none, bool_or_none, int_or_none, sanitize
from compendium_v2.publishers.helpers import valid_url, merge_string from compendium_v2.publishers.helpers import valid_url, merge_string
logger = logging.getLogger(__name__)
def map_budget_entry(year: int, nren: NREN, answers: Dict[str, Any]): def map_budget_entry(year: int, nren: NREN, answers: Dict[str, Any]):
budget = answers.get("budget") budget = answers.get("budget")
...@@ -79,6 +82,9 @@ def map_nren_staff(year: int, nren: NREN, answers: Dict[str, Any]): ...@@ -79,6 +82,9 @@ def map_nren_staff(year: int, nren: NREN, answers: Dict[str, Any]):
def map_parent_orgs(nren: NREN, year: int, answers: Dict[str, Any]): def map_parent_orgs(nren: NREN, year: int, answers: Dict[str, Any]):
has_parent = answers.get("parent_organization") == "Yes" has_parent = answers.get("parent_organization") == "Yes"
parent = answers.get("parent_organization_name", "").strip() parent = answers.get("parent_organization_name", "").strip()
if len(parent) > 256:
logger.warning(f"Parent organization name too long for NREN {nren.id}")
parent = parent[:256]
if has_parent and parent: if has_parent and parent:
return { return {
'nren_id': nren.id, 'nren_id': nren.id,
...@@ -98,6 +104,14 @@ def map_sub_orgs(nren: NREN, year: int, answers: Dict[str, Any]): ...@@ -98,6 +104,14 @@ def map_sub_orgs(nren: NREN, year: int, answers: Dict[str, Any]):
name = sub.get("suborganization_name", "").strip() name = sub.get("suborganization_name", "").strip()
if not name: if not name:
continue continue
if len(name) > 256:
logger.warning(f"Suborganization name too long for NREN {nren.name} ({year})")
logger.warning(name)
name = name[:256]
if len(role) > 256:
logger.warning(f"Organization role too long for NREN {nren.name} ({year})")
logger.warning(role)
role = role[:256]
yield { yield {
'nren_id': nren.id, 'nren_id': nren.id,
'year': year, 'year': year,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment