diff --git a/compendium_v2/db/presentation_models.py b/compendium_v2/db/presentation_models.py index 261fffc65b0c30330bcf9bb0fd26cbe18107bae2..8ed7b290ab3a861a20c1d7b9411290c46eddda45 100644 --- a/compendium_v2/db/presentation_models.py +++ b/compendium_v2/db/presentation_models.py @@ -18,8 +18,7 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial logger = logging.getLogger(__name__) -str128 = Annotated[str, 128] -str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)] +str256 = Annotated[str, mapped_column(String(256))] str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)] int_pk = Annotated[int, mapped_column(primary_key=True)] int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)] @@ -65,8 +64,8 @@ class NREN(PresentationModel): """ __tablename__ = 'nren' id: Mapped[int_pk] - name: Mapped[str128] - country: Mapped[str128] + name: Mapped[str256] + country: Mapped[str256] def to_dict(self, download=False): return { @@ -222,7 +221,7 @@ class ParentOrganization(PresentationModel): nren_id: Mapped[int_pk_fkNREN] nren: Mapped[NREN] = relationship(lazy='joined') year: Mapped[int_pk] - organization: Mapped[str128] + organization: Mapped[str256] def compare_dict(self): return { @@ -252,8 +251,8 @@ class SubOrganization(PresentationModel): nren_id: Mapped[int_pk_fkNREN] nren: Mapped[NREN] = relationship(lazy='joined') year: Mapped[int_pk] - organization: Mapped[str128_pk] - role: Mapped[str128_pk] + organization: Mapped[str256_pk] + role: Mapped[str256_pk] def compare_dict(self): return { @@ -1280,8 +1279,8 @@ class Service(PresentationModel): .. autoenum:: compendium_v2.db.presentation_model_enums.ServiceCategory """ __tablename__ = 'service' - name_key: Mapped[str128_pk] - name: Mapped[str128] + name_key: Mapped[str256_pk] + name: Mapped[str256] category: Mapped[ServiceCategory] description: Mapped[str] @@ -1297,9 +1296,9 @@ class NRENService(PresentationModel): nren_id: Mapped[int_pk_fkNREN] nren: Mapped[NREN] = relationship(lazy='joined') year: Mapped[int_pk] - service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True) + service_key: Mapped[str256] = mapped_column(ForeignKey("service.name_key"), primary_key=True) service: Mapped[Service] = relationship(lazy='joined') - product_name: Mapped[str128] + product_name: Mapped[str256] additional_information: Mapped[str] official_description: Mapped[str] diff --git a/compendium_v2/migrations/versions/4b531785f8d4_use_256_strings.py b/compendium_v2/migrations/versions/4b531785f8d4_use_256_strings.py new file mode 100644 index 0000000000000000000000000000000000000000..493140ec740f9551bed33145fb9cef70792d775b --- /dev/null +++ b/compendium_v2/migrations/versions/4b531785f8d4_use_256_strings.py @@ -0,0 +1,100 @@ +"""Use 256 strings + +Revision ID: 4b531785f8d4 +Revises: 8298b45f8e3d +Create Date: 2025-01-22 14:54:42.695483 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4b531785f8d4' +down_revision = '8298b45f8e3d' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('nren', schema=None) as batch_op: + batch_op.alter_column( + 'name', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + + with op.batch_alter_table('nren_service', schema=None) as batch_op: + batch_op.alter_column( + 'service_key', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + + with op.batch_alter_table('parent_organization', schema=None) as batch_op: + batch_op.alter_column( + 'organization', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + + with op.batch_alter_table('service', schema=None) as batch_op: + batch_op.alter_column( + 'name_key', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + + with op.batch_alter_table('sub_organization', schema=None) as batch_op: + batch_op.alter_column( + 'organization', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + batch_op.alter_column( + 'role', + existing_type=sa.VARCHAR(length=128), + type_=sa.String(length=256), + existing_nullable=False) + + +def downgrade(): + with op.batch_alter_table('sub_organization', schema=None) as batch_op: + batch_op.alter_column( + 'role', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) + batch_op.alter_column( + 'organization', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) + + with op.batch_alter_table('service', schema=None) as batch_op: + batch_op.alter_column( + 'name_key', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) + + with op.batch_alter_table('parent_organization', schema=None) as batch_op: + batch_op.alter_column( + 'organization', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) + + with op.batch_alter_table('nren_service', schema=None) as batch_op: + batch_op.alter_column( + 'service_key', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) + + with op.batch_alter_table('nren', schema=None) as batch_op: + batch_op.alter_column( + 'name', + existing_type=sa.String(length=256), + type_=sa.VARCHAR(length=128), + existing_nullable=False) diff --git a/compendium_v2/publishers/year/map_2023.py b/compendium_v2/publishers/year/map_2023.py index 399040141c50d8c696df297e4f4488bde303db1a..44a959b57c1a2cc0b72627dd83d6710f88658068 100644 --- a/compendium_v2/publishers/year/map_2023.py +++ b/compendium_v2/publishers/year/map_2023.py @@ -1,3 +1,4 @@ +import logging from decimal import Decimal from typing import List, Dict, Any, Type from collections import defaultdict @@ -16,6 +17,8 @@ from compendium_v2.db.survey_models import SurveyResponse from compendium_v2.publishers.utils import float_or_zero, float_or_none, bool_or_none, int_or_none, sanitize from compendium_v2.publishers.helpers import valid_url, merge_string +logger = logging.getLogger(__name__) + def map_budget_entry(year: int, nren: NREN, answers: Dict[str, Any]): budget = answers.get("budget") @@ -79,6 +82,9 @@ def map_nren_staff(year: int, nren: NREN, answers: Dict[str, Any]): def map_parent_orgs(nren: NREN, year: int, answers: Dict[str, Any]): has_parent = answers.get("parent_organization") == "Yes" parent = answers.get("parent_organization_name", "").strip() + if len(parent) > 256: + logger.warning(f"Parent organization name too long for NREN {nren.id}") + parent = parent[:256] if has_parent and parent: return { 'nren_id': nren.id, @@ -98,6 +104,14 @@ def map_sub_orgs(nren: NREN, year: int, answers: Dict[str, Any]): name = sub.get("suborganization_name", "").strip() if not name: continue + if len(name) > 256: + logger.warning(f"Suborganization name too long for NREN {nren.name} ({year})") + logger.warning(name) + name = name[:256] + if len(role) > 256: + logger.warning(f"Organization role too long for NREN {nren.name} ({year})") + logger.warning(role) + role = role[:256] yield { 'nren_id': nren.id, 'year': year,