Finished release 0.81.

Changelog.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.81] - 2025-01-22
+- Harden get_response_data for moving responses from a previous survey to a new survey year.
+- Moved response data now satisfies the dependencies of the new survey. If a question would be hidden in the new survey, the data for that question is removed.
+- Changed string lengths to minimum 256 and updated the survey publisher to handle max string lengths without erroring.
+
 ## [0.80] - 2025-01-21
 - Add missing legacy_publisher module due to missing init file
 

compendium-frontend/.babelrc

@@ -11,6 +11,6 @@
           "regenerator": true
         }
       ],
-      "@babel/plugin-proposal-class-properties"
+      "@babel/plugin-transform-class-properties"
     ]
 }

compendium-frontend/README.md

@@ -2,6 +2,8 @@
 
 ## development environment
 
+# Requires Node.js v22.13.0 LTS or later
+
 From this folder, run:
 
 ```bash

compendium-frontend/package.json

@@ -2,36 +2,36 @@
   "name": "compendium-v2",
   "version": "1.0.0",
   "devDependencies": {
-    "@babel/core": "~7.24",
-    "@babel/plugin-proposal-class-properties": "~7.18",
-    "@babel/plugin-transform-runtime": "~7.24",
-    "@babel/preset-env": "~7.24.7",
-    "@babel/preset-react": "~7.24",
-    "@babel/preset-typescript": "~7.24",
-    "@babel/runtime": "~7.24",
-    "@types/react": "~18.3",
-    "@types/react-dom": "~18.3",
+    "@babel/core": "~7.26.0",
+    "@babel/plugin-transform-class-properties": "~7.25.9",
+    "@babel/plugin-transform-runtime": "~7.25.9",
+    "@babel/preset-env": "~7.26.0",
+    "@babel/preset-react": "~7.26.3",
+    "@babel/preset-typescript": "~7.26.0",
+    "@babel/runtime": "~7.26.0",
+    "@types/react": "~19.0.7",
+    "@types/react-dom": "~19.0.3",
     "@types/react-router-dom": "~5.3.3",
-    "@types/webpack": "~5.28",
-    "@typescript-eslint/eslint-plugin": "~6.21",
-    "@typescript-eslint/parser": "~6.21",
-    "babel-loader": "~9.1.3",
-    "css-loader": "~6.11.0",
-    "date-fns": "~3.6.0",
-    "eslint": "^8.57.0",
-    "eslint-plugin-react": "~7.34.3",
-    "eslint-plugin-react-hooks": "~4.6.2",
+    "@types/webpack": "~5.28.5",
+    "@typescript-eslint/eslint-plugin": "~8.21.0",
+    "@typescript-eslint/parser": "~8.21.0",
+    "babel-loader": "~9.2.1",
+    "css-loader": "~7.1.2",
+    "date-fns": "~4.1.0",
+    "eslint": "~9.18.0",
+    "eslint-plugin-react": "~7.37.4",
+    "eslint-plugin-react-hooks": "~5.1.0",
     "fork-ts-checker-webpack-plugin": "~9.0.2",
-    "mini-css-extract-plugin": "~2.9.0",
-    "sass": "~1.77.6",
-    "sass-loader": "~13.3.3",
-    "style-loader": "~3.3.4",
+    "mini-css-extract-plugin": "~2.9.2",
+    "sass": "~1.83.4",
+    "sass-loader": "~16.0.4",
+    "style-loader": "~4.0.0",
     "ts-node": "~10.9.2",
-    "typescript": "~5.5.3",
+    "typescript": "~5.7.3",
     "url-loader": "~4.1.1",
-    "webpack": "~5.92.1",
-    "webpack-cli": "~5.1.4",
-    "webpack-dev-server": "~4.15.2"
+    "webpack": "~5.97.1",
+    "webpack-cli": "~6.0.1",
+    "webpack-dev-server": "~5.2.0"
   },
   "scripts": {
     "start": "webpack serve --mode development --open --port 4000",
@@ -40,17 +40,17 @@
   "dependencies": {
     "bootstrap": "~5.3.3",
     "cartesian-product-multiple-arrays": "~1.0.9",
-    "chart.js": "~4.4.3",
+    "chart.js": "~4.4.7",
     "chartjs-plugin-datalabels": "~2.2.0",
     "html-to-image": "~1.11.11",
-    "react": "~18.3.1",
-    "react-bootstrap": "~2.10.4",
-    "react-chartjs-2": "~5.2.0",
-    "react-dom": "~18.3.1",
-    "react-hot-toast": "~2.4.1",
-    "react-icons": "~5.2.1",
-    "react-router-dom": "~6.24.1",
-    "survey-react-ui": "~1.11.5",
+    "react": "~19.0.0",
+    "react-bootstrap": "~2.10.8",
+    "react-chartjs-2": "~5.3.0",
+    "react-dom": "~19.0.0",
+    "react-hot-toast": "~2.5.1",
+    "react-icons": "~5.4.0",
+    "react-router-dom": "~7.1.3",
+    "survey-react-ui": "~1.12.19",
     "xlsx": "~0.18.5",
     "lodash": "~4.17.21"
   },
@@ -60,6 +60,6 @@
   "license": "ISC",
   "repository": {
     "type": "git",
-    "url": "https://gitlab.geant.org/geant-swd/compendium-v2.git"
+    "url": "https://gitlab.software.geant.org/geant-swd/compendium-v2.git"
   }
-}
+}
\ No newline at end of file

compendium-frontend/src/components/sidebar/LinkWithHighlight.tsx

@@ -9,7 +9,7 @@ interface inputProps {
 
 function LinkWithHighlight({ to, children }: inputProps): ReactElement {
     const currentPageIsTo = window.location.pathname === to;
-    const ref = useRef() as React.MutableRefObject<HTMLAnchorElement>;
+    const ref = useRef<HTMLAnchorElement>(null);
 
     if (currentPageIsTo && ref.current) {
         // scroll the link into view in the sidebar

compendium-frontend/src/main.scss

-@import 'scss/fonts';
-@import 'scss/base/text';
-@import 'scss/layout/components';
+@use 'scss/fonts';
+@use 'scss/base/text';
+@use 'scss/layout/components';
+@use "scss/abstracts/variables";
 
 .nav-link-entry {
   border-radius: 2px;
@@ -140,7 +141,7 @@
 .imageoption {
   padding: 0.5rem;
   cursor: pointer;
-  color: $dark-teal;
+  color: variables.$dark-teal;
   font-weight: bold;
 }
 

compendium-frontend/src/providers/ChartContainerProvider.tsx

 import React, { createContext, useRef, ReactNode, RefObject } from 'react';
 
-const ChartContainerContext = createContext<RefObject<HTMLDivElement> | null>(null);
+const ChartContainerContext = createContext<RefObject<HTMLDivElement | null> | null>(null);
 
 interface ChartContainerProviderProps {
     children: ReactNode;

compendium-frontend/src/scss/base/_text.scss

-@import '../abstracts/variables';
+@use '../abstracts/variables';
 
 .regular-17pt {
     font-family: "Open Sans", sans-serif;
@@ -41,12 +41,12 @@
 }
 
 .dark-teal {
-    color: $dark-teal
+    color: variables.$dark-teal
 }
 
 .geant-header {
     @extend .bold-caps-20pt;
-    color: $dark-teal;
+    color: variables.$dark-teal;
 }
 
 .bold-grey-12pt {

compendium-frontend/src/scss/layout/Login.scss

+@use "../abstracts/variables";
+
 .btn-login {
     --bs-btn-color: #fff;
     --bs-btn-border-color: #6c757d;
@@ -6,13 +8,13 @@
     // active
     --bs-btn-active-color: #fff;
     // see https://stackoverflow.com/a/76213205 for why this syntax is necessary
-    --bs-btn-active-bg: #{$yellow-orange};
-    --bs-btn-active-border-color: #{$yellow-orange};
+    --bs-btn-active-bg: #{variables.$yellow-orange};
+    --bs-btn-active-border-color: #{variables.$yellow-orange};
 
     // hover
     --bs-btn-hover-color: rgb(0, 63, 95);
-    --bs-btn-hover-bg: #{$yellow-orange};
-    --bs-btn-hover-border-color: #{$yellow-orange};
+    --bs-btn-hover-bg: #{variables.$yellow-orange};
+    --bs-btn-hover-border-color: #{variables.$yellow-orange};
 
 
     // disabled
@@ -20,5 +22,5 @@
     --bs-btn-disabled-bg: transparent;
     // --bs-btn-disabled-border-color: #6c757d;
 
-    border: 2px solid $yellow-orange;
+    border: 2px solid variables.$yellow-orange;
 }
\ No newline at end of file

compendium-frontend/src/scss/layout/SectionNavigation.scss

-@import '../abstracts/variables';
+@use '../abstracts/variables';
 
 .btn-nav-box {
     --bs-btn-color: rgb(0, 63, 95);
@@ -8,13 +8,13 @@
     // active
     --bs-btn-active-color: #fff;
     // see https://stackoverflow.com/a/76213205 for why this syntax is necessary
-    --bs-btn-active-bg: #{$yellow-orange};
-    --bs-btn-active-border-color: #{$yellow-orange};
+    --bs-btn-active-bg: #{variables.$yellow-orange};
+    --bs-btn-active-border-color: #{variables.$yellow-orange};
 
     // hover
     --bs-btn-hover-color: rgb(0, 63, 95);
-    --bs-btn-hover-bg: #{$yellow-orange};
-    --bs-btn-hover-border-color: #{$yellow-orange};
+    --bs-btn-hover-bg: #{variables.$yellow-orange};
+    --bs-btn-hover-border-color: #{variables.$yellow-orange};
 
 
     // disabled
@@ -22,5 +22,5 @@
     --bs-btn-disabled-bg: transparent;
     // --bs-btn-disabled-border-color: #6c757d;
 
-    border: 2px solid $yellow-orange;
+    border: 2px solid variables.$yellow-orange;
 }
\ No newline at end of file

compendium-frontend/src/scss/layout/Sidebar.scss

+@use "../abstracts/variables";
+@use "../base/text";
+
 #sidebar {
     overflow-y: scroll;
     overflow-x:hidden;
@@ -26,7 +29,7 @@
     margin-left: 0;
     background-color: #fff;
     box-shadow: 0 2px 10px rgba(0, 0, 0, 0.25);
-    border: $yellow-orange 2px solid;
+    border: variables.$yellow-orange 2px solid;
     pointer-events: auto;
     width: 28rem;
 
@@ -37,16 +40,16 @@
     }
 
     a:hover {
-        color: $yellow-orange;
+        color: variables.$yellow-orange;
         text-decoration: none;
     }
 }
 
 .sidebar-wrapper>nav.survey {
-    border: $dark-teal 2px solid;
+    border: variables.$dark-teal 2px solid;
 
     a:hover {
-        color: $teal-blue;
+        color: variables.$teal-blue;
     }
 }
 
@@ -58,7 +61,7 @@ nav.no-sidebar {
 
 .toggle-btn {
     @extend .bold-caps-16pt;
-    background-color: $yellow-orange;
+    background-color: variables.$yellow-orange;
     color: white;
     height: 3.5rem;
     cursor: pointer;
@@ -73,7 +76,7 @@ nav.no-sidebar {
 
 .toggle-btn-survey {
     @extend .toggle-btn;
-    background-color: $dark-teal;
+    background-color: variables.$dark-teal;
 }
 
 .toggle-btn-wrapper {

compendium-frontend/src/scss/layout/_components.scss

 @use "sass:list";
 
-@import '../abstracts/variables';
-@import '../layout/Sidebar';
-@import '../layout/SectionNavigation';
-@import '../layout/Login';
+@use '../abstracts/variables';
+@use '../layout/Sidebar';
+@use '../layout/SectionNavigation';
+@use '../layout/Login';
 
 $year-colors-muted: (
   rgba(206, 61, 91, var(--muted-alpha)),
@@ -41,7 +41,7 @@ $year-colors: (
 
 .rounded-border {
   border-radius: 25px;
-  border: 1px solid $light-ash-grey
+  border: 1px solid variables.$light-ash-grey
 }
 
 .card {
@@ -57,7 +57,7 @@ $year-colors: (
 .grey-container {
   @extend .grow;
   max-width: 100vw;
-  background-color: $light-off-white;
+  background-color: variables.$light-off-white;
 }
 
 .wordwrap {
@@ -81,21 +81,21 @@ $year-colors: (
 }
 
 .compendium-data-header {
-  background-color: $yellow;
+  background-color: variables.$yellow;
   color: white;
   padding: 10px;
 }
 
 .compendium-data-banner {
-  background-color: $light-beige;
-  color: $dark-teal;
+  background-color: variables.$light-beige;
+  color: variables.$dark-teal;
   padding: 5px;
   padding-top: 25px;
 }
 
 .collapsible-box {
   margin: 1rem;
-  border: 2px solid $yellow-orange;
+  border: 2px solid variables.$yellow-orange;
   padding: 10px;
   width: 80rem;
   max-width: 97%;
@@ -368,7 +368,7 @@ $service-check-colors: (
   }
 
   thead th {
-    color: $dark-teal;
+    color: variables.$dark-teal;
     background-color: white;
     padding: 12px;
     font-weight: bold;

compendium-frontend/src/survey/survey.scss

-@import '../main.scss';
+@use '../main.scss';
 
 .sv-multipletext__cell {
 	padding: .5rem;

compendium-frontend/webpack.config.ts

@@ -66,12 +66,24 @@ const config: Configuration = {
     port: 4000,
     // Allow SPA urls to work with dev-server
     historyApiFallback: true,
-    proxy: {
-      "/api": "http://127.0.0.1:5000",
-      "/login": "http://127.0.0.1:5000",
-      "/logout": "http://127.0.0.1:5000",
-      "/authorize": "http://127.0.0.1:5000",
-    },
+    proxy: [
+      {
+        path: "/api",
+        target: "http://127.0.0.1:5000"
+      },
+      {
+        path: "/login",
+        target: "http://127.0.0.1:5000"
+      },
+      {
+        path: "/logout",
+        target: "http://127.0.0.1:5000"
+      },
+      {
+        path: "/authorize",
+        target: "http://127.0.0.1:5000"
+      },
+    ]
   },
   devtool: isProduction ? undefined : "inline-source-map",
   plugins: [

compendium_v2/db/presentation_models.py

@@ -18,8 +18,7 @@ from compendium_v2.db.presentation_model_enums import CarryMechanism, Commercial
 
 logger = logging.getLogger(__name__)
 
-str128 = Annotated[str, 128]
-str128_pk = Annotated[str, mapped_column(String(128), primary_key=True)]
+str256 = Annotated[str, mapped_column(String(256))]
 str256_pk = Annotated[str, mapped_column(String(256), primary_key=True)]
 int_pk = Annotated[int, mapped_column(primary_key=True)]
 int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
@@ -65,8 +64,8 @@ class NREN(PresentationModel):
     """
     __tablename__ = 'nren'
     id: Mapped[int_pk]
-    name: Mapped[str128]
-    country: Mapped[str128]
+    name: Mapped[str256]
+    country: Mapped[str256]
 
     def to_dict(self, download=False):
         return {
@@ -222,7 +221,7 @@ class ParentOrganization(PresentationModel):
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    organization: Mapped[str128]
+    organization: Mapped[str256]
 
     def compare_dict(self):
         return {
@@ -252,8 +251,8 @@ class SubOrganization(PresentationModel):
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    organization: Mapped[str128_pk]
-    role: Mapped[str128_pk]
+    organization: Mapped[str256_pk]
+    role: Mapped[str256_pk]
 
     def compare_dict(self):
         return {
@@ -1280,8 +1279,8 @@ class Service(PresentationModel):
     .. autoenum:: compendium_v2.db.presentation_model_enums.ServiceCategory
     """
     __tablename__ = 'service'
-    name_key: Mapped[str128_pk]
-    name: Mapped[str128]
+    name_key: Mapped[str256_pk]
+    name: Mapped[str256]
     category: Mapped[ServiceCategory]
     description: Mapped[str]
 
@@ -1297,9 +1296,9 @@ class NRENService(PresentationModel):
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    service_key: Mapped[str128] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
+    service_key: Mapped[str256] = mapped_column(ForeignKey("service.name_key"), primary_key=True)
     service: Mapped[Service] = relationship(lazy='joined')
-    product_name: Mapped[str128]
+    product_name: Mapped[str256]
     additional_information: Mapped[str]
     official_description: Mapped[str]
 

compendium_v2/helpers.py

 """
 Utilities used around the codebase.
 """
+import re
+
 from flask import request
 from flask_login import current_user  # type: ignore
-from typing import Type, Sequence, TypeVar
+from typing import Type, Sequence, TypeVar, Dict, Any
 from itertools import chain
 
 from compendium_v2.db import db
@@ -45,3 +47,231 @@ def extract_model_data(entry_model, download=False):
         return
 
     yield entry_model.to_dict()
+
+
+def get_survey_question_dependency_graph(survey: Dict[str, Any]):
+    """
+    This function takes a survey in JSON format, walks through the questions and builds a dependency graph of which
+    questions depend on which other questions in various ways.
+    Mostly centered around visibleIf logic, to make sure that when moving data from one survey to the next, only
+    keys that are actually used are moved.
+
+    E.g. if a question is not answered, but a hidden question depends on it, the data for the hidden question
+    should not be moved to the next survey response "prefilled" automatically.
+    """
+
+    pages = survey.get('pages', [])
+    if not pages:
+        return {}
+
+    dependencies: Dict[str, Any] = {}
+
+    def _extract_dependencies(visibleIf: str):
+        """
+        Extract the key that this question depends on. There's a few cases covered:
+
+        It's usually structured like:
+        {network_automation} = 'stringvalue'
+        {network_automation} = ['stringvalue in array']
+        {network_automation} = 'yes' or {network_automation} = 'planned'
+
+        This function extracts the key that the question depends on, as well as the required value.
+        In case of multiple tests, a tuple is returned with the key and the accepted values.
+
+        """
+
+        if not visibleIf:
+            return None
+
+        assert isinstance(visibleIf, str), f"visibleIf should be a string, got {visibleIf}"
+
+        is_or = False
+        if ' or ' in visibleIf:
+            is_or = True
+            groups = visibleIf.split(' or ')
+        elif ' and ' in visibleIf:
+            groups = visibleIf.split(' and ')
+        else:
+            groups = []
+
+        if len(groups) > 1:
+            groups = [group.strip() for group in groups]
+            operation = 'or' if is_or else 'and'
+            return operation, tuple(_extract_dependencies(group) for group in groups)
+
+        match = re.match(r'{(.+?)}\s*=\s*(.+)', visibleIf)
+        if not match:
+            raise ValueError(f"Could not parse visibleIf: {visibleIf}")
+
+        key, value = match.groups()
+        key = key.strip().replace('row.', '')
+        value = value.strip()
+
+        if value.startswith('[') and value.endswith(']'):
+            # "['yes']" -> ['yes'] as a python list
+            value = [value[1:-1].replace("'", "").replace('"', "")]
+        else:
+            # "'yes'" -> "yes"
+            value = value.replace("'", "").replace('"', "")
+
+        return key, value
+
+    def _add_element_dependencies(question):
+        name = question.get('name')
+
+        if not name:
+            raise ValueError(f"Question {question} does not have a name, cannot determine dependencies")
+
+        deps = dependencies.get(name, [])
+
+        visible_if = question.get('visibleIf')
+        if not visible_if:
+            return
+
+        dependent_key = _extract_dependencies(visible_if)
+        if dependent_key:
+            deps.append(dependent_key)
+
+        if deps:
+            dependencies[name] = deps
+
+    def _process_element(element):
+        if element.get('type') in ['comment', 'html']:
+            return
+        if element.get('type') == 'panel':
+            panel_elements = element.get('elements', [])
+            if not panel_elements:
+                return
+
+            for panel_element in panel_elements:
+                _process_element(panel_element)
+        if element.get('type') == 'matrixdropdown':
+            columns = element.get('columns', [])
+            if not columns:
+                return
+
+            for column in columns:
+                if not column.get('visibleIf'):
+                    continue
+                column['name'] = f"{element.get('name')}[..]{column.get('name')}"
+                _add_element_dependencies(column)
+        else:
+            _add_element_dependencies(element)
+
+    for page in pages:
+        elements = page.get('elements', [])
+        if not elements:
+            continue
+
+        for element in elements:
+            _process_element(element)
+
+    return dependencies
+
+
+def clean_answers_with_dependencies(data: Dict[str, Any], survey: Dict[str, Any]):
+    """
+    Given a survey and a set of answers, clean the answers based on the dependencies in the survey.
+    This is used to clean up the answers before moving them to the next survey, to make sure that only the
+    necessary answers are moved.
+    """
+
+    if not survey:
+        raise ValueError("Provided survey is empty, cannot clean answers")
+
+    dependencies = get_survey_question_dependency_graph(survey)
+
+    cleaned_data = data.copy()
+
+    for key, value in dependencies.items():
+
+        if '[..]' in key:
+            # matrixdropdown
+            matrix_key, column_key = key.split('[..]')
+            if matrix_key not in data:
+                continue
+
+            matrix_data = data.get(matrix_key)
+            if not matrix_data:
+                continue
+
+            for column, row in list(matrix_data.items()):
+                if column_key not in row:
+                    continue
+
+                is_valid = column_key in row
+                for expected_key, values in value:
+                    if not (expected_key in row and row[expected_key] == values):
+                        is_valid = False
+                        break
+
+                if is_valid:
+                    cleaned_data.setdefault(matrix_key, {})[column][column_key] = row[column_key]
+                else:
+                    del cleaned_data[matrix_key][column][column_key]
+
+            continue
+        if key not in data:
+            continue
+
+        data_value = data.get(key)
+        # check that the data_value satisfies the dependency
+
+        if not isinstance(value, list):
+            raise ValueError(f"Expected dependencies to be a list, got {value}")
+
+        # multiple dependencies, all must be satisfied
+        is_valid = True
+        for expected_key, values in value:
+            actual_value = data.get(expected_key)
+            if isinstance(actual_value, str):
+                actual_value = actual_value.lower()  # survey is case insensitive
+            if isinstance(values, str):
+                values = values.lower()
+
+            if expected_key in ['or', 'and']:
+                # special handling for when a question is shown if one or more conditions are met
+
+                if expected_key == 'or':
+                    or_valid = False
+                    # if any of the conditions are met, the question is shown
+                    for _or_expected_key, _or_values in values:
+                        _or_data = data.get(_or_expected_key)
+                        if isinstance(_or_values, str):
+                            _or_values = _or_values.lower()
+                        if isinstance(_or_data, str):
+                            _or_data = _or_data.lower()
+
+                        if _or_expected_key in data and _or_data == _or_values:
+                            or_valid = True
+                            break
+                    if not or_valid:
+                        is_valid = False
+                        break
+                elif expected_key == 'and':
+                    and_valid = True
+                    # all conditions must be met for the question to be shown
+                    for _and_expected_key, _and_values in values:
+                        _and_data = data.get(_and_expected_key)
+                        if isinstance(_and_values, str):
+                            _and_values = _and_values.lower()
+                        if isinstance(_and_data, str):
+                            _and_data = _and_data.lower()
+
+                        if not (_and_expected_key in data and _and_data == _and_values):
+                            and_valid = False
+                            break
+                    if not and_valid:
+                        is_valid = False
+                        break
+
+            elif not (expected_key in data and actual_value == values):
+                is_valid = False
+                break
+
+        if is_valid:
+            cleaned_data[key] = data_value
+        else:
+            del cleaned_data[key]
+
+    return cleaned_data

compendium_v2/migrations/versions/4b531785f8d4_use_256_strings.py

+"""Use 256 strings
+
+Revision ID: 4b531785f8d4
+Revises: 8298b45f8e3d
+Create Date: 2025-01-22 14:54:42.695483
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4b531785f8d4'
+down_revision = '8298b45f8e3d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('nren', schema=None) as batch_op:
+        batch_op.alter_column(
+            'name',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+
+    with op.batch_alter_table('nren_service', schema=None) as batch_op:
+        batch_op.alter_column(
+            'service_key',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+
+    with op.batch_alter_table('parent_organization', schema=None) as batch_op:
+        batch_op.alter_column(
+            'organization',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+
+    with op.batch_alter_table('service', schema=None) as batch_op:
+        batch_op.alter_column(
+            'name_key',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+
+    with op.batch_alter_table('sub_organization', schema=None) as batch_op:
+        batch_op.alter_column(
+            'organization',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+        batch_op.alter_column(
+            'role',
+            existing_type=sa.VARCHAR(length=128),
+            type_=sa.String(length=256),
+            existing_nullable=False)
+
+
+def downgrade():
+    with op.batch_alter_table('sub_organization', schema=None) as batch_op:
+        batch_op.alter_column(
+            'role',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)
+        batch_op.alter_column(
+            'organization',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)
+
+    with op.batch_alter_table('service', schema=None) as batch_op:
+        batch_op.alter_column(
+            'name_key',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)
+
+    with op.batch_alter_table('parent_organization', schema=None) as batch_op:
+        batch_op.alter_column(
+            'organization',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)
+
+    with op.batch_alter_table('nren_service', schema=None) as batch_op:
+        batch_op.alter_column(
+            'service_key',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)
+
+    with op.batch_alter_table('nren', schema=None) as batch_op:
+        batch_op.alter_column(
+            'name',
+            existing_type=sa.String(length=256),
+            type_=sa.VARCHAR(length=128),
+            existing_nullable=False)

compendium_v2/migrations/versions/cb8aea87cab5_service_matrix_merge_software_licenses_.py

@@ -8,8 +8,6 @@ Create Date: 2024-12-17 16:44:51.834601
 """
 from alembic import op
 import sqlalchemy as sa
-from collections import defaultdict
-from compendium_v2.publishers.helpers import merge_string
 
 
 # revision identifiers, used by Alembic.

compendium_v2/publishers/legacy_publisher/survey_publisher_legacy.py

+"""
+survey_publisher_legacy
+=========================
+
+This script is used to import data from the legacy datasources into the new survey system.
+Legacy data is used to generate a new survey for each NREN, for each year that we have data for.
+The generated surveys are based on the 2024 survey.
+
+"""
 import click
 from itertools import chain
 from collections import defaultdict