add survey model to db and fix docs and testcases

README.md

@@ -60,6 +60,12 @@ survey-publisher-v1
 survey-publisher-2022
 ```
 
+To import the 2022 data to the new datamodel for prefilling the 2023 surveys:
+```bash
+pip install --editable .
+conversion
+```
+
 ## Creating a db migration after editing the sqlalchemy models
 
 ```bash
@@ -73,3 +79,19 @@ Flask-migrate sets `compare_type=True` by default.
 Note that starting the application applies all upgrades.
 This also happens when running `flask db` commands such as `flask db downgrade`,
 so if you want to downgrade 2 or more versions you need to do so in one command, eg by specifying the revision number.
+
+## Editing a newly created survey
+
+A new survey can be created and inspected from the survey admin page.
+If you need to make edits, do the following:
+
+```bash
+pip install --editable .
+dump_survey_model
+```
+
+This will create a json dump of the new survey in the `migrations/surveymodels` folder.
+Create a new branch, commit this file in git, and make the edits, which should of course also be committed.
+Create an empty db migration with `flask db revision`, copy the upgrade command from
+`b3e1c5149181_insert_survey_model_2022.py`, set the year, and copy the contents of the json file into the migration.
+Merge this back and deploy.

compendium_v2/conversion/conversion.py

@@ -19,7 +19,7 @@ from compendium_v2.db import db
 from compendium_v2.config import load
 from compendium_v2.survey_db import model as survey_model
 from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus
+from compendium_v2.db.survey_model import Survey, SurveyResponse, ResponseStatus
 
 from compendium_v2.conversion import mapping
 
@@ -119,12 +119,7 @@ def _cli(app):
             SurveyResponse.survey_year == 2022
         ))
 
-        db.session.execute(delete(Survey).where(
-            Survey.year == 2022
-        ))
-
-        survey = Survey(year=2022, survey={}, status=SurveyStatus.published)
-        db.session.add(survey)
+        survey = db.session.scalar(select(Survey).where(Survey.year == 2022))
 
         for nren, answers in nren_surveys.items():
             survey_dict = convert_answers(answers)

compendium_v2/migrations/dump_survey_model.py

+"""
+dump survey model
+=========================
+
+This module dumps a survey model to file to check into git,
+so that changes can be tracked.
+The dumped survey model should afterwards be committed back
+to the database with a migration script.
+
+See b3e1c5149181_insert_survey_model_2022.py for such a script.
+Create a revision with `flask db revision` and then just copy paste the new model and year.
+
+If the --year option is not supplied, the newest survey will be dumped.
+
+"""
+import logging
+import click
+import json
+import os
+
+from sqlalchemy import select
+
+import compendium_v2
+from compendium_v2.environment import setup_logging
+from compendium_v2.db import db
+from compendium_v2.config import load
+from compendium_v2.db.survey_model import Survey, SurveyStatus
+
+setup_logging()
+
+logger = logging.getLogger('dump_survey_model')
+
+
+def _cli(app, year):
+
+    query = select(Survey).where(Survey.year == year) if year else select(Survey).order_by(Survey.year.desc()).limit(1)
+
+    with app.app_context():
+        survey = db.session.scalar(query)
+        if not survey:
+            raise Exception('Survey not found')
+
+        if survey.status != SurveyStatus.closed:
+            raise Exception(
+                'Make sure the survey has status closed when youre going to update it, especially in production!'
+            )
+
+        filename = "survey_model_" + str(survey.year) + ".json"
+        dump_file = os.path.join(os.path.dirname(__file__), 'surveymodels', filename)
+        with open(dump_file, "w") as out:
+            json.dump(survey.survey, out, indent=1)
+
+
+@click.command()
+@click.option('--config', type=click.STRING, default='config.json')
+@click.option('--year', type=click.INT)
+def cli(config, year):
+    app_config = load(open(config, 'r'))
+    app = compendium_v2._create_app_with_db(app_config)
+    _cli(app, year)
+
+
+if __name__ == "__main__":
+    cli()

compendium_v2/migrations/surveymodels/survey_model_2022.json

@@ -130,7 +130,7 @@
        "type": "multipletext",
        "name": "staff_employment_type",
        "title": "What is the number of staff engaged in the NREN activities? ",
-       "description": "Please distinguish permanent\f and subcontracted using the boxes below. Please put answer in Full Time Equivalent (FTE's).",
+       "description": "Please distinguish permanent and subcontracted using the boxes below. Please put answer in Full Time Equivalent (FTE's).",
        "items": [
         {
          "name": "permanent_fte",
@@ -256,7 +256,7 @@
        "name": "ec_project_names",
        "visibleIf": "{ec_projects} = 'Yes'",
        "indent": 1,
-       "title": "Please list the name of the projects\n",
+       "title": "Please list the name of the projects",
        "description": "Use one line per project.",
        "columns": [
         {

compendium_v2/migrations/versions/5c9016cc6348_update_2022_survey_status.py

+"""update 2022 survey status
+
+Revision ID: 5c9016cc6348
+Revises: b3e1c5149181
+Create Date: 2023-07-21 22:57:16.847885
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '5c9016cc6348'
+down_revision = 'b3e1c5149181'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("UPDATE survey SET status = 'published' WHERE year = 2022")
+
+
+def downgrade():
+    pass

compendium_v2/static/survey-bundle.js.LICENSE.txt

+/*!
+	Copyright (c) 2018 Jed Watson.
+	Licensed under the MIT License (MIT), see
+	http://jedwatson.github.io/classnames
+*/
+
 /*!
  * Signature Pad v2.3.2
  * https://github.com/szimek/signature_pad
@@ -2054,6 +2060,16 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
 /**
  * @license React
  * react.production.min.js

docs/source/publishers.rst → docs/source/commands.rst

 .. api intro
 
-Publishers
+Commands
 ===============
 
 
@@ -11,3 +11,9 @@ Publishers
 
 .. automodule:: compendium_v2.publishers.survey_publisher_2022
     :members:
+
+.. automodule:: compendium_v2.conversion.conversion
+    :members:
+
+.. automodule:: compendium_v2.migrations.dump_survey_model
+    :members:

docs/source/index.rst

@@ -11,4 +11,4 @@ a React web application that consumes and renders the json data.
 
    api
    development
-   publishers
+   commands

setup.py

@@ -31,6 +31,7 @@ setup(
             'survey-publisher-v1=compendium_v2.publishers.survey_publisher_v1:cli',  # noqa
             'survey-publisher-2022=compendium_v2.publishers.survey_publisher_2022:cli',  # noqa
             'conversion=compendium_v2.conversion.conversion:cli',  # noqa
+            'dump_survey_model=compendium_v2.migrations.dump_survey_model:cli',  # noqa
         ]
     },
     license='MIT',

test/test_conversion.py

@@ -2,7 +2,7 @@ from sqlalchemy import select
 
 from compendium_v2.db import db
 from compendium_v2.db.model import NREN
-from compendium_v2.db.survey_model import Survey, SurveyResponse
+from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus
 from compendium_v2.conversion.conversion import _cli, convert_answers
 
 
@@ -17,6 +17,7 @@ def mock_query_nren(_):
 def test_queries(app_with_survey_db, mocker):
 
     with app_with_survey_db.app_context():
+        db.session.add(Survey(year=2022, status=SurveyStatus.closed, survey={}))
         db.session.add(NREN(name='Restena', country='country'))
         db.session.commit()
 
@@ -26,9 +27,6 @@ def test_queries(app_with_survey_db, mocker):
     _cli(app_with_survey_db)
 
     with app_with_survey_db.app_context():
-        surveys = db.session.scalars(select(Survey)).all()
-        assert len(surveys) == 1
-        assert surveys[0].year == 2022
 
         responses = db.session.scalars(select(SurveyResponse).order_by(SurveyResponse.nren_id)).all()
         assert len(responses) == 1

test/test_dump_survey_model.py

+import pytest
+
+from compendium_v2.db import db
+from compendium_v2.db.survey_model import Survey, SurveyStatus
+from compendium_v2.migrations.dump_survey_model import _cli
+
+
+def test_dump_status_check(app_with_survey_db):
+
+    with app_with_survey_db.app_context():
+        db.session.add(Survey(year=2022, status=SurveyStatus.published, survey={}))
+        db.session.commit()
+
+    with pytest.raises(Exception, match=r'^Make sure the survey has status closed'):
+        _cli(app_with_survey_db, None)