Merge branch 'feature/COMP-205_create_survey_datamodel' into 'develop'

Feature/comp 205 create survey datamodel

See merge request !43

compendium_v2/db/survey_model.py

+import logging
+
+from typing import Dict, Any
+from typing_extensions import Annotated
+
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.types import JSON
+
+
+from compendium_v2.db import db
+from compendium_v2.db.model import NREN
+
+
+logger = logging.getLogger(__name__)
+
+
+int_pk = Annotated[int, mapped_column(primary_key=True)]
+int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
+int_pk_fkSurvey = Annotated[int, mapped_column(ForeignKey("survey.year"), primary_key=True)]
+json = Annotated[Dict[str, Any], mapped_column(JSON)]
+
+
+# Unfortunately flask-sqlalchemy doesnt fully support DeclarativeBase yet.
+# See https://github.com/pallets-eco/flask-sqlalchemy/issues/1140
+# mypy: disable-error-code="name-defined"
+
+class Survey(db.Model):
+    __tablename__ = 'survey'
+    year: Mapped[int_pk]
+    survey: Mapped[json]
+    # status column?
+
+
+class SurveyResponse(db.Model):
+    __tablename__ = 'survey_response'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    survey_year: Mapped[int_pk_fkSurvey]
+    survey: Mapped[Survey] = relationship(lazy='joined')
+    answers: Mapped[json]
+    # completed column?? I think we need that..

compendium_v2/migrations/versions/366171c7ba9e_add_survey_and_response_tables.py

+"""Add survey and response tables
+
+Revision ID: 366171c7ba9e
+Revises: 42a826af0431
+Create Date: 2023-06-15 11:33:28.549679
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '366171c7ba9e'
+down_revision = '42a826af0431'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'survey',
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('survey', sa.JSON(), nullable=False),
+        sa.PrimaryKeyConstraint('year', name=op.f('pk_survey'))
+    )
+    op.create_table(
+        'survey_response',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('survey_year', sa.Integer(), nullable=False),
+        sa.Column('answers', sa.JSON(), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_survey_response_nren_id_nren')),
+        sa.ForeignKeyConstraint(['survey_year'], ['survey.year'], name=op.f('fk_survey_response_survey_year_survey')),
+        sa.PrimaryKeyConstraint('nren_id', 'survey_year', name=op.f('pk_survey_response'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('survey_response')
+    op.drop_table('survey')
+    # ### end Alembic commands ###

compendium_v2/routes/api.py

@@ -10,6 +10,7 @@ from compendium_v2.routes.staff import routes as staff_routes
 from compendium_v2.routes.organization import routes as org_routes
 from compendium_v2.routes.ec_projects import routes as ec_routes
 from compendium_v2.routes.policy import routes as policy
+from compendium_v2.routes.survey import routes as survey
 
 routes = Blueprint('compendium-v2-api', __name__)
 routes.register_blueprint(budget_routes, url_prefix='/budget')
@@ -19,6 +20,7 @@ routes.register_blueprint(staff_routes, url_prefix='/staff')
 routes.register_blueprint(org_routes, url_prefix='/organization')
 routes.register_blueprint(ec_routes, url_prefix='/ec-project')
 routes.register_blueprint(policy, url_prefix='/policy')
+routes.register_blueprint(survey, url_prefix='/survey')
 
 logger = logging.getLogger(__name__)
 

compendium_v2/routes/survey.py

+import json
+import logging
+from pathlib import Path
+from typing import Any, List, Optional
+
+from flask import Blueprint, jsonify, request
+from sqlalchemy import select
+
+from compendium_v2.db import db
+from compendium_v2.db.model import NREN
+from compendium_v2.db.survey_model import Survey, SurveyResponse
+from compendium_v2.routes import common
+
+
+routes = Blueprint('survey', __name__)
+logger = logging.getLogger(__name__)
+
+
+# TODO (partial) schemas
+
+
+@routes.route('/open', methods=['GET'])
+@common.require_accepts_json
+def open_survey() -> Any:
+
+    # just a hardcoded year and nren for development for now
+    nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one()
+    year = 1988
+
+    survey = db.session.scalar(select(Survey).where(Survey.year == year))
+    if survey is None or survey.survey == {}:
+        # TODO remove this at some point, its just convenient for now while we are changing the survey model a lot
+        p = Path(__file__).with_name('survey_model.json')
+        with p.open('r') as f:
+            survey = json.load(f)
+
+    # TODO add some magic strings in the json (like the year) and interpolate them here
+
+    data: Optional[dict] = None
+    page = 0
+    unvalidated: List[str] = []  # or should we keep track of what _was_ validated?
+
+    response = db.session.scalar(
+        select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
+    )
+    previous_response = db.session.scalar(
+        select(SurveyResponse).where(SurveyResponse.survey_year == year - 1).where(SurveyResponse.nren_id == nren.id)
+    )
+
+    if response:
+        data = response.answers["data"]
+        page = response.answers["page"]
+        unvalidated = response.answers["unvalidated"]
+    elif previous_response:
+        data = previous_response.answers["data"]
+        unvalidated = ["TODO everything?"]
+
+    open_survey: dict = {
+        "model": survey,
+        "data": data,
+        "page": page,
+        "unvalidated": unvalidated
+    }
+
+    return jsonify(open_survey)
+
+
+@routes.route('/save', methods=['POST'])
+@common.require_accepts_json
+def save_survey() -> Any:
+
+    # just a hardcoded year and nren for development for now
+    nren = db.session.execute(select(NREN).order_by(NREN.id).limit(1)).scalar_one()
+    year = 1988
+    survey = db.session.scalar(select(Survey).where(Survey.year == year))
+    if survey is None:
+        survey = Survey(year=year, survey={})
+        db.session.add(survey)
+
+    response = db.session.scalar(
+        select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
+    )
+    if response is None:
+        response = SurveyResponse(survey_year=year, nren_id=nren.id)
+        db.session.add(response)
+
+    save_survey = request.json
+    if not save_survey:
+        raise Exception("Invalid format")
+
+    response.answers = {
+        "data": save_survey["data"],
+        "page": save_survey["page"],
+        "unvalidated": save_survey["unvalidated"]
+    }
+
+    db.session.commit()
+
+    return {'success': True}

survey-frontend/src/survey_model.json → compendium_v2/routes/survey_model.json

@@ -318,7 +318,6 @@
        "type": "dropdown",
        "name": "country",
        "title": "Country:",
-       "isRequired": true,
        "choices": ["Afghanistan","Albania","Algeria","Angola","Argentina","Armenia","Australia","Austria","Azerbaijan","Bahamas","Bahrain","Bangladesh","Barbados","Belarus","Belgium","Belize","Benin","Bermuda","Bhutan","Bolivia","Bosnia And Herzegowina","Botswana","Brazil","Brunei Darussalam","Bulgaria","Burkina Faso","Burundi","Cambodia","Cameroon","Canada","Cape Verde","Chad","Chile","China","Colombia","Congo","Congo, The Democratic Republic Of The","Costa Rica","Cote D'ivoire","Croatia","Cuba","Cyprus","Czech Republic","Denmark","Dominican Republic","Ecuador","Egypt","El Salvador","Estonia","Ethiopia","Fiji","Finland","France","Gabon","Gambia","Georgia","Germany","Ghana","Greece","Grenada","Guatemala","Guinea","Guyana","Haiti","Honduras","Hong Kong","Hungary","Iceland","India","Indonesia","Iran (islamic Republic Of)","Iraq","Ireland","Israel","Italy","Jamaica","Japan","Jordan","Kazakhstan","Kenya","Korea, Republic Of","Kuwait","Kyrgyzstan","Lao People's Democratic Republic","Latvia","Lebanon","Lesotho","Liberia","Libyan Arab Jamahiriya","Lithuania","Luxembourg","Macedonia, North","Madagascar","Malawi","Malaysia","Mali","Malta","Mauritania","Mauritius","Mexico","Micronesia, Federated States Of","Moldova, Republic Of","Mongolia","Montenegro","Morocco","Mozambique","Myanmar","Namibia","Nepal","Netherlands","New Zealand","Nicaragua","Niger","Nigeria","Norway","Oman","Pakistan","Palestine","Panama","Papua New Guinea","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russian Federation","Rwanda","Samoa","Saudi Arabia","Senegal","Serbia","Serbia","Singapore","Slovakia (Slovak Republic)","Slovenia","Somalia","South Africa","South Sudan","Spain","Sri Lanka","Sudan","Suriname","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Tajikistan","Tanzania, United Republic Of","Thailand","Togo","Trinidad And Tobago","Tunisia","Turkey","Turkmenistan","Uganda","Ukraine","United Arab Emirates","United Kingdom","United Kingdom","United States","Uruguay","Uzbekistan","Venezuela","Vietnam","Virgin Islands (british)","Yemen","Zambia","Zimbabwe"]
       },
       {
@@ -635,82 +634,83 @@
          "name": "user_types",
          "title": "User types",
          "cellType": "checkbox",
+         "showInMultipleColumns": true,
          "choices": [
           {
-           "value": "universities",
-           "text": "Universities & Other (ISCED 6-8)"
+           "value": "network_services",
+           "text": "Network services"
           },
           {
-           "value": "further_education",
-           "text": "Further education (ISCED 4-5)"
+           "value": "isp_support",
+           "text": "ISP support"
           },
           {
-           "value": "secondary_schools",
-           "text": "Secondary schools (ISCED 2-3)"
+           "value": "security",
+           "text": "Security"
           },
           {
-           "value": "primary_schools",
-           "text": "Primary schools (ISCED 1)"
+           "value": "identity",
+           "text": "Identity/T&I"
           },
           {
-           "value": "institutes",
-           "text": "Research Institutes"
+           "value": "collaboration",
+           "text": "Collaboration"
           },
           {
-           "value": "cultural",
-           "text": "Libraries, Museums, Archives, Cultural institutions"
+           "value": "multimedia",
+           "text": "Multimedia"
           },
           {
-           "value": "hospitals",
-           "text": "Non-university public Hospitals"
+           "value": "storage_and_hosting",
+           "text": "Storage and Hosting"
           },
           {
-           "value": "government",
-           "text": "Government departments (national, regional, local)"
-          },
-          {
-           "value": "iros",
-           "text": "International (virtual) research organisations"
-          },
-          {
-           "value": "for_profit_orgs",
-           "text": "For-profit organisations"
+           "value": "professional_services",
+           "text": "Professional services"
           }
          ]
         }
        ],
        "rows": [
         {
-         "value": "network_services",
-         "text": "Network services"
+         "value": "universities",
+         "text": "Universities & Other (ISCED 6-8)"
         },
         {
-         "value": "isp_support",
-         "text": "ISP support"
+         "value": "further_education",
+         "text": "Further education (ISCED 4-5)"
         },
         {
-         "value": "security",
-         "text": "Security"
+         "value": "secondary_schools",
+         "text": "Secondary schools (ISCED 2-3)"
         },
         {
-         "value": "identity",
-         "text": "Identity/T&I"
+         "value": "primary_schools",
+         "text": "Primary schools (ISCED 1)"
         },
         {
-         "value": "collaboration",
-         "text": "Collaboration"
+         "value": "institutes",
+         "text": "Research Institutes"
         },
         {
-         "value": "multimedia",
-         "text": "Multimedia"
+         "value": "cultural",
+         "text": "Libraries, Museums, Archives, Cultural institutions"
+        },
+        {
+         "value": "hospitals",
+         "text": "Non-university public Hospitals"
+        },
+        {
+         "value": "government",
+         "text": "Government departments (national, regional, local)"
         },
         {
-         "value": "storage_and_hosting",
-         "text": "Storage and Hosting"
+         "value": "iros",
+         "text": "International (virtual) research organisations"
         },
         {
-         "value": "professional_services",
-         "text": "Professional services"
+         "value": "for_profit_orgs",
+         "text": "For-profit organisations"
         }
        ]
       },

compendium_v2/static/survey-bundle.js.LICENSE.txt

@@ -1062,6 +1062,8 @@
 
 /*! react-dom */
 
+/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
+
 /*! signature_pad */
 
 /*! survey-core */

survey-frontend/src/SurveyComponent.tsx

+import React, { useState, useEffect } from "react";
+import { Model } from "survey-core";
+import { Survey } from "survey-react-ui";
+import "survey-core/modern.min.css";
+
+
+function SurveyComponent() {
+
+    const [surveyModel, setSurveyModel] = useState<Model>();
+
+    // const surveyComplete = useCallback((sender) => {
+    //   console.log(sender.data);
+    // }, []);
+
+    async function getModel()
+    {
+        const response = await fetch('/api/survey/open');
+        const json = await response.json();
+        const survey = new Model(json['model']);
+
+        if (json['data'] !== null)
+        {
+            survey.data = json['data'];
+        }
+
+        // survey.onComplete.add((sender, options) => {
+        //     console.log(JSON.stringify(sender.data, null, 3));
+        // });
+
+        // TODO also use data and page info
+        // TODO also manage the list of (un)validated answers from last year
+
+        survey.onComplete.add((sender, options) => {
+
+            console.log(sender.data)
+
+            options.showSaveInProgress();
+            const xhr = new XMLHttpRequest();
+            xhr.open("POST", "/api/survey/save");
+            xhr.setRequestHeader("Content-Type", "application/json; charset=utf-8");
+            xhr.onload = xhr.onerror = function () {
+              if (xhr.status == 200) {
+                // Display the "Success" message (pass a string value to display a custom message)
+                options.showSaveSuccess();
+                // Alternatively, you can clear all messages:
+                // options.clearSaveMessages();
+              } else {
+                // Display the "Error" message (pass a string value to display a custom message)
+                options.showSaveError();
+              }
+            };
+
+            const saveData = {
+                data: sender.data,
+                page: 2,
+                unvalidated: []
+            }
+            xhr.send(JSON.stringify(saveData));
+        });
+        survey.onPartialSend.add((sender, options) => {
+            console.log(sender.data)
+
+            // TODO same as above
+
+        });
+
+        survey.onGetQuestionTitleActions.add((_, opt) => {
+          // opt.question TODO check what we can do with this..
+          console.log(opt.question.title, opt.question.value, opt.question.validators);
+          opt.titleActions = [
+            {
+              title: 'Validate pre-filled value',
+              innerCss: 'validate-pre-filled-value',
+              action: () => {
+                console.log('verified!')
+              },
+            },
+          ];
+        });
+
+        setSurveyModel(survey);
+
+    }
+
+    useEffect(() => {
+        getModel();
+    }, []);
+
+
+    if (surveyModel)
+    {
+        return (<Survey model={surveyModel} />);
+    }
+    else
+    {
+        return (<span>loading...</span>);
+    }
+
+}
+
+export default SurveyComponent;
\ No newline at end of file

survey-frontend/src/index.tsx

 
-import React, { useCallback } from 'react';
+import React from 'react';
 import { createRoot } from 'react-dom/client';
+import SurveyComponent from './SurveyComponent';
 
-//import 'bootstrap/dist/css/bootstrap.min.css';
-
-import 'survey-core/modern.min.css';
-import surveyJson from './survey_model.json';
-import { Model } from 'survey-core';
-import { Survey } from 'survey-react-ui';
 
 const container = document.getElementById('root') as HTMLElement;
 const root = createRoot(container); 
 
-const survey = new Model(surveyJson);
-// const surveyComplete = useCallback((sender) => {
-//   console.log(sender.data);
-// }, []);
-
-survey.onComplete.add((sender) => { console.log(sender.data) });
-
 
 root.render(
   <React.StrictMode>
-   <Survey model={survey} />
+   <SurveyComponent />
   </React.StrictMode>
 )
\ No newline at end of file