survey.py

import json
import logging
from enum import Enum
from pathlib import Path
from typing import Any, Optional, TypedDict, List, Dict

from flask import Blueprint, jsonify, request
from sqlalchemy import select
from sqlalchemy.orm import joinedload, load_only

from compendium_v2.db import db
from compendium_v2.db.model import NREN
from compendium_v2.db.survey_model import Survey, SurveyResponse, SurveyStatus, ResponseStatus
from compendium_v2.routes import common


routes = Blueprint('survey', __name__)
logger = logging.getLogger(__name__)


LIST_SURVEYS_RESPONSE_SCHEMA = {
    '$schema': 'http://json-schema.org/draft-07/schema#',
    'definitions': {
        'response': {
            'type': 'object',
            'properties': {
                'nren': {'type': 'string'},
                'status': {'type': 'string'},
            },
            'required': ['nren', 'status'],
            'additionalProperties': False
        },
        'survey': {
            'type': 'object',
            'properties': {
                'year': {'type': 'string'},
                'status': {'type': 'string'},
                'responses': {'type': 'array', 'items': {'$ref': '#/definitions/response'}},
            },
            'required': ['year', 'status', 'responses'],
            'additionalProperties': False
        }
    },
    'type': 'array',
    'items': {'$ref': '#/definitions/survey'}
}


SURVEY_RESPONSE_SCHEMA = {
    '$schema': 'http://json-schema.org/draft-07/schema#',
    'type': 'object',
    'properties': {
        'model': {'type': 'object'},
        'data': {'type': 'object'},
        'page': {'type': 'number'},
        'verification_status': {'type': 'array', 'items': {'type': 'string'}}
    },
    'required': ['year', 'status', 'responses'],
    'additionalProperties': False
}


class VerificationStatus(str, Enum):
    New = "new"                   # a question that was not answered last year
    Answered = "answered"         # a question that was not answered last year but has an answer now
    Unverified = "unverified"     # a question that has its answered copied from last year
    Verified = "verified"         # a question for which last years answer was verified
    Edited = "edited"             # a question for which last years answer was edited


# TODO admin only
@routes.route('/list', methods=['GET'])
@common.require_accepts_json
def list_surveys() -> Any:
    """
    retrieve a list of surveys and responses, including their status

    response will be formatted as:

    .. asjson::
        compendium_v2.routes.survey.LIST_SURVEYS_RESPONSE_SCHEMA

    """
    surveys = db.session.scalars(
        select(Survey).options(
            load_only(Survey.year, Survey.status),
            joinedload(Survey.responses).load_only(SurveyResponse.status)
            .joinedload(SurveyResponse.nren).load_only(NREN.name)
        ).order_by(Survey.year.desc())
    ).unique()

    def response_key(response):
        return response.status.value + response.nren.name.lower()

    class SurveyDict(TypedDict):
        year: int
        status: str
        responses: List[Dict[str, str]]

    entries: List[SurveyDict] = [
        {
            "year": entry.year,
            "status": entry.status.value,
            "responses": [
                {"nren": r.nren.name, "status": r.status.value}
                for r in sorted(entry.responses, key=response_key)
            ]
        }
        for entry in surveys
    ]

    # add in nrens without a response if the survey is open
    nren_names = set([name for name in db.session.scalars(select(NREN.name))])
    for entry in entries:
        if entry["status"] != SurveyStatus.open.value:
            continue
        nrens_with_responses = set([r["nren"] for r in entry["responses"]])
        for nren_name in sorted(nren_names.difference(nrens_with_responses), key=str.lower):
            entry["responses"].append({"nren": nren_name, "status": "not started"})

    return jsonify(entries)


# TODO admin only
@routes.route('/new', methods=['POST'])
@common.require_accepts_json
def start_new_survey() -> Any:
    """
    endpoint to initiate a new survey

    :returns: ``{'success': True}`` or a 400 or 404 status with a descriptive message
    """
    all_surveys = db.session.scalars(select(Survey).options(load_only(Survey.status)))
    if any([survey.status != SurveyStatus.published for survey in all_surveys]):
        return "All earlier surveys should be published before starting a new one", 400

    last_survey = db.session.scalar(
        select(Survey).order_by(Survey.year.desc()).limit(1)
    )

    if not last_survey:
        return "No survey found", 404

    new_year = last_survey.year + 1
    new_survey = last_survey.survey
    new_survey = Survey(year=new_year, survey=new_survey, status=SurveyStatus.closed)
    db.session.add(new_survey)
    db.session.commit()

    return {'success': True}


# TODO admin only
@routes.route('/open/<int:year>', methods=['POST'])
@common.require_accepts_json
def open_survey(year) -> Any:
    """
    endpoint to open a survey to the nrens

    :returns: ``{'success': True}`` or a 400 or 404 status with a descriptive message
    """
    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if not survey:
        return "Survey not found", 404

    if survey.status != SurveyStatus.closed:
        return "Survey is not closed and can therefore not be opened", 400

    all_surveys = db.session.scalars(select(Survey))
    if any([s.status == SurveyStatus.open for s in all_surveys]):
        return "There already is an open survey", 400

    survey.status = SurveyStatus.open
    db.session.commit()

    return {'success': True}


# TODO admin only
@routes.route('/close/<int:year>', methods=['POST'])
@common.require_accepts_json
def close_survey(year) -> Any:
    """
    endpoint to close a survey to the nrens

    :returns: ``{'success': True}`` or a 400 or 404 status with a descriptive message
    """
    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if not survey:
        return "Survey not found", 404

    if survey.status != SurveyStatus.open:
        return "Survey is not open and can therefore not be closed", 400

    survey.status = SurveyStatus.closed
    db.session.commit()

    return {'success': True}


# TODO admin only
@routes.route('/publish/<int:year>', methods=['POST'])
@common.require_accepts_json
def publish_survey(year) -> Any:
    """
    endpoint to publish a survey to the compendium website

    :returns: ``{'success': True}`` or a 400 or 404 status with a descriptive message
    """
    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if not survey:
        return "Survey not found", 404

    if survey.status not in [SurveyStatus.closed, SurveyStatus.published]:
        return "Survey is not closed or published and can therefore not be published", 400

    if any([response.status != ResponseStatus.checked for response in survey.responses]):
        return "There are responses that arent checked yet", 400

    # TODO call new survey_publisher with all responses and the year

    survey.status = SurveyStatus.published
    db.session.commit()

    return {'success': True}


# TODO admin only
@routes.route('/try/<int:year>', methods=['GET'])
@common.require_accepts_json
def try_survey(year) -> Any:
    """
    Get a survey without any associated nren for trying out the survey.
    The survey will behave exactly as when an NREN opens it.

    response will be formatted as:

    .. asjson::
        compendium_v2.routes.survey.SURVEY_RESPONSE_SCHEMA

    """
    return get_survey(year, all_visible=False)


# TODO admin only
@routes.route('/inspect/<int:year>', methods=['GET'])
@common.require_accepts_json
def inspect_survey(year) -> Any:
    """
    Get a survey without any associated nren for inspecting all questions.
    All questions are made visible and any VisibleIf condition added to the title.

    response will be formatted as:

    .. asjson::
        compendium_v2.routes.survey.SURVEY_RESPONSE_SCHEMA

    """
    return get_survey(year, all_visible=True)


def get_survey(year, all_visible) -> Any:
    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if not survey:
        return "Survey not found", 404

    survey_json = prepare_survey_model(survey.survey, year, all_visible)

    def visible_visitor(object, items):
        for key, value in items:
            if type(value) == dict:
                visible_visitor(value, value.items())
            elif type(value) == list:
                visible_visitor(value, enumerate(value))
            elif key == 'visibleIf':
                object['title'] = object['title'] + ' (visibleif: [' + value.replace('{', '#').replace('}', '#') + '])'
                object[key] = 'true'

    if all_visible:
        visible_visitor(survey, survey.items())

    return jsonify({
        "model": survey_json,
        "data": None,
        "page": 0,
        "verification_status": {}
    })


@routes.route('/load/<int:year>/<string:nren_name>', methods=['GET'])
@common.require_accepts_json
def load_survey(year, nren_name) -> Any:
    """
    Get a survey for an nren.
    If the survey was saved before, that data will be loaded.
    If the survey was not saved before and the survey was completed last year,
    the data from last year will be prefilled.

    response will be formatted as:

    .. asjson::
        compendium_v2.routes.survey.SURVEY_RESPONSE_SCHEMA

    """
    nren = db.session.scalar(select(NREN).filter(NREN.name == nren_name))
    if not nren:
        return "NREN not found", 404

    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if not survey:
        return "Survey not found", 404

    # TODO validation (if not admin) on year (is survey open?) and nren (logged in user is part of nren?)

    survey_json = prepare_survey_model(survey.survey, year)
    data: Optional[dict] = None
    page = 0
    verification_status = {}

    response = db.session.scalar(
        select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
    )
    previous_response = db.session.scalar(
        select(SurveyResponse).where(SurveyResponse.survey_year == year-1).where(SurveyResponse.nren_id == nren.id)
    )

    if response:
        data = response.answers["data"]
        page = response.answers["page"]
        verification_status = response.answers["verification_status"]
    elif previous_response:
        # TODO add a 'migration' hook here for updating data per year
        previous_response_data: dict = previous_response.answers["data"]
        data = previous_response_data
        verification_status = {question_name: VerificationStatus.Unverified for question_name in data.keys()}

    return jsonify({
        "model": survey_json,
        "data": data,
        "page": page,
        "verification_status": verification_status
    })


def prepare_survey_model(survey, year, all_visible=False):
    if survey is None or survey == {}:
        # TODO remove this at some point, its just convenient for now while we are changing the survey model a lot
        # or is it really?? we can also keep the surveys on disk, which makes it really easy to diff, etc
        p = Path(__file__).with_name('survey_model.json')
        with p.open('r') as f:
            survey = json.load(f)

    return survey


@routes.route('/save/<int:year>/<string:nren_name>', methods=['POST'])
@common.require_accepts_json
def save_survey(year, nren_name) -> Any:
    """
    endpoint to save a survey response

    :returns: ``{'success': True}`` or a 400 or 404 status with a descriptive message
    """

    nren = db.session.scalar(select(NREN).filter(NREN.name == nren_name))
    if nren is None:
        return "NREN not found", 404

    survey = db.session.scalar(select(Survey).where(Survey.year == year))
    if survey is None:
        return "Survey not found", 404

    # TODO validation (if not admin) on year (is survey open?) and nren (logged in user is part of nren?)

    response = db.session.scalar(
        select(SurveyResponse).where(SurveyResponse.survey_year == year).where(SurveyResponse.nren_id == nren.id)
    )
    if response is None:
        response = SurveyResponse(survey_year=year, nren_id=nren.id, status=ResponseStatus.started)
        db.session.add(response)

    save_survey = request.json
    if not save_survey:
        raise Exception("Invalid format")

    response.answers = {
        "data": save_survey["data"],
        "page": save_survey["page"],
        "verification_status": save_survey["verification_status"]
    }

    # TODO set status on complete or checked when necessary
    # if admin: completed: set to checked   / not completed: leave as is (started if new)
    # if nren:  completed: set to completed / not completed: leave as is (started if new)

    db.session.commit()

    return {'success': True}