From 86a50f99327a00d3c2e9264e6a8740b801a521d0 Mon Sep 17 00:00:00 2001 From: Bjarke Madsen <bjarke@nordu.net> Date: Mon, 26 Dec 2022 08:11:01 +0100 Subject: [PATCH] Refactor & fix tests --- compendium_v2/environment.py | 2 +- compendium_v2/routes/api.py | 4 +- compendium_v2/routes/budget.py | 76 ++++++++++ compendium_v2/routes/common.py | 2 - compendium_v2/routes/data_entry.py | 231 ----------------------------- mypy.ini | 5 + test/conftest.py | 95 ++++++------ test/errors.log | 0 test/info.log | 0 test/test_routes.py | 28 +--- tox.ini | 5 +- 11 files changed, 134 insertions(+), 314 deletions(-) create mode 100644 compendium_v2/routes/budget.py delete mode 100644 compendium_v2/routes/data_entry.py create mode 100644 mypy.ini delete mode 100644 test/errors.log delete mode 100644 test/info.log diff --git a/compendium_v2/environment.py b/compendium_v2/environment.py index 8b9d025e..55cfae65 100644 --- a/compendium_v2/environment.py +++ b/compendium_v2/environment.py @@ -31,7 +31,7 @@ LOGGING_DEFAULT_CONFIG = { }, 'root': { - 'level': 'WARNING', + 'level': 'INFO', 'handlers': ['console'] } } diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py index 3c07cef3..d1820425 100644 --- a/compendium_v2/routes/api.py +++ b/compendium_v2/routes/api.py @@ -15,10 +15,10 @@ import logging from flask import Blueprint from compendium_v2.routes import common -from compendium_v2.routes.data_entry import routes as data_entry_routes +from compendium_v2.routes.budget import routes as budget_routes routes = Blueprint('compendium-v2-api', __name__) -routes.register_blueprint(data_entry_routes, url_prefix='/data-entries/') +routes.register_blueprint(budget_routes, url_prefix='/budget') logger = logging.getLogger(__name__) diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py new file mode 100644 index 00000000..58da60c6 --- /dev/null +++ b/compendium_v2/routes/budget.py @@ -0,0 +1,76 @@ +import logging +from typing import Any + +from flask import Blueprint, jsonify, current_app + +from compendium_v2 import db +from compendium_v2.db import model +from compendium_v2.routes import common + +routes = Blueprint('budget', __name__) + + +@routes.before_request +def before_request(): + config = current_app.config['CONFIG_PARAMS'] + dsn = config['SQLALCHEMY_DATABASE_URI'] + db.init_db_model(dsn) + + +logger = logging.getLogger(__name__) + +col_pal = ['#fd7f6f', '#7eb0d5', '#b2e061', + '#bd7ebe', '#ffb55a', '#ffee65', + '#beb9db', '#fdcce5', '#8bd3c7'] + +BUDGET_RESPONSE_SCHEMA = { + '$schema': 'http://json-schema.org/draft-07/schema#', + + 'definitions': { + 'budget': { + 'type': 'object', + 'properties': { + 'id': {'type': 'number'}, + 'name': {'type': 'string'}, + 'type': {'type': 'string'}, + 'description': {'type': 'string'}, + 'url': {'type': 'string'} + }, + 'required': ['id', 'name', 'type', 'description', 'url'], + 'additionalProperties': False + } + }, + + 'type': 'array', + 'items': {'$ref': '#/definitions/budget'} +} + + +@routes.route('/', methods=['GET']) +@common.require_accepts_json +def budget_view() -> Any: + """ + handler for /api/budget/ requests + + response will be formatted as: + + .. asjson:: + compendium_v2.routes.data_entry.BUDGET_RESPONSE_SCHEMA + + :return: + """ + with db.session_scope() as session: + data = session.query(model.BudgetEntry).all() + + def _extract_data(entry: model.BudgetEntry): + return { + 'id': entry.id, + 'type': entry.budget_type, + 'name': entry.name, + 'description': entry.description, + 'url': '' + } + + entries = [_extract_data(entry) for entry in data] + + return jsonify(entries) diff --git a/compendium_v2/routes/common.py b/compendium_v2/routes/common.py index e6de1a89..62b83f72 100644 --- a/compendium_v2/routes/common.py +++ b/compendium_v2/routes/common.py @@ -7,8 +7,6 @@ import logging from flask import Response, request logger = logging.getLogger(__name__) -_DECODE_TYPE_XML = 'xml' -_DECODE_TYPE_JSON = 'json' def require_accepts_json(f): diff --git a/compendium_v2/routes/data_entry.py b/compendium_v2/routes/data_entry.py deleted file mode 100644 index f38e8f62..00000000 --- a/compendium_v2/routes/data_entry.py +++ /dev/null @@ -1,231 +0,0 @@ -from typing import Any - -from flask import Blueprint, abort, jsonify, url_for - -from compendium_v2.db import db -from compendium_v2.db.models import (DataEntryItem, DataEntrySection, - DataSourceType) -from compendium_v2.db.survey import get_budget_by_nren, get_budget_by_year -from compendium_v2.routes import common - -routes = Blueprint('data-entry', __name__) - -col_pal = ['#fd7f6f', '#7eb0d5', '#b2e061', - '#bd7ebe', '#ffb55a', '#ffee65', - '#beb9db', '#fdcce5', '#8bd3c7'] - -DATA_ENTRY_SECTIONS_LIST_SCHEMA = { - '$schema': 'http://json-schema.org/draft-07/schema#', - - 'definitions': { - 'section': { - 'type': 'object', - 'properties': { - 'id': {'type': 'number'}, - 'name': {'type': 'string'}, - 'description': {'type': 'string'}, - 'url': {'type': 'string'} - }, - 'required': ['id', 'name', 'description', 'url'], - 'additionalProperties': False - } - }, - - 'type': 'array', - 'items': {'$ref': '#/definitions/section'} -} - -DATA_ENTRY_SECTIONS_DETAIL_SCHEMA = { - '$schema': 'http://json-schema.org/draft-07/schema#', - - 'definitions': { - 'item': { - 'type': 'object', - 'properties': { - 'id': {'type': 'number'}, - 'title': {'type': 'string'}, - 'url': {'type': 'string'} - }, - 'required': ['id', 'title', 'url'], - 'additionalProperties': False - } - }, - - 'type': 'object', - 'properties': { - 'name': {'type': 'string'}, - 'description': {'type': 'string'}, - 'items': { - 'type': 'array', - 'items': {'$ref': '#/definitions/item'} - } - }, - 'required': ['name', 'description', 'items'], - 'additionalProperties': False -} - -DATA_ENTRY_ITEM_DETAIL_SCHEMA = { - '$schema': 'http://json-schema.org/draft-07/schema#', - - 'definitions': { - 'settings': { - 'type': 'object', - 'properties': { - - }, - 'additionalProperties': False - }, - 'dataset': { - 'type': 'object', - 'properties': { - 'data': { - 'type': 'array', - 'items': {'type': ['number', 'null']} - }, - 'backgroundColor': { - 'type': 'string' - }, - 'label': { - 'type': 'string' - } - } - }, - 'data': { - 'type': 'object', - 'properties': { - 'labels': { - 'type': 'array', - 'items': {'type': 'string'}}, - 'datasets': { - 'type': 'array', - 'items': { - '$ref': '#/definitions/dataset' - } - } - }, - 'required': ['labels', 'datasets'], - 'additionalProperties': False - } - }, - - 'type': 'object', - 'properties': { - 'id': {'type': 'number'}, - 'title': {'type': 'string'}, - 'description': {'type': 'string'}, - 'settings': { - 'type': 'object', - '$ref': '#/definitions/settings' - }, - 'data': { - 'type': 'object', - '$ref': '#/definitions/data' - } - }, - 'required': ['id', 'title', 'description', 'settings', 'data'], - 'additionalProperties': False -} - - -def load_data(data_source_id: DataSourceType) -> Any: - response_data = {} - if data_source_id == DataSourceType.BUDGETS_BY_YEAR: - response_data = get_budget_by_year() - if data_source_id == DataSourceType.BUDGETS_BY_NREN: - response_data = get_budget_by_nren() - - # Enrich response data - # Add the colour formatting - for index, dataset in enumerate(response_data['datasets']): - dataset['backgroundColor'] = col_pal[index % len(col_pal)] - dataset['borderColor'] = col_pal[index % len(col_pal)] - - return response_data - - -@routes.route('/item/<int:item_id>', methods=['GET']) -@common.require_accepts_json -def item_view(item_id): - """ - handler for /api/data-entries/item/<item_id> requests - - response will be formatted as: - - .. asjson:: - compendium_v2.routes.data_entry.DATA_ENTRY_ITEM_DETAIL_SCHEMA - - :return: - """ - de_item = db.get_or_404(DataEntryItem, item_id) - # Confirm that only active sections can be loaded - if not de_item.is_active: - return abort(404) - - return jsonify({ - 'id': de_item.id, - 'title': de_item.title, - 'description': de_item.description, - 'settings': {}, - 'data': load_data(de_item.data_source) - }) - - -@routes.route('/sections/<int:section_id>', methods=['GET']) -@common.require_accepts_json -def section_view(section_id): - """ - handler for /api/data-entries/sections/<section_id> requests - - response will be formatted as: - - .. asjson:: - compendium_v2.routes.data_entry.DATA_ENTRY_SECTIONS_DETAIL_SCHEMA - - :return: - """ - de_section = db.get_or_404(DataEntrySection, section_id) - # Confirm that only active sections can be loaded - if not de_section.is_active: - return abort(404) - - items = [ - { - 'id': item.id, - 'url': url_for('.item_view', item_id=item.id), - 'title': item.title - } for item in de_section.items if item.is_active - ] - response_section = { - 'name': de_section.name, - 'description': de_section.description, - 'items': items - } - return jsonify(response_section) - - -@routes.route('/sections', methods=['GET']) -@common.require_accepts_json -def sections_view(): - """ - handler for /api/data-entries/sections requests - - response will be formatted as: - - .. asjson:: - compendium_v2.routes.data_entry.DATA_ENTRY_SECTIONS_LIST_SCHEMA - - :return: - """ - model_sections = db.session.execute( - db.select(DataEntrySection) - .filter(DataEntrySection.is_active) - .order_by(DataEntrySection.sort_order) - ).scalars() - - de_sections = [{'id': de_section.id, - 'name': de_section.name, - 'description': de_section.description, - 'url': url_for('.section_view', section_id=de_section.id) - } for de_section in model_sections] - - return jsonify(list(de_sections)) diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..cc7d0c82 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +python_version = 3.8 +disallow_untyped_defs = False +ignore_missing_imports = False +exclude = ['env/'] \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index f055fda3..bd3d6628 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,70 +3,63 @@ import os import tempfile import pytest - import compendium_v2 -from compendium_v2.db import db, db_survey -from compendium_v2.db.models import DataEntryItem, DataEntrySection -from compendium_v2.db.survey import AnnualBudgetEntry +from compendium_v2 import db +from compendium_v2.db import model +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool -def create_test_presentation_data(app): - with app.app_context(): - db.engine.execute("ATTACH DATABASE ':memory:' AS presentation") - db.create_all() - db.session.add( - DataEntrySection(id=1, - name='Section 1', - description='The first section', - is_active=True, - sort_order=1)) - db.session.add( - DataEntryItem(id=1, - title=' Sec:1 Item 1', - description='First Item in the first section', - is_active=True, - sort_order=1, - data_source='BUDGETS_BY_YEAR', - section_id=1), - ) - db.session.commit() +@pytest.fixture +def dummy_config(): + yield { + 'SQLALCHEMY_DATABASE_URI': 'sqlite://', + } -def create_test_survey_data(app): - with app.app_context(): - db_survey.engine.execute("ATTACH DATABASE ':memory:' AS survey") - db_survey.create_all() - db_survey.session.add( - AnnualBudgetEntry(id=1, country_code='AA', budget='1', year=2020)) - db_survey.session.add( - AnnualBudgetEntry(id=2, country_code='AA', budget='2', year=2021)) - db_survey.session.add( - AnnualBudgetEntry(id=3, country_code='AA', budget='3', year=2022)) - db_survey.session.add( - AnnualBudgetEntry(id=4, country_code='BB', budget='5', year=2021)) - db_survey.session.add( - AnnualBudgetEntry(id=5, country_code='BB', budget='6', year=2022)) - db_survey.session.commit() +@pytest.fixture +def mocked_db(mocker): + # cf. https://stackoverflow.com/a/33057675 + engine = create_engine( + 'sqlite://', + connect_args={'check_same_thread': False}, + poolclass=StaticPool, + echo=False) + model.base_schema.metadata.create_all(engine) + mocker.patch( + 'compendium_v2.db._SESSION_MAKER', + sessionmaker(bind=engine)) + mocker.patch( + 'compendium_v2.db.init_db_model', + lambda dsn: None) + mocker.patch( + 'compendium_v2.migrate_database', + lambda config: None) @pytest.fixture -def data_config_filename(): - test_config_data = { - 'SQLALCHEMY_DATABASE_URI': 'sqlite://', - } +def create_test_presentation_data(): + with db.session_scope() as session: + session.add( + model.BudgetEntry( + id=1, + budget_type=model.BudgetType.NREN.name, + description='Test description') + ) + + +@pytest.fixture +def data_config_filename(dummy_config): with tempfile.NamedTemporaryFile() as f: - f.write(json.dumps(test_config_data).encode('utf-8')) + f.write(json.dumps(dummy_config).encode('utf-8')) f.flush() yield f.name @pytest.fixture -def client(data_config_filename): +def client(data_config_filename, mocked_db): os.environ['SETTINGS_FILENAME'] = data_config_filename - app = compendium_v2.create_app() - test_client = app.test_client() - create_test_presentation_data(app) - create_test_survey_data(app) - - yield test_client + with compendium_v2.create_app().test_client() as c: + yield c diff --git a/test/errors.log b/test/errors.log deleted file mode 100644 index e69de29b..00000000 diff --git a/test/info.log b/test/info.log deleted file mode 100644 index e69de29b..00000000 diff --git a/test/test_routes.py b/test/test_routes.py index 292de119..a572857d 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -3,9 +3,7 @@ import json import jsonschema import pytest -from compendium_v2.routes.data_entry import (DATA_ENTRY_ITEM_DETAIL_SCHEMA, - DATA_ENTRY_SECTIONS_DETAIL_SCHEMA, - DATA_ENTRY_SECTIONS_LIST_SCHEMA) +from compendium_v2.routes.budget import BUDGET_RESPONSE_SCHEMA from compendium_v2.routes.default import VERSION_SCHEMA @@ -28,28 +26,10 @@ def test_version_request(client): jsonschema.validate(result, VERSION_SCHEMA) -def test_api_data_entry_sections(client): +def test_budget_response(client): rv = client.get( - 'api/data-entries/sections', + '/api/budget/', headers={'Accept': ['application/json']}) assert rv.status_code == 200 result = json.loads(rv.data.decode('utf-8')) - jsonschema.validate(result, DATA_ENTRY_SECTIONS_LIST_SCHEMA) - - -def test_api_data_entry_sections_detail(client): - rv = client.get( - 'api/data-entries/sections/1', - headers={'Accept': ['application/json']}) - assert rv.status_code == 200 - result = json.loads(rv.data.decode('utf-8')) - jsonschema.validate(result, DATA_ENTRY_SECTIONS_DETAIL_SCHEMA) - - -def test_api_data_entry_item_detail(client): - rv = client.get( - 'api/data-entries/item/1', - headers={'Accept': ['application/json']}) - assert rv.status_code == 200 - result = json.loads(rv.data.decode('utf-8')) - jsonschema.validate(result, DATA_ENTRY_ITEM_DETAIL_SCHEMA) + jsonschema.validate(result, BUDGET_RESPONSE_SCHEMA) diff --git a/tox.ini b/tox.ini index 6d5a03b7..5744c648 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] envlist = py39 - [flake8] exclude = venv,.tox,webapp @@ -16,9 +15,9 @@ commands = coverage run --source compendium_v2 -m pytest {posargs} coverage xml coverage html - coverage report --fail-under 85 + coverage report --fail-under 75 flake8 # Disable mypy in tox until build server supports python 3.9 -# mypy . + # mypy compendium_v2/**/*.py test/*.py sphinx-build -M html docs/source docs/build -- GitLab