diff --git a/.gitignore b/.gitignore index 5dfdc26d817763827bde9217e70957178a5e1795..e76380f586e34782f660ba54116e9e0ffd1ee1b4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ htmlcov/ node_modules # sphinx -/docs/build/ \ No newline at end of file +/docs/build/ +/config.json diff --git a/compendium_v2/config.py b/compendium_v2/config.py index 49b88b7e224a60bce85b6d0f5886186b5f5c1e50..30ee63881344dc13080fb3664d838532302b85a8 100644 --- a/compendium_v2/config.py +++ b/compendium_v2/config.py @@ -20,8 +20,15 @@ CONFIG_SCHEMA = { }, 'additionalProperties': False }, + 'SURVEY_DATABASE_URI': { + 'type': 'string', + 'properties': { + 'database-uri': {'$ref': '#definitions/database-uri'} + }, + 'additionalProperties': False + } }, - 'required': ['SQLALCHEMY_DATABASE_URI'], + 'required': ['SQLALCHEMY_DATABASE_URI', 'SURVEY_DATABASE_URI'], 'additionalProperties': False } diff --git a/compendium_v2/db/model.py b/compendium_v2/db/model.py index 1f257eb1f1f40bd1b33bebea920d745e7b815982..1cc54238c5c108206fbf4edbca2f30f9eef85131 100644 --- a/compendium_v2/db/model.py +++ b/compendium_v2/db/model.py @@ -17,18 +17,11 @@ def _enum_names(enum_class): return [x.name for x in list(enum_class)] -class BudgetType(enum.Enum): - YEARLY = 1 - NREN = 2 class BudgetEntry(base_schema): __tablename__ = 'budgets' - id = sa.Column(sa.Integer, primary_key=True) - budget_type = sa.Column( - 'budget_type', - sa.Enum(*_enum_names(BudgetType), name='budget_type'), - nullable=False) - - name = sa.Column(sa.String(128)) - description = sa.Column(sa.String(2048)) + id = sa.Column(sa.Sequence('budgetentry_seq_id_seq'), nullable=False) + nren = sa.Column(sa.String(128), primary_key=True) + budget = sa.Column(sa.String(128), nullable=True) + year = sa.Column(sa.Integer, primary_key=True) diff --git a/compendium_v2/migrations/versions/95577456fcfd_initial_db.py b/compendium_v2/migrations/versions/95577456fcfd_initial_db.py deleted file mode 100644 index 76857bdae5584b90f1c810c2edb23f6476474634..0000000000000000000000000000000000000000 --- a/compendium_v2/migrations/versions/95577456fcfd_initial_db.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Initial DB - -Revision ID: 95577456fcfd -Revises: -Create Date: 2022-12-26 08:08:09.711624 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '95577456fcfd' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - op.create_table( - 'budgets', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('budget_type', sa.Enum( - 'YEARLY', 'NREN', name='budget_type'), nullable=False), - sa.Column('name', sa.String(length=128), nullable=True), - sa.Column('description', sa.String( - length=2048), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - - -def downgrade(): - op.drop_table('budgets') diff --git a/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py b/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py new file mode 100644 index 0000000000000000000000000000000000000000..8ff0e48cc2f8f5c0de79a22e16aea14d46b71369 --- /dev/null +++ b/compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py @@ -0,0 +1,36 @@ +"""Initial DB + +Revision ID: cbcd21fcc151 +Revises: +Create Date: 2023-02-07 15:56:22.086064 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cbcd21fcc151' +down_revision = None +branch_labels = None +depends_on = None + +budget_id_seq = sa.Sequence('budgetentry_seq_id_seq') # represents the sequence + + + +def upgrade(): + op.execute(sa.schema.CreateSequence(budget_id_seq)) # create the sequence + op.create_table('budgets', + sa.Column('id', sa.Integer, budget_id_seq, nullable=False, server_default=budget_id_seq.next_value()), + sa.Column('nren', sa.String(length=128), nullable=False), + sa.Column('budget', sa.String(length=128), nullable=True), + sa.Column('year', sa.Integer, nullable=False), + sa.PrimaryKeyConstraint('nren', 'year') + ) + + +def downgrade(): + op.execute( + sa.schema.DropSequence(sa.Sequence('budgetentry_seq_id_seq'))) + op.drop_table('budgets') diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py index 58da60c6da7f0397c545e4f3a5921de467c14362..7c5090ade591a0eb77736d5226bac9b8208d94bb 100644 --- a/compendium_v2/routes/budget.py +++ b/compendium_v2/routes/budget.py @@ -1,10 +1,12 @@ import logging +from collections import defaultdict from typing import Any from flask import Blueprint, jsonify, current_app -from compendium_v2 import db +from compendium_v2 import db, survey_db from compendium_v2.db import model +from compendium_v2.survey_db import model as survey_model from compendium_v2.routes import common routes = Blueprint('budget', __name__) @@ -13,8 +15,10 @@ routes = Blueprint('budget', __name__) @routes.before_request def before_request(): config = current_app.config['CONFIG_PARAMS'] - dsn = config['SQLALCHEMY_DATABASE_URI'] - db.init_db_model(dsn) + dsn_prn = config['SQLALCHEMY_DATABASE_URI'] + db.init_db_model(dsn_prn) + dsn_survey = config['SURVEY_DATABASE_URI'] + survey_db.init_db_model(dsn_survey) logger = logging.getLogger(__name__) @@ -31,12 +35,11 @@ BUDGET_RESPONSE_SCHEMA = { 'type': 'object', 'properties': { 'id': {'type': 'number'}, - 'name': {'type': 'string'}, - 'type': {'type': 'string'}, - 'description': {'type': 'string'}, - 'url': {'type': 'string'} + 'NREN': {'type': 'string'}, + 'BUDGET': {'type': 'string'}, + 'BUDGET_YEAR': {'type': 'string'}, }, - 'required': ['id', 'name', 'type', 'description', 'url'], + 'required': ['id'], 'additionalProperties': False } }, @@ -59,18 +62,44 @@ def budget_view() -> Any: :return: """ - with db.session_scope() as session: - data = session.query(model.BudgetEntry).all() + + + + with survey_db.session_scope() as survey_session, \ + db.session_scope() as session: + + _entries = session.query(model.BudgetEntry) + + inserted = defaultdict(dict) + + + for entry in _entries: + inserted[entry.nren][entry.year] = entry.budget + + data = survey_session.query(survey_model.Nrens) + for nren in data: + for budget in nren.budgets: + abbrev = nren.abbreviation + year = budget.year + + if inserted.get(abbrev, {}).get(year): + continue + else: + inserted[abbrev][year] = True + entry = model.BudgetEntry( + nren=abbrev, budget=budget.budget, year=year) + session.add(entry) def _extract_data(entry: model.BudgetEntry): return { 'id': entry.id, - 'type': entry.budget_type, - 'name': entry.name, - 'description': entry.description, - 'url': '' + 'NREN': entry.nren, + 'BUDGET': entry.budget, + 'BUDGET_YEAR': entry.year, } - entries = [_extract_data(entry) for entry in data] + with db.session_scope() as session: + entries = sorted([_extract_data(entry) + for entry in session.query(model.BudgetEntry)], key=lambda d: (d['BUDGET_YEAR'], d['NREN'])) return jsonify(entries) diff --git a/compendium_v2/survey_db/__init__.py b/compendium_v2/survey_db/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2a07293a14277e62923a2abcbdc449204e661af8 --- /dev/null +++ b/compendium_v2/survey_db/__init__.py @@ -0,0 +1,45 @@ +import contextlib +import logging +from typing import Optional, Union, Callable, Iterator + +from sqlalchemy import create_engine +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import sessionmaker, Session + +logger = logging.getLogger(__name__) +_SESSION_MAKER: Union[None, sessionmaker] = None + + +@contextlib.contextmanager +def session_scope( + callback_before_close: Optional[Callable] = None) -> Iterator[Session]: + # best practice is to keep session scope separate from data processing + # cf. https://docs.sqlalchemy.org/en/13/orm/session_basics.html + + assert _SESSION_MAKER + session = _SESSION_MAKER() + try: + yield session + session.commit() + if callback_before_close: + callback_before_close() + except SQLAlchemyError: + logger.error('caught sql layer exception, rolling back') + session.rollback() + raise # re-raise, will be handled by main consumer + finally: + session.close() + + +def postgresql_dsn(db_username, db_password, db_hostname, db_name, port=5432): + return (f'postgresql://{db_username}:{db_password}' + f'@{db_hostname}:{port}/{db_name}') + + +def init_db_model(dsn): + global _SESSION_MAKER + + # cf. https://docs.sqlalchemy.org/en + # /latest/orm/extensions/automap.html + engine = create_engine(dsn, pool_size=10, max_overflow=0) + _SESSION_MAKER = sessionmaker(bind=engine) diff --git a/compendium_v2/survey_db/model.py b/compendium_v2/survey_db/model.py new file mode 100644 index 0000000000000000000000000000000000000000..ef05e057c0c3f60480fe92bf3832d87bc69ced70 --- /dev/null +++ b/compendium_v2/survey_db/model.py @@ -0,0 +1,47 @@ +import enum +import logging +import sqlalchemy as sa + +from typing import Any + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +logger = logging.getLogger(__name__) + +# https://github.com/python/mypy/issues/2477 +base_schema: Any = declarative_base() + + +def _enum_names(enum_class): + return [x.name for x in list(enum_class)] + + +class Budgets(base_schema): + __tablename__ = 'budgets' + id = sa.Column(sa.Integer, primary_key=True) + budget = sa.Column(sa.String) + year = sa.Column(sa.Integer) + country_code = sa.Column('country_code', sa.String, + sa.ForeignKey('nrens.country_code')) + nren = relationship('Nrens', back_populates='budgets') + + +class Nrens(base_schema): + __tablename__ = 'nrens' + id = sa.Column(sa.Integer, primary_key=True) + abbreviation = sa.Column(sa.String) + country_code = sa.Column(sa.String) + budgets = relationship('Budgets', back_populates='nren') + + +# class BudgetEntry(base_schema): +# # Session = sessionmaker(bind=engine) +# # session = Session() +# +# __tablename__ = 'budgets' +# session.query(Budget).all() +# id = sa.Column(sa.Integer, autoincrement=True) +# nren = sa.Column(sa.String(128), primary_key=True) +# budget = sa.Column(sa.String(128), nullable=True) +# year = sa.Column(sa.String(128), primary_key=True) diff --git a/docker-compose.yml b/docker-compose.yml index 8ac10f23829d11b9a0553ef7945d350221199bf9..e51a090bbe0d95a592a787dd2dd1e1c8e9335c33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,4 +12,4 @@ services: ports: - "65000:5432" volumes: - - ./build/db:/var/lib/postgresql \ No newline at end of file + - ./build/db:/var/lib/postgresql diff --git a/requirements.txt b/requirements.txt index 704282cf5576b0fa4d2f75484b90491bd99aa7ec..8aa1fa7649c3e753d1ae8512d71bda5131f28340 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ types-docutils types-jsonschema types-Flask-Cors types-setuptools -types-sqlalchemy \ No newline at end of file +types-sqlalchemy diff --git a/webapp/src/pages/DataAnalysis.tsx b/webapp/src/pages/DataAnalysis.tsx index 78cd7ebc3d0e082f00f7ea81ea34aee5efc0f82d..ce181ba74e3ccc1b8f4b182ff83c016d29a5f4dc 100644 --- a/webapp/src/pages/DataAnalysis.tsx +++ b/webapp/src/pages/DataAnalysis.tsx @@ -65,7 +65,7 @@ function DataAnalysis(): ReactElement { return; } - api<BudgetMatrix>('/api/data-entries/item/' + selectedDataEntry,{ + api<BudgetMatrix>('/api/budget' + selectedDataEntry,{ referrerPolicy: "unsafe-url", headers: { "Access-Control-Allow-Origin": "*",