BE Changes Initial

.gitignore

@@ -31,4 +31,5 @@ htmlcov/
 node_modules
 
 # sphinx
-/docs/build/
\ No newline at end of file
+/docs/build/
+/config.json

compendium_v2/config.py

@@ -20,8 +20,15 @@ CONFIG_SCHEMA = {
             },
             'additionalProperties': False
         },
+        'SURVEY_DATABASE_URI': {
+            'type': 'string',
+            'properties': {
+                'database-uri': {'$ref': '#definitions/database-uri'}
+            },
+            'additionalProperties': False
+        }
     },
-    'required': ['SQLALCHEMY_DATABASE_URI'],
+    'required': ['SQLALCHEMY_DATABASE_URI', 'SURVEY_DATABASE_URI'],
     'additionalProperties': False
 }
 

compendium_v2/db/model.py

@@ -17,18 +17,11 @@ def _enum_names(enum_class):
     return [x.name for x in list(enum_class)]
 
 
-class BudgetType(enum.Enum):
-    YEARLY = 1
-    NREN = 2
 
 
 class BudgetEntry(base_schema):
     __tablename__ = 'budgets'
-    id = sa.Column(sa.Integer, primary_key=True)
-    budget_type = sa.Column(
-        'budget_type',
-        sa.Enum(*_enum_names(BudgetType), name='budget_type'),
-        nullable=False)
-
-    name = sa.Column(sa.String(128))
-    description = sa.Column(sa.String(2048))
+    id = sa.Column(sa.Integer, autoincrement=True)
+    nren = sa.Column(sa.String(128), primary_key=True)
+    budget = sa.Column(sa.String(128), nullable=True)
+    year = sa.Column(sa.String(128), primary_key=True)

compendium_v2/migrations/versions/95577456fcfd_initial_db.py → compendium_v2/migrations/versions/366dfd333305_initial_db_setup.py

-"""Initial DB
+"""Initial DB setup
 
-Revision ID: 95577456fcfd
-Revises:
-Create Date: 2022-12-26 08:08:09.711624
+Revision ID: 366dfd333305
+Revises: 
+Create Date: 2023-02-07 11:37:56.576183
 
 """
 from alembic import op
@@ -10,22 +10,19 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '95577456fcfd'
+revision = '366dfd333305'
 down_revision = None
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    op.create_table(
-        'budgets',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('budget_type', sa.Enum(
-            'YEARLY', 'NREN', name='budget_type'), nullable=False),
-        sa.Column('name', sa.String(length=128), nullable=True),
-        sa.Column('description', sa.String(
-            length=2048), nullable=True),
-        sa.PrimaryKeyConstraint('id')
+    op.create_table('budgets',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=True),
+    sa.Column('nren', sa.String(length=128), nullable=False),
+    sa.Column('budget', sa.String(length=128), nullable=True),
+    sa.Column('year', sa.String(length=128), nullable=False),
+    sa.PrimaryKeyConstraint('nren', 'year')
     )
 
 

compendium_v2/routes/budget.py

@@ -3,8 +3,8 @@ from typing import Any
 
 from flask import Blueprint, jsonify, current_app
 
-from compendium_v2 import db
-from compendium_v2.db import model
+from compendium_v2 import db, survey_db
+from compendium_v2.survey_db import model as survey_model
 from compendium_v2.routes import common
 
 routes = Blueprint('budget', __name__)
@@ -13,8 +13,11 @@ routes = Blueprint('budget', __name__)
 @routes.before_request
 def before_request():
     config = current_app.config['CONFIG_PARAMS']
-    dsn = config['SQLALCHEMY_DATABASE_URI']
-    db.init_db_model(dsn)
+    dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+    db.init_db_model(dsn_prn)
+    dsn_survey = config['SURVEY_DATABASE_URI']
+    survey_db.init_db_model(dsn_survey)
+
 
 
 logger = logging.getLogger(__name__)
@@ -31,12 +34,11 @@ BUDGET_RESPONSE_SCHEMA = {
             'type': 'object',
             'properties': {
                 'id': {'type': 'number'},
-                'name': {'type': 'string'},
-                'type': {'type': 'string'},
-                'description': {'type': 'string'},
-                'url': {'type': 'string'}
+                'NREN': {'type': 'string'},
+                'BUDGET': {'type': 'string'},
+                'BUDGET_YEAR': {'type': 'string'},
             },
-            'required': ['id', 'name', 'type', 'description', 'url'],
+            'required': ['id'],
             'additionalProperties': False
         }
     },
@@ -59,18 +61,24 @@ def budget_view() -> Any:
 
     :return:
     """
-    with db.session_scope() as session:
-        data = session.query(model.BudgetEntry).all()
-
-    def _extract_data(entry: model.BudgetEntry):
-        return {
-            'id': entry.id,
-            'type': entry.budget_type,
-            'name': entry.name,
-            'description': entry.description,
-            'url': ''
-        }
-
-    entries = [_extract_data(entry) for entry in data]
-
-    return jsonify(entries)
+    try:
+        with survey_db.session_scope() as session:
+            data = session.query(survey_model.Nrens).join(survey_model.Budgets,
+                                                          survey_model.Budgets.country_code == survey_model.Nrens.country_code)
+            for nren in data:
+                print(nren)
+    except Exception as e:
+        print(e)
+
+
+    # def _extract_data(entry: model.BudgetEntry):
+    #     return {
+    #         'id': entry.id,
+    #         'NREN': entry.NREN,
+    #         'BUDGET': entry.BUDGET,
+    #         'BUDGET_YEAR': entry.BUDGET_YEAR,
+    #     }
+
+    # entries = [_extract_data(entry) for entry in data]
+    #
+    # return jsonify(entries)

compendium_v2/survey_db/__init__.py

+import contextlib
+import logging
+from typing import Optional, Union, Callable, Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import sessionmaker, Session
+
+logger = logging.getLogger(__name__)
+_SESSION_MAKER: Union[None, sessionmaker] = None
+
+
+@contextlib.contextmanager
+def session_scope(
+        callback_before_close: Optional[Callable] = None) -> Iterator[Session]:
+    # best practice is to keep session scope separate from data processing
+    # cf. https://docs.sqlalchemy.org/en/13/orm/session_basics.html
+
+    assert _SESSION_MAKER
+    session = _SESSION_MAKER()
+    try:
+        yield session
+        session.commit()
+        if callback_before_close:
+            callback_before_close()
+    except SQLAlchemyError:
+        logger.error('caught sql layer exception, rolling back')
+        session.rollback()
+        raise  # re-raise, will be handled by main consumer
+    finally:
+        session.close()
+
+
+def postgresql_dsn(db_username, db_password, db_hostname, db_name, port=5432):
+    return (f'postgresql://{db_username}:{db_password}'
+            f'@{db_hostname}:{port}/{db_name}')
+
+
+def init_db_model(dsn):
+    global _SESSION_MAKER
+
+    # cf. https://docs.sqlalchemy.org/en
+    #        /latest/orm/extensions/automap.html
+    engine = create_engine(dsn, pool_size=10, max_overflow=0)
+    _SESSION_MAKER = sessionmaker(bind=engine)

compendium_v2/survey_db/model.py

+import enum
+import logging
+import sqlalchemy as sa
+from sqlalchemy.orm import sessionmaker
+
+from typing import Any
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+logger = logging.getLogger(__name__)
+
+# https://github.com/python/mypy/issues/2477
+base_schema: Any = declarative_base()
+
+
+def _enum_names(enum_class):
+    return [x.name for x in list(enum_class)]
+
+
+class Budgets(base_schema):
+    __tablename__ = 'budgets'
+    id = sa.Column(sa.Integer, primary_key=True)
+    budget = sa.Column(sa.String)
+    year = sa.Column(sa.Integer)
+    country_code = sa.Column(sa.String)
+    # nren = relationship(
+    #     'Nrens', foreign_keys=[country_code],
+    #     primaryjoin='Nrens.country_code == Budgets.country_code',
+    #     back_populates='budgets')
+
+
+class Nrens(base_schema):
+    __tablename__ = 'nrens'
+    id = sa.Column(sa.Integer, primary_key=True)
+    abbreviation = sa.Column(sa.String)
+    country_code = sa.Column(sa.String)
+    # budgets = relationship(
+    #     'Budgets', foreign_keys=[country_code],
+    #     primaryjoin='Nrens.country_code == Budgets.country_code')
+
+
+# class BudgetEntry(base_schema):
+#     # Session = sessionmaker(bind=engine)
+#     # session = Session()
+#
+#     __tablename__ = 'budgets'
+#     session.query(Budget).all()
+#     id = sa.Column(sa.Integer, autoincrement=True)
+#     nren = sa.Column(sa.String(128), primary_key=True)
+#     budget = sa.Column(sa.String(128), nullable=True)
+#     year = sa.Column(sa.String(128), primary_key=True)

docker-compose.yml

@@ -12,4 +12,4 @@ services:
       ports:
           - "65000:5432"
       volumes:
-          - ./build/db:/var/lib/postgresql
\ No newline at end of file
+          - ./build/db:/var/lib/postgresql

requirements.txt

@@ -18,4 +18,4 @@ types-docutils
 types-jsonschema
 types-Flask-Cors
 types-setuptools
-types-sqlalchemy
\ No newline at end of file
+types-sqlalchemy

webapp/src/pages/DataAnalysis.tsx

@@ -65,7 +65,7 @@ function DataAnalysis(): ReactElement {
                 return;
             }
 
-            api<BudgetMatrix>('/api/data-entries/item/' + selectedDataEntry,{
+            api<BudgetMatrix>('/api/budget' + selectedDataEntry,{
                 referrerPolicy: "unsafe-url",
                 headers: {
                     "Access-Control-Allow-Origin": "*",