Merge branch 'saket_be' into frontend-work

.gitignore

@@ -31,4 +31,5 @@ htmlcov/
 node_modules
 
 # sphinx
-/docs/build/
+/docs/build/
\ No newline at end of file
+/config.json

compendium_v2/config.py

@@ -20,8 +20,15 @@ CONFIG_SCHEMA = {
             },
             'additionalProperties': False
         },
+        'SURVEY_DATABASE_URI': {
+            'type': 'string',
+            'properties': {
+                'database-uri': {'$ref': '#definitions/database-uri'}
+            },
+            'additionalProperties': False
+        }
     },
-    'required': ['SQLALCHEMY_DATABASE_URI'],
+    'required': ['SQLALCHEMY_DATABASE_URI', 'SURVEY_DATABASE_URI'],
     'additionalProperties': False
 }
 

compendium_v2/db/model.py

@@ -17,18 +17,11 @@ def _enum_names(enum_class):
     return [x.name for x in list(enum_class)]
 
 
-class BudgetType(enum.Enum):
-    YEARLY = 1
-    NREN = 2
 
 
 class BudgetEntry(base_schema):
     __tablename__ = 'budgets'
-    id = sa.Column(sa.Integer, primary_key=True)
-    budget_type = sa.Column(
-        'budget_type',
-        sa.Enum(*_enum_names(BudgetType), name='budget_type'),
-        nullable=False)
-
-    name = sa.Column(sa.String(128))
-    description = sa.Column(sa.String(2048))
+    id = sa.Column(sa.Sequence('budgetentry_seq_id_seq'), nullable=False)
+    nren = sa.Column(sa.String(128), primary_key=True)
+    budget = sa.Column(sa.String(128), nullable=True)
+    year = sa.Column(sa.Integer, primary_key=True)

compendium_v2/migrations/versions/95577456fcfd_initial_db.py

-"""Initial DB
-
-Revision ID: 95577456fcfd
-Revises:
-Create Date: 2022-12-26 08:08:09.711624
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '95577456fcfd'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        'budgets',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('budget_type', sa.Enum(
-            'YEARLY', 'NREN', name='budget_type'), nullable=False),
-        sa.Column('name', sa.String(length=128), nullable=True),
-        sa.Column('description', sa.String(
-            length=2048), nullable=True),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-
-def downgrade():
-    op.drop_table('budgets')

compendium_v2/migrations/versions/cbcd21fcc151_initial_db.py

+"""Initial DB
+
+Revision ID: cbcd21fcc151
+Revises: 
+Create Date: 2023-02-07 15:56:22.086064
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'cbcd21fcc151'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+budget_id_seq = sa.Sequence('budgetentry_seq_id_seq') # represents the sequence
+
+
+
+def upgrade():
+    op.execute(sa.schema.CreateSequence(budget_id_seq)) # create the sequence
+    op.create_table('budgets',
+    sa.Column('id', sa.Integer, budget_id_seq, nullable=False, server_default=budget_id_seq.next_value()),
+    sa.Column('nren', sa.String(length=128), nullable=False),
+    sa.Column('budget', sa.String(length=128), nullable=True),
+    sa.Column('year', sa.Integer, nullable=False),
+    sa.PrimaryKeyConstraint('nren', 'year')
+    )
+
+
+def downgrade():
+    op.execute(
+      sa.schema.DropSequence(sa.Sequence('budgetentry_seq_id_seq')))
+    op.drop_table('budgets')

compendium_v2/routes/budget.py

 import logging
+from collections import defaultdict
 from typing import Any
 
 from flask import Blueprint, jsonify, current_app
 
-from compendium_v2 import db
+from compendium_v2 import db, survey_db
 from compendium_v2.db import model
+from compendium_v2.survey_db import model as survey_model
 from compendium_v2.routes import common
 
 routes = Blueprint('budget', __name__)
@@ -13,8 +15,10 @@ routes = Blueprint('budget', __name__)
 @routes.before_request
 def before_request():
     config = current_app.config['CONFIG_PARAMS']
-    dsn = config['SQLALCHEMY_DATABASE_URI']
-    db.init_db_model(dsn)
+    dsn_prn = config['SQLALCHEMY_DATABASE_URI']
+    db.init_db_model(dsn_prn)
+    dsn_survey = config['SURVEY_DATABASE_URI']
+    survey_db.init_db_model(dsn_survey)
 
 
 logger = logging.getLogger(__name__)
@@ -31,12 +35,11 @@ BUDGET_RESPONSE_SCHEMA = {
             'type': 'object',
             'properties': {
                 'id': {'type': 'number'},
-                'name': {'type': 'string'},
-                'type': {'type': 'string'},
-                'description': {'type': 'string'},
-                'url': {'type': 'string'}
+                'NREN': {'type': 'string'},
+                'BUDGET': {'type': 'string'},
+                'BUDGET_YEAR': {'type': 'string'},
             },
-            'required': ['id', 'name', 'type', 'description', 'url'],
+            'required': ['id'],
             'additionalProperties': False
         }
     },
@@ -59,18 +62,44 @@ def budget_view() -> Any:
 
     :return:
     """
-    with db.session_scope() as session:
-        data = session.query(model.BudgetEntry).all()
+
+   
+
+    with survey_db.session_scope() as survey_session, \
+            db.session_scope() as session:
+        
+        _entries = session.query(model.BudgetEntry)
+
+        inserted = defaultdict(dict)
+
+
+        for entry in _entries:
+            inserted[entry.nren][entry.year] = entry.budget
+        
+        data = survey_session.query(survey_model.Nrens)
+        for nren in data:
+            for budget in nren.budgets:
+                abbrev = nren.abbreviation
+                year = budget.year
+
+                if inserted.get(abbrev, {}).get(year):
+                    continue
+                else:
+                    inserted[abbrev][year] = True
+                entry = model.BudgetEntry(
+                    nren=abbrev, budget=budget.budget, year=year)
+                session.add(entry)
 
     def _extract_data(entry: model.BudgetEntry):
         return {
             'id': entry.id,
-            'type': entry.budget_type,
-            'name': entry.name,
-            'description': entry.description,
-            'url': ''
+            'NREN': entry.nren,
+            'BUDGET': entry.budget,
+            'BUDGET_YEAR': entry.year,
         }
 
-    entries = [_extract_data(entry) for entry in data]
+    with db.session_scope() as session:
+        entries = sorted([_extract_data(entry)
+                   for entry in session.query(model.BudgetEntry)], key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
 
     return jsonify(entries)

compendium_v2/survey_db/__init__.py

+import contextlib
+import logging
+from typing import Optional, Union, Callable, Iterator
+
+from sqlalchemy import create_engine
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import sessionmaker, Session
+
+logger = logging.getLogger(__name__)
+_SESSION_MAKER: Union[None, sessionmaker] = None
+
+
+@contextlib.contextmanager
+def session_scope(
+        callback_before_close: Optional[Callable] = None) -> Iterator[Session]:
+    # best practice is to keep session scope separate from data processing
+    # cf. https://docs.sqlalchemy.org/en/13/orm/session_basics.html
+
+    assert _SESSION_MAKER
+    session = _SESSION_MAKER()
+    try:
+        yield session
+        session.commit()
+        if callback_before_close:
+            callback_before_close()
+    except SQLAlchemyError:
+        logger.error('caught sql layer exception, rolling back')
+        session.rollback()
+        raise  # re-raise, will be handled by main consumer
+    finally:
+        session.close()
+
+
+def postgresql_dsn(db_username, db_password, db_hostname, db_name, port=5432):
+    return (f'postgresql://{db_username}:{db_password}'
+            f'@{db_hostname}:{port}/{db_name}')
+
+
+def init_db_model(dsn):
+    global _SESSION_MAKER
+
+    # cf. https://docs.sqlalchemy.org/en
+    #        /latest/orm/extensions/automap.html
+    engine = create_engine(dsn, pool_size=10, max_overflow=0)
+    _SESSION_MAKER = sessionmaker(bind=engine)

compendium_v2/survey_db/model.py

+import enum
+import logging
+import sqlalchemy as sa
+
+from typing import Any
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+logger = logging.getLogger(__name__)
+
+# https://github.com/python/mypy/issues/2477
+base_schema: Any = declarative_base()
+
+
+def _enum_names(enum_class):
+    return [x.name for x in list(enum_class)]
+
+
+class Budgets(base_schema):
+    __tablename__ = 'budgets'
+    id = sa.Column(sa.Integer, primary_key=True)
+    budget = sa.Column(sa.String)
+    year = sa.Column(sa.Integer)
+    country_code = sa.Column('country_code', sa.String,
+                             sa.ForeignKey('nrens.country_code'))
+    nren = relationship('Nrens', back_populates='budgets')
+
+
+class Nrens(base_schema):
+    __tablename__ = 'nrens'
+    id = sa.Column(sa.Integer, primary_key=True)
+    abbreviation = sa.Column(sa.String)
+    country_code = sa.Column(sa.String)
+    budgets = relationship('Budgets', back_populates='nren')
+
+
+# class BudgetEntry(base_schema):
+#     # Session = sessionmaker(bind=engine)
+#     # session = Session()
+#
+#     __tablename__ = 'budgets'
+#     session.query(Budget).all()
+#     id = sa.Column(sa.Integer, autoincrement=True)
+#     nren = sa.Column(sa.String(128), primary_key=True)
+#     budget = sa.Column(sa.String(128), nullable=True)
+#     year = sa.Column(sa.String(128), primary_key=True)

docker-compose.yml

@@ -12,4 +12,4 @@ services:
       ports:
           - "65000:5432"
       volumes:
-          - ./build/db:/var/lib/postgresql
+          - ./build/db:/var/lib/postgresql
\ No newline at end of file

requirements.txt

@@ -18,4 +18,4 @@ types-docutils
 types-jsonschema
 types-Flask-Cors
 types-setuptools
-types-sqlalchemy
+types-sqlalchemy
\ No newline at end of file

webapp/src/pages/DataAnalysis.tsx

@@ -65,7 +65,7 @@ function DataAnalysis(): ReactElement {
                 return;
             }
 
-            api<BudgetMatrix>('/api/data-entries/item/' + selectedDataEntry,{
+            api<BudgetMatrix>('/api/budget' + selectedDataEntry,{
                 referrerPolicy: "unsafe-url",
                 headers: {
                     "Access-Control-Allow-Origin": "*",