Added data models and first alembic migration

compendium_v2/__init__.py

@@ -8,6 +8,7 @@ from flask import Flask
 from flask_cors import CORS  # for debugging
 
 from compendium_v2 import config, environment
+from compendium_v2.db import db, db_survey, migrate
 
 
 def create_app():
@@ -27,6 +28,12 @@ def create_app():
 
     app.secret_key = 'super secret session key'
     app.config['CONFIG_PARAMS'] = app_config
+    app.config['SQLALCHEMY_DATABASE_URI'] = \
+        app_config['SQLALCHEMY_DATABASE_URI']
+
+    db.init_app(app)
+    db_survey.init_app(app)
+    migrate.init_app(app, db)
 
     from compendium_v2.routes import default
     app.register_blueprint(default.routes, url_prefix='/')

compendium_v2/db/__init__.py

+import os
+from typing import Any
+
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy import MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+MIGRATION_DIR = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '..', 'migrations'))
+
+# https://github.com/python/mypy/issues/2477a
+base_schema: Any = declarative_base(metadata=MetaData(schema='presentation'))
+db = SQLAlchemy(model_class=base_schema)
+
+base_survey_schema: Any = declarative_base(metadata=MetaData(schema='survey'))
+db_survey = SQLAlchemy(model_class=base_survey_schema)
+
+migrate = Migrate(directory=MIGRATION_DIR)

compendium_v2/db/models.py

+import enum
+import logging
+
+from sqlalchemy.orm import relationship
+
+from compendium_v2.db import base_schema, db
+
+logger = logging.getLogger(__name__)
+
+
+class DataEntrySection(db.Model):
+    __tablename__ = 'data_entry_sections'
+    id = db.Column(db.Integer, primary_key=True)
+
+    name = db.Column(db.String(128))
+    description = db.Column(db.String(2048))
+
+    is_active = db.Column(db.Boolean)
+    sort_order = db.Column(db.Integer)
+
+    items = relationship('DataEntryItem')
+
+
+class DataSourceType(enum.Enum):
+    BUDGETS_BY_YEAR = 1
+    BUDGETS_BY_NREN = 2
+
+
+data_entry_settings_assoc_table = db.Table(
+    'data_entry_settings_assoc_table',
+    base_schema.metadata,
+    db.Column('item_id', db.ForeignKey('data_entry_items.id')),
+    db.Column('setting_id', db.ForeignKey('data_entry_settings.id')))
+
+
+class DataEntryItem(db.Model):
+    __tablename__ = 'data_entry_items'
+    id = db.Column(db.Integer, primary_key=True)
+
+    title = db.Column(db.String(128))
+    description = db.Column(db.String(2048))
+
+    is_active = db.Column(db.Boolean)
+    sort_order = db.Column(db.Integer)
+
+    data_source = db.Column(db.Enum(DataSourceType))
+
+    section_id = db.Column(db.Integer, db.ForeignKey('data_entry_sections.id'))
+    section = relationship('DataEntrySection', back_populates='items')
+
+    settings = relationship('DataEntrySettings',
+                            secondary=data_entry_settings_assoc_table)
+
+
+class SettingType(enum.Enum):
+    COLOUR_PALETTE = 1
+    CHART_TYPE = 2
+    HELP_ITEM = 3
+
+
+class DataEntrySettings(db.Model):
+    __tablename__ = 'data_entry_settings'
+    id = db.Column(db.Integer, primary_key=True)
+    setting_type = db.Column(db.Enum(SettingType))
+    setting_value = db.Column(db.String(512))

compendium_v2/db/survey.py

+import logging
+
+from compendium_v2.db import db_survey
+
+logger = logging.getLogger(__name__)
+
+
+class AnnualBudgetEntry(db_survey.Model):
+    __tablename__ = 'budgets'
+    id = db_survey.Column(db_survey.Integer, primary_key=True)
+    region = db_survey.Column(db_survey.String(7))
+    country = db_survey.Column(db_survey.Text())
+    budget = db_survey.Column(db_survey.Text())
+    year = db_survey.Column(db_survey.Integer())
+    country_code = db_survey.Column(db_survey.Text())
+    region_name = db_survey.Column(db_survey.Text())
+
+
+def get_budget_by_year():
+    budget_data = db_survey.session.execute(
+        db_survey.select(AnnualBudgetEntry)
+
+    ).scalars()
+
+    annual_data = {
+
+    }
+    seen_countries = set()
+
+    for line_item in budget_data:
+        li_year = line_item.year
+        li_country_code = line_item.country_code
+        if li_year not in annual_data:
+            annual_data[li_year] = {}
+        seen_countries.add(li_country_code)
+        annual_data[li_year][li_country_code] = line_item.budget
+
+    sorted_countries = sorted(seen_countries)
+    response_data = {
+        'labels': sorted_countries,
+        'datasets': []
+    }
+
+    for year in sorted(annual_data.keys()):
+        dataset = {
+            'label': str(year),
+            'data': []
+        }
+        for country in sorted_countries:
+            budget_amount = annual_data[year].get(country)
+            dataset['data'].append(float(budget_amount)
+                                   if budget_amount else None)
+
+        response_data['datasets'].append(dataset)
+
+    return response_data

compendium_v2/migrations/README

+Single-database configuration for Flask.

compendium_v2/migrations/alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

compendium_v2/migrations/env.py

+from __future__ import with_statement
+
+import logging
+from logging.config import fileConfig
+
+from alembic import context
+from flask import current_app
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# from compendium_v2.db.models import DataEntryItem, DataEntrySection
+from compendium_v2.db import base_schema
+
+target_metadata = base_schema.metadata
+
+config.set_main_option(
+    'sqlalchemy.url',
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+        '%', '%%'))
+target_db = current_app.extensions['migrate'].db
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option('sqlalchemy.url')
+    context.configure(
+        url=url, target_metadata=get_metadata(), literal_binds=True
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    connectable = current_app.extensions['migrate'].db.get_engine()
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=get_metadata(),
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

compendium_v2/migrations/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

compendium_v2/migrations/versions/20221128_1223_1e8ba780b977_initial_data_entry_models.py

+"""Initial Data Entry models
+
+Revision ID: 1e8ba780b977
+Revises: 
+Create Date: 2022-11-28 12:23:36.478734
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '1e8ba780b977'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('data_entry_sections',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('name', sa.String(length=128), nullable=True),
+                    sa.Column('description', sa.String(length=2048), nullable=True),
+                    sa.Column('sort_order', sa.Integer(), nullable=True),
+                    sa.Column('is_active', sa.Boolean(), nullable=True),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema='presentation'
+                    )
+    op.create_table('data_entry_settings',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('setting_type',
+                              sa.Enum('COLOUR_PALLETE', 'CHART_TYPE', 'HELP_ITEM', name='settingtype'),
+                              nullable=True),
+                    sa.Column('setting_value', sa.String(length=512), nullable=True),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema='presentation'
+                    )
+    op.create_table('data_entry_items',
+                    sa.Column('id', sa.Integer(), nullable=False),
+                    sa.Column('title', sa.String(length=128), nullable=True),
+                    sa.Column('description', sa.String(length=2048), nullable=True),
+                    sa.Column('sort_order', sa.Integer(), nullable=True),
+                    sa.Column('is_visible', sa.Boolean(), nullable=True),
+                    sa.Column('is_active', sa.Boolean(), nullable=True),
+                    sa.Column('data_source',
+                              sa.Enum('BUDGETS_BY_YEAR', 'BUDGETS_BY_NREN', name='datasourcetype'),
+                              nullable=True),
+                    sa.Column('section_id', sa.Integer(), nullable=True),
+                    sa.ForeignKeyConstraint(['section_id'], ['presentation.data_entry_sections.id'], ),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema='presentation'
+                    )
+    op.create_table('data_entry_settings_assoc_table',
+                    sa.Column('item_id', sa.Integer(), nullable=True),
+                    sa.Column('setting_id', sa.Integer(), nullable=True),
+                    sa.ForeignKeyConstraint(['item_id'], ['presentation.data_entry_items.id'], ),
+                    sa.ForeignKeyConstraint(['setting_id'], ['presentation.data_entry_settings.id'], ),
+                    schema='presentation'
+                    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('data_entry_settings_assoc_table', schema='presentation')
+    op.drop_table('data_entry_items', schema='presentation')
+    op.drop_table('data_entry_settings', schema='presentation')
+    op.drop_table('data_entry_sections', schema='presentation')
+    # ### end Alembic commands ###

requirements.txt

+alembic
 jsonschema
 Flask
 Flask-Cors
 Flask-SQLAlchemy
-
+Flask-Migrate
+psycopg2
 Sphinx
 sphinx-rtd-theme
+SQLAlchemy

test/conftest.py

@@ -5,13 +5,55 @@ import tempfile
 import pytest
 
 import compendium_v2
+from compendium_v2.db import db, db_survey
+from compendium_v2.db.models import DataEntryItem, DataEntrySection
+from compendium_v2.db.survey import AnnualBudgetEntry
+
+
+def create_test_presentation_data(app):
+    with app.app_context():
+        db.engine.execute("ATTACH DATABASE ':memory:' AS presentation")
+        db.create_all()
+        db.session.add(
+            DataEntrySection(id=1,
+                             name='Section 1',
+                             description='The first section',
+                             is_active=True,
+                             sort_order=1))
+        db.session.add(
+            DataEntryItem(id=1,
+                          title=' Sec:1 Item 1',
+                          description='First Item in the first section',
+                          is_active=True,
+                          sort_order=1,
+                          data_source='BUDGETS_BY_YEAR',
+                          section_id=1),
+        )
+        db.session.commit()
+
+
+def create_test_survey_data(app):
+    with app.app_context():
+        db_survey.engine.execute("ATTACH DATABASE ':memory:' AS survey")
+
+        db_survey.create_all()
+        db_survey.session.add(
+            AnnualBudgetEntry(id=1, country_code='AA', budget='1', year=2020))
+        db_survey.session.add(
+            AnnualBudgetEntry(id=2, country_code='AA', budget='2', year=2021))
+        db_survey.session.add(
+            AnnualBudgetEntry(id=3, country_code='AA', budget='3', year=2022))
+        db_survey.session.add(
+            AnnualBudgetEntry(id=4, country_code='BB', budget='5', year=2021))
+        db_survey.session.add(
+            AnnualBudgetEntry(id=5, country_code='BB', budget='6', year=2022))
+        db_survey.session.commit()
 
 
 @pytest.fixture
 def data_config_filename():
     test_config_data = {
-        'SQLALCHEMY_DATABASE_URI': 'test-uri',
-
+        'SQLALCHEMY_DATABASE_URI': 'sqlite://',
     }
     with tempfile.NamedTemporaryFile() as f:
         f.write(json.dumps(test_config_data).encode('utf-8'))
@@ -22,5 +64,9 @@ def data_config_filename():
 @pytest.fixture
 def client(data_config_filename):
     os.environ['SETTINGS_FILENAME'] = data_config_filename
-    with compendium_v2.create_app().test_client() as c:
-        yield c
+    app = compendium_v2.create_app()
+    test_client = app.test_client()
+    create_test_presentation_data(app)
+    create_test_survey_data(app)
+
+    yield test_client

tox.ini

@@ -3,7 +3,7 @@ envlist = py39
 
 
 [flake8]
-exclude = ./.tox,./webapp
+exclude = ./.tox,./webapp,./compendium_v2/migrations
 
 [testenv]
 deps =