diff --git a/compendium_v2/__init__.py b/compendium_v2/__init__.py
index 28dcaa14370e16802a25d735cbaffb3b6a175dc6..6f4c755e4e8328a02003578a23651d1679973231 100644
--- a/compendium_v2/__init__.py
+++ b/compendium_v2/__init__.py
@@ -8,6 +8,7 @@ from flask import Flask
from flask_cors import CORS # for debugging
from compendium_v2 import config, environment
+from compendium_v2.db import db, db_survey, migrate
def create_app():
@@ -27,6 +28,12 @@ def create_app():
app.secret_key = 'super secret session key'
app.config['CONFIG_PARAMS'] = app_config
+ app.config['SQLALCHEMY_DATABASE_URI'] = \
+ app_config['SQLALCHEMY_DATABASE_URI']
+
+ db.init_app(app)
+ db_survey.init_app(app)
+ migrate.init_app(app, db)
from compendium_v2.routes import default
app.register_blueprint(default.routes, url_prefix='/')
diff --git a/compendium_v2/db/__init__.py b/compendium_v2/db/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1011335c995ccd82747e61a551da3267b8a3a4b3
--- /dev/null
+++ b/compendium_v2/db/__init__.py
@@ -0,0 +1,19 @@
+import os
+from typing import Any
+
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy import MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+MIGRATION_DIR = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), '..', 'migrations'))
+
+# https://github.com/python/mypy/issues/2477a
+base_schema: Any = declarative_base(metadata=MetaData(schema='presentation'))
+db = SQLAlchemy(model_class=base_schema)
+
+base_survey_schema: Any = declarative_base(metadata=MetaData(schema='survey'))
+db_survey = SQLAlchemy(model_class=base_survey_schema)
+
+migrate = Migrate(directory=MIGRATION_DIR)
diff --git a/compendium_v2/db/models.py b/compendium_v2/db/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..bcd0a897f2a9476c645d3b8b37bbc7234dbec579
--- /dev/null
+++ b/compendium_v2/db/models.py
@@ -0,0 +1,65 @@
+import enum
+import logging
+
+from sqlalchemy.orm import relationship
+
+from compendium_v2.db import base_schema, db
+
+logger = logging.getLogger(__name__)
+
+
+class DataEntrySection(db.Model):
+ __tablename__ = 'data_entry_sections'
+ id = db.Column(db.Integer, primary_key=True)
+
+ name = db.Column(db.String(128))
+ description = db.Column(db.String(2048))
+
+ is_active = db.Column(db.Boolean)
+ sort_order = db.Column(db.Integer)
+
+ items = relationship('DataEntryItem')
+
+
+class DataSourceType(enum.Enum):
+ BUDGETS_BY_YEAR = 1
+ BUDGETS_BY_NREN = 2
+
+
+data_entry_settings_assoc_table = db.Table(
+ 'data_entry_settings_assoc_table',
+ base_schema.metadata,
+ db.Column('item_id', db.ForeignKey('data_entry_items.id')),
+ db.Column('setting_id', db.ForeignKey('data_entry_settings.id')))
+
+
+class DataEntryItem(db.Model):
+ __tablename__ = 'data_entry_items'
+ id = db.Column(db.Integer, primary_key=True)
+
+ title = db.Column(db.String(128))
+ description = db.Column(db.String(2048))
+
+ is_active = db.Column(db.Boolean)
+ sort_order = db.Column(db.Integer)
+
+ data_source = db.Column(db.Enum(DataSourceType))
+
+ section_id = db.Column(db.Integer, db.ForeignKey('data_entry_sections.id'))
+ section = relationship('DataEntrySection', back_populates='items')
+
+ settings = relationship('DataEntrySettings',
+ secondary=data_entry_settings_assoc_table)
+
+
+class SettingType(enum.Enum):
+ COLOUR_PALETTE = 1
+ CHART_TYPE = 2
+ HELP_ITEM = 3
+
+
+class DataEntrySettings(db.Model):
+ __tablename__ = 'data_entry_settings'
+ id = db.Column(db.Integer, primary_key=True)
+ setting_type = db.Column(db.Enum(SettingType))
+ setting_value = db.Column(db.String(512))
diff --git a/compendium_v2/db/survey.py b/compendium_v2/db/survey.py
new file mode 100644
index 0000000000000000000000000000000000000000..90e6d101eb1a12bebb9d1820bb6692a9d3633226
--- /dev/null
+++ b/compendium_v2/db/survey.py
@@ -0,0 +1,56 @@
+import logging
+
+from compendium_v2.db import db_survey
+
+logger = logging.getLogger(__name__)
+
+
+class AnnualBudgetEntry(db_survey.Model):
+ __tablename__ = 'budgets'
+ id = db_survey.Column(db_survey.Integer, primary_key=True)
+ region = db_survey.Column(db_survey.String(7))
+ country = db_survey.Column(db_survey.Text())
+ budget = db_survey.Column(db_survey.Text())
+ year = db_survey.Column(db_survey.Integer())
+ country_code = db_survey.Column(db_survey.Text())
+ region_name = db_survey.Column(db_survey.Text())
+
+
+def get_budget_by_year():
+ budget_data = db_survey.session.execute(
+ db_survey.select(AnnualBudgetEntry)
+
+ ).scalars()
+
+ annual_data = {
+
+ }
+ seen_countries = set()
+
+ for line_item in budget_data:
+ li_year = line_item.year
+ li_country_code = line_item.country_code
+ if li_year not in annual_data:
+ annual_data[li_year] = {}
+ seen_countries.add(li_country_code)
+ annual_data[li_year][li_country_code] = line_item.budget
+
+ sorted_countries = sorted(seen_countries)
+ response_data = {
+ 'labels': sorted_countries,
+ 'datasets': []
+ }
+
+ for year in sorted(annual_data.keys()):
+ dataset = {
+ 'label': str(year),
+ 'data': []
+ }
+ for country in sorted_countries:
+ budget_amount = annual_data[year].get(country)
+ dataset['data'].append(float(budget_amount)
+ if budget_amount else None)
+
+ response_data['datasets'].append(dataset)
+
+ return response_data
diff --git a/compendium_v2/migrations/README b/compendium_v2/migrations/README
new file mode 100644
index 0000000000000000000000000000000000000000..0e048441597444a7e2850d6d7c4ce15550f79bda
--- /dev/null
+++ b/compendium_v2/migrations/README
@@ -0,0 +1 @@
+Single-database configuration for Flask.
diff --git a/compendium_v2/migrations/alembic.ini b/compendium_v2/migrations/alembic.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3bc59edd53dc6ec5d20d16df02243f44ad04a1eb
--- /dev/null
+++ b/compendium_v2/migrations/alembic.ini
@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/compendium_v2/migrations/env.py b/compendium_v2/migrations/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc238e276b173b781aad505ecb452f8e819e41c3
--- /dev/null
+++ b/compendium_v2/migrations/env.py
@@ -0,0 +1,100 @@
+from __future__ import with_statement
+
+import logging
+from logging.config import fileConfig
+
+from alembic import context
+from flask import current_app
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# from compendium_v2.db.models import DataEntryItem, DataEntrySection
+from compendium_v2.db import base_schema
+
+target_metadata = base_schema.metadata
+
+config.set_main_option(
+ 'sqlalchemy.url',
+ str(current_app.extensions['migrate'].db.get_engine().url).replace(
+ '%', '%%'))
+target_db = current_app.extensions['migrate'].db
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def get_metadata():
+ if hasattr(target_db, 'metadatas'):
+ return target_db.metadatas[None]
+ return target_db.metadata
+
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL
+ and not an Engine, though an Engine is acceptable
+ here as well. By skipping the Engine creation
+ we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the
+ script output.
+
+ """
+ url = config.get_main_option('sqlalchemy.url')
+ context.configure(
+ url=url, target_metadata=get_metadata(), literal_binds=True
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine
+ and associate a connection with the context.
+
+ """
+
+ # this callback is used to prevent an auto-migration from being generated
+ # when there are no changes to the schema
+ # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+ def process_revision_directives(context, revision, directives):
+ if getattr(config.cmd_opts, 'autogenerate', False):
+ script = directives[0]
+ if script.upgrade_ops.is_empty():
+ directives[:] = []
+ logger.info('No changes in schema detected.')
+
+ connectable = current_app.extensions['migrate'].db.get_engine()
+
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection,
+ target_metadata=get_metadata(),
+ process_revision_directives=process_revision_directives,
+ **current_app.extensions['migrate'].configure_args
+ )
+
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/compendium_v2/migrations/script.py.mako b/compendium_v2/migrations/script.py.mako
new file mode 100644
index 0000000000000000000000000000000000000000..2c0156303a8df3ffdc9de87765bf801bf6bea4a5
--- /dev/null
+++ b/compendium_v2/migrations/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}
diff --git a/compendium_v2/migrations/versions/20221128_1223_1e8ba780b977_initial_data_entry_models.py b/compendium_v2/migrations/versions/20221128_1223_1e8ba780b977_initial_data_entry_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..774d35a429a82538881b74dc8ffb7340f9955261
--- /dev/null
+++ b/compendium_v2/migrations/versions/20221128_1223_1e8ba780b977_initial_data_entry_models.py
@@ -0,0 +1,69 @@
+"""Initial Data Entry models
+
+Revision ID: 1e8ba780b977
+Revises:
+Create Date: 2022-11-28 12:23:36.478734
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '1e8ba780b977'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('data_entry_sections',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('name', sa.String(length=128), nullable=True),
+ sa.Column('description', sa.String(length=2048), nullable=True),
+ sa.Column('sort_order', sa.Integer(), nullable=True),
+ sa.Column('is_active', sa.Boolean(), nullable=True),
+ sa.PrimaryKeyConstraint('id'),
+ schema='presentation'
+ )
+ op.create_table('data_entry_settings',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('setting_type',
+ sa.Enum('COLOUR_PALLETE', 'CHART_TYPE', 'HELP_ITEM', name='settingtype'),
+ nullable=True),
+ sa.Column('setting_value', sa.String(length=512), nullable=True),
+ sa.PrimaryKeyConstraint('id'),
+ schema='presentation'
+ )
+ op.create_table('data_entry_items',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('title', sa.String(length=128), nullable=True),
+ sa.Column('description', sa.String(length=2048), nullable=True),
+ sa.Column('sort_order', sa.Integer(), nullable=True),
+ sa.Column('is_visible', sa.Boolean(), nullable=True),
+ sa.Column('is_active', sa.Boolean(), nullable=True),
+ sa.Column('data_source',
+ sa.Enum('BUDGETS_BY_YEAR', 'BUDGETS_BY_NREN', name='datasourcetype'),
+ nullable=True),
+ sa.Column('section_id', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['section_id'], ['presentation.data_entry_sections.id'], ),
+ sa.PrimaryKeyConstraint('id'),
+ schema='presentation'
+ )
+ op.create_table('data_entry_settings_assoc_table',
+ sa.Column('item_id', sa.Integer(), nullable=True),
+ sa.Column('setting_id', sa.Integer(), nullable=True),
+ sa.ForeignKeyConstraint(['item_id'], ['presentation.data_entry_items.id'], ),
+ sa.ForeignKeyConstraint(['setting_id'], ['presentation.data_entry_settings.id'], ),
+ schema='presentation'
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('data_entry_settings_assoc_table', schema='presentation')
+ op.drop_table('data_entry_items', schema='presentation')
+ op.drop_table('data_entry_settings', schema='presentation')
+ op.drop_table('data_entry_sections', schema='presentation')
+ # ### end Alembic commands ###
diff --git a/requirements.txt b/requirements.txt
index 44ec01cc39cfa390f03b13817b138e032a6dabc7..5b9fb4a4e06eef8261fcdc0d88ebcc09c10408a6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,10 @@
+alembic
jsonschema
Flask
Flask-Cors
Flask-SQLAlchemy
-
+Flask-Migrate
+psycopg2
Sphinx
sphinx-rtd-theme
+SQLAlchemy
diff --git a/test/conftest.py b/test/conftest.py
index 43d9b16a936f7ad89d7ec6602b5592c26564a46f..f055fda3a6231bb6641b953b2c0f246f61e52712 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -5,13 +5,55 @@ import tempfile
import pytest
import compendium_v2
+from compendium_v2.db import db, db_survey
+from compendium_v2.db.models import DataEntryItem, DataEntrySection
+from compendium_v2.db.survey import AnnualBudgetEntry
+
+
+def create_test_presentation_data(app):
+ with app.app_context():
+ db.engine.execute("ATTACH DATABASE ':memory:' AS presentation")
+ db.create_all()
+ db.session.add(
+ DataEntrySection(id=1,
+ name='Section 1',
+ description='The first section',
+ is_active=True,
+ sort_order=1))
+ db.session.add(
+ DataEntryItem(id=1,
+ title=' Sec:1 Item 1',
+ description='First Item in the first section',
+ is_active=True,
+ sort_order=1,
+ data_source='BUDGETS_BY_YEAR',
+ section_id=1),
+ )
+ db.session.commit()
+
+
+def create_test_survey_data(app):
+ with app.app_context():
+ db_survey.engine.execute("ATTACH DATABASE ':memory:' AS survey")
+
+ db_survey.create_all()
+ db_survey.session.add(
+ AnnualBudgetEntry(id=1, country_code='AA', budget='1', year=2020))
+ db_survey.session.add(
+ AnnualBudgetEntry(id=2, country_code='AA', budget='2', year=2021))
+ db_survey.session.add(
+ AnnualBudgetEntry(id=3, country_code='AA', budget='3', year=2022))
+ db_survey.session.add(
+ AnnualBudgetEntry(id=4, country_code='BB', budget='5', year=2021))
+ db_survey.session.add(
+ AnnualBudgetEntry(id=5, country_code='BB', budget='6', year=2022))
+ db_survey.session.commit()
@pytest.fixture
def data_config_filename():
test_config_data = {
- 'SQLALCHEMY_DATABASE_URI': 'test-uri',
-
+ 'SQLALCHEMY_DATABASE_URI': 'sqlite://',
}
with tempfile.NamedTemporaryFile() as f:
f.write(json.dumps(test_config_data).encode('utf-8'))
@@ -22,5 +64,9 @@ def data_config_filename():
@pytest.fixture
def client(data_config_filename):
os.environ['SETTINGS_FILENAME'] = data_config_filename
- with compendium_v2.create_app().test_client() as c:
- yield c
+ app = compendium_v2.create_app()
+ test_client = app.test_client()
+ create_test_presentation_data(app)
+ create_test_survey_data(app)
+
+ yield test_client
diff --git a/tox.ini b/tox.ini
index b02bc9b33be56dd2970839b2336646ccba7d7eac..74a219d33af3ae0bd808ebf0d7fe2a9f42b5893d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,7 @@ envlist = py39
[flake8]
-exclude = ./.tox,./webapp
+exclude = ./.tox,./webapp,./compendium_v2/migrations
[testenv]
deps =