use flask-migrate for the migrations

README.md

@@ -63,11 +63,13 @@ survey-publisher-2022
 ## Creating a db migration after editing the sqlalchemy models
 
 ```bash
-cd compendium_v2
-alembic revision --autogenerate -m "description"
+flask db migrate -m "description"
 ```
 
 Then go to the created migration file to make any necessary additions, for example to migrate data.
 Also see https://alembic.sqlalchemy.org/en/latest/autogenerate.html#what-does-autogenerate-detect-and-what-does-it-not-detect
+Flask-migrate sets `compare_type=True` by default.
 
 Note that starting the application applies all upgrades.
+This also happens when running `flask db` commands such as `flask db downgrade`,
+so if you want to downgrade 2 or more versions you need to do so in one command, eg by specifying the revision number.

compendium_v2/__init__.py

@@ -6,15 +6,11 @@ import os
 
 from flask import Flask
 from flask_cors import CORS  # for debugging
+# the currently available stubs for flask_migrate are old (they depend on sqlalchemy 1.4 types)
+from flask_migrate import Migrate, upgrade  # type: ignore
 
 from compendium_v2 import config, environment
 from compendium_v2.db import db
-from compendium_v2.migrations import migration_utils
-
-
-def migrate_database(config: dict) -> None:
-    dsn = config['SQLALCHEMY_DATABASE_URI']
-    migration_utils.upgrade(dsn)
 
 
 def _create_app(app_config) -> Flask:
@@ -56,11 +52,14 @@ def create_app() -> Flask:
 
     app = _create_app_with_db(app_config)
 
+    Migrate(app, db, directory=os.path.join(os.path.dirname(__file__), 'migrations'))
+
     logging.info('Flask app initialized')
 
     environment.setup_logging()
 
     # run migrations on startup
-    migrate_database(app_config)
+    with app.app_context():
+        upgrade()
 
     return app

compendium_v2/alembic.ini

-# A generic, single database configuration.
-
-# only needed for generating new revision scripts
-[alembic]
-# make sure the right line is un / commented depending on which schema you want
-# a migration for
-script_location = migrations
-# script_location = cachedb_migrations
-# change this to run migrations from the command line
-sqlalchemy.url = postgresql://compendium:compendium321@localhost:65000/compendium

compendium_v2/migrations/README

+Single-database configuration for Flask.

compendium_v2/migrations/alembic.ini

+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

compendium_v2/migrations/env.py

 import logging
+from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from flask import current_app
 
 from alembic import context
-from compendium_v2.db import metadata_obj
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -12,13 +11,34 @@ config = context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
-logging.basicConfig(level=logging.INFO)
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+
+def get_engine():
+    try:
+        # this works with Flask-SQLAlchemy<3 and Alchemical
+        return current_app.extensions['migrate'].db.get_engine()
+    except TypeError:
+        # this works with Flask-SQLAlchemy>=3
+        return current_app.extensions['migrate'].db.engine
+
+
+def get_engine_url():
+    try:
+        return get_engine().url.render_as_string(hide_password=False).replace(
+            '%', '%%')
+    except AttributeError:
+        return str(get_engine().url).replace('%', '%%')
+
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = metadata_obj
+config.set_main_option('sqlalchemy.url', get_engine_url())
+target_db = current_app.extensions['migrate'].db
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -26,6 +46,12 @@ target_metadata = metadata_obj
 # ... etc.
 
 
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -40,10 +66,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url,
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
+        url=url, target_metadata=get_metadata(), literal_binds=True
     )
 
     with context.begin_transaction():
@@ -57,15 +80,25 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section),
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
-    )
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    connectable = get_engine()
 
     with connectable.connect() as connection:
         context.configure(
-            connection=connection, target_metadata=target_metadata
+            connection=connection,
+            target_metadata=get_metadata(),
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
         )
 
         with context.begin_transaction():

compendium_v2/migrations/migration_utils.py

-import logging
-import os
-
-from alembic.config import Config
-from alembic import command
-
-logger = logging.getLogger(__name__)
-DEFAULT_MIGRATIONS_DIRECTORY = os.path.dirname(__file__)
-
-
-def upgrade(dsn, migrations_directory=DEFAULT_MIGRATIONS_DIRECTORY):
-    """
-    migrate db to head version
-
-    cf. https://stackoverflow.com/a/43530495,
-        https://stackoverflow.com/a/54402853
-
-    :param dsn: dsn string, passed to alembic
-    :param migrations_directory: full path to migrations directory
-        (default is this directory)
-    :return:
-    """
-    alembic_config = Config()
-    alembic_config.set_main_option('script_location', migrations_directory)
-    alembic_config.set_main_option('sqlalchemy.url', dsn)
-    command.upgrade(alembic_config, 'head')
-
-
-def postgresql_dsn(db_username, db_password, db_hostname, db_name, port=5432):
-    return (f'postgresql://{db_username}:{db_password}'
-            f'@{db_hostname}:{port}/{db_name}')
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    upgrade(postgresql_dsn(
-        db_username='compendium',
-        db_password='compendium321',
-        db_hostname='localhost',
-        db_name='compendium',
-        port=65000))

requirements.txt

@@ -3,6 +3,7 @@ click~=8.1
 jsonschema~=4.17
 flask~=2.2
 flask-cors~=3.0
+flask-migrate~=4.0
 flask-sqlalchemy~=3.0
 openpyxl~=3.1
 psycopg2-binary~=2.9

setup.py

@@ -15,6 +15,7 @@ setup(
         'jsonschema~=4.17',
         'flask~=2.2',
         'flask-cors~=3.0',
+        'flask-migrate~=4.0',
         'flask-sqlalchemy~=3.0',
         'openpyxl~=3.1',
         'psycopg2-binary~=2.9',