diff --git a/compendium_v2/__init__.py b/compendium_v2/__init__.py
index 33868c28bf74ea7320e1bfeae682b43611f0bb73..622565f2f7a3a5d39c1df9b562e11bd336dfc230 100644
--- a/compendium_v2/__init__.py
+++ b/compendium_v2/__init__.py
@@ -17,6 +17,22 @@ def migrate_database(config: dict) -> None:
migration_utils.upgrade(dsn)
+def _create_app(app_config) -> Flask:
+ # used by sphinx to create documentation without config and db migrations
+ app = Flask(__name__)
+ CORS(app)
+
+ app.config['CONFIG_PARAMS'] = app_config
+
+ from compendium_v2.routes import default
+ app.register_blueprint(default.routes, url_prefix='/')
+
+ from compendium_v2.routes import api
+ app.register_blueprint(api.routes, url_prefix='/api')
+
+ return app
+
+
def create_app() -> Flask:
"""
overrides default settings with those found
@@ -25,22 +41,12 @@ def create_app() -> Flask:
:return: a new flask app instance
"""
- assert 'SETTINGS_FILENAME' in os.environ, \
- "environment variable 'SETTINGS_FILENAME' is required"
+ assert 'SETTINGS_FILENAME' in os.environ, "environment variable 'SETTINGS_FILENAME' is required"
with open(os.environ['SETTINGS_FILENAME']) as f:
app_config = config.load(f)
- app = Flask(__name__)
- CORS(app)
-
- app.config['CONFIG_PARAMS'] = app_config
-
- from compendium_v2.routes import default
- app.register_blueprint(default.routes, url_prefix='/')
-
- from compendium_v2.routes import api
- app.register_blueprint(api.routes, url_prefix='/api')
+ app = _create_app(app_config)
logging.info('Flask app initialized')
diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/background_task/parse_excel_data.py
index d599f477a46cebb163f19a87d4559aaafcef8543..ef8e2cd61d023df86d53cf534d93db83e84a7709 100644
--- a/compendium_v2/background_task/parse_excel_data.py
+++ b/compendium_v2/background_task/parse_excel_data.py
@@ -1,6 +1,6 @@
+import logging
import openpyxl
import os
-import logging
from compendium_v2.db.model import FeeType
from compendium_v2.environment import setup_logging
@@ -9,9 +9,7 @@ setup_logging()
logger = logging.getLogger(__name__)
-EXCEL_FILE = os.path.join(
- os.path.dirname(__file__), "xlsx",
- "2021_Organisation_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
def fetch_budget_excel_data():
@@ -32,12 +30,7 @@ def fetch_budget_excel_data():
if budget is not None:
budget = round(budget / 1000000, 2)
if budget > 200:
- logger.info(
- f'{nren} has budget set to '
- f'>200M EUR for {year}. ({budget})')
-
- # process the data (e.g. save to database)
- # print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
+ logger.info(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
yield nren.upper(), budget, year
@@ -52,17 +45,12 @@ def fetch_funding_excel_data():
def hard_number_convert(s, source_name, nren, year):
if s is None:
- logger.info(
- f'Invalid Value :{nren} has empty value for {source_name}.'
- + f'for year ({year})')
+ logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year})')
return float(0)
- """ Returns True if string is a number. """
try:
return float(s)
except ValueError:
- logger.info(
- f'Invalid Value :{nren} has empty value for {source_name}.'
- + f'for year ({year}) with value ({s})')
+ logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year}) with value ({s})')
return float(0)
# iterate over the rows in the worksheet
@@ -76,7 +64,6 @@ def fetch_funding_excel_data():
gov_public_bodies = ws.cell(row=row, column=col_start + 3).value
other_european_funding = ws.cell(row=row, column=col_start + 4).value
other = ws.cell(row=row, column=col_start + 5).value
- print(nren, client_institution, commercial, geant_subsidy, gov_public_bodies, other_european_funding, other)
client_institution = hard_number_convert(client_institution, "client institution", nren, year)
commercial = hard_number_convert(commercial, "commercial", nren, year)
@@ -89,10 +76,7 @@ def fetch_funding_excel_data():
# process the data (e.g. save to database)
if nren is not None:
- yield (nren.upper(), year, client_institution,
- european_funding,
- gov_public_bodies,
- commercial, other)
+ yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
def create_points_for_year_from_2018(start_row, end_row, year, col_start):
for row in range(start_row, end_row):
@@ -112,10 +96,7 @@ def fetch_funding_excel_data():
# process the data (e.g. save to database)
if nren is not None:
- yield (nren.upper(), year, client_institution,
- european_funding,
- gov_public_bodies,
- commercial, other)
+ yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
# For 2016
yield from create_points_for_year_until_2017(8, 50, 2016, 43, 45)
@@ -147,26 +128,22 @@ def fetch_charging_structure_excel_data():
# extract the data from the row
nren = ws.cell(row=row, column=col_start).value
charging_structure = ws.cell(row=row, column=col_start + 1).value
- logger.info(
- f'NREN: {nren}, Charging Structure: {charging_structure},'
- f' Year: {year}')
+ logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
if charging_structure is not None:
if "do not charge" in charging_structure:
- charging_structure = FeeType.no_charge.value
+ charging_structure = FeeType.no_charge
elif "combination" in charging_structure:
- charging_structure = FeeType.combination.value
+ charging_structure = FeeType.combination
elif "flat" in charging_structure:
- charging_structure = FeeType.flat_fee.value
+ charging_structure = FeeType.flat_fee
elif "usage-based" in charging_structure:
- charging_structure = FeeType.usage_based_fee.value
+ charging_structure = FeeType.usage_based_fee
elif "Other" in charging_structure:
- charging_structure = FeeType.other.value
+ charging_structure = FeeType.other
else:
charging_structure = None
- logger.info(
- f'NREN: {nren}, Charging Structure: {charging_structure},'
- f' Year: {year}')
+ logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
yield nren.upper(), year, charging_structure
diff --git a/compendium_v2/config.py b/compendium_v2/config.py
index 30ee63881344dc13080fb3664d838532302b85a8..5cf1a69b3527529672a25ed6af7130fd4a14cb0e 100644
--- a/compendium_v2/config.py
+++ b/compendium_v2/config.py
@@ -1,32 +1,13 @@
import json
-
import jsonschema
+
CONFIG_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'type': 'object',
-
- 'definitions': {
- 'database-uri': {
- 'type': 'string'
- }
- },
-
'properties': {
- 'SQLALCHEMY_DATABASE_URI': {
- 'type': 'string',
- 'properties': {
- 'database-uri': {'$ref': '#definitions/database-uri'}
- },
- 'additionalProperties': False
- },
- 'SURVEY_DATABASE_URI': {
- 'type': 'string',
- 'properties': {
- 'database-uri': {'$ref': '#definitions/database-uri'}
- },
- 'additionalProperties': False
- }
+ 'SQLALCHEMY_DATABASE_URI': {'type': 'string', 'format': 'database-uri'},
+ 'SURVEY_DATABASE_URI': {'type': 'string', 'format': 'database-uri'}
},
'required': ['SQLALCHEMY_DATABASE_URI', 'SURVEY_DATABASE_URI'],
'additionalProperties': False
diff --git a/compendium_v2/db/model.py b/compendium_v2/db/model.py
index d1b04a0ac1a74b669a79bbe09233226431d15943..676727437984fcf4b0160d91060dfcc343c376c8 100644
--- a/compendium_v2/db/model.py
+++ b/compendium_v2/db/model.py
@@ -1,12 +1,13 @@
import logging
-import sqlalchemy as sa
+from decimal import Decimal
from enum import Enum
+from typing import Optional
+from typing_extensions import Annotated
-from typing import Any
+from sqlalchemy import MetaData, String
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+from sqlalchemy.schema import ForeignKey
-from sqlalchemy import MetaData
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
logger = logging.getLogger(__name__)
@@ -20,37 +21,42 @@ convention = {
metadata_obj = MetaData(naming_convention=convention)
-# https://github.com/python/mypy/issues/2477
-base_schema: Any = declarative_base(metadata=metadata_obj)
+str128 = Annotated[str, 128]
+int_pk = Annotated[int, mapped_column(primary_key=True)]
+int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=True)]
-class NREN(base_schema):
+class Base(DeclarativeBase):
+ metadata = metadata_obj
+ type_annotation_map = {
+ str128: String(128),
+ }
+
+
+class NREN(Base):
__tablename__ = 'nren'
- id = sa.Column(sa.Integer, primary_key=True)
- name = sa.Column(sa.String(128), nullable=False)
+ id: Mapped[int_pk]
+ name: Mapped[str128]
-class BudgetEntry(base_schema):
+class BudgetEntry(Base):
__tablename__ = 'budgets'
- nren_id = sa.Column(
- sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- budget = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ budget: Mapped[Decimal]
-class FundingSource(base_schema):
+class FundingSource(Base):
__tablename__ = 'funding_source'
- nren_id = sa.Column(
- sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- client_institutions = sa.Column(
- sa.Numeric(asdecimal=False), nullable=False)
- european_funding = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- gov_public_bodies = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- commercial = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- other = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ client_institutions: Mapped[Decimal]
+ european_funding: Mapped[Decimal]
+ gov_public_bodies: Mapped[Decimal]
+ commercial: Mapped[Decimal]
+ other: Mapped[Decimal]
class FeeType(Enum):
@@ -61,49 +67,45 @@ class FeeType(Enum):
other = "other"
-class ChargingStructure(base_schema):
+class ChargingStructure(Base):
__tablename__ = 'charging_structure'
- nren_id = sa.Column(
- sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- fee_type = sa.Column('fee_type', sa.Enum("flat_fee", "usage_based_fee",
- "combination", "no_charge",
- "other",
- name="fee_type"), nullable=True)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ fee_type: Mapped[Optional[FeeType]]
-class NrenStaff(base_schema):
+class NrenStaff(Base):
__tablename__ = 'nren_staff'
- nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- permanent_fte = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- subcontracted_fte = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- technical_fte = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
- non_technical_fte = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ permanent_fte: Mapped[Decimal]
+ subcontracted_fte: Mapped[Decimal]
+ technical_fte: Mapped[Decimal]
+ non_technical_fte: Mapped[Decimal]
-class ParentOrganization(base_schema):
+class ParentOrganization(Base):
__tablename__ = 'parent_organization'
- nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- organization = sa.Column(sa.String(128), nullable=False)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ organization: Mapped[str128]
-class SubOrganization(base_schema):
+class SubOrganization(Base):
__tablename__ = 'sub_organization'
- nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- organization = sa.Column(sa.String(128), primary_key=True)
- role = sa.Column(sa.String(128), nullable=False)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(lazy='joined')
+ year: Mapped[int_pk]
+ organization: Mapped[str128] = mapped_column(primary_key=True)
+ role: Mapped[str128]
-class ECProject(base_schema):
+class ECProject(Base):
__tablename__ = 'ec_project'
- nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
- nren = relationship(NREN, lazy='joined')
- year = sa.Column(sa.Integer, primary_key=True)
- project = sa.Column(sa.String(256), primary_key=True)
+ nren_id: Mapped[int_pk_fkNREN]
+ nren: Mapped[NREN] = relationship(NREN, lazy='joined')
+ year: Mapped[int_pk]
+ project: Mapped[str] = mapped_column(String(256), primary_key=True)
diff --git a/compendium_v2/environment.py b/compendium_v2/environment.py
index 55cfae65ac4f75532215fc03b551c93aa522a489..527687c42d497d6ec64a15c4b1eabfad9244db1c 100644
--- a/compendium_v2/environment.py
+++ b/compendium_v2/environment.py
@@ -7,8 +7,7 @@ LOGGING_DEFAULT_CONFIG = {
'disable_existing_loggers': False,
'formatters': {
'simple': {
- 'format': '%(asctime)s - %(name)s '
- '(%(lineno)d) - %(levelname)s - %(message)s'
+ 'format': '%(asctime)s - %(name)s (%(lineno)d) - %(levelname)s - %(message)s'
}
},
diff --git a/compendium_v2/migrations/README b/compendium_v2/migrations/README
deleted file mode 100644
index 0e048441597444a7e2850d6d7c4ce15550f79bda..0000000000000000000000000000000000000000
--- a/compendium_v2/migrations/README
+++ /dev/null
@@ -1 +0,0 @@
-Single-database configuration for Flask.
diff --git a/compendium_v2/migrations/env.py b/compendium_v2/migrations/env.py
index 5ae69c65ba9b2c844a16e54c9f03d4b0297ae68b..0307be3377e7eb332145d5822b4fb126b6f96e59 100644
--- a/compendium_v2/migrations/env.py
+++ b/compendium_v2/migrations/env.py
@@ -4,7 +4,7 @@ from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
-from compendium_v2.db.model import base_schema
+from compendium_v2.db.model import metadata_obj
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
@@ -18,7 +18,7 @@ logging.basicConfig(level=logging.INFO)
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
-target_metadata = base_schema.metadata
+target_metadata = metadata_obj
# other values from the config, defined by the needs of env.py,
# can be acquired:
diff --git a/compendium_v2/publishers/survey_publisher_2022.py b/compendium_v2/publishers/survey_publisher_2022.py
index 6334fae31446d600bfd0ece3dccc50d436524a2b..710c899d4378a7e5a9bd76246789176fb2d57e54 100644
--- a/compendium_v2/publishers/survey_publisher_2022.py
+++ b/compendium_v2/publishers/survey_publisher_2022.py
@@ -1,3 +1,11 @@
+"""
+survey_publisher_2022
+=========================
+
+This module loads the survey data from 2022 from the survey database.
+Registered as click cli command when installing compendium-v2.
+
+"""
import logging
import click
import enum
@@ -136,15 +144,11 @@ def transfer_budget():
try:
budget = float(_budget.replace('"', '').replace(',', ''))
except ValueError:
- logger.info(
- f'{nren_name} has no budget for 2022. Skipping.'
- f' ({_budget}))')
+ logger.info(f'{nren_name} has no budget for 2022. Skipping. ({_budget}))')
continue
if budget > 200:
- logger.info(
- f'{nren_name} has budget set to >200M EUR for 2022.'
- f' ({budget})')
+ logger.info(f'{nren_name} has budget set to >200M EUR for 2022. ({budget})')
if nren_name not in nren_dict:
logger.info(f'{nren_name} unknown. Skipping.')
@@ -172,9 +176,7 @@ def transfer_funding_sources():
value = float(_value.replace('"', '').replace(',', ''))
except ValueError:
name = source.name
- logger.info(
- f'{nren_name} has invalid value for {name}.'
- + f' ({_value}))')
+ logger.info(f'{nren_name} has invalid value for {name}. ({_value}))')
value = 0
nren_info = sourcedata.setdefault(
@@ -196,8 +198,7 @@ def transfer_funding_sources():
funding_source = model.FundingSource(
nren=nren_dict[nren_name],
year=2022,
- client_institutions=nren_info[
- FundingSource.CLIENT_INSTITUTIONS],
+ client_institutions=nren_info[FundingSource.CLIENT_INSTITUTIONS],
european_funding=nren_info[FundingSource.EUROPEAN_FUNDING],
gov_public_bodies=nren_info[FundingSource.GOV_PUBLIC_BODIES],
commercial=nren_info[FundingSource.COMMERCIAL],
@@ -241,11 +242,8 @@ def transfer_staff_data():
employed = nren_info[StaffQuestion.PERMANENT_FTE] + nren_info[StaffQuestion.SUBCONTRACTED_FTE]
technical = nren_info[StaffQuestion.TECHNICAL_FTE] + nren_info[StaffQuestion.NON_TECHNICAL_FTE]
- if not math.isclose(
- employed,
- technical,
- abs_tol=0.01,
- ):
+
+ if not math.isclose(employed, technical, abs_tol=0.01):
logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
f' ({employed} != {technical})')
@@ -354,15 +352,15 @@ def transfer_charging_structure():
continue
if "do not charge" in value:
- charging_structure = FeeType.no_charge.value
+ charging_structure = FeeType.no_charge
elif "combination" in value:
- charging_structure = FeeType.combination.value
+ charging_structure = FeeType.combination
elif "flat" in value:
- charging_structure = FeeType.flat_fee.value
+ charging_structure = FeeType.flat_fee
elif "usage-based" in value:
- charging_structure = FeeType.usage_based_fee.value
+ charging_structure = FeeType.usage_based_fee
elif "Other" in value:
- charging_structure = FeeType.other.value
+ charging_structure = FeeType.other
else:
charging_structure = None
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_v1.py
index 297ec047f1dba15b44487efeaca9dfcb5b484346..d07f1f42f32f0cf9315faa3e0d0c08c7dfab4792 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_v1.py
@@ -1,3 +1,12 @@
+"""
+survey_publisher_v1
+=========================
+
+This module loads the survey data from before 2022 from and excel file.
+Missing info is filled in from the survey db for some questions.
+Registered as click cli command when installing compendium-v2.
+
+"""
import logging
import math
import click
@@ -29,10 +38,7 @@ def db_budget_migration():
year = budget.year
if float(budget.budget) > 200:
- logger.info(
- f'Incorrect Data: '
- f'{abbrev} has budget set to '
- f'>200M EUR for {year}. ({budget.budget})')
+ logger.info(f'Incorrect Data: {abbrev} has budget set to >200M EUR for {year}. ({budget.budget})')
if abbrev not in nren_dict:
logger.info(f'{abbrev} unknown. Skipping.')
@@ -53,8 +59,7 @@ def db_budget_migration():
logger.info(f'{abbrev} unknown. Skipping.')
continue
- budget_entry = model.BudgetEntry(
- nren=nren_dict[abbrev], budget=budget, year=year)
+ budget_entry = model.BudgetEntry(nren=nren_dict[abbrev], budget=budget, year=year)
session.merge(budget_entry)
session.commit()
@@ -71,13 +76,10 @@ def db_funding_migration():
gov_public_bodies,
commercial, other) in data:
- _data = [client_institution, european_funding,
- gov_public_bodies, commercial, other]
+ _data = [client_institution, european_funding, gov_public_bodies, commercial, other]
total = sum(_data)
if not math.isclose(total, 100, abs_tol=0.01):
- logger.info(
- f'{abbrev} funding sources for {year}'
- f' do not sum to 100% ({total})')
+ logger.info(f'{abbrev} funding sources for {year} do not sum to 100% ({total})')
if abbrev not in nren_dict:
logger.info(f'{abbrev} unknown. Skipping.')
diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py
index e4fd41042a065ed283dd21b2b3da84f0e749bd43..ec39ab773ee426a387b49911a7399dd3b82662d2 100644
--- a/compendium_v2/routes/api.py
+++ b/compendium_v2/routes/api.py
@@ -1,15 +1,3 @@
-"""
-API Endpoints
-=========================
-
-.. contents:: :local:
-
-/api/
----------------------
-
-
-
-"""
import logging
from flask import Blueprint
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index eefeb017223196e526bc959fff4a45206cd5992e..0b2c238414bda0f4460762bb48a27464a2101f3c 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -57,7 +57,7 @@ def charging_structure_view() -> Any:
return {
'NREN': entry.nren.name,
'YEAR': int(entry.year),
- 'FEE_TYPE': entry.fee_type,
+ 'FEE_TYPE': entry.fee_type.value if entry.fee_type is not None else None,
}
with db.session_scope() as session:
diff --git a/compendium_v2/routes/default.py b/compendium_v2/routes/default.py
index eaf1420685079ac49119b3a84355df8f49d752c8..9d2ed2f9bef6e6e8d742a7d8ad02d41d99f90f5d 100644
--- a/compendium_v2/routes/default.py
+++ b/compendium_v2/routes/default.py
@@ -1,15 +1,3 @@
-"""
-Default Endpoints
-=========================
-
-.. contents:: :local:
-
-/version
----------------------
-
-.. autofunction:: compendium_v2.routes.default.version
-
-"""
import pkg_resources
from flask import Blueprint, jsonify, render_template, Response
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 0e504355593efd1a37c2ee490af810a1732d71a5..ed0e26c8809b0b3f8187af5a8afe25ed2ec16010 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -62,7 +62,7 @@ def funding_source_view() -> Any:
def _extract_data(entry: model.FundingSource):
return {
'NREN': entry.nren.name,
- 'YEAR': int(entry.year),
+ 'YEAR': entry.year,
'CLIENT_INSTITUTIONS': float(entry.client_institutions),
'EUROPEAN_FUNDING': float(entry.european_funding),
'GOV_PUBLIC_BODIES': float(entry.gov_public_bodies),
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 0a57d57a87d0486d09d37d9f70fe1abc77d37d72..73e79c4836bb42e2959ae41a22c1b094946e0f29 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -61,10 +61,10 @@ def staff_view() -> Any:
return {
'nren': entry.nren.name,
'year': entry.year,
- 'permanent_fte': entry.permanent_fte,
- 'subcontracted_fte': entry.subcontracted_fte,
- 'technical_fte': entry.technical_fte,
- 'non_technical_fte': entry.non_technical_fte
+ 'permanent_fte': float(entry.permanent_fte),
+ 'subcontracted_fte': float(entry.subcontracted_fte),
+ 'technical_fte': float(entry.technical_fte),
+ 'non_technical_fte': float(entry.non_technical_fte)
}
with db.session_scope() as session:
diff --git a/compendium_v2/survey_db/model.py b/compendium_v2/survey_db/model.py
index 45c27ddf82a2854799190b438c31576d3c5126dc..a908aa201aaae8c194614ece4ab067aa1664a43b 100644
--- a/compendium_v2/survey_db/model.py
+++ b/compendium_v2/survey_db/model.py
@@ -1,30 +1,28 @@
import logging
-import sqlalchemy as sa
+from typing import List, Optional
-from typing import Any
-
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+from sqlalchemy.schema import ForeignKey
logger = logging.getLogger(__name__)
-# https://github.com/python/mypy/issues/2477
-base_schema: Any = declarative_base()
+
+class Base(DeclarativeBase):
+ pass
-class Budgets(base_schema):
+class Budgets(Base):
__tablename__ = 'budgets'
- id = sa.Column(sa.Integer, primary_key=True)
- budget = sa.Column(sa.String)
- year = sa.Column(sa.Integer)
- country_code = sa.Column('country_code', sa.String,
- sa.ForeignKey('nrens.country_code'))
- nren = relationship('Nrens', back_populates='budgets')
+ id: Mapped[int] = mapped_column(primary_key=True)
+ budget: Mapped[Optional[str]]
+ year: Mapped[Optional[int]]
+ country_code: Mapped[Optional[str]] = mapped_column(ForeignKey('nrens.country_code'))
+ nren: Mapped[Optional['Nrens']] = relationship(back_populates='budgets')
-class Nrens(base_schema):
+class Nrens(Base):
__tablename__ = 'nrens'
- id = sa.Column(sa.Integer, primary_key=True)
- abbreviation = sa.Column(sa.String)
- country_code = sa.Column(sa.String)
- budgets = relationship('Budgets', back_populates='nren')
+ id: Mapped[int] = mapped_column(primary_key=True)
+ abbreviation: Mapped[Optional[str]]
+ country_code: Mapped[Optional[str]]
+ budgets: Mapped[List['Budgets']] = relationship(back_populates='nren')
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 1db2969abd99b080c252bb06be8b75e5b7572c82..bd4f24693c5a6e754886afa585b3e4d04945c0f3 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -9,14 +9,8 @@ Responses to valid requests are returned as JSON messages.
The server will therefore return an error unless
`application/json` is in the `Accept` request header field.
-HTTP communication and JSON grammar details are
-beyond the scope of this document.
-Please refer to [RFC 2616](https://tools.ietf.org/html/rfc2616)
-and www.json.org for more details.
-
-.. contents:: :local:
-
-.. automodule:: compendium_v2.routes.default
-
-.. automodule:: compendium_v2.routes.api
+.. qrefflask:: compendium_v2:_create_app(None)
+ :autoquickref:
+.. autoflask:: compendium_v2:_create_app(None)
+ :undoc-static:
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 48b40e921dbb72df8f82c3604f36a250bd557c14..62ba0d569569f45347cb9c6a3974ab948a79b3f7 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -73,7 +73,9 @@ release = '0.0'
extensions = [
'sphinx_rtd_theme',
'sphinx.ext.autodoc',
- 'sphinx.ext.coverage'
+ 'sphinx.ext.coverage',
+ 'sphinxcontrib.autohttp.flask',
+ 'sphinxcontrib.autohttp.flaskqref'
]
# Add any paths that contain templates here, relative to this directory.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 11a4f0a350ea8385478ed15053622c3051f13aaa..8a97f8f6cd34c7c6df1d3997d21b46fc04f798c9 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -11,3 +11,4 @@ a React web application that consumes and renders the json data.
api
development
+ publishers
diff --git a/docs/source/publishers.rst b/docs/source/publishers.rst
new file mode 100644
index 0000000000000000000000000000000000000000..85818b3f9a1370bacff99e806de11ca1951248bc
--- /dev/null
+++ b/docs/source/publishers.rst
@@ -0,0 +1,13 @@
+.. api intro
+
+Publishers
+===============
+
+
+.. contents:: :local:
+
+.. automodule:: compendium_v2.publishers.survey_publisher_v1
+ :members:
+
+.. automodule:: compendium_v2.publishers.survey_publisher_2022
+ :members:
diff --git a/requirements.txt b/requirements.txt
index 782ca2f83404d84121f60e8351c2cf8c83e5e380..f409d9fa4cb574651214e8f455436d4298aef041 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,19 +6,19 @@ flask-cors
psycopg2-binary
cryptography
SQLAlchemy
+openpyxl
pytest
pytest-mock
python-dotenv
sphinx
sphinx-rtd-theme
+sphinxcontrib-httpdomain
tox
mypy
types-docutils
types-jsonschema
types-Flask-Cors
+types-openpyxl
types-setuptools
-types-sqlalchemy
-
-openpyxl
diff --git a/test/conftest.py b/test/conftest.py
index a2507e0fea6df752d96c351fa535762ab8831462..79186212deb7f5bf06fdd61df781349e5574ddff 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -18,10 +18,7 @@ import csv
def _test_data_csv(filename):
- data_filename = os.path.join(
- os.path.dirname(__file__),
- 'data',
- filename)
+ data_filename = os.path.join(os.path.dirname(__file__), 'data', filename)
yield from csv.DictReader(open(data_filename, "r"))
@@ -40,13 +37,9 @@ def mocked_survey_db(mocker):
connect_args={'check_same_thread': False},
poolclass=StaticPool,
echo=False)
- survey_model.base_schema.metadata.create_all(engine)
- mocker.patch(
- 'compendium_v2.survey_db._SESSION_MAKER',
- sessionmaker(bind=engine))
- mocker.patch(
- 'compendium_v2.survey_db.init_db_model',
- lambda dsn: None)
+ survey_model.Base.metadata.create_all(engine)
+ mocker.patch('compendium_v2.survey_db._SESSION_MAKER', sessionmaker(bind=engine))
+ mocker.patch('compendium_v2.survey_db.init_db_model', lambda dsn: None)
@pytest.fixture
@@ -57,16 +50,10 @@ def mocked_db(mocker):
connect_args={'check_same_thread': False},
poolclass=StaticPool,
echo=False)
- model.base_schema.metadata.create_all(engine)
- mocker.patch(
- 'compendium_v2.db._SESSION_MAKER',
- sessionmaker(bind=engine))
- mocker.patch(
- 'compendium_v2.db.init_db_model',
- lambda dsn: None)
- mocker.patch(
- 'compendium_v2.migrate_database',
- lambda config: None)
+ model.Base.metadata.create_all(engine)
+ mocker.patch('compendium_v2.db._SESSION_MAKER', sessionmaker(bind=engine))
+ mocker.patch('compendium_v2.db.init_db_model', lambda dsn: None)
+ mocker.patch('compendium_v2.migrate_database', lambda config: None)
@pytest.fixture
@@ -74,8 +61,7 @@ def test_budget_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("BudgetTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {
- nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
+ nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
@@ -83,12 +69,8 @@ def test_budget_data():
budget = row["budget"]
year = row["year"]
- session.add(
- model.BudgetEntry(
- nren=nren,
- budget=float(budget),
- year=int(year))
- )
+ session.add(model.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
+
with survey_db.session_scope() as session:
data = _test_data_csv("BudgetTestData.csv")
nrens = set()
@@ -101,19 +83,10 @@ def test_budget_data():
nrens.add(nren)
- budgets_data.append(
- survey_model.Budgets(
- budget=budget,
- year=year,
- country_code=country_code
- )
- )
+ budgets_data.append(survey_model.Budgets(budget=budget, year=year, country_code=country_code))
+
for nren in nrens:
- session.add(
- survey_model.Nrens(
- abbreviation=nren,
- country_code=nren
- ))
+ session.add(survey_model.Nrens(abbreviation=nren, country_code=nren))
session.add_all(budgets_data)
@@ -123,8 +96,7 @@ def test_funding_source_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("FundingSourceTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {
- nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
+ nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
@@ -167,8 +139,7 @@ def test_staff_data():
data = list(_generate_rows())
- nren_dict = {
- nren_name: model.NREN(name=nren_name) for nren_name in [d['nren'] for d in data]}
+ nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in [d['nren'] for d in data]}
session.add_all(nren_dict.values())
for row in data:
@@ -211,8 +182,7 @@ def test_charging_structure_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("ChargingStructureTestData.csv")]
nren_names = set([row["nren"] for row in data])
- nren_dict = {
- nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
+ nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
@@ -264,27 +234,16 @@ def test_organization_data():
year = org["year"]
name = org["name"]
- session.add(
- model.ParentOrganization(
- nren=nren,
- year=year,
- organization=name
- )
- )
+ session.add(model.ParentOrganization(nren=nren, year=year, organization=name))
+
for sub_org in sub_org_data:
nren = nren_dict[sub_org["nren"]]
year = sub_org["year"]
name = sub_org["name"]
role = sub_org["role"]
- session.add(
- model.SubOrganization(
- nren=nren,
- year=year,
- organization=name,
- role=role
- )
- )
+ session.add(model.SubOrganization(nren=nren, year=year, organization=name, role=role))
+
session.commit()
@@ -318,11 +277,6 @@ def test_ec_project_data():
year = ec_project["year"]
project = ec_project["project"]
- session.add(
- model.ECProject(
- nren=nren,
- year=year,
- project=project
- )
- )
+ session.add(model.ECProject(nren=nren, year=year, project=project))
+
session.commit()
diff --git a/test/test_survey_publisher_2022.py b/test/test_survey_publisher_2022.py
index 0ec89e6ed4f9b148d93cb546b27ff0ccd6bd59e7..ec8802edec3d0a3f16d7e62cb01277dcce554ed8 100644
--- a/test/test_survey_publisher_2022.py
+++ b/test/test_survey_publisher_2022.py
@@ -182,36 +182,23 @@ def test_publisher(client, mocker, dummy_config):
('nren3', '["project3"]'),
]
- mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget',
- get_rows_as_tuples)
-
- mocker.patch(
- 'compendium_v2.publishers.survey_publisher_2022.query_funding_sources',
- funding_source_data)
-
- mocker.patch(
- 'compendium_v2.publishers.survey_publisher_2022.query_question',
- question_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget', get_rows_as_tuples)
+ mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_funding_sources', funding_source_data)
+ mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question', question_data)
with db.session_scope() as session:
- nren_names = [
- 'Nren1', 'Nren2', 'Nren3', 'Nren4',
- 'SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH'
- ]
- session.add_all(
- [model.NREN(name=nren_name) for nren_name in nren_names])
+ nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH']
+ session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
with db.session_scope() as session:
- budgets = session.query(model.BudgetEntry).order_by(
- model.BudgetEntry.nren_id.asc()).all()
+ budgets = session.query(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc()).all()
assert len(budgets) == 3
assert budgets[0].nren.name.lower() == 'nren1'
assert budgets[0].budget == 100
- funding_sources = session.query(model.FundingSource).order_by(
- model.FundingSource.nren_id.asc()).all()
+ funding_sources = session.query(model.FundingSource).order_by(model.FundingSource.nren_id.asc()).all()
assert len(funding_sources) == 3
assert funding_sources[0].nren.name.lower() == 'nren1'
assert funding_sources[0].client_institutions == 10
@@ -228,8 +215,7 @@ def test_publisher(client, mocker, dummy_config):
assert funding_sources[2].european_funding == 30
assert funding_sources[2].other == 30
- staff_data = session.query(model.NrenStaff).order_by(
- model.NrenStaff.nren_id.asc()).all()
+ staff_data = session.query(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc()).all()
assert len(staff_data) == 3
assert staff_data[0].nren.name.lower() == 'nren1'
@@ -250,8 +236,7 @@ def test_publisher(client, mocker, dummy_config):
assert staff_data[2].permanent_fte == 30
assert staff_data[2].subcontracted_fte == 0
- _org_data = session.query(model.ParentOrganization).order_by(
- model.ParentOrganization.nren_id.asc()).all()
+ _org_data = session.query(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc()).all()
assert len(_org_data) == 2
assert _org_data[0].nren.name.lower() == 'nren1'
@@ -264,14 +249,13 @@ def test_publisher(client, mocker, dummy_config):
model.ChargingStructure.nren_id.asc()).all()
assert len(charging_structures) == 3
assert charging_structures[0].nren.name.lower() == 'nren1'
- assert charging_structures[0].fee_type == 'no_charge'
+ assert charging_structures[0].fee_type == model.FeeType.no_charge
assert charging_structures[1].nren.name.lower() == 'nren2'
- assert charging_structures[1].fee_type == 'usage_based_fee'
+ assert charging_structures[1].fee_type == model.FeeType.usage_based_fee
assert charging_structures[2].nren.name.lower() == 'nren3'
- assert charging_structures[2].fee_type == 'other'
+ assert charging_structures[2].fee_type == model.FeeType.other
- _ec_data = session.query(model.ECProject).order_by(
- model.ECProject.nren_id.asc()).all()
+ _ec_data = session.query(model.ECProject).order_by(model.ECProject.nren_id.asc()).all()
assert len(_ec_data) == 3
assert _ec_data[0].nren.name.lower() == 'nren2'
diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_v1.py
index 79c809316d713da2223af1c77c069b5cec06f85e..6908b69dd374938ea82532470e653dc35426df2b 100644
--- a/test/test_survey_publisher_v1.py
+++ b/test/test_survey_publisher_v1.py
@@ -4,19 +4,15 @@ from compendium_v2 import db
from compendium_v2.db import model
from compendium_v2.publishers.survey_publisher_v1 import _cli
-EXCEL_FILE = os.path.join(
- os.path.dirname(__file__), "data",
- "2021_Organisation_DataSeries.xlsx")
+EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
def test_publisher(client, mocker, dummy_config):
- mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE',
- EXCEL_FILE)
+ mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
with db.session_scope() as session:
nren_names = ['SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH']
- session.add_all(
- [model.NREN(name=nren_name) for nren_name in nren_names])
+ session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
@@ -25,6 +21,5 @@ def test_publisher(client, mocker, dummy_config):
assert budget_count
funding_source_count = session.query(model.FundingSource.year).count()
assert funding_source_count
- charging_structure_count = session.query(model.ChargingStructure.year)\
- .count()
+ charging_structure_count = session.query(model.ChargingStructure.year).count()
assert charging_structure_count