Skip to content
Snippets Groups Projects
Commit db20d475 authored by Remco Tukker's avatar Remco Tukker
Browse files

small readability improvements

parent 3c3fb6ac
No related branches found
No related tags found
1 merge request!12Feature/comp 114 python code quality
Hide whitespace changes
Inline Side-by-side
compendium_v2/__init__.py
+ 1
− 2
View file @ db20d475
......@@ -25,8 +25,7 @@ def create_app() -> Flask:
:return: a new flask app instance
"""
assert 'SETTINGS_FILENAME' in os.environ, \
"environment variable 'SETTINGS_FILENAME' is required"
assert 'SETTINGS_FILENAME' in os.environ, "environment variable 'SETTINGS_FILENAME' is required"
with open(os.environ['SETTINGS_FILENAME']) as f:
app_config = config.load(f)
......
compendium_v2/publishers/survey_publisher_2022.py
+ 6
− 16
View file @ db20d475
......@@ -136,15 +136,11 @@ def transfer_budget():
try:
budget = float(_budget.replace('"', '').replace(',', ''))
except ValueError:
logger.info(
f'{nren_name} has no budget for 2022. Skipping.'
f' ({_budget}))')
logger.info(f'{nren_name} has no budget for 2022. Skipping. ({_budget}))')
continue
if budget > 200:
logger.info(
f'{nren_name} has budget set to >200M EUR for 2022.'
f' ({budget})')
logger.info(f'{nren_name} has budget set to >200M EUR for 2022. ({budget})')
if nren_name not in nren_dict:
logger.info(f'{nren_name} unknown. Skipping.')
......@@ -172,9 +168,7 @@ def transfer_funding_sources():
value = float(_value.replace('"', '').replace(',', ''))
except ValueError:
name = source.name
logger.info(
f'{nren_name} has invalid value for {name}.'
+ f' ({_value}))')
logger.info(f'{nren_name} has invalid value for {name}. ({_value}))')
value = 0
nren_info = sourcedata.setdefault(
......@@ -196,8 +190,7 @@ def transfer_funding_sources():
funding_source = model.FundingSource(
nren=nren_dict[nren_name],
year=2022,
client_institutions=nren_info[
FundingSource.CLIENT_INSTITUTIONS],
client_institutions=nren_info[FundingSource.CLIENT_INSTITUTIONS],
european_funding=nren_info[FundingSource.EUROPEAN_FUNDING],
gov_public_bodies=nren_info[FundingSource.GOV_PUBLIC_BODIES],
commercial=nren_info[FundingSource.COMMERCIAL],
......@@ -241,11 +234,8 @@ def transfer_staff_data():
employed = nren_info[StaffQuestion.PERMANENT_FTE] + nren_info[StaffQuestion.SUBCONTRACTED_FTE]
technical = nren_info[StaffQuestion.TECHNICAL_FTE] + nren_info[StaffQuestion.NON_TECHNICAL_FTE]
if not math.isclose(
employed,
technical,
abs_tol=0.01,
):
if not math.isclose(employed, technical, abs_tol=0.01):
logger.info(f'{nren_name} FTE do not equal across employed/technical categories.'
f' ({employed} != {technical})')
......
compendium_v2/publishers/survey_publisher_v1.py
+ 4
− 11
View file @ db20d475
......@@ -29,10 +29,7 @@ def db_budget_migration():
year = budget.year
if float(budget.budget) > 200:
logger.info(
f'Incorrect Data: '
f'{abbrev} has budget set to '
f'>200M EUR for {year}. ({budget.budget})')
logger.info(f'Incorrect Data: {abbrev} has budget set to >200M EUR for {year}. ({budget.budget})')
if abbrev not in nren_dict:
logger.info(f'{abbrev} unknown. Skipping.')
......@@ -53,8 +50,7 @@ def db_budget_migration():
logger.info(f'{abbrev} unknown. Skipping.')
continue
budget_entry = model.BudgetEntry(
nren=nren_dict[abbrev], budget=budget, year=year)
budget_entry = model.BudgetEntry(nren=nren_dict[abbrev], budget=budget, year=year)
session.merge(budget_entry)
session.commit()
......@@ -71,13 +67,10 @@ def db_funding_migration():
gov_public_bodies,
commercial, other) in data:
_data = [client_institution, european_funding,
gov_public_bodies, commercial, other]
_data = [client_institution, european_funding, gov_public_bodies, commercial, other]
total = sum(_data)
if not math.isclose(total, 100, abs_tol=0.01):
logger.info(
f'{abbrev} funding sources for {year}'
f' do not sum to 100% ({total})')
logger.info(f'{abbrev} funding sources for {year} do not sum to 100% ({total})')
if abbrev not in nren_dict:
logger.info(f'{abbrev} unknown. Skipping.')
......
test/conftest.py
+ 21
− 67
View file @ db20d475
......@@ -18,10 +18,7 @@ import csv
def _test_data_csv(filename):
data_filename = os.path.join(
os.path.dirname(__file__),
'data',
filename)
data_filename = os.path.join(os.path.dirname(__file__), 'data', filename)
yield from csv.DictReader(open(data_filename, "r"))
......@@ -41,12 +38,8 @@ def mocked_survey_db(mocker):
poolclass=StaticPool,
echo=False)
survey_model.Base.metadata.create_all(engine)
mocker.patch(
'compendium_v2.survey_db._SESSION_MAKER',
sessionmaker(bind=engine))
mocker.patch(
'compendium_v2.survey_db.init_db_model',
lambda dsn: None)
mocker.patch('compendium_v2.survey_db._SESSION_MAKER', sessionmaker(bind=engine))
mocker.patch('compendium_v2.survey_db.init_db_model', lambda dsn: None)
@pytest.fixture
......@@ -58,15 +51,9 @@ def mocked_db(mocker):
poolclass=StaticPool,
echo=False)
model.Base.metadata.create_all(engine)
mocker.patch(
'compendium_v2.db._SESSION_MAKER',
sessionmaker(bind=engine))
mocker.patch(
'compendium_v2.db.init_db_model',
lambda dsn: None)
mocker.patch(
'compendium_v2.migrate_database',
lambda config: None)
mocker.patch('compendium_v2.db._SESSION_MAKER', sessionmaker(bind=engine))
mocker.patch('compendium_v2.db.init_db_model', lambda dsn: None)
mocker.patch('compendium_v2.migrate_database', lambda config: None)
@pytest.fixture
......@@ -74,8 +61,7 @@ def test_budget_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("BudgetTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......@@ -83,12 +69,8 @@ def test_budget_data():
budget = row["budget"]
year = row["year"]
session.add(
model.BudgetEntry(
nren=nren,
budget=float(budget),
year=int(year))
)
session.add(model.BudgetEntry(nren=nren, budget=float(budget), year=int(year)))
with survey_db.session_scope() as session:
data = _test_data_csv("BudgetTestData.csv")
nrens = set()
......@@ -101,19 +83,10 @@ def test_budget_data():
nrens.add(nren)
budgets_data.append(
survey_model.Budgets(
budget=budget,
year=year,
country_code=country_code
)
)
budgets_data.append(survey_model.Budgets(budget=budget, year=year, country_code=country_code))
for nren in nrens:
session.add(
survey_model.Nrens(
abbreviation=nren,
country_code=nren
))
session.add(survey_model.Nrens(abbreviation=nren, country_code=nren))
session.add_all(budgets_data)
......@@ -123,8 +96,7 @@ def test_funding_source_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("FundingSourceTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......@@ -167,8 +139,7 @@ def test_staff_data():
data = list(_generate_rows())
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in [d['nren'] for d in data]}
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in [d['nren'] for d in data]}
session.add_all(nren_dict.values())
for row in data:
......@@ -211,8 +182,7 @@ def test_charging_structure_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("ChargingStructureTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......@@ -264,27 +234,16 @@ def test_organization_data():
year = org["year"]
name = org["name"]
session.add(
model.ParentOrganization(
nren=nren,
year=year,
organization=name
)
)
session.add(model.ParentOrganization(nren=nren, year=year, organization=name))
for sub_org in sub_org_data:
nren = nren_dict[sub_org["nren"]]
year = sub_org["year"]
name = sub_org["name"]
role = sub_org["role"]
session.add(
model.SubOrganization(
nren=nren,
year=year,
organization=name,
role=role
)
)
session.add(model.SubOrganization(nren=nren, year=year, organization=name, role=role))
session.commit()
......@@ -318,11 +277,6 @@ def test_ec_project_data():
year = ec_project["year"]
project = ec_project["project"]
session.add(
model.ECProject(
nren=nren,
year=year,
project=project
)
)
session.add(model.ECProject(nren=nren, year=year, project=project))
session.commit()
test/test_survey_publisher_2022.py
+ 10
− 26
View file @ db20d475
......@@ -182,36 +182,23 @@ def test_publisher(client, mocker, dummy_config):
('nren3', '["project3"]'),
]
mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget',
get_rows_as_tuples)
mocker.patch(
'compendium_v2.publishers.survey_publisher_2022.query_funding_sources',
funding_source_data)
mocker.patch(
'compendium_v2.publishers.survey_publisher_2022.query_question',
question_data)
mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_budget', get_rows_as_tuples)
mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_funding_sources', funding_source_data)
mocker.patch('compendium_v2.publishers.survey_publisher_2022.query_question', question_data)
with db.session_scope() as session:
nren_names = [
'Nren1', 'Nren2', 'Nren3', 'Nren4',
'SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH'
]
session.add_all(
[model.NREN(name=nren_name) for nren_name in nren_names])
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH']
session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
with db.session_scope() as session:
budgets = session.query(model.BudgetEntry).order_by(
model.BudgetEntry.nren_id.asc()).all()
budgets = session.query(model.BudgetEntry).order_by(model.BudgetEntry.nren_id.asc()).all()
assert len(budgets) == 3
assert budgets[0].nren.name.lower() == 'nren1'
assert budgets[0].budget == 100
funding_sources = session.query(model.FundingSource).order_by(
model.FundingSource.nren_id.asc()).all()
funding_sources = session.query(model.FundingSource).order_by(model.FundingSource.nren_id.asc()).all()
assert len(funding_sources) == 3
assert funding_sources[0].nren.name.lower() == 'nren1'
assert funding_sources[0].client_institutions == 10
......@@ -228,8 +215,7 @@ def test_publisher(client, mocker, dummy_config):
assert funding_sources[2].european_funding == 30
assert funding_sources[2].other == 30
staff_data = session.query(model.NrenStaff).order_by(
model.NrenStaff.nren_id.asc()).all()
staff_data = session.query(model.NrenStaff).order_by(model.NrenStaff.nren_id.asc()).all()
assert len(staff_data) == 3
assert staff_data[0].nren.name.lower() == 'nren1'
......@@ -250,8 +236,7 @@ def test_publisher(client, mocker, dummy_config):
assert staff_data[2].permanent_fte == 30
assert staff_data[2].subcontracted_fte == 0
_org_data = session.query(model.ParentOrganization).order_by(
model.ParentOrganization.nren_id.asc()).all()
_org_data = session.query(model.ParentOrganization).order_by(model.ParentOrganization.nren_id.asc()).all()
assert len(_org_data) == 2
assert _org_data[0].nren.name.lower() == 'nren1'
......@@ -270,8 +255,7 @@ def test_publisher(client, mocker, dummy_config):
assert charging_structures[2].nren.name.lower() == 'nren3'
assert charging_structures[2].fee_type == model.FeeType.other
_ec_data = session.query(model.ECProject).order_by(
model.ECProject.nren_id.asc()).all()
_ec_data = session.query(model.ECProject).order_by(model.ECProject.nren_id.asc()).all()
assert len(_ec_data) == 3
assert _ec_data[0].nren.name.lower() == 'nren2'
......
test/test_survey_publisher_v1.py
+ 4
− 9
View file @ db20d475
......@@ -4,19 +4,15 @@ from compendium_v2 import db
from compendium_v2.db import model
from compendium_v2.publishers.survey_publisher_v1 import _cli
EXCEL_FILE = os.path.join(
os.path.dirname(__file__), "data",
"2021_Organisation_DataSeries.xlsx")
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "data", "2021_Organisation_DataSeries.xlsx")
def test_publisher(client, mocker, dummy_config):
mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE',
EXCEL_FILE)
mocker.patch('compendium_v2.background_task.parse_excel_data.EXCEL_FILE', EXCEL_FILE)
with db.session_scope() as session:
nren_names = ['SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH']
session.add_all(
[model.NREN(name=nren_name) for nren_name in nren_names])
session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
......@@ -25,6 +21,5 @@ def test_publisher(client, mocker, dummy_config):
assert budget_count
funding_source_count = session.query(model.FundingSource.year).count()
assert funding_source_count
charging_structure_count = session.query(model.ChargingStructure.year)\
.count()
charging_structure_count = session.query(model.ChargingStructure.year).count()
assert charging_structure_count
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment