diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/background_task/parse_excel_data.py index 993ece9873e5150fcb3035e6e06b05553ce8e052..51ec2efee92c84855c34f30050d92bbd8513b043 100644 --- a/compendium_v2/background_task/parse_excel_data.py +++ b/compendium_v2/background_task/parse_excel_data.py @@ -266,3 +266,53 @@ def fetch_staff_function_excel_data(): # For 2021 yield from create_points_for_year(2021, 3, 5) + + +def fetch_ecproject_excel_data(): + # load the xlsx file + wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True) + + # select the active worksheet + sheet_name = "7. EC Projects" + ws = wb[sheet_name] + + start_row = 6 + + def create_points_for_year(year, start_column, end_row): + for row in range(start_row, end_row): + # extract the data from the row + nren = ws.cell(row=row, column=start_column).value + if nren is None: + continue + project = ws.cell(row=row, column=start_column + 1).value + if project is None: + continue + yield nren.upper(), year, project + + yield from create_points_for_year(2017, 13, 165) + + yield from create_points_for_year(2018, 10, 165) + + yield from create_points_for_year(2019, 7, 165) + + yield from create_points_for_year(2020, 4, 180) + + yield from create_points_for_year(2021, 1, 173) + + +def fetch_organization_excel_data(): + # load the xlsx file + wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True) + + # select the active worksheet + sheet_name = "Organization" + ws = wb[sheet_name] + + # iterate over the rows in the worksheet + for row in range(5, 48): + # extract the data from the row + nren = ws.cell(row=row, column=2).value + parent_org = ws.cell(row=row, column=4).value + + if parent_org not in [None, 'NA', 'N/A']: + yield nren.upper(), 2021, parent_org diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_v1.py index ede965474cccca04b20a9ebb0f494d65a2413c70..ba0e4e2a9952a4b32f6bb74a2088488fdb1852e4 100644 --- a/compendium_v2/publishers/survey_publisher_v1.py +++ b/compendium_v2/publishers/survey_publisher_v1.py @@ -2,7 +2,7 @@ survey_publisher_v1 ========================= -This module loads the survey data from before 2022 from and excel file. +This module loads the survey data from before 2022 from an excel file. Missing info is filled in from the survey db for some questions. Registered as click cli command when installing compendium-v2. @@ -38,10 +38,10 @@ def db_budget_migration(): year = budget.year if float(budget.budget) > 200: - logger.warn(f'Incorrect Data: {abbrev} has budget set to >200M EUR for {year}. ({budget.budget})') + logger.warning(f'Incorrect Data: {abbrev} has budget set >200M EUR for {year}. ({budget.budget})') if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping.') + logger.warning(f'{abbrev} unknown. Skipping.') continue budget_entry = model.BudgetEntry( @@ -56,11 +56,11 @@ def db_budget_migration(): for abbrev, budget, year in exceldata: if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping.') + logger.warning(f'{abbrev} unknown. Skipping.') continue if budget > 200: - logger.warn(f'{nren} has budget set to >200M EUR for {year}. ({budget})') + logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})') budget_entry = model.BudgetEntry(nren=nren_dict[abbrev], budget=budget, year=year) session.merge(budget_entry) @@ -82,10 +82,10 @@ def db_funding_migration(): _data = [client_institution, european_funding, gov_public_bodies, commercial, other] total = sum(_data) if not math.isclose(total, 100, abs_tol=0.01) and total != 0: - logger.warn(f'{abbrev} funding sources for {year} do not sum to 100% ({total})') + logger.warning(f'{abbrev} funding sources for {year} do not sum to 100% ({total})') if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping.') + logger.warning(f'{abbrev} unknown. Skipping.') continue budget_entry = model.FundingSource( @@ -109,7 +109,7 @@ def db_charging_structure_migration(): for (abbrev, year, charging_structure) in data: if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping.') + logger.warning(f'{abbrev} unknown. Skipping.') continue charging_structure_entry = model.ChargingStructure( @@ -127,7 +127,7 @@ def db_staffing_migration(): nren_staff_map = {} for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data: if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping staff data.') + logger.warning(f'{abbrev} unknown. Skipping staff data.') continue nren = nren_dict[abbrev] @@ -144,7 +144,7 @@ def db_staffing_migration(): function_data = parse_excel_data.fetch_staff_function_excel_data() for (abbrev, year, technical_fte, non_technical_fte) in function_data: if abbrev not in nren_dict: - logger.warn(f'{abbrev} unknown. Skipping staff function data.') + logger.warning(f'{abbrev} unknown. Skipping staff function data.') continue nren = nren_dict[abbrev] @@ -166,7 +166,7 @@ def db_staffing_migration(): employed = nren_staff_model.permanent_fte + nren_staff_model.subcontracted_fte technical = nren_staff_model.technical_fte + nren_staff_model.non_technical_fte if not math.isclose(employed, technical, abs_tol=0.01) and employed != 0 and technical != 0: - logger.warn(f'{nren_staff_model.nren.name} in {nren_staff_model.year}:' + logger.warning(f'{nren_staff_model.nren.name} in {nren_staff_model.year}:' f' FTE do not equal across employed/technical categories ({employed} != {technical})') session.merge(nren_staff_model) @@ -174,12 +174,46 @@ def db_staffing_migration(): session.commit() +def db_ecprojects_migration(): + with db.session_scope() as session: + nren_dict = helpers.get_uppercase_nren_dict(session) + + ecproject_data = parse_excel_data.fetch_ecproject_excel_data() + for (abbrev, year, project) in ecproject_data: + if abbrev not in nren_dict: + logger.warning(f'{abbrev} unknown. Skipping.') + continue + + nren = nren_dict[abbrev] + ecproject_entry = model.ECProject(nren=nren, nren_id=nren.id, year=year, project=project) + session.merge(ecproject_entry) + session.commit() + + +def db_organizations_migration(): + with db.session_scope() as session: + nren_dict = helpers.get_uppercase_nren_dict(session) + + organization_data = parse_excel_data.fetch_organization_excel_data() + for (abbrev, year, org) in organization_data: + if abbrev not in nren_dict: + logger.warning(f'{abbrev} unknown. Skipping.') + continue + + nren = nren_dict[abbrev] + org_entry = model.ParentOrganization(nren=nren, nren_id=nren.id, year=year, organization=org) + session.merge(org_entry) + session.commit() + + def _cli(config): helpers.init_db(config) db_budget_migration() db_funding_migration() db_charging_structure_migration() db_staffing_migration() + db_ecprojects_migration() + db_organizations_migration() @click.command() diff --git a/test/test_survey_publisher_v1.py b/test/test_survey_publisher_v1.py index b712812ba0caf5adbc31e5aac44fb7cff2a36aff..c7ebc2d909e15f3be872d01d52a30ebba94af024 100644 --- a/test/test_survey_publisher_v1.py +++ b/test/test_survey_publisher_v1.py @@ -68,3 +68,24 @@ def test_publisher(client, mocker, dummy_config): assert kifu_data[5].subcontracted_fte == 3 assert kifu_data[5].technical_fte == 133 assert kifu_data[5].non_technical_fte == 45 + + ecproject_data = session.query(model.ECProject).all() + # test a couple of random entries + surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017] + assert len(surf2017) == 1 + assert surf2017[0].project == 'Asterics and Magic' + + asnetam2018 = [x for x in ecproject_data if x.nren.name == 'ASNET-AM' and x.year == 2018] + assert len(asnetam2018) == 1 + assert asnetam2018[0].project == 'EaPConnect' + + kifu2019 = [x for x in ecproject_data if x.nren.name == 'KIFU' and x.year == 2019] + assert len(kifu2019) == 4 + assert kifu2019[3].project == 'SuperHeroes for Science' + + parent_data = session.query(model.ParentOrganization).all() + # test a random entry + asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021] + assert len(asnet2021) == 1 + assert asnet2021[0].organization\ + == 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia'