diff --git a/test/test_survey_publisher_legacy_excel.py b/test/test_survey_publisher_legacy_excel.py deleted file mode 100644 index 57c936f21af553c6214a35defac709f61d0244b7..0000000000000000000000000000000000000000 --- a/test/test_survey_publisher_legacy_excel.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import openpyxl - -from sqlalchemy import select, func -from compendium_v2 import db -from compendium_v2.db import presentation_models -from compendium_v2.publishers.survey_publisher_legacy_excel import _cli - - -def test_excel_publisher(app_with_survey_db, mocker, nren_services): - nren_services(app_with_survey_db) - EXCEL_FILE_ORGANISATION = openpyxl.load_workbook(os.path.join(os.path.dirname( - __file__), "data", "2021_Organisation_DataSeries.xlsx"), data_only=True, read_only=True) - EXCEL_FILE_USERS = openpyxl.load_workbook(os.path.join(os.path.dirname( - __file__), "data", "2022_Connected_Users_DataSeries.xlsx"), data_only=True, read_only=True) - EXCEL_FILE_NETWORKS = openpyxl.load_workbook(os.path.join(os.path.dirname( - __file__), "data", "2022_Networks_DataSeries.xlsx"), data_only=True, read_only=True) - EXCEL_FILE_NREN_SERVICES = openpyxl.load_workbook(os.path.join(os.path.dirname( - __file__), "data", "NREN-Services-prefills_2023_Recovered.xlsx"), data_only=True, read_only=True) - mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_ORGANISATION', EXCEL_FILE_ORGANISATION) - mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_USERS', EXCEL_FILE_USERS) - mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NETWORKS', EXCEL_FILE_NETWORKS) - mocker.patch('compendium_v2.publishers.excel_parser.EXCEL_FILE_NREN_SERVICES', EXCEL_FILE_NREN_SERVICES) - - with app_with_survey_db.app_context(): - nren_names = ['SURF', 'KIFU', 'University of Malta', 'ASNET-AM', 'SIKT', 'LAT', 'RASH', 'AzScienceNet', 'GRNET', - 'CSC', 'PSNC'] - db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names]) - db.session.commit() - - _cli(app_with_survey_db) - - with app_with_survey_db.app_context(): - budget_count = db.session.scalar(select(func.count(presentation_models.BudgetEntry.year))) - assert budget_count - funding_source_count = db.session.scalar(select(func.count(presentation_models.FundingSource.year))) - assert funding_source_count - charging_structure_count = db.session.scalar(select(func.count(presentation_models.ChargingStructure.year))) - assert charging_structure_count - staff_data = db.session.scalars(select(presentation_models.NrenStaff).order_by( - presentation_models.NrenStaff.year.asc()) - ).all() - - # data should only be saved for the NRENs we have saved in the database - staff_data_nrens = set([staff.nren.name for staff in staff_data]) - assert len(staff_data_nrens) == len(nren_names) - 1 # no UoM data - - kifu_data = [staff for staff in staff_data if staff.nren.name == 'KIFU'] - # check that the data is saved correctly for KIFU, it should be OK for the rest then.. - assert len(kifu_data) == 6 - - assert kifu_data[0].year == 2016 - assert kifu_data[0].permanent_fte == 100 - assert kifu_data[0].subcontracted_fte == 2 - assert kifu_data[0].technical_fte == 0 - assert kifu_data[0].non_technical_fte == 0 - - assert kifu_data[1].year == 2017 - assert kifu_data[1].permanent_fte == 80 - assert kifu_data[1].subcontracted_fte == 2 - assert kifu_data[1].technical_fte == 0 - assert kifu_data[1].non_technical_fte == 0 - - assert kifu_data[2].year == 2018 - assert kifu_data[2].permanent_fte == 80 - assert kifu_data[2].subcontracted_fte == 3 - assert kifu_data[2].technical_fte == 0 - assert kifu_data[2].non_technical_fte == 0 - - assert kifu_data[3].year == 2019 - assert kifu_data[3].permanent_fte == 148 - assert kifu_data[3].subcontracted_fte == 4 - assert kifu_data[3].technical_fte == 117 - assert kifu_data[3].non_technical_fte == 33 - - assert kifu_data[4].year == 2020 - assert kifu_data[4].permanent_fte == 190 - assert kifu_data[4].subcontracted_fte == 3 - assert kifu_data[4].technical_fte == 133 - assert kifu_data[4].non_technical_fte == 60 - - assert kifu_data[5].year == 2021 - assert kifu_data[5].permanent_fte == 178 - assert kifu_data[5].subcontracted_fte == 3 - assert kifu_data[5].technical_fte == 133 - assert kifu_data[5].non_technical_fte == 45 - - ecproject_data = db.session.scalars(select(presentation_models.ECProject)).all() - # test a couple of random entries - surf2017 = [x for x in ecproject_data if x.nren.name == 'SURF' and x.year == 2017] - assert len(surf2017) == 1 - assert surf2017[0].project == 'Asterics and Magic' - - asnetam2018 = [x for x in ecproject_data if x.nren.name == 'ASNET-AM' and x.year == 2018] - assert len(asnetam2018) == 1 - assert asnetam2018[0].project == 'EaPConnect' - - kifu2019 = [x for x in ecproject_data if x.nren.name == 'KIFU' and x.year == 2019] - assert len(kifu2019) == 4 - assert kifu2019[3].project == 'SuperHeroes for Science' - - parent_data = db.session.scalars(select(presentation_models.ParentOrganization)).all() - # test a random entry - asnet2021 = [x for x in parent_data if x.nren.name == 'ASNET-AM' and x.year == 2021] - assert len(asnet2021) == 1 - assert asnet2021[0].organization\ - == 'Institute for Informatics and Automation Problems of the National Academy of Sciences of Armenia' - - service_data = db.session.scalars(select(presentation_models.Service)).all() - assert len(service_data) > 70 - - nren_service_data = db.session.scalars(select(presentation_models.NRENService)).all() - # test a random entry - sikt2022 = [x for x in nren_service_data - if x.nren.name == 'SIKT' and x.year == 2022 and x.service.name == 'Journal access'] - assert len(sikt2022) == 1 - assert sikt2022[0].additional_information.startswith("Sikt negotiates license a") diff --git a/test/test_survey_publisher_old_db_2022.py b/test/test_survey_publisher_old_db_2022.py deleted file mode 100644 index ed5e2e5f5fb5c9c1bcff9484e3b49f9e56bc8d3d..0000000000000000000000000000000000000000 --- a/test/test_survey_publisher_old_db_2022.py +++ /dev/null @@ -1,309 +0,0 @@ -from sqlalchemy import select - -from compendium_v2.db import db, presentation_model_enums, presentation_models -from compendium_v2.publishers.survey_publisher_old_db_2022 import _cli, FundingSource, \ - StaffQuestion, OrgQuestion, ChargingStructure, ECQuestion - - -def org_data(question): - """ - This function defines test data for the org questions. - - The following data is defined for the appropriate questions as modeled in Compendium: - - nren1,CYNET-CSIRT,cert team - nren1,DFN-CERT,CERT - nren2,Educampus Services,MIS shared services for third level. - nren3,VilniusTech,Technical centre - nren3,KU,Technical centre - nren3,VDU,Technical centre - nren3,VU,Technical centre - nren3,org_data -org_data -org_data -org_data -org_dataKTU,"NOC, administrative authority" - """ - - if question == OrgQuestion.PARENT_ORG_NAME: - return [ - ('nren1', 'Org1'), - ('nren3', 'Org3'), - ] - - if str(question.name).endswith('1_NAME'): - return [ - ('nren1', 'CYNET-CSIRT'), - ('nren2', 'Educampus Services'), - ('nren3', 'VilniusTech'), - ] - - if str(question.name).endswith('2_NAME'): - return [ - ('nren1', 'DFN-CERT'), - ('nren3', 'KU'), - ] - - if str(question.name).endswith('3_NAME'): - return [ - ('nren3', 'VDU'), - ] - - if str(question.name).endswith('4_NAME'): - return [ - ('nren3', 'VU'), - ] - - if str(question.name).endswith('5_NAME'): - return [ - ('nren3', 'KTU'), - ] - - if str(question.name).endswith('1_CHOICE'): - return [ - ('nren1', 'other'), - ('nren2', 'other'), - ('nren3', 'Technical centre'), - ] - - if str(question.name).endswith('2_CHOICE'): - return [ - ('nren1', 'other'), - ('nren3', 'Technical centre'), - ] - - if str(question.name).endswith('3_CHOICE'): - return [ - ('nren3', 'Technical centre'), - ] - - if str(question.name).endswith('4_CHOICE'): - return [ - ('nren3', 'Technical centre'), - ] - - if str(question.name).endswith('5_CHOICE'): - return [ - ('nren3', 'other'), - ] - - if str(question.name).endswith('1_ROLE'): - return [ - ('nren1', 'cert team'), - ('nren2', 'MIS shared services for third level.') - ] - - if str(question.name).endswith('2_ROLE'): - return [ - ('nren1', 'CERT'), - ] - - if str(question.name).endswith('3_ROLE'): - return [] - - if str(question.name).endswith('4_ROLE'): - return [] - - if str(question.name).endswith('5_ROLE'): - return [ - ('nren3', 'NOC, administrative authority') - ] - - -def test_publisher(app_with_survey_db, mocker): - global org_data - - def get_rows_as_tuples(*args, **kwargs): - return [ - ('nren1', '100'), - ('nren2', '200'), - ('nren3', '300'), - ('nren4', 'abcd') - ] - - def funding_source_data(): - yield FundingSource.CLIENT_INSTITUTIONS, [ - ('nren1', '10'), - ('nren2', '80'), - ('nren3', '30'), - ] - yield FundingSource.EUROPEAN_FUNDING, [ - ('nren1', '50'), - ('nren2', '20'), - ('nren3', '30'), - ] - yield FundingSource.OTHER, [ - ('nren1', '40'), - ('nren2', 'abc'), - ('nren3', '30'), - ] - - def question_data(question): - if question == StaffQuestion.NON_TECHNICAL_FTE: - return [ - ('nren1', '10'), - ('nren2', '80'), - ('nren3', '30'), - ] - - if question == StaffQuestion.TECHNICAL_FTE: - return [ - ('nren1', '50'), - ('nren2', '20'), - ('nren3', '30'), - ] - - if question == StaffQuestion.PERMANENT_FTE: - return [ - ('nren1', '60'), - ('nren2', 'abc'), - ('nren3', '30'), - ] - - if question == StaffQuestion.SUBCONTRACTED_FTE: - return [ - ('nren1', '0'), - ('nren2', '0'), - ('nren3', '0'), - ] - if question in OrgQuestion: - return org_data(question) - - if question == ChargingStructure.charging_structure: - return [ - ('nren1', 'We do not charge them directly'), - ('nren2', 'We charge a usage-based fee'), - ('nren3', 'Other'), - ] - - if question in ECQuestion: - return [ - ('nren1', '[""'), - ('nren2', '["project1", "project2"]'), - ('nren3', '["project3"]'), - ] - - def question_id_data(question_id, year): - if question_id in [ - 16469, 16064, 15720, 15305, 14910, 16471, 16066, 15722, 15307, 14912, 16473, 16378, - 16475, 16068, 15724, 15309, 14914, 16477, 16070, 15726, 15311, 14916, 16479, 16072, 15728, 15575, - 16481, 16074, 15730, 15577, 16761]: - return [ - ('nren1', f'www.nren.com/somepolicy{year}.pdf'), - ('nren2', 'policyemail@nren.com'), - ('nren3', 'n.a. online'), - ] - - def institutions_urls_data(question_id): - if question_id == 16507: - return [ - (87483, 'ANA', 2013, "http://www.rash.al/index.php/network/points-of-presence-pop"), - (163286, 'ANA', 2014, "http://www.rash.al/index.php/network/points-of-presence-pop"), - ] - else: - return [] - - mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_budget', get_rows_as_tuples) - mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_funding_sources', funding_source_data) - mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question', question_data) - mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.query_question_id', question_id_data) - mocker.patch('compendium_v2.publishers.survey_publisher_old_db_2022.recursive_query', institutions_urls_data) - - nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'University of Malta', 'ASNET-AM', - 'SIKT', 'LAT', 'RASH', 'AzScienceNet', 'GRNET', 'CSC', 'PSNC'] - with app_with_survey_db.app_context(): - db.session.add_all([presentation_models.NREN(name=nren_name, country='country') for nren_name in nren_names]) - db.session.commit() - - _cli(app_with_survey_db) - - with app_with_survey_db.app_context(): - budgets = db.session.scalars( - select(presentation_models.BudgetEntry).order_by(presentation_models.BudgetEntry.nren_id.asc()) - ).all() - assert len(budgets) == 3 - assert budgets[0].nren.name.lower() == 'nren1' - assert budgets[0].budget == 100 - - funding_sources = db.session.scalars( - select(presentation_models.FundingSource).order_by(presentation_models.FundingSource.nren_id.asc()) - ).all() - assert len(funding_sources) == 2 - assert funding_sources[0].nren.name.lower() == 'nren1' - assert funding_sources[0].client_institutions == 10 - assert funding_sources[0].european_funding == 50 - assert funding_sources[0].other == 40 - - assert funding_sources[1].nren.name.lower() == 'nren2' - assert funding_sources[1].client_institutions == 80 - assert funding_sources[1].european_funding == 20 - assert funding_sources[1].other == 0 - - staff_data = db.session.scalars( - select(presentation_models.NrenStaff).order_by(presentation_models.NrenStaff.nren_id.asc()) - ).all() - - assert len(staff_data) == 1 - assert staff_data[0].nren.name.lower() == 'nren1' - assert staff_data[0].non_technical_fte == 10 - assert staff_data[0].technical_fte == 50 - assert staff_data[0].permanent_fte == 60 - assert staff_data[0].subcontracted_fte == 0 - - _org_data = db.session.scalars( - select(presentation_models.ParentOrganization).order_by( - presentation_models.ParentOrganization.nren_id.asc()) - ).all() - - assert len(_org_data) == 2 - assert _org_data[0].nren.name.lower() == 'nren1' - assert _org_data[0].organization == 'Org1' - - assert _org_data[1].nren.name.lower() == 'nren3' - assert _org_data[1].organization == 'Org3' - - charging_structures = db.session.scalars( - select(presentation_models.ChargingStructure).order_by(presentation_models.ChargingStructure.nren_id.asc()) - ).all() - assert len(charging_structures) == 3 - assert charging_structures[0].nren.name.lower() == 'nren1' - assert charging_structures[0].fee_type == presentation_model_enums.FeeType.no_charge - assert charging_structures[1].nren.name.lower() == 'nren2' - assert charging_structures[1].fee_type == presentation_model_enums.FeeType.usage_based_fee - assert charging_structures[2].nren.name.lower() == 'nren3' - assert charging_structures[2].fee_type == presentation_model_enums.FeeType.other - - _ec_data = db.session.scalars( - select(presentation_models.ECProject).order_by(presentation_models.ECProject.nren_id.asc()) - ).all() - - assert len(_ec_data) == 3 - assert _ec_data[0].nren.name.lower() == 'nren2' - assert _ec_data[0].project == 'project1' - - assert _ec_data[1].nren.name.lower() == 'nren2' - assert _ec_data[1].project == 'project2' - - assert _ec_data[2].nren.name.lower() == 'nren3' - assert _ec_data[2].project == 'project3' - - policy_data = db.session.scalars( - select(presentation_models.Policy).order_by(presentation_models.Policy.nren_id.asc()) - ).all() - policy_data_2020 = [p for p in policy_data if p.year == 2020] - policy_data_2022 = [p for p in policy_data if p.year == 2022] - assert len(policy_data_2020) == 2 - assert len(policy_data_2022) == 2 - assert policy_data_2020[0].strategic_plan == 'www.nren.com/somepolicy2020.pdf' - assert policy_data_2020[1].strategic_plan == 'policyemail@nren.com' - - _institution_urls_data = db.session.scalars( - select(presentation_models.InstitutionURLs).order_by(presentation_models.InstitutionURLs.nren_id.asc()) - ).all() - assert len(_institution_urls_data) == 2 - assert _institution_urls_data[0].nren.name.lower() == 'rash' - assert _institution_urls_data[0].year == 2013 - assert _institution_urls_data[0].urls == ["http://www.rash.al/index.php/network/points-of-presence-pop"] - assert _institution_urls_data[1].nren.name.lower() == 'rash' - assert _institution_urls_data[1].year == 2014 - assert _institution_urls_data[1].urls == ["http://www.rash.al/index.php/network/points-of-presence-pop"]