untested v1 staffing migration code

compendium_v2/background_task/parse_excel_data.py

@@ -175,3 +175,119 @@ def fetch_charging_structure_excel_data():
 
     # For 2019
     yield from create_points_for_year(3, 45, 2019, 6)
+
+
+def fetch_staffing_excel_data():
+    # load the xlsx file
+    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+
+    # select the active worksheet
+    sheet_name = "4. Staff"
+    ws = wb[sheet_name]
+
+    start_row = 18
+    end_row = 61
+
+    def convert_number(value, nren, year, description):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            return 0
+
+    def create_points_for_year(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            permanent = ws.cell(row=row, column=start_column).value
+            permanent = convert_number(permanent, nren, year, "permanent ftes")
+            subcontracted = ws.cell(row=row, column=start_column + 1).value
+            subcontracted = convert_number(subcontracted, nren, year, "subcontractor ftes")
+            if permanent + subcontracted > 0:
+                yield nren.upper(), year, permanent, subcontracted
+
+    # For 2016
+    yield from create_points_for_year(2016, 53, 55)
+
+    # For 2017
+    yield from create_points_for_year(2017, 43, 46)
+
+    # For 2018
+    yield from create_points_for_year(2018, 33, 36)
+
+    # For 2019
+    yield from create_points_for_year(2019, 23, 26)
+
+    # For 2020
+    yield from create_points_for_year(2020, 13, 16)
+
+    # For 2021
+    yield from create_points_for_year(2021, 2, 5)
+
+
+def fetch_staff_function_excel_data():
+    # load the xlsx file
+    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+
+    # select the active worksheet
+    sheet_name = "5. Staff by Function"
+    ws = wb[sheet_name]
+
+    start_row = 14
+    end_row = 58
+
+    def convert_number(value, nren, year, description):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            return 0
+
+    def read_cell_number(row, column, nren, year, description):
+        value = ws.cell(row=row, column=column).value
+        return convert_number(value, nren, year, description)
+
+    def create_points_for_year_until_2019(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            if nren is None:
+                continue
+            admin = read_cell_number(row, start_column, nren, year, "admin and finance ftes")
+            communication = read_cell_number(row, start_column + 1, nren, year, "communication ftes")
+            infosec = read_cell_number(row, start_column + 2, nren, year, "infosec ftes")
+            it = read_cell_number(row, start_column + 3, nren, year, "it and software dev ftes")
+            noc = read_cell_number(row, start_column + 4, nren, year, "NOC and engineering ftes")
+            others = read_cell_number(row, start_column + 5, nren, year, "other ftes")
+
+            technical = infosec + it + noc
+            non_technical = admin + communication + others
+
+            if technical + non_technical > 0:
+                yield nren.upper(), year, technical, non_technical
+
+    def create_points_for_year(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            if nren is None:
+                continue
+            technical = read_cell_number(row, start_column, nren, year, "technical ftes")
+            non_technical = read_cell_number(row, start_column + 1, nren, year, "non-technical ftes")
+            if technical + non_technical > 0:
+                yield nren.upper(), year, technical, non_technical
+
+    # For 2017
+    yield from create_points_for_year_until_2019(2017, 41, 43)
+
+    # For 2018
+    yield from create_points_for_year_until_2019(2018, 31, 33)
+
+    # For 2019
+    yield from create_points_for_year_until_2019(2019, 20, 22)
+
+    # For 2020
+    yield from create_points_for_year(2020, 12, 14)
+
+    # For 2021
+    yield from create_points_for_year(2021, 3, 5)

compendium_v2/publishers/survey_publisher_v1.py

@@ -113,11 +113,68 @@ def db_charging_structure_migration():
         session.commit()
 
 
+def db_staffing_migration():
+    with db.session_scope() as session:
+        nren_dict = helpers.get_uppercase_nren_dict(session)
+
+        staff_data = parse_excel_data.fetch_staffing_excel_data()
+
+        nren_staff_map = {}
+        for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
+            if abbrev not in nren_dict:
+                logger.info(f'{abbrev} unknown. Skipping staff data.')
+                continue
+
+            nren = nren_dict[abbrev]
+            nren_staff_map[(nren.id, year)] = model.NrenStaff(
+                nren=nren,
+                nren_id=nren.id,
+                year=year,
+                permanent_fte=permanent_fte,
+                subcontracted_fte=subcontracted_fte,
+                technical_fte=0,
+                non_technical_fte=0
+            )
+
+        function_data = parse_excel_data.fetch_staff_function_excel_data()
+        for (abbrev, year, technical_fte, non_technical_fte) in function_data:
+            if abbrev not in nren_dict:
+                logger.info(f'{abbrev} unknown. Skipping staff function data.')
+                continue
+
+            nren = nren_dict[abbrev]
+            if (nren.id, year) in nren_staff_map:
+                nren_staff_map[(nren.id, year)].technical_fte = technical_fte
+                nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
+            else:
+                nren_staff_map[(nren.id, year)] = model.NrenStaff(
+                    nren=nren,
+                    nren_id=nren.id,
+                    year=year,
+                    permanent_fte=0,
+                    subcontracted_fte=0,
+                    technical_fte=technical_fte,
+                    non_technical_fte=non_technical_fte
+                )
+
+        for nren_staff_model in nren_staff_map.values():
+            employed = nren_staff_model.permanent_fte + nren_staff_model.subcontracted_fte
+            technical = nren_staff_model.technical_fte + nren_staff_model.non_technical_fte
+            if not math.isclose(employed, technical, abs_tol=0.01):
+                logger.info(f'{nren_staff_model.nren.name} in {nren_staff_model.year}:'
+                            f' FTE do not equal across employed/technical categories ({employed} != {technical})')
+
+            session.merge(nren_staff_model)
+
+        session.commit()
+
+
 def _cli(config):
     helpers.init_db(config)
     db_budget_migration()
     db_funding_migration()
     db_charging_structure_migration()
+    db_staffing_migration()
 
 
 @click.command()