Merge branch 'feature/COMP-130_staffing_data_v1_publisher' into 'develop'

publisher v1 staffing migration

See merge request !13

compendium_v2/background_task/parse_excel_data.py

@@ -152,3 +152,119 @@ def fetch_charging_structure_excel_data():
 
     # For 2019
     yield from create_points_for_year(3, 45, 2019, 6)
+
+
+def fetch_staffing_excel_data():
+    # load the xlsx file
+    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+
+    # select the active worksheet
+    sheet_name = "4. Staff"
+    ws = wb[sheet_name]
+
+    start_row = 18
+    end_row = 61
+
+    def convert_number(value, nren, year, description):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            return 0
+
+    def create_points_for_year(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            permanent = ws.cell(row=row, column=start_column).value
+            permanent = convert_number(permanent, nren, year, "permanent ftes")
+            subcontracted = ws.cell(row=row, column=start_column + 1).value
+            subcontracted = convert_number(subcontracted, nren, year, "subcontractor ftes")
+            if permanent + subcontracted > 0:
+                yield nren.upper(), year, permanent, subcontracted
+
+    # For 2016
+    yield from create_points_for_year(2016, 53, 55)
+
+    # For 2017
+    yield from create_points_for_year(2017, 43, 46)
+
+    # For 2018
+    yield from create_points_for_year(2018, 33, 36)
+
+    # For 2019
+    yield from create_points_for_year(2019, 23, 26)
+
+    # For 2020
+    yield from create_points_for_year(2020, 13, 16)
+
+    # For 2021
+    yield from create_points_for_year(2021, 2, 5)
+
+
+def fetch_staff_function_excel_data():
+    # load the xlsx file
+    wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
+
+    # select the active worksheet
+    sheet_name = "5. Staff by Function"
+    ws = wb[sheet_name]
+
+    start_row = 14
+    end_row = 58
+
+    def convert_number(value, nren, year, description):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            return 0
+
+    def read_cell_number(row, column, nren, year, description):
+        value = ws.cell(row=row, column=column).value
+        return convert_number(value, nren, year, description)
+
+    def create_points_for_year_until_2019(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            if nren is None:
+                continue
+            admin = read_cell_number(row, start_column, nren, year, "admin and finance ftes")
+            communication = read_cell_number(row, start_column + 1, nren, year, "communication ftes")
+            infosec = read_cell_number(row, start_column + 2, nren, year, "infosec ftes")
+            it = read_cell_number(row, start_column + 3, nren, year, "it and software dev ftes")
+            noc = read_cell_number(row, start_column + 4, nren, year, "NOC and engineering ftes")
+            others = read_cell_number(row, start_column + 5, nren, year, "other ftes")
+
+            technical = infosec + it + noc
+            non_technical = admin + communication + others
+
+            if technical + non_technical > 0:
+                yield nren.upper(), year, technical, non_technical
+
+    def create_points_for_year(year, nren_column, start_column):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=nren_column).value
+            if nren is None:
+                continue
+            technical = read_cell_number(row, start_column, nren, year, "technical ftes")
+            non_technical = read_cell_number(row, start_column + 1, nren, year, "non-technical ftes")
+            if technical + non_technical > 0:
+                yield nren.upper(), year, technical, non_technical
+
+    # For 2017
+    yield from create_points_for_year_until_2019(2017, 41, 43)
+
+    # For 2018
+    yield from create_points_for_year_until_2019(2018, 31, 33)
+
+    # For 2019
+    yield from create_points_for_year_until_2019(2019, 20, 22)
+
+    # For 2020
+    yield from create_points_for_year(2020, 12, 14)
+
+    # For 2021
+    yield from create_points_for_year(2021, 3, 5)

compendium_v2/publishers/survey_publisher_v1.py

@@ -115,11 +115,68 @@ def db_charging_structure_migration():
         session.commit()
 
 
+def db_staffing_migration():
+    with db.session_scope() as session:
+        nren_dict = helpers.get_uppercase_nren_dict(session)
+
+        staff_data = parse_excel_data.fetch_staffing_excel_data()
+
+        nren_staff_map = {}
+        for (abbrev, year, permanent_fte, subcontracted_fte) in staff_data:
+            if abbrev not in nren_dict:
+                logger.info(f'{abbrev} unknown. Skipping staff data.')
+                continue
+
+            nren = nren_dict[abbrev]
+            nren_staff_map[(nren.id, year)] = model.NrenStaff(
+                nren=nren,
+                nren_id=nren.id,
+                year=year,
+                permanent_fte=permanent_fte,
+                subcontracted_fte=subcontracted_fte,
+                technical_fte=0,
+                non_technical_fte=0
+            )
+
+        function_data = parse_excel_data.fetch_staff_function_excel_data()
+        for (abbrev, year, technical_fte, non_technical_fte) in function_data:
+            if abbrev not in nren_dict:
+                logger.info(f'{abbrev} unknown. Skipping staff function data.')
+                continue
+
+            nren = nren_dict[abbrev]
+            if (nren.id, year) in nren_staff_map:
+                nren_staff_map[(nren.id, year)].technical_fte = technical_fte
+                nren_staff_map[(nren.id, year)].non_technical_fte = non_technical_fte
+            else:
+                nren_staff_map[(nren.id, year)] = model.NrenStaff(
+                    nren=nren,
+                    nren_id=nren.id,
+                    year=year,
+                    permanent_fte=0,
+                    subcontracted_fte=0,
+                    technical_fte=technical_fte,
+                    non_technical_fte=non_technical_fte
+                )
+
+        for nren_staff_model in nren_staff_map.values():
+            employed = nren_staff_model.permanent_fte + nren_staff_model.subcontracted_fte
+            technical = nren_staff_model.technical_fte + nren_staff_model.non_technical_fte
+            if not math.isclose(employed, technical, abs_tol=0.01):
+                logger.info(f'{nren_staff_model.nren.name} in {nren_staff_model.year}:'
+                            f' FTE do not equal across employed/technical categories ({employed} != {technical})')
+
+            session.merge(nren_staff_model)
+
+        session.commit()
+
+
 def _cli(config):
     helpers.init_db(config)
     db_budget_migration()
     db_funding_migration()
     db_charging_structure_migration()
+    db_staffing_migration()
 
 
 @click.command()

compendium_v2/static/index.html

@@ -6,6 +6,6 @@
 </head>
 <body>
   <div id="root"></div>
-  <script src="bundle.js"></script>
+  <script src="/bundle.js"></script>
 </body>
 </html>
\ No newline at end of file

compendium_v2/templates/index.html

@@ -5,6 +5,6 @@
 </head>
 <body>
   <div id="root"></div>
-  <script src="static/bundle.js"></script>
+  <script src="/static/bundle.js"></script>
 </body>
 </html>
\ No newline at end of file

test/test_survey_publisher_v1.py

@@ -23,3 +23,48 @@ def test_publisher(client, mocker, dummy_config):
         assert funding_source_count
         charging_structure_count = session.query(model.ChargingStructure.year).count()
         assert charging_structure_count
+        staff_data = session.query(model.NrenStaff).order_by(model.NrenStaff.year.asc()).all()
+
+        # data should only be saved for the NRENs we have saved in the database
+        staff_data_nrens = set([staff.nren.name for staff in staff_data])
+        assert len(staff_data_nrens) == len(nren_names) - 1  # no UoM data
+
+        kifu_data = [staff for staff in staff_data if staff.nren.name == 'KIFU']
+        # check that the data is saved correctly for KIFU, it should be OK for the rest then..
+        assert len(kifu_data) == 6
+
+        assert kifu_data[0].year == 2016
+        assert kifu_data[0].permanent_fte == 100
+        assert kifu_data[0].subcontracted_fte == 2
+        assert kifu_data[0].technical_fte == 0
+        assert kifu_data[0].non_technical_fte == 0
+
+        assert kifu_data[1].year == 2017
+        assert kifu_data[1].permanent_fte == 80
+        assert kifu_data[1].subcontracted_fte == 2
+        assert kifu_data[1].technical_fte == 0
+        assert kifu_data[1].non_technical_fte == 0
+
+        assert kifu_data[2].year == 2018
+        assert kifu_data[2].permanent_fte == 80
+        assert kifu_data[2].subcontracted_fte == 3
+        assert kifu_data[2].technical_fte == 0
+        assert kifu_data[2].non_technical_fte == 0
+
+        assert kifu_data[3].year == 2019
+        assert kifu_data[3].permanent_fte == 148
+        assert kifu_data[3].subcontracted_fte == 4
+        assert kifu_data[3].technical_fte == 117
+        assert kifu_data[3].non_technical_fte == 33
+
+        assert kifu_data[4].year == 2020
+        assert kifu_data[4].permanent_fte == 190
+        assert kifu_data[4].subcontracted_fte == 3
+        assert kifu_data[4].technical_fte == 133
+        assert kifu_data[4].non_technical_fte == 60
+
+        assert kifu_data[5].year == 2021
+        assert kifu_data[5].permanent_fte == 178
+        assert kifu_data[5].subcontracted_fte == 3
+        assert kifu_data[5].technical_fte == 133
+        assert kifu_data[5].non_technical_fte == 45

webapp/src/helpers/dataconversion.tsx

@@ -179,53 +179,104 @@ export function createOrganisationDataLookup(organisationEntries: Organisation[]
 }
 
 
-export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean) => {
+export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean, selectedYear: Number) => {
+
+    let categories;
+    if (roles) {
+        categories = [
+            "Technical FTE",
+            "Non-technical FTE"
+        ]
+    } else {
+        categories = [
+            "Permanent FTE",
+            "Subcontracted FTE"
+        ]
+    }
+
     function CreateDataLookup(data: NrenStaff[]) {
+        const fields = {
+            "Technical FTE": "technical_fte",
+            "Non-technical FTE": "non_technical_fte",
+            "Permanent FTE": "permanent_fte",
+            "Subcontracted FTE": "subcontracted_fte"
+        }
+
         const dataLookup = new Map<string, Map<string, number>>();
 
         data.forEach((item: NrenStaff) => {
-            const lookupKey = `${item.nren}/${item.year}`
+            if (selectedYear !== item.year) {
+                // only include data for the selected year
+                return;
+            }
+            const nren = item.nren;
 
-            let NrenStaffMap = dataLookup.get(lookupKey)
-            if (!NrenStaffMap) {
-                NrenStaffMap = new Map<string, number>();
+            let categoryMap = dataLookup.get(nren)
+            if (!categoryMap) {
+                categoryMap = new Map<string, number>();
             }
-            const total = item.non_technical_fte + item.technical_fte
-            const technical_percentage = Math.round(((item.technical_fte / total) || 0) * 100)
-            const non_technical_percentage = Math.round(((item.non_technical_fte / total) || 0) * 100)
-
-            const contractor_total = item.subcontracted_fte + item.permanent_fte
-            const permanent_percentage = Math.round(((item.permanent_fte / contractor_total) || 0) * 100)
-            const subcontracted_percentage = Math.round(((item.subcontracted_fte / contractor_total) || 0) * 100)
-            NrenStaffMap.set("Technical FTE", technical_percentage)
-            NrenStaffMap.set("Non-technical FTE", non_technical_percentage)
-            NrenStaffMap.set("Permanent FTE", permanent_percentage)
-            NrenStaffMap.set("Subcontracted FTE", subcontracted_percentage)
-            dataLookup.set(lookupKey, NrenStaffMap)
+
+
+            // get the values for the two categories
+            const [category1, category2] = categories
+            const [category1Field, category2Field] = [fields[category1], fields[category2]]
+
+            const category1Value = item[category1Field]
+            const category2Value = item[category2Field]
+
+
+            // calculate the percentages
+            const total = category1Value + category2Value
+
+            let category1_percentage = ((category1Value / total) || 0) * 100
+            let category2_percentage = ((category2Value / total) || 0) * 100
+
+            // round to 2 decimal places
+            category1_percentage = Math.round(Math.floor(category1_percentage * 100)) / 100
+            category2_percentage = Math.round(Math.floor(category2_percentage * 100)) / 100
+
+            categoryMap.set(category1, category1_percentage)
+            categoryMap.set(category2, category2_percentage)
+            dataLookup.set(nren, categoryMap)
         })
         return dataLookup
     }
 
     const dataLookup = CreateDataLookup(data)
 
-    const labelsYear = [...new Set(data.map((item: NrenStaff) => item.year))];
-    const labelsNREN = [...new Set(data.map((item: NrenStaff) => item.nren))];
+    const labelsYear = [selectedYear];
+    const labelsNREN = [...new Set(data.map((item: NrenStaff) => item.nren))].sort((nrenA, nrenB) => {
+        const categoryMapNrenA = dataLookup.get(nrenA)
+        const categoryMapNrenB = dataLookup.get(nrenB)
+        if (categoryMapNrenA && categoryMapNrenB) {
 
-    let categories;
+            const [category1, category2] = categories
 
-    if (roles) {
-        categories = [
-            "Technical FTE",
-            "Non-technical FTE"
-        ]
-    } else {
-        categories = [
-            "Permanent FTE",
-            "Subcontracted FTE"
-        ]
-        
-    }
+            const nrenAData = {
+                category1: categoryMapNrenA.get(category1)!,
+                category2: categoryMapNrenA.get(category2)!
+            }
+
+            const nrenBData = {
+                category1: categoryMapNrenB.get(category1)!,
+                category2: categoryMapNrenB.get(category2)!
+            }
 
+            if (nrenAData.category1 === nrenBData.category1) {
+                return nrenBData.category2 - nrenAData.category1;
+            }
+            return nrenBData.category1 - nrenAData.category1;
+        } else {
+            // put NRENs with no data at the end
+            if (categoryMapNrenA) {
+                return -1
+            }
+            if (categoryMapNrenB) {
+                return 1
+            }
+            return 0
+        }
+    });
 
     const categoriesPerYear = cartesianProduct(categories, labelsYear)
 
@@ -233,28 +284,25 @@ export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean) => {
 
         let color = ""
         if (category === "Technical FTE") {
-            // color = dark blue
             color = 'rgba(40, 40, 250, 0.8)'
         } else if (category === "Permanent FTE") {
             color = 'rgba(159, 129, 235, 1)'
         } else if (category === "Subcontracted FTE") {
             color = 'rgba(173, 216, 229, 1)'
-        }
-        else {
-            // light blue
+        } else if (category === "Non-technical FTE") {
             color = 'rgba(116, 216, 242, 0.54)'
         }
+
         return {
             backgroundColor: color,
             label: `${category} (${year})`,
             data: labelsNREN.map(nren => {
                 // ensure that the data is in the same order as the labels
-                const lookupKey = `${nren}/${year}`
-                const dataForYear = dataLookup.get(lookupKey)
-                if (!dataForYear) {
+                const dataForNren = dataLookup.get(nren)
+                if (!dataForNren) {
                     return 0
                 }
-                return dataForYear.get(category) ?? 0
+                return dataForNren.get(category) ?? 0
             }),
             stack: year,
             borderRadius: 10,
@@ -268,47 +316,8 @@ export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean) => {
 
     const dataset: NrenStaffDataset = {
         datasets: nrenStaffDataset,
-        labels: labelsNREN.map(l => l.toString())
-    }
-
-
-    function sortDatasetByTechnicalFTE(dataset: NrenStaffDataset) {
-        // sort dataset by Technical FTE value descending and ensure labels are in the same order
-        const sortedDataset = { ...dataset }
-
-        const uniqueYears = [...new Set(dataset.datasets.map(d => d.stack))]
-
-        for (const year of uniqueYears) {
-            const technicalFTEIndex = dataset.datasets.findIndex(d => d.label === `Technical FTE (${year})` || d.label === `Permanent FTE (${year})`)
-            if (technicalFTEIndex === -1) {
-                continue
-            }
-            const nonTechnicalFTEIndex = dataset.datasets.findIndex(d => d.label === `Non-technical FTE (${year})` || d.label === `Subcontracted FTE (${year})`)
-            if (nonTechnicalFTEIndex === -1) {
-                continue
-            }
-
-            const datasetForYearTechnical = dataset.datasets[technicalFTEIndex];
-            const datasetForYearNonTechnical = dataset.datasets[nonTechnicalFTEIndex];
-
-            const dataByYear = datasetForYearTechnical.data.map((d, i) => ({
-                datatechnical: d,
-                datanontechnical: datasetForYearNonTechnical.data[i],
-                label: dataset.labels[i],
-            }))
-
-            function sortfunc(a: any, b: any) {
-                return b.datatechnical - a.datatechnical;
-            }
-            const sortedDataByYear = dataByYear.sort(sortfunc)
-
-            sortedDataset.datasets[technicalFTEIndex].data = sortedDataByYear.map(d => d.datatechnical)
-            sortedDataset.datasets[nonTechnicalFTEIndex].data = sortedDataByYear.map(d => d.datanontechnical)
-            sortedDataset.labels = sortedDataByYear.map(d => d.label)
-        }
-        return sortedDataset
+        labels: labelsNREN
     }
 
-    const sortedDataset = sortDatasetByTechnicalFTE(dataset)
-    return sortedDataset;
+    return dataset;
 }

webapp/src/pages/StaffGraph.tsx

@@ -98,14 +98,14 @@ interface inputProps {
     roles?: boolean
 }
 
-function StaffGraph({ filterSelection, setFilterSelection, roles=false }: inputProps) {
+function StaffGraph({ filterSelection, setFilterSelection, roles = false }: inputProps) {
     const [staffData, setStaffData] = useState<NrenStaff[]>();
 
     const { years, nrens } = useMemo(
         () => getYearsAndNrens(staffData || []),
         [staffData]
     );
-    const nrenStaffDataset = createNRENStaffDataset(staffData || [], roles);
+    const nrenStaffDataset = createNRENStaffDataset(staffData || [], roles, filterSelection.selectedYears[0]);
 
     nrenStaffDataset.datasets.forEach(dataset => {
         dataset.hidden = !filterSelection.selectedYears.includes(parseInt(dataset.stack));