fix remit excel parsing, and update HEANET funding source data

compendium_v2/publishers/excel_parser.py

@@ -40,7 +40,7 @@ def fetch_funding_excel_data():
 
     def hard_number_convert(s, source_name, nren, year):
         if s is None:
-            logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year})')
+            # logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year})')
             return float(0)
         try:
             return float(s)
@@ -72,6 +72,9 @@ def fetch_funding_excel_data():
 
             # process the data (e.g. save to database)
             if nren is not None:
+                if nren.upper() == 'HEANET':
+                    other = other + commercial
+                    commercial = 0
                 yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
 
     def create_points_for_year_from_2018(sheet, start_row, end_row, year, nren_col, col_start):
@@ -93,6 +96,9 @@ def fetch_funding_excel_data():
 
             # process the data (e.g. save to database)
             if nren is not None:
+                if nren.upper() == 'HEANET':
+                    other = other + commercial
+                    commercial = 0
                 yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
 
     # For 2016
@@ -123,7 +129,6 @@ def fetch_charging_structure_excel_data():
             # extract the data from the row
             nren = ws.cell(row=row, column=col_start).value
             charging_structure = ws.cell(row=row, column=col_start + 1).value
-            logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
             if charging_structure is not None:
                 if "do not charge" in charging_structure:
                     charging_structure = FeeType.no_charge
@@ -138,8 +143,6 @@ def fetch_charging_structure_excel_data():
                 else:
                     charging_structure = None
 
-                logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
-
                 yield nren.upper(), year, charging_structure
 
     def create_points_for_2019(start_row, end_row, year, col_start):
@@ -165,8 +168,6 @@ def fetch_charging_structure_excel_data():
                 else:
                     charging_structure = None
 
-                logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
-
                 yield nren.upper(), year, charging_structure
 
     # For 2021
@@ -222,13 +223,10 @@ def fetch_staff_function_excel_data():
         try:
             return float(value)
         except (TypeError, ValueError):
-            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            logger.info(
+                f'NREN: {nren} year: {year} has {"NO DATA" if value == "" else value} for {description}; set to 0.')
             return 0
 
-    def read_cell_number(row, column, nren, year, description):
-        value = ws.cell(row=row, column=column).value
-        return convert_number(value, nren, year, description)
-
     def create_points_for_year_until_2019(year, nren_column, start_column):
         offset = abs(start_column - nren_column)
         for row in ws.iter_rows(min_row=start_row, max_row=end_row, min_col=nren_column, max_col=start_column + 5):
@@ -451,21 +449,26 @@ def fetch_remit_excel_data():
     def create_points_for_year(year, start_column, cut=False):
         category_row = [c.value for c in list(ws.iter_rows(
             min_row=8, max_row=8, min_col=start_column+2, max_col=start_column + 21 + (2 if cut else 0)))[0]]
-        if cut:
-            category_row = category_row[:4] + category_row[6:]
         categories = [get_category(cat) for cat in category_row[::2]]
+        if cut:
+            # remove the 3rd value, which is the nren name column
+            categories = categories[:2] + categories[3:]
 
         for row in ws.iter_rows(min_row=9, max_row=52, min_col=start_column+1,
                                 max_col=start_column + 21 + (2 if cut else 0)):
-            if cut:
-                row = row[:4] + row[6:]
 
             nren_name = row[0].value
             if not nren_name:
                 continue
             nren_name = nren_name.upper()
             every_second_value = [c.value for c in row][::2]
-            remit = [get_remit(val) for val in every_second_value[1:]]
+            # remove the first value, which is the nren name
+            every_second_value = every_second_value[1:]
+            if cut:
+                # remove the 3rd value, which is another nren name
+                every_second_value = every_second_value[:2] + every_second_value[3:]
+
+            remit = [get_remit(val) for val in every_second_value]
 
             values = dict(zip(categories, remit))