Skip to content
Snippets Groups Projects

Refactor

Merged Bjarke Madsen requested to merge refactor into develop
1 file
+ 19
16
Compare changes
  • Side-by-side
  • Inline
@@ -40,7 +40,7 @@ def fetch_funding_excel_data():
def hard_number_convert(s, source_name, nren, year):
if s is None:
logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year})')
# logger.info(f'Invalid Value :{nren} has empty value for {source_name} for year ({year})')
return float(0)
try:
return float(s)
@@ -72,6 +72,9 @@ def fetch_funding_excel_data():
# process the data (e.g. save to database)
if nren is not None:
if nren.upper() == 'HEANET':
other = other + commercial
commercial = 0
yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
def create_points_for_year_from_2018(sheet, start_row, end_row, year, nren_col, col_start):
@@ -93,6 +96,9 @@ def fetch_funding_excel_data():
# process the data (e.g. save to database)
if nren is not None:
if nren.upper() == 'HEANET':
other = other + commercial
commercial = 0
yield (nren.upper(), year, client_institution, european_funding, gov_public_bodies, commercial, other)
# For 2016
@@ -123,7 +129,6 @@ def fetch_charging_structure_excel_data():
# extract the data from the row
nren = ws.cell(row=row, column=col_start).value
charging_structure = ws.cell(row=row, column=col_start + 1).value
logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
if charging_structure is not None:
if "do not charge" in charging_structure:
charging_structure = FeeType.no_charge
@@ -138,8 +143,6 @@ def fetch_charging_structure_excel_data():
else:
charging_structure = None
logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
yield nren.upper(), year, charging_structure
def create_points_for_2019(start_row, end_row, year, col_start):
@@ -165,8 +168,6 @@ def fetch_charging_structure_excel_data():
else:
charging_structure = None
logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
yield nren.upper(), year, charging_structure
# For 2021
@@ -222,13 +223,10 @@ def fetch_staff_function_excel_data():
try:
return float(value)
except (TypeError, ValueError):
logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
logger.info(
f'NREN: {nren} year: {year} has {"NO DATA" if value == "" else value} for {description}; set to 0.')
return 0
def read_cell_number(row, column, nren, year, description):
value = ws.cell(row=row, column=column).value
return convert_number(value, nren, year, description)
def create_points_for_year_until_2019(year, nren_column, start_column):
offset = abs(start_column - nren_column)
for row in ws.iter_rows(min_row=start_row, max_row=end_row, min_col=nren_column, max_col=start_column + 5):
@@ -451,21 +449,26 @@ def fetch_remit_excel_data():
def create_points_for_year(year, start_column, cut=False):
category_row = [c.value for c in list(ws.iter_rows(
min_row=8, max_row=8, min_col=start_column+2, max_col=start_column + 21 + (2 if cut else 0)))[0]]
if cut:
category_row = category_row[:4] + category_row[6:]
categories = [get_category(cat) for cat in category_row[::2]]
if cut:
# remove the 3rd value, which is the nren name column
categories = categories[:2] + categories[3:]
for row in ws.iter_rows(min_row=9, max_row=52, min_col=start_column+1,
max_col=start_column + 21 + (2 if cut else 0)):
if cut:
row = row[:4] + row[6:]
nren_name = row[0].value
if not nren_name:
continue
nren_name = nren_name.upper()
every_second_value = [c.value for c in row][::2]
remit = [get_remit(val) for val in every_second_value[1:]]
# remove the first value, which is the nren name
every_second_value = every_second_value[1:]
if cut:
# remove the 3rd value, which is another nren name
every_second_value = every_second_value[:2] + every_second_value[3:]
remit = [get_remit(val) for val in every_second_value]
values = dict(zip(categories, remit))
Loading