Skip to content
Snippets Groups Projects

Refactor

Merged Bjarke Madsen requested to merge refactor into develop
1 file
+ 15
5
Compare changes
  • Side-by-side
  • Inline
@@ -35,7 +35,7 @@ def db_budget_migration(nren_dict):
data = db.session.scalars(select(survey_model.Nrens))
db.session.execute(delete(presentation_models.BudgetEntry).where(
presentation_models.BudgetEntry.year < 2022))
inserts = []
inserts = defaultdict(dict)
for nren in data:
for budget in nren.budgets:
abbrev = nren.abbreviation.upper()
@@ -59,7 +59,7 @@ def db_budget_migration(nren_dict):
'budget': float(budget.budget),
'year': year
}
inserts.append(budget_entry)
inserts[nren_dict[abbrev].id][year] = budget_entry
# Import the data from excel sheet to database
exceldata = excel_parser.fetch_budget_excel_data()
@@ -79,8 +79,9 @@ def db_budget_migration(nren_dict):
'budget': budget,
'year': year
}
inserts.append(budget_entry)
db.session.bulk_insert_mappings(presentation_models.BudgetEntry, inserts)
inserts[nren_dict[abbrev].id][year] = budget_entry
all_inserts = list(itertools.chain([i for y in inserts.values() for i in y.values()]))
db.session.bulk_insert_mappings(presentation_models.BudgetEntry, all_inserts)
db.session.commit()
@@ -200,6 +201,7 @@ def db_ecprojects_migration(nren_dict):
db.session.execute(delete(presentation_models.ECProject).where(
presentation_models.ECProject.year < 2022))
ecproject_data = excel_parser.fetch_ecproject_excel_data()
inserted = set()
inserts = []
for (abbrev, year, project) in ecproject_data:
if abbrev not in nren_dict:
@@ -207,6 +209,10 @@ def db_ecprojects_migration(nren_dict):
continue
nren = nren_dict[abbrev]
if (nren.id, year, project) in inserted:
logger.warning(f'{nren.name} has duplicate EC project {project} in {year}. Skipping.')
continue
inserted.add((nren.id, year, project))
inserts.append({
'nren_id': nren.id,
'year': year,
@@ -245,8 +251,12 @@ def db_traffic_volume_migration(nren_dict):
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
if nren.name == 'CESNET':
# COMP-447: correct CESNET traffic data for 2019
if year == 2019:
to_customers = 222766
inserts.append({
'nren_id': nren.id,
'year': year,
Loading