deduplicate inserts & update CESNET data

compendium_v2/publishers/survey_publisher_legacy_excel.py

@@ -35,7 +35,7 @@ def db_budget_migration(nren_dict):
     data = db.session.scalars(select(survey_model.Nrens))
     db.session.execute(delete(presentation_models.BudgetEntry).where(
         presentation_models.BudgetEntry.year < 2022))
-    inserts = []
+    inserts = defaultdict(dict)
     for nren in data:
         for budget in nren.budgets:
             abbrev = nren.abbreviation.upper()
@@ -59,7 +59,7 @@ def db_budget_migration(nren_dict):
                 'budget': float(budget.budget),
                 'year': year
             }
-            inserts.append(budget_entry)
+            inserts[nren_dict[abbrev].id][year] = budget_entry
 
     # Import the data from excel sheet to database
     exceldata = excel_parser.fetch_budget_excel_data()
@@ -79,8 +79,9 @@ def db_budget_migration(nren_dict):
             'budget': budget,
             'year': year
         }
-        inserts.append(budget_entry)
-    db.session.bulk_insert_mappings(presentation_models.BudgetEntry, inserts)
+        inserts[nren_dict[abbrev].id][year] = budget_entry
+    all_inserts = list(itertools.chain([i for y in inserts.values() for i in y.values()]))
+    db.session.bulk_insert_mappings(presentation_models.BudgetEntry, all_inserts)
     db.session.commit()
 
 
@@ -200,6 +201,7 @@ def db_ecprojects_migration(nren_dict):
     db.session.execute(delete(presentation_models.ECProject).where(
         presentation_models.ECProject.year < 2022))
     ecproject_data = excel_parser.fetch_ecproject_excel_data()
+    inserted = set()
     inserts = []
     for (abbrev, year, project) in ecproject_data:
         if abbrev not in nren_dict:
@@ -207,6 +209,10 @@ def db_ecprojects_migration(nren_dict):
             continue
 
         nren = nren_dict[abbrev]
+        if (nren.id, year, project) in inserted:
+            logger.warning(f'{nren.name} has duplicate EC project {project} in {year}. Skipping.')
+            continue
+        inserted.add((nren.id, year, project))
         inserts.append({
             'nren_id': nren.id,
             'year': year,
@@ -245,8 +251,12 @@ def db_traffic_volume_migration(nren_dict):
         if abbrev not in nren_dict:
             logger.warning(f'{abbrev} unknown. Skipping.')
             continue
-
+        
         nren = nren_dict[abbrev]
+        if nren.name == 'CESNET':
+            # COMP-447: correct CESNET traffic data for 2019
+            if year == 2019:
+                to_customers = 222766
         inserts.append({
             'nren_id': nren.id,
             'year': year,