fix publishers for primary keys and use second column of charging data in 2019

compendium_v2/background_task/parse_excel_data.py

@@ -121,7 +121,7 @@ def fetch_charging_structure_excel_data():
     ws = wb[sheet_name]
 
     # iterate over the rows in the worksheet
-    def create_points_for_year(start_row, end_row, year, col_start):
+    def create_points_for_2021(start_row, end_row, year, col_start):
         for row in range(start_row, end_row):
             # extract the data from the row
             nren = ws.cell(row=row, column=col_start).value
@@ -145,11 +145,38 @@ def fetch_charging_structure_excel_data():
 
                 yield nren.upper(), year, charging_structure
 
+    def create_points_for_2019(start_row, end_row, year, col_start):
+        for row in range(start_row, end_row):
+            # extract the data from the row
+            nren = ws.cell(row=row, column=col_start).value
+            charging_structure = ws.cell(row=row, column=col_start + 1).value
+            structure_2 = ws.cell(row=row, column=col_start + 2).value
+            logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, {structure_2}, Year: {year}')
+            if charging_structure is not None:
+                if structure_2 not in [None, '']:
+                    charging_structure = FeeType.other
+                elif "do not charge" in charging_structure:
+                    charging_structure = FeeType.no_charge
+                elif "combination" in charging_structure:
+                    charging_structure = FeeType.combination
+                elif "flat" in charging_structure:
+                    charging_structure = FeeType.flat_fee
+                elif "usage-based" in charging_structure:
+                    charging_structure = FeeType.usage_based_fee
+                elif "Other" in charging_structure:
+                    charging_structure = FeeType.other
+                else:
+                    charging_structure = None
+
+                logger.info(f'NREN: {nren}, Charging Structure: {charging_structure}, Year: {year}')
+
+                yield nren.upper(), year, charging_structure
+
     # For 2021
-    yield from create_points_for_year(3, 46, 2021, 2)
+    yield from create_points_for_2021(3, 46, 2021, 2)
 
     # For 2019
-    yield from create_points_for_year(3, 46, 2019, 6)
+    yield from create_points_for_2019(3, 46, 2019, 6)
 
 
 def fetch_staffing_excel_data():

compendium_v2/publishers/survey_publisher_2022.py

@@ -156,6 +156,7 @@ def transfer_budget():
 
             budget_entry = model.BudgetEntry(
                 nren=nren_dict[nren_name],
+                nren_id=nren_dict[nren_name].id,
                 budget=budget,
                 year=2022,
             )
@@ -197,6 +198,7 @@ def transfer_funding_sources():
 
             funding_source = model.FundingSource(
                 nren=nren_dict[nren_name],
+                nren_id=nren_dict[nren_name].id,
                 year=2022,
                 client_institutions=nren_info[FundingSource.CLIENT_INSTITUTIONS],
                 european_funding=nren_info[FundingSource.EUROPEAN_FUNDING],
@@ -248,6 +250,7 @@ def transfer_staff_data():
                             f' ({employed} != {technical})')
 
             staff_data = model.NrenStaff(
+                nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
                 permanent_fte=nren_info[StaffQuestion.PERMANENT_FTE],
@@ -281,6 +284,7 @@ def transfer_nren_parent_org():
                 continue
 
             parent_org = model.ParentOrganization(
+                nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
                 organization=value,
@@ -329,6 +333,7 @@ def transfer_nren_sub_org():
         for nren_name, suborgs in lookup.items():
             for suborg_name, role in suborgs:
                 suborg = model.SubOrganization(
+                    nren=nren_dict[nren_name],
                     nren_id=nren_dict[nren_name].id,
                     year=2022,
                     organization=suborg_name,
@@ -365,6 +370,7 @@ def transfer_charging_structure():
                 charging_structure = None
 
             charging_structure = model.ChargingStructure(
+                nren=nren_dict[nren_name],
                 nren_id=nren_dict[nren_name].id,
                 year=2022,
                 fee_type=charging_structure,
@@ -408,6 +414,7 @@ def transfer_ec_projects():
                 val = val.split('(contract n')[0]
 
                 ec_project = model.ECProject(
+                    nren=nren_dict[nren_name],
                     nren_id=nren_dict[nren_name].id,
                     year=2022,
                     project=str(val).strip()

compendium_v2/publishers/survey_publisher_v1.py

@@ -46,6 +46,7 @@ def db_budget_migration():
 
                 budget_entry = model.BudgetEntry(
                     nren=nren_dict[abbrev],
+                    nren_id=nren_dict[abbrev].id,
                     budget=float(budget.budget),
                     year=year
                 )
@@ -62,7 +63,12 @@ def db_budget_migration():
             if budget > 200:
                 logger.warning(f'{nren} has budget set to >200M EUR for {year}. ({budget})')
 
-            budget_entry = model.BudgetEntry(nren=nren_dict[abbrev], budget=budget, year=year)
+            budget_entry = model.BudgetEntry(
+                nren=nren_dict[abbrev],
+                nren_id=nren_dict[abbrev].id,
+                budget=budget,
+                year=year
+            )
             session.merge(budget_entry)
         session.commit()
 
@@ -90,6 +96,7 @@ def db_funding_migration():
 
             budget_entry = model.FundingSource(
                 nren=nren_dict[abbrev],
+                nren_id=nren_dict[abbrev].id,
                 year=year,
                 client_institutions=client_institution,
                 european_funding=european_funding,
@@ -113,7 +120,11 @@ def db_charging_structure_migration():
                 continue
 
             charging_structure_entry = model.ChargingStructure(
-                nren=nren_dict[abbrev], year=year, fee_type=charging_structure)
+                nren=nren_dict[abbrev],
+                nren_id=nren_dict[abbrev].id,
+                year=year,
+                fee_type=charging_structure
+            )
             session.merge(charging_structure_entry)
         session.commit()