From dc7076eb872b99972cacb1245473783a9a8e6e2a Mon Sep 17 00:00:00 2001
From: Remco Tukker <remco.tukker@geant.org>
Date: Fri, 19 May 2023 12:52:24 +0200
Subject: [PATCH] clean up the api schemas and frontend interfaces

---
 compendium_v2/routes/budget.py         | 20 ++++----
 compendium_v2/routes/charging.py       | 20 ++++----
 compendium_v2/routes/ec_projects.py    |  2 +-
 compendium_v2/routes/funding.py        | 38 +++++++--------
 compendium_v2/routes/organization.py   |  4 +-
 compendium_v2/routes/policy.py         |  3 +-
 compendium_v2/routes/staff.py          |  3 +-
 webapp/src/Schema.tsx                  | 64 +++++++++-----------------
 webapp/src/helpers/dataconversion.tsx  | 30 ++++++------
 webapp/src/pages/ChargingStructure.tsx | 10 ++--
 webapp/src/pages/DataAnalysis.tsx      |  4 +-
 webapp/src/pages/FundingSource.tsx     |  4 +-
 12 files changed, 92 insertions(+), 110 deletions(-)

diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py
index cd097657..3a7c2774 100644
--- a/compendium_v2/routes/budget.py
+++ b/compendium_v2/routes/budget.py
@@ -19,12 +19,12 @@ BUDGET_RESPONSE_SCHEMA = {
         'budget': {
             'type': 'object',
             'properties': {
-                'NREN': {'type': 'string'},
-                'NREN_COUNTRY': {'type': 'string'},
-                'BUDGET': {'type': 'number'},
-                'BUDGET_YEAR': {'type': 'integer'},
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'budget': {'type': 'number'},
+                'budget_year': {'type': 'integer'},
             },
-            'required': ['NREN', 'BUDGET', 'BUDGET_YEAR'],
+            'required': ['nren', 'nren_country', 'budget', 'budget_year'],
             'additionalProperties': False
         }
     },
@@ -50,14 +50,14 @@ def budget_view() -> Any:
 
     def _extract_data(entry: BudgetEntry):
         return {
-            'NREN': entry.nren.name,
-            'NREN_COUNTRY': entry.nren.country,
-            'BUDGET': float(entry.budget),
-            'BUDGET_YEAR': entry.year,
+            'nren': entry.nren.name,
+            'nren_country': entry.nren.country,
+            'budget': float(entry.budget),
+            'budget_year': entry.year,
         }
 
     entries = sorted(
         [_extract_data(entry) for entry in db.session.scalars(select(BudgetEntry))],
-        key=lambda d: (d['BUDGET_YEAR'], d['NREN'])
+        key=lambda d: (d['budget_year'], d['nren'])
     )
     return jsonify(entries)
diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py
index 24f49dde..e8c265db 100644
--- a/compendium_v2/routes/charging.py
+++ b/compendium_v2/routes/charging.py
@@ -19,12 +19,12 @@ CHARGING_STRUCTURE_RESPONSE_SCHEMA = {
         'charging': {
             'type': 'object',
             'properties': {
-                'NREN': {'type': 'string'},
-                'NREN_COUNTRY': {'type': 'string'},
-                'YEAR': {'type': 'integer'},
-                'FEE_TYPE': {'type': ["string", "null"]},
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'year': {'type': 'integer'},
+                'fee_type': {'type': ["string", "null"]},
             },
-            'required': ['NREN', 'YEAR'],
+            'required': ['nren', 'nren_country', 'year', 'fee_type'],
             'additionalProperties': False
         }
     },
@@ -50,14 +50,14 @@ def charging_structure_view() -> Any:
 
     def _extract_data(entry: ChargingStructure):
         return {
-            'NREN': entry.nren.name,
-            'NREN_COUNTRY': entry.nren.country,
-            'YEAR': int(entry.year),
-            'FEE_TYPE': entry.fee_type.value if entry.fee_type is not None else None,
+            'nren': entry.nren.name,
+            'nren_country': entry.nren.country,
+            'year': int(entry.year),
+            'fee_type': entry.fee_type.value if entry.fee_type is not None else None,
         }
 
     entries = sorted(
         [_extract_data(entry) for entry in db.session.scalars(select(ChargingStructure))],
-        key=lambda d: (d['NREN'], d['YEAR'])
+        key=lambda d: (d['nren'], d['year'])
     )
     return jsonify(entries)
diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py
index 346f34e4..81fa4b60 100644
--- a/compendium_v2/routes/ec_projects.py
+++ b/compendium_v2/routes/ec_projects.py
@@ -24,7 +24,7 @@ EC_PROJECTS_RESPONSE_SCHEMA = {
                 'year': {'type': 'integer'},
                 'project': {'type': 'string'}
             },
-            'required': ['nren', 'year', 'project'],
+            'required': ['nren', 'nren_country', 'year', 'project'],
             'additionalProperties': False
         }
     },
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 3955add7..fa6147eb 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -19,17 +19,17 @@ FUNDING_RESPONSE_SCHEMA = {
         'funding': {
             'type': 'object',
             'properties': {
-                'id': {'type': 'number'},
-                'NREN': {'type': 'string'},
-                'NREN_COUNTRY': {'type': 'string'},
-                'YEAR': {'type': 'integer'},
-                'CLIENT_INSTITUTIONS': {'type': 'number'},
-                'EUROPEAN_FUNDING': {'type': 'number'},
-                'GOV_PUBLIC_BODIES': {'type': 'number'},
-                'COMMERCIAL': {'type': 'number'},
-                'OTHER': {'type': 'number'}
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'year': {'type': 'integer'},
+                'client_institutions': {'type': 'number'},
+                'european_funding': {'type': 'number'},
+                'gov_public_bodies': {'type': 'number'},
+                'commercial': {'type': 'number'},
+                'other': {'type': 'number'}
             },
-            'required': ['NREN', 'YEAR'],
+            'required': ['nren', 'nren_country', 'year', 'client_institutions',
+                         'european_funding', 'gov_public_bodies', 'commercial', 'other'],
             'additionalProperties': False
         }
     },
@@ -55,18 +55,18 @@ def funding_source_view() -> Any:
 
     def _extract_data(entry: FundingSource):
         return {
-            'NREN': entry.nren.name,
-            'NREN_COUNTRY': entry.nren.country,
-            'YEAR': entry.year,
-            'CLIENT_INSTITUTIONS': float(entry.client_institutions),
-            'EUROPEAN_FUNDING': float(entry.european_funding),
-            'GOV_PUBLIC_BODIES': float(entry.gov_public_bodies),
-            'COMMERCIAL': float(entry.commercial),
-            'OTHER': float(entry.other)
+            'nren': entry.nren.name,
+            'nren_country': entry.nren.country,
+            'year': entry.year,
+            'client_institutions': float(entry.client_institutions),
+            'european_funding': float(entry.european_funding),
+            'gov_public_bodies': float(entry.gov_public_bodies),
+            'commercial': float(entry.commercial),
+            'other': float(entry.other)
         }
 
     entries = sorted(
         [_extract_data(entry) for entry in db.session.scalars(select(FundingSource))],
-        key=lambda d: (d['NREN'], d['YEAR'])
+        key=lambda d: (d['nren'], d['year'])
     )
     return jsonify(entries)
diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py
index 390af3b7..c4503207 100644
--- a/compendium_v2/routes/organization.py
+++ b/compendium_v2/routes/organization.py
@@ -24,7 +24,7 @@ ORGANIZATION_RESPONSE_SCHEMA = {
                 'year': {'type': 'integer'},
                 'name': {'type': 'string'}
             },
-            'required': ['nren', 'year', 'name'],
+            'required': ['nren', 'nren_country', 'year', 'name'],
             'additionalProperties': False
         },
         'sub_organization': {
@@ -36,7 +36,7 @@ ORGANIZATION_RESPONSE_SCHEMA = {
                 'name': {'type': 'string'},
                 'role': {'type': 'string'},
             },
-            'required': ['nren', 'year', 'name', 'role'],
+            'required': ['nren', 'nren_country', 'year', 'name', 'role'],
             'additionalProperties': False
         }
     },
diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py
index 186edcf0..b89c6741 100644
--- a/compendium_v2/routes/policy.py
+++ b/compendium_v2/routes/policy.py
@@ -27,7 +27,8 @@ POLICY_RESPONSE_SCHEMA = {
                 'privacy_notice': {'type': 'string'},
                 'data_protection': {'type': 'string'},
             },
-            'required': ['nren', 'year'],
+            'required': ['nren', 'nren_country', 'year', 'strategic_plan', 'environmental', 'equal_opportunity',
+                         'connectivity', 'acceptable_use', 'privacy_notice', 'data_protection'],
             'additionalProperties': False
         }
     },
diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py
index 720ea938..0dbbb36a 100644
--- a/compendium_v2/routes/staff.py
+++ b/compendium_v2/routes/staff.py
@@ -27,7 +27,8 @@ STAFF_RESPONSE_SCHEMA = {
                 'technical_fte': {'type': 'number'},
                 'non_technical_fte': {'type': 'number'}
             },
-            'required': ['nren', 'year', 'permanent_fte', 'subcontracted_fte', 'technical_fte', 'non_technical_fte'],
+            'required': ['nren', 'nren_country', 'year',
+                         'permanent_fte', 'subcontracted_fte', 'technical_fte', 'non_technical_fte'],
             'additionalProperties': False
         }
     },
diff --git a/webapp/src/Schema.tsx b/webapp/src/Schema.tsx
index d2c78be6..936bb30a 100644
--- a/webapp/src/Schema.tsx
+++ b/webapp/src/Schema.tsx
@@ -1,25 +1,24 @@
 export interface Nren {
-    name: string
-    nren_id: number
-    tags: string[]
+    name: string,
+    country: string
 }
 
 export interface Budget {
-    BUDGET: string,
-    BUDGET_YEAR: number,
-    NREN: string,
-    id: number
+    budget: string,
+    budget_year: number,
+    nren: string,
+    nren_country: string
 }
 
 export interface FundingSource {
-    CLIENT_INSTITUTIONS: number,
-    COMMERCIAL: number,
-    EUROPEAN_FUNDING: number,
-    GOV_PUBLIC_BODIES: number,
-    OTHER: number,
-    NREN: string,
-    YEAR: number,
-    id: number
+    client_institutions: number,
+    commercial: number,
+    european_funding: number,
+    gov_public_bodies: number,
+    other: number,
+    nren: string,
+    nren_country: string,
+    year: number
 }
 
 export interface FilterSelection {
@@ -56,9 +55,10 @@ export interface FundingSourceDataset {
 }
 
 export interface ChargingStructure {
-    NREN: string,
-    YEAR: number,
-    FEE_TYPE: (string | null),
+    nren: string,
+    nren_country: string,
+    year: number,
+    fee_type: (string | null),
 }
 
 export interface ChargingStructureDataset {
@@ -74,32 +74,9 @@ export interface ChargingStructureDataset {
     }[]
 }
 
-export interface Service {
-    compendium_id: number,
-    country_code: string,
-    country_name: string,
-    created_at: Date,
-    id: number,
-    identifier: string,
-    kpi: string[],
-    nren_abbreviation: string
-    nren_id: number,
-    public: boolean,
-    question_id: number,
-    question_style: string,
-    response_id: number,
-    short: string,
-    status: number,
-    tags: string[],
-    title: string,
-    title_detailed: string,
-    updated_at: Date,
-    url: string,
-    value: string
-}
-
 export interface NrenStaff {
     nren: string,
+    nren_country: string,
     year: number,
     permanent_fte: number,
     subcontracted_fte: number,
@@ -126,6 +103,7 @@ export interface NrenStaffDataset {
 
 export interface Organisation {
     nren: string,
+    nren_country: string,
     year: number,
     name: string,
     role?: string
@@ -133,12 +111,14 @@ export interface Organisation {
 
 export interface ECProject {
     nren: string,
+    nren_country: string,
     year: number,
     project: string,
 }
 
 export interface Policy {
     nren: string,
+    nren_country: string,
     year: number,
     acceptable_use: string,
     connectivity: string,
diff --git a/webapp/src/helpers/dataconversion.tsx b/webapp/src/helpers/dataconversion.tsx
index 772e1244..d281c258 100644
--- a/webapp/src/helpers/dataconversion.tsx
+++ b/webapp/src/helpers/dataconversion.tsx
@@ -48,17 +48,17 @@ function CreateDataLookup(data: FundingSource[]) {
     const dataLookup = new Map<string, Map<string, number>>();
 
     data.forEach((item: FundingSource) => {
-        const lookupKey = `${item.NREN}/${item.YEAR}`
+        const lookupKey = `${item.nren}/${item.year}`
 
         let fundingSourceMap = dataLookup.get(lookupKey)
         if (!fundingSourceMap) {
             fundingSourceMap = new Map<string, number>();
         }
-        fundingSourceMap.set("CLIENT INSTITUTIONS", item.CLIENT_INSTITUTIONS)
-        fundingSourceMap.set("COMMERCIAL", item.COMMERCIAL)
-        fundingSourceMap.set("EUROPEAN FUNDING", item.EUROPEAN_FUNDING)
-        fundingSourceMap.set("GOV/PUBLIC_BODIES", item.GOV_PUBLIC_BODIES)
-        fundingSourceMap.set("OTHER", item.OTHER)
+        fundingSourceMap.set("CLIENT INSTITUTIONS", item.client_institutions)
+        fundingSourceMap.set("COMMERCIAL", item.commercial)
+        fundingSourceMap.set("EUROPEAN FUNDING", item.european_funding)
+        fundingSourceMap.set("GOV/PUBLIC_BODIES", item.gov_public_bodies)
+        fundingSourceMap.set("OTHER", item.other)
         dataLookup.set(lookupKey, fundingSourceMap)
     })
     return dataLookup
@@ -68,8 +68,8 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[])
     const data = fundingSourcesData;
     const dataLookup = CreateDataLookup(data)
 
-    const labelsYear = [...new Set(data.map((item: FundingSource) => item.YEAR))];
-    const labelsNREN = [...new Set(data.map((item: FundingSource) => item.NREN))];
+    const labelsYear = [...new Set(data.map((item: FundingSource) => item.year))];
+    const labelsNREN = [...new Set(data.map((item: FundingSource) => item.nren))];
     const fundingSources = [
         "CLIENT INSTITUTIONS",
         "COMMERCIAL",
@@ -117,15 +117,15 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[])
 function createBudgetDataLookup(budgetEntries: Budget[]) {
     const dataLookup = new Map<string, number>();
     budgetEntries.forEach((item: Budget) => {
-        const lookupKey = `${item.NREN}/${item.BUDGET_YEAR}`;
-        dataLookup.set(lookupKey, Number(item.BUDGET));
+        const lookupKey = `${item.nren}/${item.budget_year}`;
+        dataLookup.set(lookupKey, Number(item.budget));
     })
     return dataLookup;
 }
 
 export function createBudgetDataset(budgetEntries: Budget[]): BasicDataset {
-    const labelsYear = [...new Set(budgetEntries.map((item) => item.BUDGET_YEAR))].sort();
-    const labelsNREN = [...new Set(budgetEntries.map((item) => item.NREN))].sort();
+    const labelsYear = [...new Set(budgetEntries.map((item) => item.budget_year))].sort();
+    const labelsNREN = [...new Set(budgetEntries.map((item) => item.nren))].sort();
 
     const dataLookup = createBudgetDataLookup(budgetEntries);
 
@@ -153,12 +153,12 @@ export function createChargingStructureDataLookup(data: ChargingStructure[]) {
     const dataLookup = new Map<string, Map<number, string>>();
 
     data.forEach(entry => {
-        let nrenEntry = dataLookup.get(entry.NREN);
+        let nrenEntry = dataLookup.get(entry.nren);
         if (!nrenEntry) {
             nrenEntry = new Map<number, string>();
         }
-        nrenEntry.set(entry.YEAR, entry.FEE_TYPE || '');
-        dataLookup.set(entry.NREN, nrenEntry);
+        nrenEntry.set(entry.year, entry.fee_type || '');
+        dataLookup.set(entry.nren, nrenEntry);
     });
     return dataLookup;
 }
diff --git a/webapp/src/pages/ChargingStructure.tsx b/webapp/src/pages/ChargingStructure.tsx
index 4948a911..ac7c925f 100644
--- a/webapp/src/pages/ChargingStructure.tsx
+++ b/webapp/src/pages/ChargingStructure.tsx
@@ -42,11 +42,11 @@ interface inputProps {
 function ChargingStructurePage({ filterSelection, setFilterSelection }: inputProps): React.ReactElement {
     const [chargingStructureData, setChargingStructureData] = useState<ChargingStructure[]>([]);
 
-    const labelsNREN = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.NREN))];
-    const labelsYear = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.YEAR))];
+    const labelsNREN = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.nren))];
+    const labelsYear = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.year))];
 
     const selectedData = (chargingStructureData || []).filter(data =>
-        filterSelection.selectedYears.includes(data.YEAR) && filterSelection.selectedNrens.includes(data.NREN)
+        filterSelection.selectedYears.includes(data.year) && filterSelection.selectedNrens.includes(data.nren)
     );
 
     const dataLookup = createChargingStructureDataLookup(selectedData);
@@ -58,8 +58,8 @@ function ChargingStructurePage({ filterSelection, setFilterSelection }: inputPro
             setChargingStructureData(_chargingStructureData);
 
             // filter fallback for when nothing is selected (only last year for all nrens)
-            const nrens = new Set(_chargingStructureData.map((item: ChargingStructure) => item.NREN));
-            const years = new Set(_chargingStructureData.map((item: ChargingStructure) => item.YEAR));
+            const nrens = new Set(_chargingStructureData.map((item: ChargingStructure) => item.nren));
+            const years = new Set(_chargingStructureData.map((item: ChargingStructure) => item.year));
             setFilterSelection(previous => {
                 const visibleYears = previous.selectedYears.filter(year => years.has(year));
                 const newSelectedYears = visibleYears.length ? previous.selectedYears : [Math.max(...years)];
diff --git a/webapp/src/pages/DataAnalysis.tsx b/webapp/src/pages/DataAnalysis.tsx
index 6549742c..7a4030c6 100644
--- a/webapp/src/pages/DataAnalysis.tsx
+++ b/webapp/src/pages/DataAnalysis.tsx
@@ -30,7 +30,7 @@ function DataAnalysis({ filterSelection, setFilterSelection }: inputProps): Reac
 
     const [budgetResponse, setBudget] = useState<Budget[]>();
 
-    const nrens = new Set((budgetResponse ?? []).map((item) => item.NREN));
+    const nrens = new Set((budgetResponse ?? []).map((item) => item.nren));
     const budgetData = createBudgetDataset(budgetResponse ?? []);
 
     budgetData.datasets.forEach(dataset => {
@@ -44,7 +44,7 @@ function DataAnalysis({ filterSelection, setFilterSelection }: inputProps): Reac
                     setBudget(budget)
 
                     // filter fallback for when nothing is selected (all nrens)
-                    const nrens = new Set(budget.map((item) => item.NREN));
+                    const nrens = new Set(budget.map((item) => item.nren));
                     setFilterSelection(previous => {
                         const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
                         const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens];
diff --git a/webapp/src/pages/FundingSource.tsx b/webapp/src/pages/FundingSource.tsx
index 8454e83a..3430b393 100644
--- a/webapp/src/pages/FundingSource.tsx
+++ b/webapp/src/pages/FundingSource.tsx
@@ -81,8 +81,8 @@ function getYearsAndNrens(sourceData: FundingSource[]) {
     const years = new Set<number>();
     const nrens = new Set<string>();
     sourceData.forEach(datapoint => {
-        years.add(datapoint.YEAR);
-        nrens.add(datapoint.NREN);
+        years.add(datapoint.year);
+        nrens.add(datapoint.nren);
     });
     return { years: years, nrens: nrens };
 }
-- 
GitLab