From dc7076eb872b99972cacb1245473783a9a8e6e2a Mon Sep 17 00:00:00 2001 From: Remco Tukker <remco.tukker@geant.org> Date: Fri, 19 May 2023 12:52:24 +0200 Subject: [PATCH] clean up the api schemas and frontend interfaces --- compendium_v2/routes/budget.py | 20 ++++---- compendium_v2/routes/charging.py | 20 ++++---- compendium_v2/routes/ec_projects.py | 2 +- compendium_v2/routes/funding.py | 38 +++++++-------- compendium_v2/routes/organization.py | 4 +- compendium_v2/routes/policy.py | 3 +- compendium_v2/routes/staff.py | 3 +- webapp/src/Schema.tsx | 64 +++++++++----------------- webapp/src/helpers/dataconversion.tsx | 30 ++++++------ webapp/src/pages/ChargingStructure.tsx | 10 ++-- webapp/src/pages/DataAnalysis.tsx | 4 +- webapp/src/pages/FundingSource.tsx | 4 +- 12 files changed, 92 insertions(+), 110 deletions(-) diff --git a/compendium_v2/routes/budget.py b/compendium_v2/routes/budget.py index cd097657..3a7c2774 100644 --- a/compendium_v2/routes/budget.py +++ b/compendium_v2/routes/budget.py @@ -19,12 +19,12 @@ BUDGET_RESPONSE_SCHEMA = { 'budget': { 'type': 'object', 'properties': { - 'NREN': {'type': 'string'}, - 'NREN_COUNTRY': {'type': 'string'}, - 'BUDGET': {'type': 'number'}, - 'BUDGET_YEAR': {'type': 'integer'}, + 'nren': {'type': 'string'}, + 'nren_country': {'type': 'string'}, + 'budget': {'type': 'number'}, + 'budget_year': {'type': 'integer'}, }, - 'required': ['NREN', 'BUDGET', 'BUDGET_YEAR'], + 'required': ['nren', 'nren_country', 'budget', 'budget_year'], 'additionalProperties': False } }, @@ -50,14 +50,14 @@ def budget_view() -> Any: def _extract_data(entry: BudgetEntry): return { - 'NREN': entry.nren.name, - 'NREN_COUNTRY': entry.nren.country, - 'BUDGET': float(entry.budget), - 'BUDGET_YEAR': entry.year, + 'nren': entry.nren.name, + 'nren_country': entry.nren.country, + 'budget': float(entry.budget), + 'budget_year': entry.year, } entries = sorted( [_extract_data(entry) for entry in db.session.scalars(select(BudgetEntry))], - key=lambda d: (d['BUDGET_YEAR'], d['NREN']) + key=lambda d: (d['budget_year'], d['nren']) ) return jsonify(entries) diff --git a/compendium_v2/routes/charging.py b/compendium_v2/routes/charging.py index 24f49dde..e8c265db 100644 --- a/compendium_v2/routes/charging.py +++ b/compendium_v2/routes/charging.py @@ -19,12 +19,12 @@ CHARGING_STRUCTURE_RESPONSE_SCHEMA = { 'charging': { 'type': 'object', 'properties': { - 'NREN': {'type': 'string'}, - 'NREN_COUNTRY': {'type': 'string'}, - 'YEAR': {'type': 'integer'}, - 'FEE_TYPE': {'type': ["string", "null"]}, + 'nren': {'type': 'string'}, + 'nren_country': {'type': 'string'}, + 'year': {'type': 'integer'}, + 'fee_type': {'type': ["string", "null"]}, }, - 'required': ['NREN', 'YEAR'], + 'required': ['nren', 'nren_country', 'year', 'fee_type'], 'additionalProperties': False } }, @@ -50,14 +50,14 @@ def charging_structure_view() -> Any: def _extract_data(entry: ChargingStructure): return { - 'NREN': entry.nren.name, - 'NREN_COUNTRY': entry.nren.country, - 'YEAR': int(entry.year), - 'FEE_TYPE': entry.fee_type.value if entry.fee_type is not None else None, + 'nren': entry.nren.name, + 'nren_country': entry.nren.country, + 'year': int(entry.year), + 'fee_type': entry.fee_type.value if entry.fee_type is not None else None, } entries = sorted( [_extract_data(entry) for entry in db.session.scalars(select(ChargingStructure))], - key=lambda d: (d['NREN'], d['YEAR']) + key=lambda d: (d['nren'], d['year']) ) return jsonify(entries) diff --git a/compendium_v2/routes/ec_projects.py b/compendium_v2/routes/ec_projects.py index 346f34e4..81fa4b60 100644 --- a/compendium_v2/routes/ec_projects.py +++ b/compendium_v2/routes/ec_projects.py @@ -24,7 +24,7 @@ EC_PROJECTS_RESPONSE_SCHEMA = { 'year': {'type': 'integer'}, 'project': {'type': 'string'} }, - 'required': ['nren', 'year', 'project'], + 'required': ['nren', 'nren_country', 'year', 'project'], 'additionalProperties': False } }, diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py index 3955add7..fa6147eb 100644 --- a/compendium_v2/routes/funding.py +++ b/compendium_v2/routes/funding.py @@ -19,17 +19,17 @@ FUNDING_RESPONSE_SCHEMA = { 'funding': { 'type': 'object', 'properties': { - 'id': {'type': 'number'}, - 'NREN': {'type': 'string'}, - 'NREN_COUNTRY': {'type': 'string'}, - 'YEAR': {'type': 'integer'}, - 'CLIENT_INSTITUTIONS': {'type': 'number'}, - 'EUROPEAN_FUNDING': {'type': 'number'}, - 'GOV_PUBLIC_BODIES': {'type': 'number'}, - 'COMMERCIAL': {'type': 'number'}, - 'OTHER': {'type': 'number'} + 'nren': {'type': 'string'}, + 'nren_country': {'type': 'string'}, + 'year': {'type': 'integer'}, + 'client_institutions': {'type': 'number'}, + 'european_funding': {'type': 'number'}, + 'gov_public_bodies': {'type': 'number'}, + 'commercial': {'type': 'number'}, + 'other': {'type': 'number'} }, - 'required': ['NREN', 'YEAR'], + 'required': ['nren', 'nren_country', 'year', 'client_institutions', + 'european_funding', 'gov_public_bodies', 'commercial', 'other'], 'additionalProperties': False } }, @@ -55,18 +55,18 @@ def funding_source_view() -> Any: def _extract_data(entry: FundingSource): return { - 'NREN': entry.nren.name, - 'NREN_COUNTRY': entry.nren.country, - 'YEAR': entry.year, - 'CLIENT_INSTITUTIONS': float(entry.client_institutions), - 'EUROPEAN_FUNDING': float(entry.european_funding), - 'GOV_PUBLIC_BODIES': float(entry.gov_public_bodies), - 'COMMERCIAL': float(entry.commercial), - 'OTHER': float(entry.other) + 'nren': entry.nren.name, + 'nren_country': entry.nren.country, + 'year': entry.year, + 'client_institutions': float(entry.client_institutions), + 'european_funding': float(entry.european_funding), + 'gov_public_bodies': float(entry.gov_public_bodies), + 'commercial': float(entry.commercial), + 'other': float(entry.other) } entries = sorted( [_extract_data(entry) for entry in db.session.scalars(select(FundingSource))], - key=lambda d: (d['NREN'], d['YEAR']) + key=lambda d: (d['nren'], d['year']) ) return jsonify(entries) diff --git a/compendium_v2/routes/organization.py b/compendium_v2/routes/organization.py index 390af3b7..c4503207 100644 --- a/compendium_v2/routes/organization.py +++ b/compendium_v2/routes/organization.py @@ -24,7 +24,7 @@ ORGANIZATION_RESPONSE_SCHEMA = { 'year': {'type': 'integer'}, 'name': {'type': 'string'} }, - 'required': ['nren', 'year', 'name'], + 'required': ['nren', 'nren_country', 'year', 'name'], 'additionalProperties': False }, 'sub_organization': { @@ -36,7 +36,7 @@ ORGANIZATION_RESPONSE_SCHEMA = { 'name': {'type': 'string'}, 'role': {'type': 'string'}, }, - 'required': ['nren', 'year', 'name', 'role'], + 'required': ['nren', 'nren_country', 'year', 'name', 'role'], 'additionalProperties': False } }, diff --git a/compendium_v2/routes/policy.py b/compendium_v2/routes/policy.py index 186edcf0..b89c6741 100644 --- a/compendium_v2/routes/policy.py +++ b/compendium_v2/routes/policy.py @@ -27,7 +27,8 @@ POLICY_RESPONSE_SCHEMA = { 'privacy_notice': {'type': 'string'}, 'data_protection': {'type': 'string'}, }, - 'required': ['nren', 'year'], + 'required': ['nren', 'nren_country', 'year', 'strategic_plan', 'environmental', 'equal_opportunity', + 'connectivity', 'acceptable_use', 'privacy_notice', 'data_protection'], 'additionalProperties': False } }, diff --git a/compendium_v2/routes/staff.py b/compendium_v2/routes/staff.py index 720ea938..0dbbb36a 100644 --- a/compendium_v2/routes/staff.py +++ b/compendium_v2/routes/staff.py @@ -27,7 +27,8 @@ STAFF_RESPONSE_SCHEMA = { 'technical_fte': {'type': 'number'}, 'non_technical_fte': {'type': 'number'} }, - 'required': ['nren', 'year', 'permanent_fte', 'subcontracted_fte', 'technical_fte', 'non_technical_fte'], + 'required': ['nren', 'nren_country', 'year', + 'permanent_fte', 'subcontracted_fte', 'technical_fte', 'non_technical_fte'], 'additionalProperties': False } }, diff --git a/webapp/src/Schema.tsx b/webapp/src/Schema.tsx index d2c78be6..936bb30a 100644 --- a/webapp/src/Schema.tsx +++ b/webapp/src/Schema.tsx @@ -1,25 +1,24 @@ export interface Nren { - name: string - nren_id: number - tags: string[] + name: string, + country: string } export interface Budget { - BUDGET: string, - BUDGET_YEAR: number, - NREN: string, - id: number + budget: string, + budget_year: number, + nren: string, + nren_country: string } export interface FundingSource { - CLIENT_INSTITUTIONS: number, - COMMERCIAL: number, - EUROPEAN_FUNDING: number, - GOV_PUBLIC_BODIES: number, - OTHER: number, - NREN: string, - YEAR: number, - id: number + client_institutions: number, + commercial: number, + european_funding: number, + gov_public_bodies: number, + other: number, + nren: string, + nren_country: string, + year: number } export interface FilterSelection { @@ -56,9 +55,10 @@ export interface FundingSourceDataset { } export interface ChargingStructure { - NREN: string, - YEAR: number, - FEE_TYPE: (string | null), + nren: string, + nren_country: string, + year: number, + fee_type: (string | null), } export interface ChargingStructureDataset { @@ -74,32 +74,9 @@ export interface ChargingStructureDataset { }[] } -export interface Service { - compendium_id: number, - country_code: string, - country_name: string, - created_at: Date, - id: number, - identifier: string, - kpi: string[], - nren_abbreviation: string - nren_id: number, - public: boolean, - question_id: number, - question_style: string, - response_id: number, - short: string, - status: number, - tags: string[], - title: string, - title_detailed: string, - updated_at: Date, - url: string, - value: string -} - export interface NrenStaff { nren: string, + nren_country: string, year: number, permanent_fte: number, subcontracted_fte: number, @@ -126,6 +103,7 @@ export interface NrenStaffDataset { export interface Organisation { nren: string, + nren_country: string, year: number, name: string, role?: string @@ -133,12 +111,14 @@ export interface Organisation { export interface ECProject { nren: string, + nren_country: string, year: number, project: string, } export interface Policy { nren: string, + nren_country: string, year: number, acceptable_use: string, connectivity: string, diff --git a/webapp/src/helpers/dataconversion.tsx b/webapp/src/helpers/dataconversion.tsx index 772e1244..d281c258 100644 --- a/webapp/src/helpers/dataconversion.tsx +++ b/webapp/src/helpers/dataconversion.tsx @@ -48,17 +48,17 @@ function CreateDataLookup(data: FundingSource[]) { const dataLookup = new Map<string, Map<string, number>>(); data.forEach((item: FundingSource) => { - const lookupKey = `${item.NREN}/${item.YEAR}` + const lookupKey = `${item.nren}/${item.year}` let fundingSourceMap = dataLookup.get(lookupKey) if (!fundingSourceMap) { fundingSourceMap = new Map<string, number>(); } - fundingSourceMap.set("CLIENT INSTITUTIONS", item.CLIENT_INSTITUTIONS) - fundingSourceMap.set("COMMERCIAL", item.COMMERCIAL) - fundingSourceMap.set("EUROPEAN FUNDING", item.EUROPEAN_FUNDING) - fundingSourceMap.set("GOV/PUBLIC_BODIES", item.GOV_PUBLIC_BODIES) - fundingSourceMap.set("OTHER", item.OTHER) + fundingSourceMap.set("CLIENT INSTITUTIONS", item.client_institutions) + fundingSourceMap.set("COMMERCIAL", item.commercial) + fundingSourceMap.set("EUROPEAN FUNDING", item.european_funding) + fundingSourceMap.set("GOV/PUBLIC_BODIES", item.gov_public_bodies) + fundingSourceMap.set("OTHER", item.other) dataLookup.set(lookupKey, fundingSourceMap) }) return dataLookup @@ -68,8 +68,8 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) const data = fundingSourcesData; const dataLookup = CreateDataLookup(data) - const labelsYear = [...new Set(data.map((item: FundingSource) => item.YEAR))]; - const labelsNREN = [...new Set(data.map((item: FundingSource) => item.NREN))]; + const labelsYear = [...new Set(data.map((item: FundingSource) => item.year))]; + const labelsNREN = [...new Set(data.map((item: FundingSource) => item.nren))]; const fundingSources = [ "CLIENT INSTITUTIONS", "COMMERCIAL", @@ -117,15 +117,15 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) function createBudgetDataLookup(budgetEntries: Budget[]) { const dataLookup = new Map<string, number>(); budgetEntries.forEach((item: Budget) => { - const lookupKey = `${item.NREN}/${item.BUDGET_YEAR}`; - dataLookup.set(lookupKey, Number(item.BUDGET)); + const lookupKey = `${item.nren}/${item.budget_year}`; + dataLookup.set(lookupKey, Number(item.budget)); }) return dataLookup; } export function createBudgetDataset(budgetEntries: Budget[]): BasicDataset { - const labelsYear = [...new Set(budgetEntries.map((item) => item.BUDGET_YEAR))].sort(); - const labelsNREN = [...new Set(budgetEntries.map((item) => item.NREN))].sort(); + const labelsYear = [...new Set(budgetEntries.map((item) => item.budget_year))].sort(); + const labelsNREN = [...new Set(budgetEntries.map((item) => item.nren))].sort(); const dataLookup = createBudgetDataLookup(budgetEntries); @@ -153,12 +153,12 @@ export function createChargingStructureDataLookup(data: ChargingStructure[]) { const dataLookup = new Map<string, Map<number, string>>(); data.forEach(entry => { - let nrenEntry = dataLookup.get(entry.NREN); + let nrenEntry = dataLookup.get(entry.nren); if (!nrenEntry) { nrenEntry = new Map<number, string>(); } - nrenEntry.set(entry.YEAR, entry.FEE_TYPE || ''); - dataLookup.set(entry.NREN, nrenEntry); + nrenEntry.set(entry.year, entry.fee_type || ''); + dataLookup.set(entry.nren, nrenEntry); }); return dataLookup; } diff --git a/webapp/src/pages/ChargingStructure.tsx b/webapp/src/pages/ChargingStructure.tsx index 4948a911..ac7c925f 100644 --- a/webapp/src/pages/ChargingStructure.tsx +++ b/webapp/src/pages/ChargingStructure.tsx @@ -42,11 +42,11 @@ interface inputProps { function ChargingStructurePage({ filterSelection, setFilterSelection }: inputProps): React.ReactElement { const [chargingStructureData, setChargingStructureData] = useState<ChargingStructure[]>([]); - const labelsNREN = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.NREN))]; - const labelsYear = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.YEAR))]; + const labelsNREN = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.nren))]; + const labelsYear = [...new Set(chargingStructureData.map((item: ChargingStructure) => item.year))]; const selectedData = (chargingStructureData || []).filter(data => - filterSelection.selectedYears.includes(data.YEAR) && filterSelection.selectedNrens.includes(data.NREN) + filterSelection.selectedYears.includes(data.year) && filterSelection.selectedNrens.includes(data.nren) ); const dataLookup = createChargingStructureDataLookup(selectedData); @@ -58,8 +58,8 @@ function ChargingStructurePage({ filterSelection, setFilterSelection }: inputPro setChargingStructureData(_chargingStructureData); // filter fallback for when nothing is selected (only last year for all nrens) - const nrens = new Set(_chargingStructureData.map((item: ChargingStructure) => item.NREN)); - const years = new Set(_chargingStructureData.map((item: ChargingStructure) => item.YEAR)); + const nrens = new Set(_chargingStructureData.map((item: ChargingStructure) => item.nren)); + const years = new Set(_chargingStructureData.map((item: ChargingStructure) => item.year)); setFilterSelection(previous => { const visibleYears = previous.selectedYears.filter(year => years.has(year)); const newSelectedYears = visibleYears.length ? previous.selectedYears : [Math.max(...years)]; diff --git a/webapp/src/pages/DataAnalysis.tsx b/webapp/src/pages/DataAnalysis.tsx index 6549742c..7a4030c6 100644 --- a/webapp/src/pages/DataAnalysis.tsx +++ b/webapp/src/pages/DataAnalysis.tsx @@ -30,7 +30,7 @@ function DataAnalysis({ filterSelection, setFilterSelection }: inputProps): Reac const [budgetResponse, setBudget] = useState<Budget[]>(); - const nrens = new Set((budgetResponse ?? []).map((item) => item.NREN)); + const nrens = new Set((budgetResponse ?? []).map((item) => item.nren)); const budgetData = createBudgetDataset(budgetResponse ?? []); budgetData.datasets.forEach(dataset => { @@ -44,7 +44,7 @@ function DataAnalysis({ filterSelection, setFilterSelection }: inputProps): Reac setBudget(budget) // filter fallback for when nothing is selected (all nrens) - const nrens = new Set(budget.map((item) => item.NREN)); + const nrens = new Set(budget.map((item) => item.nren)); setFilterSelection(previous => { const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren)); const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens]; diff --git a/webapp/src/pages/FundingSource.tsx b/webapp/src/pages/FundingSource.tsx index 8454e83a..3430b393 100644 --- a/webapp/src/pages/FundingSource.tsx +++ b/webapp/src/pages/FundingSource.tsx @@ -81,8 +81,8 @@ function getYearsAndNrens(sourceData: FundingSource[]) { const years = new Set<number>(); const nrens = new Set<string>(); sourceData.forEach(datapoint => { - years.add(datapoint.YEAR); - nrens.add(datapoint.NREN); + years.add(datapoint.year); + nrens.add(datapoint.nren); }); return { years: years, nrens: nrens }; } -- GitLab