-
Mohammad Torkashvand authoredMohammad Torkashvand authored
dataconversion.tsx 13.95 KiB
import { cartesianProduct } from 'cartesian-product-multiple-arrays';
import {
FundingSource, FundingSourceDataset, ChargingStructure, NrenAndYearDatapoint, Nren,
Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, FilterSelection
} from "../Schema";
// create a color from a string, credits https://stackoverflow.com/a/16348977
const stringToColour = function (str) {
let hash = 0;
for (let i = 0; i < str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
}
let colour = '#';
for (let i = 0; i < 3; i++) {
const value = (hash >> (i * 8)) & 0xFF;
const valueAsString = '00' + value.toString(16);
colour += valueAsString.substring(valueAsString.length - 2);
}
return colour;
}
export function getYearsAndNrens(sourceData: NrenAndYearDatapoint[]) {
const years = new Set<number>();
const nrenMap = new Map<string, Nren>();
sourceData.forEach(datapoint => {
years.add(datapoint.year);
nrenMap.set(datapoint.nren, { name: datapoint.nren, country: datapoint.nren_country });
});
return { years: years, nrens: nrenMap };
}
export async function loadDataWithFilterSelectionFallback<Datatype extends NrenAndYearDatapoint>(
url: string,
setData: React.Dispatch<React.SetStateAction<Datatype[]>>,
setFilterSelection: React.Dispatch<React.SetStateAction<FilterSelection>>
) {
const response = await fetch(url);
const data: Datatype[] = await response.json();
setData(data);
// filter fallback for when nothing is selected (only last year for all nrens)
const { years, nrens } = getYearsAndNrens(data);
setFilterSelection(previous => {
const visibleYears = previous.selectedYears.filter(year => years.has(year));
const newSelectedYears = visibleYears.length ? previous.selectedYears : [Math.max(...years)];
const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens.keys()];
return { selectedYears: newSelectedYears, selectedNrens: newSelectedNrens };
});
}
export async function loadDataWithFilterNrenSelectionFallback<Datatype extends NrenAndYearDatapoint>(
url: string,
setData: React.Dispatch<React.SetStateAction<Datatype[]>>,
setFilterSelection: React.Dispatch<React.SetStateAction<FilterSelection>>
) {
const response = await fetch(url);
const data: Datatype[] = await response.json();
setData(data);
// filter fallback for when nothing is selected (all nrens)
const { nrens } = getYearsAndNrens(data);
setFilterSelection(previous => {
const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens.keys()];
return { selectedYears: previous.selectedYears, selectedNrens: newSelectedNrens };
});
}
function getColorMap() {
const rgbToHex = (r: number, g: number, b: number) => '#' + [r, g, b].map(x => {
const hex = x.toString(16)
return hex.length === 1 ? '0' + hex : hex
}).join('')
const colorMap = new Map<string, string>();
colorMap.set("CLIENT INSTITUTIONS", rgbToHex(157, 40, 114))
colorMap.set("COMMERCIAL", rgbToHex(241, 224, 79))
colorMap.set("EUROPEAN FUNDING", rgbToHex(219, 42, 76))
colorMap.set("GOV/PUBLIC_BODIES", rgbToHex(237, 141, 24))
colorMap.set("OTHER", rgbToHex(137, 166, 121))
return colorMap
}
function CreateDataLookup(data: FundingSource[]) {
const dataLookup = new Map<string, Map<string, number>>();
data.forEach((item: FundingSource) => {
const lookupKey = `${item.nren}/${item.year}`
let fundingSourceMap = dataLookup.get(lookupKey)
if (!fundingSourceMap) {
fundingSourceMap = new Map<string, number>();
}
fundingSourceMap.set("CLIENT INSTITUTIONS", item.client_institutions)
fundingSourceMap.set("COMMERCIAL", item.commercial)
fundingSourceMap.set("EUROPEAN FUNDING", item.european_funding)
fundingSourceMap.set("GOV/PUBLIC_BODIES", item.gov_public_bodies)
fundingSourceMap.set("OTHER", item.other)
dataLookup.set(lookupKey, fundingSourceMap)
})
return dataLookup
}
export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) => {
const data = fundingSourcesData;
const dataLookup = CreateDataLookup(data)
const labelsYear = [...new Set(data.map((item: FundingSource) => item.year))];
const labelsNREN = [...new Set(data.map((item: FundingSource) => item.nren))];
const fundingSources = [
"CLIENT INSTITUTIONS",
"COMMERCIAL",
"EUROPEAN FUNDING",
"GOV/PUBLIC_BODIES",
"OTHER"
]
const fundingSourcesPerYear = cartesianProduct(fundingSources, labelsYear)
const colorMap = getColorMap();
const fundingSourceDataset = fundingSourcesPerYear.map(function ([fundingSource, year]) {
const color = colorMap.get(fundingSource)!;
return {
backgroundColor: color,
label: fundingSource + "(" + year + ")",
data: labelsNREN.map(nren => {
// ensure that the data is in the same order as the labels
const lookupKey = `${nren}/${year}`
const dataForYear = dataLookup.get(lookupKey)
if (!dataForYear) {
return 0
}
return dataForYear.get(fundingSource) ?? 0
}),
stack: year,
borderSkipped: true,
barPercentage: 0.8,
borderWidth: 0.5,
categoryPercentage: 0.8,
hidden: false,
datalabels: {
display: fundingSource == fundingSources[0],
color: 'grey',
formatter: function(value, context) {
return context.dataset.stack;
},
anchor: 'start',
align: 'end',
offset: function(context) {
return context.chart.chartArea.width;
}
}
}
})
const dataResponse: FundingSourceDataset = {
// datasets: datasetFunding,
datasets: fundingSourceDataset,
labels: labelsNREN.map(l => l.toString())
}
return dataResponse;
}
function createBudgetDataLookup(budgetEntries: Budget[]) {
const dataLookup = new Map<string, number>();
budgetEntries.forEach((item: Budget) => {
const lookupKey = `${item.nren}/${item.year}`;
dataLookup.set(lookupKey, Number(item.budget));
})
return dataLookup;
}
export function createBudgetDataset(budgetEntries: Budget[]): BasicDataset {
const labelsYear = [...new Set(budgetEntries.map((item) => item.year))].sort();
const labelsNREN = [...new Set(budgetEntries.map((item) => item.nren))].sort();
const dataLookup = createBudgetDataLookup(budgetEntries);
const sets = labelsNREN.map(nren => {
const randomColor = stringToColour(nren);
return {
backgroundColor: randomColor,
borderColor: randomColor,
data: labelsYear.map((year) => dataLookup.get(`${nren}/${year}`) ?? null),
label: nren,
hidden: false
}
});
return {
datasets: sets,
labels: labelsYear.map(year => year.toString())
};
}
export function createChargingStructureDataLookup(data: ChargingStructure[]) {
const dataLookup = new Map<string, Map<number, string>>();
data.forEach(entry => {
let nrenEntry = dataLookup.get(entry.nren);
if (!nrenEntry) {
nrenEntry = new Map<number, string>();
}
nrenEntry.set(entry.year, entry.fee_type || '');
dataLookup.set(entry.nren, nrenEntry);
});
return dataLookup;
}
export function createOrganisationDataLookup(organisationEntries: Organisation[]) {
const nrenMap = new Map<string, Map<number, Organisation[]>>();
organisationEntries.forEach(entry => {
let nrenEntry = nrenMap.get(entry.nren);
if (!nrenEntry) {
nrenEntry = new Map<number, Organisation[]>();
}
let suborgList = nrenEntry.get(entry.year);
if (!suborgList) {
suborgList = [];
}
suborgList.push(entry);
nrenEntry.set(entry.year, suborgList);
nrenMap.set(entry.nren, nrenEntry);
});
return nrenMap;
}
export function createECProjectsDataLookup(projectEntries: ECProject[]) {
const projectMap = new Map<string, Map<number, ECProject[]>>();
projectEntries.forEach(entry => {
let nrenEntry = projectMap.get(entry.nren);
if (!nrenEntry) {
nrenEntry = new Map<number, ECProject[]>();
}
let projectList = nrenEntry.get(entry.year);
if (!projectList) {
projectList = [];
}
projectList.push(entry);
nrenEntry.set(entry.year, projectList);
projectMap.set(entry.nren, nrenEntry);
});
return projectMap;
}
export function createPolicyDataLookup(policyEntries: Policy[]) {
const policyMap = new Map<string, Map<number, Policy>>();
policyEntries.forEach(entry => {
let nrenEntry = policyMap.get(entry.nren);
if (!nrenEntry) {
nrenEntry = new Map<number, Policy>();
}
nrenEntry.set(entry.year, entry);
policyMap.set(entry.nren, nrenEntry);
});
return policyMap;
}
export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean, selectedYear: number) => {
let categories;
if (roles) {
categories = [
"Technical FTE",
"Non-technical FTE"
]
} else {
categories = [
"Permanent FTE",
"Subcontracted FTE"
]
}
function CreateDataLookup(data: NrenStaff[]) {
const fields = {
"Technical FTE": "technical_fte",
"Non-technical FTE": "non_technical_fte",
"Permanent FTE": "permanent_fte",
"Subcontracted FTE": "subcontracted_fte"
}
const dataLookup = new Map<string, Map<string, number>>();
data.forEach((item: NrenStaff) => {
if (selectedYear !== item.year) {
// only include data for the selected year
return;
}
const nren = item.nren;
let categoryMap = dataLookup.get(nren)
if (!categoryMap) {
categoryMap = new Map<string, number>();
}
// get the values for the two categories
const [category1, category2] = categories
const [category1Field, category2Field] = [fields[category1], fields[category2]]
const category1Value = item[category1Field]
const category2Value = item[category2Field]
// calculate the percentages
const total = category1Value + category2Value
let category1_percentage = ((category1Value / total) || 0) * 100
let category2_percentage = ((category2Value / total) || 0) * 100
// round to 2 decimal places
category1_percentage = Math.round(Math.floor(category1_percentage * 100)) / 100
category2_percentage = Math.round(Math.floor(category2_percentage * 100)) / 100
categoryMap.set(category1, category1_percentage)
categoryMap.set(category2, category2_percentage)
dataLookup.set(nren, categoryMap)
})
return dataLookup
}
const dataLookup = CreateDataLookup(data)
const labelsYear = [selectedYear];
const labelsNREN = [...new Set(data.map((item: NrenStaff) => item.nren))].sort((nrenA, nrenB) => {
const categoryMapNrenA = dataLookup.get(nrenA)
const categoryMapNrenB = dataLookup.get(nrenB)
if (categoryMapNrenA && categoryMapNrenB) {
const [category1, category2] = categories
const nrenAData = {
category1: categoryMapNrenA.get(category1)!,
category2: categoryMapNrenA.get(category2)!
}
const nrenBData = {
category1: categoryMapNrenB.get(category1)!,
category2: categoryMapNrenB.get(category2)!
}
if (nrenAData.category1 === nrenBData.category1) {
return nrenBData.category2 - nrenAData.category1;
}
return nrenBData.category1 - nrenAData.category1;
} else {
// put NRENs with no data at the end
if (categoryMapNrenA) {
return -1
}
if (categoryMapNrenB) {
return 1
}
return 0
}
});
const categoriesPerYear = cartesianProduct(categories, labelsYear)
const nrenStaffDataset = categoriesPerYear.map(function ([category, year]) {
let color = ""
if (category === "Technical FTE") {
color = 'rgba(40, 40, 250, 0.8)'
} else if (category === "Permanent FTE") {
color = 'rgba(159, 129, 235, 1)'
} else if (category === "Subcontracted FTE") {
color = 'rgba(173, 216, 229, 1)'
} else if (category === "Non-technical FTE") {
color = 'rgba(116, 216, 242, 0.54)'
}
return {
backgroundColor: color,
label: `${category} (${year})`,
data: labelsNREN.map(nren => {
// ensure that the data is in the same order as the labels
const dataForNren = dataLookup.get(nren)
if (!dataForNren) {
return 0
}
return dataForNren.get(category) ?? 0
}),
stack: year,
borderRadius: 10,
borderSkipped: true,
barPercentage: 0.8,
borderWidth: 0.5,
categoryPercentage: 0.8,
hidden: false
}
})
const dataset: NrenStaffDataset = {
datasets: nrenStaffDataset,
labels: labelsNREN
}
return dataset;
}