Merge branch 'feature/COMP-258_publishing' into 'develop'

Create a preview mode for publishing survey data

See merge request !71

MANIFEST.in

 recursive-include compendium_v2/static *
 include compendium_v2/templates/index.html
 include compendium_v2/templates/survey-index.html
-recursive-include compendium_v2/migrations/versions *
 include compendium_v2/migrations/alembic.ini
+recursive-include compendium_v2/migrations/versions *
+recursive-include compendium_v2/migrations/surveymodels *
 recursive-include compendium_v2/background_task/xlsx *
 include compendium_v2/conversion/NREN-Services-prefills_2023_Recovered.xlsx
-include compendium_v2/routes/survey_model.json
 recursive-exclude test *

compendium-frontend/src/Providers.tsx

@@ -4,6 +4,7 @@ import SidebarProvider from "./helpers/SidebarProvider";
 import UserProvider from "./shared/UserProvider";
 import FilterSelectionProvider from "./helpers/FilterSelectionProvider";
 import ChartContainerProvider from "./helpers/ChartContainerProvider";
+import PreviewProvider from "./helpers/PreviewProvider";
 
 
 function Providers({ children }): ReactElement {
@@ -12,7 +13,9 @@ function Providers({ children }): ReactElement {
             <UserProvider>
                 <FilterSelectionProvider>
                     <ChartContainerProvider>
-                        {children}
+                        <PreviewProvider>
+                            {children}
+                        </PreviewProvider>
                     </ChartContainerProvider>
                 </FilterSelectionProvider>
             </UserProvider>

compendium-frontend/src/components/DataPage.tsx

@@ -8,6 +8,7 @@ import { Sections } from "../helpers/constants";
 import PolicySidebar from "./PolicySidebar";
 
 import { Chart as ChartJS } from 'chart.js';
+import { usePreview } from "../helpers/usePreview";
 
 ChartJS.defaults.font.size = 16;
 ChartJS.defaults.font.family = 'Open Sans';
@@ -22,11 +23,17 @@ interface inputProps {
 }
 
 function DataPage({ title, description, filter, children, category }: inputProps): ReactElement {
+    const preview = usePreview();
+    const locationWithoutPreview = window.location.origin + window.location.pathname;
+
     return (
         <>
             {category === Sections.Organisation && <OrganizationSidebar />}
             {category === Sections.Policy && <PolicySidebar />}
             <PageHeader type={'data'} />
+            { preview && <Row className="preview-banner">
+                <span>You are viewing a preview of the website which includes pre-published survey data. <a href={locationWithoutPreview}>Click here</a> to deactivate preview mode.</span>
+            </Row>}
             <SectionNavigation activeCategory={category} />
             <Container className="grow">
                 <Row>

compendium-frontend/src/helpers/PreviewProvider.tsx

+import React, { Dispatch, SetStateAction, createContext, useState } from 'react';
+
+interface Props {
+    children: React.ReactNode;
+}
+
+const PreviewContext = createContext<{
+    preview: boolean;
+    setPreview: Dispatch<SetStateAction<boolean>>;
+}>({
+    preview: false,
+    setPreview: () => {}
+});
+
+
+const PreviewProvider: React.FC<Props> = ({ children }) => {
+    const [preview, setPreview] = useState<boolean>(false);
+
+    return (
+        <PreviewContext.Provider value={{ preview, setPreview }}>
+            {children}
+        </PreviewContext.Provider>
+    );
+};
+
+export { PreviewContext };
+export default PreviewProvider;
\ No newline at end of file

compendium-frontend/src/helpers/dataconversion.tsx

 import { cartesianProduct } from 'cartesian-product-multiple-arrays';
 import {
-    FundingSource, FundingSourceDataset, ChargingStructure, NrenAndYearDatapoint, Nren,
-    Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, FilterSelection
+    FundingSource, FundingSourceDataset, ChargingStructure,
+    Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy
 } from "../Schema";
 
 // create a color from a string, credits https://stackoverflow.com/a/16348977
@@ -19,54 +19,6 @@ const stringToColour = function (str) {
     return colour;
 }
 
-export function getYearsAndNrens(sourceData: NrenAndYearDatapoint[]) {
-    const years = new Set<number>();
-    const nrenMap = new Map<string, Nren>();
-    sourceData.forEach(datapoint => {
-        years.add(datapoint.year);
-        nrenMap.set(datapoint.nren, { name: datapoint.nren, country: datapoint.nren_country });
-    });
-    return { years: years, nrens: nrenMap };
-}
-
-export async function loadDataWithFilterSelectionFallback<Datatype extends NrenAndYearDatapoint>(
-    url: string,
-    setData: React.Dispatch<React.SetStateAction<Datatype[]>>,
-    setFilterSelection: React.Dispatch<React.SetStateAction<FilterSelection>>
-) {
-    const response = await fetch(url);
-    const data: Datatype[] = await response.json();
-    setData(data);
-
-    // filter fallback for when nothing is selected (only last year for all nrens)
-    const { years, nrens } = getYearsAndNrens(data);
-    setFilterSelection(previous => {
-        const visibleYears = previous.selectedYears.filter(year => years.has(year));
-        const newSelectedYears = visibleYears.length ? previous.selectedYears : [Math.max(...years)];
-        const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
-        const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens.keys()];
-        return { selectedYears: newSelectedYears, selectedNrens: newSelectedNrens };
-    });
-}
-
-export async function loadDataWithFilterNrenSelectionFallback<Datatype extends NrenAndYearDatapoint>(
-    url: string,
-    setData: React.Dispatch<React.SetStateAction<Datatype[]>>,
-    setFilterSelection: React.Dispatch<React.SetStateAction<FilterSelection>>
-) {
-    const response = await fetch(url);
-    const data: Datatype[] = await response.json();
-    setData(data);
-
-    // filter fallback for when nothing is selected (all nrens)
-    const { nrens } = getYearsAndNrens(data);
-    setFilterSelection(previous => {
-        const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
-        const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens.keys()];
-        return { selectedYears: previous.selectedYears, selectedNrens: newSelectedNrens };
-    });
-}
-
 function getColorMap() {
     const rgbToHex = (r: number, g: number, b: number) => '#' + [r, g, b].map(x => {
         const hex = x.toString(16)

compendium-frontend/src/helpers/useData.tsx

+import { useEffect, useMemo, useState } from "react";
+import { Nren, NrenAndYearDatapoint } from "../Schema";
+import { usePreview } from "./usePreview";
+
+
+function getYearsAndNrens(sourceData: NrenAndYearDatapoint[]) {
+    const years = new Set<number>();
+    const nrenMap = new Map<string, Nren>();
+    sourceData.forEach(datapoint => {
+        years.add(datapoint.year);
+        nrenMap.set(datapoint.nren, { name: datapoint.nren, country: datapoint.nren_country });
+    });
+    return { years: years, nrens: nrenMap };
+}
+
+
+export function useData<Datatype extends NrenAndYearDatapoint>(url, setFilterSelection) {
+
+    const [data, setData] = useState<Datatype[]>([]);
+
+    const preview = usePreview();
+    const loadUrl = url + (preview ? '?preview' : '');
+
+    useEffect(() => {
+        fetch(loadUrl)
+            .then(response => response.json())
+            .then(json => {
+                setData(json);
+                // filter fallback for when nothing is selected (only last year for all nrens)
+                const { years, nrens } = getYearsAndNrens(json);
+                setFilterSelection(previous => {
+                    const visibleYears = previous.selectedYears.filter(year => years.has(year));
+                    const newSelectedYears = visibleYears.length ? previous.selectedYears : [Math.max(...years)];
+                    const visibleNrens = previous.selectedNrens.filter(nren => nrens.has(nren));
+                    const newSelectedNrens = visibleNrens.length ? previous.selectedNrens : [...nrens.keys()];
+                    return { selectedYears: newSelectedYears, selectedNrens: newSelectedNrens };
+                });
+            });
+    }, [loadUrl, setFilterSelection]);
+
+    const { years, nrens } = useMemo(
+        () => getYearsAndNrens(data),
+        [data]
+    );
+
+    return {data: data, years: years, nrens: nrens};
+
+}

compendium-frontend/src/helpers/usePreview.tsx

+import { useContext, useEffect } from "react";
+import { PreviewContext } from "./PreviewProvider";
+import { useSearchParams } from "react-router-dom";
+
+
+export function usePreview() {
+    const { preview, setPreview } = useContext(PreviewContext);
+    const [searchParams] = useSearchParams();
+
+    const previewParameter = searchParams.get('preview');
+
+    useEffect(() => {
+        if (previewParameter !== null) {
+            setPreview(true);
+        }
+    }, [previewParameter, setPreview]);
+
+    return preview;
+}
\ No newline at end of file

compendium-frontend/src/main.scss

@@ -88,4 +88,10 @@
   display: flex;
   flex-direction: column;
   min-height: 100vh;
+}
+
+.preview-banner {
+  background-color: pink;
+  text-align: center;
+  padding: 2em;
 }
\ No newline at end of file

compendium-frontend/src/pages/Budget.tsx

-import React, { ReactElement, useContext, useEffect, useMemo, useState } from 'react';
+import React, { ReactElement, useContext } from 'react';
 import { Row } from "react-bootstrap";
 
 import { Budget } from "../Schema";
-import { createBudgetDataset, getYearsAndNrens, loadDataWithFilterNrenSelectionFallback } from "../helpers/dataconversion";
+import { createBudgetDataset } from "../helpers/dataconversion";
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter";
 import LineGraph from "../components/graphing/LineGraph";
@@ -11,16 +11,12 @@ import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
 
 function BudgetPage(): ReactElement {
 
-    const [budgetResponse, setBudget] = useState<Budget[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
-
-    const { nrens } = useMemo(
-        () => getYearsAndNrens(budgetResponse),
-        [budgetResponse]
-    );
+    const {data: budgetResponse, nrens} = useData<Budget>('/api/budget/', setFilterSelection);
 
     const budgetData = createBudgetDataset(budgetResponse);
 
@@ -28,10 +24,6 @@ function BudgetPage(): ReactElement {
         dataset.hidden = !filterSelection.selectedNrens.includes(dataset.label);
     });
 
-    useEffect(() => {
-        loadDataWithFilterNrenSelectionFallback('/api/budget/', setBudget, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium-frontend/src/pages/ChargingStructure.tsx

-import React, { useContext, useMemo, useState } from "react";
+import React, { useContext } from "react";
 import {Row, Table} from "react-bootstrap";
 import { Chart as ChartJS, CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend } from 'chart.js';
 
 import { ChargingStructure } from "../Schema";
-import { createChargingStructureDataLookup, getYearsAndNrens, loadDataWithFilterSelectionFallback } from "../helpers/dataconversion";
+import { createChargingStructureDataLookup } from "../helpers/dataconversion";
 import ColorPill from "../components/ColorPill";
 import DataPage from "../components/DataPage";
 import Filter from "../components/graphing/Filter";
@@ -12,6 +12,7 @@ import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from "../helpers/FilterSelectionProvider";
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from "../helpers/useData";
 
 ChartJS.register(
     CategoryScale,
@@ -23,13 +24,8 @@ ChartJS.register(
 );
 
 function ChargingStructurePage(): React.ReactElement {
-    const [chargingStructureData, setChargingStructureData] = useState<ChargingStructure[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
-
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(chargingStructureData),
-        [chargingStructureData]
-    );
+    const {data: chargingStructureData, years, nrens} = useData<ChargingStructure>('/api/charging/', setFilterSelection);
 
     const selectedData = (chargingStructureData).filter(data =>
         filterSelection.selectedYears.includes(data.year) && filterSelection.selectedNrens.includes(data.nren)
@@ -37,10 +33,6 @@ function ChargingStructurePage(): React.ReactElement {
 
     const dataLookup = createChargingStructureDataLookup(selectedData);
 
-    React.useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/charging/', setChargingStructureData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium-frontend/src/pages/CompendiumData.tsx

@@ -5,8 +5,11 @@ import PageHeader from "../components/global/PageHeader"
 import Banner from "../components/global/Banner";
 import { Link } from "react-router-dom";
 import { Sections } from "../helpers/constants";
+import { usePreview } from "../helpers/usePreview";
 
 function CompendiumData(): ReactElement {
+    usePreview();
+
     return (
         <main className="grow">
             <PageHeader type={'data'}/>

compendium-frontend/src/pages/ECProjects.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import {Row, Table} from "react-bootstrap";
 
 import { ECProject } from "../Schema";
-import { createECProjectsDataLookup, getYearsAndNrens, loadDataWithFilterSelectionFallback } from '../helpers/dataconversion';
+import { createECProjectsDataLookup } from '../helpers/dataconversion';
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import {ExportType, Sections} from '../helpers/constants';
@@ -10,6 +10,7 @@ import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
 
 
 function getJSXFromMap(data: Map<string, Map<number, ECProject[]>>) {
@@ -31,23 +32,14 @@ function getJSXFromMap(data: Map<string, Map<number, ECProject[]>>) {
 }
 
 function ECProjects() {
-    const [projectData, setProjectData] = useState<ECProject[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
-
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(projectData),
-        [projectData]
-    );
+    const {data: projectData, years, nrens} = useData<ECProject>('/api/ec-project/', setFilterSelection);
 
     const selectedData = (projectData).filter(project =>
         filterSelection.selectedYears.includes(project.year) && filterSelection.selectedNrens.includes(project.nren)
     );
     const projectDataByYear = createECProjectsDataLookup(selectedData);
 
-    useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/ec-project/', setProjectData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium-frontend/src/pages/FundingSource.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import { Bar } from 'react-chartjs-2';
 import { Col, Row } from "react-bootstrap";
 import { Chart as ChartJS } from 'chart.js';
 import ChartDataLabels from 'chartjs-plugin-datalabels';
 
 import { FundingSource } from "../Schema";
-import { createFundingSourceDataset, getYearsAndNrens, loadDataWithFilterSelectionFallback } from "../helpers/dataconversion";
+import { createFundingSourceDataset } from "../helpers/dataconversion";
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import {ExportType, Sections} from '../helpers/constants';
@@ -13,8 +13,8 @@ import ColorBadge from '../components/ColorBadge';
 import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
-import { ChartContainerContext } from "../helpers/ChartContainerProvider";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
 
 export const chartOptions = {
   maintainAspectRatio: false,
@@ -109,13 +109,9 @@ function FundingSourceLegend() {
 }
 
 function FundingSourcePage() {
-  const [fundingSourceData, setFundingSourceData] = useState<FundingSource[]>([]);
   const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+  const {data: fundingSourceData, years, nrens} = useData<FundingSource>('/api/funding/', setFilterSelection);
 
-  const { years, nrens } = useMemo(
-    () => getYearsAndNrens(fundingSourceData),
-    [fundingSourceData]
-  );
   const fundingSourceDataset = createFundingSourceDataset(fundingSourceData);
 
   fundingSourceDataset.datasets.forEach(dataset => {
@@ -132,10 +128,6 @@ function FundingSourcePage() {
   });
   fundingSourceDataset.labels = fundingSourceDataset.labels.filter((e) => filterSelection.selectedNrens.includes(e));
 
-  useEffect(() => {
-    loadDataWithFilterSelectionFallback('/api/funding/', setFundingSourceData, setFilterSelection);
-  }, [setFilterSelection]);
-
   const filterNode = <Filter
     filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
     filterSelection={filterSelection}

compendium-frontend/src/pages/ParentOrganisation.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import {Row, Table} from "react-bootstrap";
 
 import { Organisation } from "../Schema";
-import { createOrganisationDataLookup, getYearsAndNrens, loadDataWithFilterSelectionFallback } from "../helpers/dataconversion";
+import { createOrganisationDataLookup } from "../helpers/dataconversion";
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import {ExportType, Sections} from '../helpers/constants';
@@ -10,6 +10,7 @@ import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
 
 
 function getJSXFromMap(data: Map<string, Map<number, Organisation[]>>) {
@@ -24,23 +25,14 @@ function getJSXFromMap(data: Map<string, Map<number, Organisation[]>>) {
     })
 }
 function ParentOrganisation() {
-    const [organisationData, setOrganisationData] = useState<Organisation[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+    const {data: organisationData, years, nrens} = useData<Organisation>('/api/organization/parent', setFilterSelection);
 
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(organisationData),
-        [organisationData]
-    );
-
-    const selectedData = (organisationData).filter(data =>
+    const selectedData = organisationData.filter(data =>
         filterSelection.selectedYears.includes(data.year) && filterSelection.selectedNrens.includes(data.nren)
     );
     const organisationDataset = createOrganisationDataLookup(selectedData);
 
-    useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/organization/parent', setOrganisationData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium-frontend/src/pages/Policy.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import { Table } from "react-bootstrap";
 
 import { Policy } from "../Schema";
-import { createPolicyDataLookup, getYearsAndNrens, loadDataWithFilterSelectionFallback } from '../helpers/dataconversion';
+import { createPolicyDataLookup } from '../helpers/dataconversion';
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import { Sections } from '../helpers/constants';
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
+import { useData } from '../helpers/useData';
 
 function getJSXFromMap(data: Map<string, Map<number, Policy>>) {
     const policies = [
@@ -43,23 +44,14 @@ function getJSXFromMap(data: Map<string, Map<number, Policy>>) {
 }
 
 function PolicyPage() {
-    const [policyData, setProjectData] = useState<Policy[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+    const {data: policyData, years, nrens} = useData<Policy>('/api/policy/', setFilterSelection);
 
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(policyData),
-        [policyData]
-    );
-
-    const selectedData = (policyData).filter(project =>
+    const selectedData = policyData.filter(project =>
         filterSelection.selectedYears.includes(project.year) && filterSelection.selectedNrens.includes(project.nren)
     );
     const policyDataByYear = createPolicyDataLookup(selectedData);
 
-    useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/policy/', setProjectData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium-frontend/src/pages/StaffGraph.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import { Bar } from 'react-chartjs-2';
 import { Chart as ChartJS, CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend } from 'chart.js';
 
 import { NrenStaff } from "../Schema";
-import { createNRENStaffDataset, getYearsAndNrens, loadDataWithFilterSelectionFallback } from "../helpers/dataconversion";
+import { createNRENStaffDataset } from "../helpers/dataconversion";
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import {ExportType, Sections} from '../helpers/constants';
@@ -13,6 +13,7 @@ import {Row} from "react-bootstrap";
 import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
+import { useData } from '../helpers/useData';
 
 ChartJS.register(
     CategoryScale,
@@ -107,13 +108,9 @@ interface inputProps {
 }
 
 function StaffGraph({ roles = false }: inputProps) {
-    const [staffData, setStaffData] = useState<NrenStaff[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+    const {data: staffData, years, nrens} = useData<NrenStaff>('/api/staff/', setFilterSelection);
 
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(staffData),
-        [staffData]
-    );
     const nrenStaffDataset = createNRENStaffDataset(staffData, roles, filterSelection.selectedYears[0]);
 
     nrenStaffDataset.datasets.forEach(dataset => {
@@ -130,10 +127,6 @@ function StaffGraph({ roles = false }: inputProps) {
     });
     nrenStaffDataset.labels = nrenStaffDataset.labels.filter((e) => filterSelection.selectedNrens.includes(e));
 
-    useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/staff/', setStaffData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         max1year
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}

compendium-frontend/src/pages/SubOrganisation.tsx

-import React, { useContext, useEffect, useMemo, useState } from 'react';
+import React, { useContext } from 'react';
 import {Row, Table} from "react-bootstrap";
 
 import { Organisation } from "../Schema";
-import { createOrganisationDataLookup, getYearsAndNrens, loadDataWithFilterSelectionFallback } from "../helpers/dataconversion";
+import { createOrganisationDataLookup } from "../helpers/dataconversion";
 import DataPage from '../components/DataPage';
 import Filter from "../components/graphing/Filter"
 import {ExportType, Sections} from '../helpers/constants';
@@ -10,6 +10,7 @@ import DownloadDataButton from "../components/DownloadDataButton";
 import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
 import DownloadImageChartButton from "../components/DownloadImageChartButton";
 import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
 
 function getJSXFromMap(data: Map<string, Map<number, Organisation[]>>) {
     return Array.from(data.entries()).map(([nren, nrenMap]) => {
@@ -30,23 +31,14 @@ function getJSXFromMap(data: Map<string, Map<number, Organisation[]>>) {
 }
 
 function SubOrganisation() {
-    const [organisationData, setOrganisationData] = useState<Organisation[]>([]);
     const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+    const {data: organisationData, years, nrens} = useData<Organisation>('/api/organization/sub', setFilterSelection);
 
-    const { years, nrens } = useMemo(
-        () => getYearsAndNrens(organisationData),
-        [organisationData]
-    );
-
-    const selectedData = (organisationData).filter(data =>
+    const selectedData = organisationData.filter(data =>
         filterSelection.selectedYears.includes(data.year) && filterSelection.selectedNrens.includes(data.nren)
     );
     const organisationDataset = createOrganisationDataLookup(selectedData);
 
-    useEffect(() => {
-        loadDataWithFilterSelectionFallback('/api/organization/sub', setOrganisationData, setFilterSelection);
-    }, [setFilterSelection]);
-
     const filterNode = <Filter
         filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
         filterSelection={filterSelection}

compendium_v2/conversion/conversion.py

@@ -180,7 +180,7 @@ def _cli(app):
                 survey_year=2022,
                 survey=survey,
                 answers=survey_dict,
-                status=ResponseStatus.checked
+                status=ResponseStatus.completed
             )
             db.session.add(response)
 

compendium_v2/db/model.py

@@ -28,6 +28,11 @@ int_pk_fkNREN = Annotated[int, mapped_column(ForeignKey("nren.id"), primary_key=
 # mypy: disable-error-code="name-defined"
 
 
+class PreviewYear(db.Model):
+    __tablename__ = 'preview_year'
+    year: Mapped[int_pk]
+
+
 class NREN(db.Model):
     __tablename__ = 'nren'
     id: Mapped[int_pk]

compendium_v2/db/survey_model.py

@@ -35,15 +35,13 @@ json = Annotated[Dict[str, Any], mapped_column(JSON)]
 class SurveyStatus(Enum):
     closed = "closed"
     open = "open"
+    preview = "preview"
     published = "published"
 
 
-# TODO The checked status is currently not used
-# see if we want to keep the checked survey status or control what is published in some other way..
 class ResponseStatus(Enum):
     started = "started"
     completed = "completed"
-    checked = "checked"
 
 
 RESPONSE_NOT_STARTED = "not started"