Merge branch 'feature/COMP-274_traffic_volume_page' into 'develop'

Feature/comp 274 traffic volume page

See merge request !74

compendium-frontend/src/App.tsx

@@ -13,6 +13,7 @@ import ParentOrganisation from "./pages/ParentOrganisation";
 import ECProjects from "./pages/ECProjects";
 import Providers from "./Providers";
 import PolicyPage from "./pages/Policy";
+import TrafficVolumePage from "./pages/TrafficVolumePerNren";
 import ConnectedInstitutionsURLs from "./pages/ConnectedInstitutionsURLs";
 
 
@@ -26,6 +27,7 @@ const router = createBrowserRouter([
   { path: "/parentorganisation", element: <ParentOrganisation />},
   { path: "/ec-projects", element: <ECProjects />},
   { path: "/policy", element: <PolicyPage />},
+  { path: "/traffic-volume", element: <TrafficVolumePage />},
   { path: "/data", element: <CompendiumData />},
   { path: "/institutions-urls", element: <ConnectedInstitutionsURLs />},
   { path: "*", element: <Landing />},

compendium-frontend/src/Schema.tsx

@@ -18,6 +18,13 @@ export interface Budget extends NrenAndYearDatapoint {
     budget: string
 }
 
+export interface TrafficVolume extends NrenAndYearDatapoint {
+    from_customers: number,
+    to_customers: number,
+    from_external: number,
+    to_external: number
+}
+
 export interface FundingSource extends NrenAndYearDatapoint {
     client_institutions: number,
     commercial: number,

compendium-frontend/src/components/DataPage.tsx

@@ -9,6 +9,7 @@ import PolicySidebar from "./PolicySidebar";
 
 import { Chart as ChartJS } from 'chart.js';
 import { usePreview } from "../helpers/usePreview";
+import NetworkSidebar from "./NetworkSidebar";
 import ConnectedUsersSidebar from "./ConnectedUsersSidebar";
 
 ChartJS.defaults.font.size = 16;
@@ -31,6 +32,7 @@ function DataPage({ title, description, filter, children, category }: inputProps
         <>
             {category === Sections.Organisation && <OrganizationSidebar />}
             {category === Sections.Policy && <PolicySidebar />}
+            {category === Sections.Network && <NetworkSidebar />}
             {category === Sections.ConnectedUsers && <ConnectedUsersSidebar />}
             <PageHeader type={'data'} />
             { preview && <Row className="preview-banner">

compendium-frontend/src/components/NetworkSidebar.tsx

+import React from 'react';
+import { Link } from 'react-router-dom';
+import { Row } from 'react-bootstrap';
+import Sidebar from './SideBar';
+
+const PolicySidebar = () => {
+    return (
+        <Sidebar>
+            <h5>Network</h5>
+            <Row>
+                <Link to="/traffic-volume" className="link-text-underline">
+                    <span>Traffic volume</span>
+                </Link>
+            </Row>
+        </Sidebar>
+    )
+}
+
+export default PolicySidebar
\ No newline at end of file

compendium-frontend/src/components/SectionNavigation.tsx

@@ -32,10 +32,9 @@ const SectionNavigation = ({ activeCategory }: inputProps) => {
                         <span>{Sections.ConnectedUsers}</span>
                     </Button>
                     <Button
-                        onClick={() => navigate(activeCategory === Sections.Network ? '.' : '.')}
+                        onClick={() => navigate(activeCategory === Sections.Network ? '.' : '/traffic-volume')}
                         variant={'nav-box'}
-                        active={activeCategory === Sections.Network}
-                        disabled={true}>
+                        active={activeCategory === Sections.Network}>
                         <span>{Sections.Network}</span>
                     </Button>
                     <Button

compendium-frontend/src/helpers/dataconversion.tsx

 import { cartesianProduct } from 'cartesian-product-multiple-arrays';
 import {
-    FundingSource, FundingSourceDataset, ChargingStructure,
-    Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, ConnectedInstitutionURLs
+    FundingSource, FundingSourceDataset, ChargingStructure, ConnectedInstitutionURLs,
+    Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, TrafficVolume
 } from "../Schema";
 
 // create a color from a string, credits https://stackoverflow.com/a/16348977
@@ -54,6 +54,50 @@ function CreateDataLookup(data: FundingSource[]) {
     return dataLookup
 }
 
+export const createTrafficVolumeDataset = (fundingSourcesData: TrafficVolume[]) => {
+    const data = fundingSourcesData;
+    const dataLookup = new Map<string, number>();
+    data.forEach((item: TrafficVolume) => {
+        const lookupKey = `${item.nren}/${item.year}`;
+        console.log(lookupKey );
+        dataLookup.set(lookupKey, item.from_customers);  // we ignore the rest of the data for now..
+    })
+
+    const labelsYear = [...new Set(data.map((item: TrafficVolume) => item.year))];
+    const labelsNREN = [...new Set(data.map((item: TrafficVolume) => item.nren))];
+
+    const sets = labelsYear.map(year => {
+        return {
+            backgroundColor: 'rgba(40, 40, 250, 0.8)',
+            data: labelsNREN.map((nren) => dataLookup.get(`${nren}/${year}`) ?? null),
+            label: year.toString(),
+            borderSkipped: true,
+            barPercentage: 0.8,
+            borderWidth: 0.5,
+            categoryPercentage: 0.8,
+            hidden: false,
+            datalabels: {
+                display: true,
+                color: 'grey',
+                formatter: function(value, context) {
+                    return context.dataset.label;
+                },
+                anchor: 'start',
+                align: 'end',
+                offset: function(context) {
+                    return context.chart.chartArea.width;
+                }
+            }
+        }
+    });
+
+    const dataResponse: BasicDataset = {
+        datasets: sets,
+        labels: labelsNREN.map(l => l.toString())
+    }
+    return dataResponse;
+}
+
 export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) => {
     const data = fundingSourcesData;
     const dataLookup = CreateDataLookup(data)

compendium-frontend/src/pages/CompendiumData.tsx

@@ -98,7 +98,11 @@ function CompendiumData(): ReactElement {
                         </CollapsibleBox>
                         <CollapsibleBox title={Sections.Network} startCollapsed>
                             <div className="collapsible-column">
-                                <h5>Coming Soon</h5>
+                                <Row>
+                                    <Link to="/traffic-volume" className="link-text-underline">
+                                        <span>Total yearly traffic volume per NREN</span>
+                                    </Link>
+                                </Row>
                             </div>
                         </CollapsibleBox>
                         <CollapsibleBox title={Sections.Services} startCollapsed>

compendium-frontend/src/pages/TrafficVolumePerNren.tsx

+import React, { useContext } from 'react';
+import { Bar } from 'react-chartjs-2';
+import { Col, Row } from "react-bootstrap";
+import { Chart as ChartJS } from 'chart.js';
+import ChartDataLabels from 'chartjs-plugin-datalabels';
+
+import { TrafficVolume } from "../Schema";
+import { createTrafficVolumeDataset } from "../helpers/dataconversion";
+import DataPage from '../components/DataPage';
+import Filter from "../components/graphing/Filter"
+import { ExportType, Sections } from '../helpers/constants';
+import DownloadDataButton from "../components/DownloadDataButton";
+import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
+import DownloadImageChartButton from "../components/DownloadImageChartButton";
+import ChartContainer from "../components/graphing/ChartContainer";
+import { useData } from '../helpers/useData';
+
+export const chartOptions = {
+  maintainAspectRatio: false,
+  layout: {
+    padding: {
+      right: 60
+    }
+  },
+  animation: {
+    duration: 0,
+  },
+  plugins: {
+    legend: {
+      display: false
+    }
+  },
+  scales: {
+    x: {
+      position: "top" as const
+    },
+    xBottom: {
+      grid: {
+        drawOnChartArea: false
+      },
+      afterDataLimits: function (axis) {
+        const indices = Object.keys(ChartJS.instances)
+
+        // initial values should be far outside possible range
+        let max = -99999999
+        let min = 99999999
+
+        for (const index of indices) {
+          if (ChartJS.instances[index] && axis.chart.scales.xBottom) {
+            min = Math.min(ChartJS.instances[index].scales.x.min, min);
+            max = Math.max(ChartJS.instances[index].scales.x.max, max);
+          }
+        }
+
+        axis.chart.scales.xBottom.options.min = min;
+        axis.chart.scales.xBottom.options.max = max;
+        axis.chart.scales.xBottom.min = min;
+        axis.chart.scales.xBottom.max = max;
+      },
+    },
+    y: {
+      ticks: {
+        autoSkip: false
+      },
+    }
+  },
+  indexAxis: "y" as const,
+};
+
+function TrafficVolumePage() {
+  const { filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
+  const { data: trafficVolumeData, years, nrens } = useData<TrafficVolume>('/api/traffic/', setFilterSelection);
+
+  const trafficVolumeDataset = createTrafficVolumeDataset(trafficVolumeData);
+
+  trafficVolumeDataset.datasets.forEach(dataset => {
+    dataset.hidden = !filterSelection.selectedYears.includes(parseInt(dataset.label));
+  });
+
+  // remove the datapoints and labels for the nrens that aren't selected
+  // unfortunately we cannot just hide them because chart.js doesn't want
+  // to create a stack from a single dataset
+  trafficVolumeDataset.datasets.forEach(dataset => {
+    dataset.data = dataset.data.filter((e, i) => {
+      return filterSelection.selectedNrens.includes(trafficVolumeDataset.labels[i]);
+    });
+  });
+  trafficVolumeDataset.labels = trafficVolumeDataset.labels.filter((e) => filterSelection.selectedNrens.includes(e));
+
+  const filterNode = <Filter
+    filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
+    filterSelection={filterSelection}
+    setFilterSelection={setFilterSelection}
+  />
+
+  const numNrens = filterSelection.selectedNrens.length;
+  const numYears = filterSelection.selectedYears.length;
+  const heightPerBar = 2; // every added bar should give this much additional height
+
+  console.log(trafficVolumeDataset)
+
+  // set a minimum height of 20rem, additional years need some more space
+  const height = numNrens * numYears * heightPerBar + 5;
+  return (
+    <DataPage title="Traffic Volume Of NRENs per Year"
+      description='Total yearly traffic volume in terabyte per NREN'
+      category={Sections.Network} filter={filterNode}>
+      <>
+        <Row>
+          <DownloadDataButton data={trafficVolumeData} filename="traffic_volume_of_nren_per_year.csv" exportType={ExportType.CSV} />
+          <DownloadDataButton data={trafficVolumeData} filename="traffic_volume_of_nren_per_year.xlsx" exportType={ExportType.EXCEL} />
+        </Row>
+        <DownloadImageChartButton filename="traffic_volume_of_nren_per_year" />
+        <ChartContainer>
+          <div className="chart-container" style={{ 'height': `${height}rem` }}>
+            <Bar
+              plugins={[ChartDataLabels]}
+              data={trafficVolumeDataset}
+              options={chartOptions}
+            />
+          </div>
+        </ChartContainer>
+      </>
+    </DataPage>
+  );
+}
+export default TrafficVolumePage;

compendium_v2/background_task/parse_excel_data.py

@@ -10,6 +10,7 @@ setup_logging()
 logger = logging.getLogger(__name__)
 
 EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
+NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
 
 
 def fetch_budget_excel_data():
@@ -347,3 +348,47 @@ def fetch_organization_excel_data():
 
         if parent_org not in [None, 'NA', 'N/A']:
             yield nren.upper(), 2021, parent_org
+
+
+def fetch_traffic_excel_data():
+    # load the xlsx file
+    wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
+
+    # select the active worksheet
+    sheet_name = "Estimated_Traffic TByte"
+    ws = wb[sheet_name]
+
+    rows = list(ws.rows)
+
+    def convert_number(value, nren, year, description):
+        if value is None or value == '--' or value == 'No data':
+            return 0
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
+            return 0
+
+    def create_points_for_year(year, start_column):
+        for i in range(6, 49):
+            nren_name = rows[i][start_column].value
+            if not nren_name:
+                continue
+            nren_name = nren_name.upper()
+
+            from_external = convert_number(rows[i][start_column + 1].value, nren_name, year, 'from_external')
+            to_external = convert_number(rows[i][start_column + 2].value, nren_name, year, 'to_external')
+            from_customer = convert_number(rows[i][start_column + 3].value, nren_name, year, 'from_customer')
+            to_customer = convert_number(rows[i][start_column + 4].value, nren_name, year, 'to_customer')
+            if from_external == 0 and to_external == 0 and from_customer == 0 and to_customer == 0:
+                continue
+
+            yield nren_name, year, from_external, to_external, from_customer, to_customer
+
+    yield from create_points_for_year(2016, 38)
+    yield from create_points_for_year(2017, 32)
+    yield from create_points_for_year(2018, 26)
+    yield from create_points_for_year(2019, 20)
+    yield from create_points_for_year(2020, 14)
+    yield from create_points_for_year(2021, 8)
+    yield from create_points_for_year(2022, 2)

compendium_v2/db/model.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import logging
 from decimal import Decimal
 from enum import Enum
-from typing import Optional
+from typing import List, Optional
 from typing_extensions import Annotated
 
 from sqlalchemy import String, JSON
@@ -126,10 +126,21 @@ class Policy(db.Model):
     data_protection: Mapped[str]
 
 
+class TrafficVolume(db.Model):
+    __tablename__ = 'traffic_volume'
+    nren_id: Mapped[int_pk_fkNREN]
+    nren: Mapped[NREN] = relationship(lazy='joined')
+    year: Mapped[int_pk]
+    to_customers: Mapped[Decimal]
+    from_customers: Mapped[Decimal]
+    to_external: Mapped[Decimal]
+    from_external: Mapped[Decimal]
+
+
 class InstitutionURLs(db.Model):
     __tablename__ = 'institution_urls'
 
     nren_id: Mapped[int_pk_fkNREN]
     nren: Mapped[NREN] = relationship(lazy='joined')
     year: Mapped[int_pk]
-    urls: Mapped[list[str]] = mapped_column(JSON)
+    urls: Mapped[List[str]] = mapped_column(JSON)

compendium_v2/migrations/versions/3730c7f1ea1b_add_traffic_volume_table.py

+"""add traffic volume table
+
+Revision ID: 3730c7f1ea1b
+Revises: d6f581374e8f
+Create Date: 2023-09-01 10:26:29.089050
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3730c7f1ea1b'
+down_revision = 'f2879a6b15c8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'traffic_volume',
+        sa.Column('nren_id', sa.Integer(), nullable=False),
+        sa.Column('year', sa.Integer(), nullable=False),
+        sa.Column('to_customers', sa.Numeric(), nullable=False),
+        sa.Column('from_customers', sa.Numeric(), nullable=False),
+        sa.Column('to_external', sa.Numeric(), nullable=False),
+        sa.Column('from_external', sa.Numeric(), nullable=False),
+        sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_volume_nren_id_nren')),
+        sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_volume'))
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('traffic_volume')
+    # ### end Alembic commands ###

compendium_v2/publishers/survey_publisher_v1.py

@@ -204,6 +204,27 @@ def db_organizations_migration(nren_dict):
     db.session.commit()
 
 
+def db_traffic_volume_migration(nren_dict):
+    traffic_data = parse_excel_data.fetch_traffic_excel_data()
+    for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
+        if abbrev not in nren_dict:
+            logger.warning(f'{abbrev} unknown. Skipping.')
+            continue
+
+        nren = nren_dict[abbrev]
+        traffic_entry = model.TrafficVolume(
+            nren=nren,
+            nren_id=nren.id,
+            year=year,
+            from_customers=from_customers,
+            to_customers=to_customers,
+            from_external=from_external,
+            to_external=to_external
+        )
+        db.session.merge(traffic_entry)
+    db.session.commit()
+
+
 def _cli(config, app):
     with app.app_context():
         nren_dict = helpers.get_uppercase_nren_dict()
@@ -213,6 +234,7 @@ def _cli(config, app):
         db_staffing_migration(nren_dict)
         db_ecprojects_migration(nren_dict)
         db_organizations_migration(nren_dict)
+        db_traffic_volume_migration(nren_dict)
 
 
 @click.command()

compendium_v2/routes/api.py

@@ -14,6 +14,7 @@ from compendium_v2.routes.survey import routes as survey
 from compendium_v2.routes.user import routes as user_routes
 from compendium_v2.routes.nren import routes as nren_routes
 from compendium_v2.routes.response import routes as response_routes
+from compendium_v2.routes.traffic import routes as traffic_routes
 from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
 
 routes = Blueprint('compendium-v2-api', __name__)
@@ -28,6 +29,7 @@ routes.register_blueprint(survey, url_prefix='/survey')
 routes.register_blueprint(user_routes, url_prefix='/user')
 routes.register_blueprint(nren_routes, url_prefix='/nren')
 routes.register_blueprint(response_routes, url_prefix='/response')
+routes.register_blueprint(traffic_routes, url_prefix='/traffic')
 routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
 
 logger = logging.getLogger(__name__)

compendium_v2/routes/traffic.py

+import logging
+
+from flask import Blueprint, jsonify
+
+from compendium_v2.routes import common
+from compendium_v2.db.model import TrafficVolume
+from typing import Any
+
+
+routes = Blueprint('traffic', __name__)
+logger = logging.getLogger(__name__)
+
+TRAFFIC_RESPONSE_SCHEMA = {
+    '$schema': 'http://json-schema.org/draft-07/schema#',
+
+    'definitions': {
+        'traffic': {
+            'type': 'object',
+            'properties': {
+                'nren': {'type': 'string'},
+                'nren_country': {'type': 'string'},
+                'year': {'type': 'integer'},
+                'to_customers': {'type': 'number'},
+                'from_customers': {'type': 'number'},
+                'to_external': {'type': 'number'},
+                'from_external': {'type': 'number'}
+            },
+            'required': ['nren', 'nren_country', 'year', 'to_customers',
+                         'from_customers', 'to_external', 'from_external'],
+            'additionalProperties': False
+        }
+    },
+
+    'type': 'array',
+    'items': {'$ref': '#/definitions/traffic'}
+}
+
+
+@routes.route('/', methods=['GET'])
+@common.require_accepts_json
+def traffic_volume_view() -> Any:
+    """
+    handler for /api/traffic/ requests
+
+    response will be formatted as:
+
+    .. asjson::
+        compendium_v2.routes.traffic.TRAFFIC_RESPONSE_SCHEMA
+
+    :return:
+    """
+
+    def _extract_data(entry: TrafficVolume):
+        return {
+            'nren': entry.nren.name,
+            'nren_country': entry.nren.country,
+            'year': entry.year,
+            'to_customers': float(entry.to_customers),
+            'from_customers': float(entry.from_customers),
+            'to_external': float(entry.to_external),
+            'from_external': float(entry.from_external)
+        }
+
+    entries = sorted(
+        [_extract_data(entry) for entry in common.get_data(TrafficVolume)],
+        key=lambda d: (d['nren'], d['year'])
+    )
+    return jsonify(entries)

test/conftest.py

@@ -333,6 +333,30 @@ def test_policy_data(app):
         db.session.commit()
 
 
+@pytest.fixture
+def test_traffic_data(app):
+    with app.app_context():
+        nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
+        nren_names = set(ny[0] for ny in nrens_and_years)
+        nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
+        db.session.add_all(nren_dict.values())
+
+        for (nren_name, year) in nrens_and_years:
+            nren = nren_dict[nren_name]
+
+            db.session.add(
+                model.TrafficVolume(
+                    nren=nren,
+                    year=year,
+                    from_customers=2.23,
+                    to_customers=5.2,
+                    from_external=0,
+                    to_external=1000
+                )
+            )
+        db.session.commit()
+
+
 @pytest.fixture
 def test_institution_urls_data(app):
     def _create_and_save_nrens(nren_names):

test/test_traffic.py

+import json
+import jsonschema
+from compendium_v2.routes.traffic import TRAFFIC_RESPONSE_SCHEMA
+
+
+def test_staff_response(client, test_traffic_data):
+    rv = client.get(
+        '/api/traffic/',
+        headers={'Accept': ['application/json']})
+    assert rv.status_code == 200
+    result = json.loads(rv.data.decode('utf-8'))
+    jsonschema.validate(result, TRAFFIC_RESPONSE_SCHEMA)
+    assert result