Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • geant-swd/compendium-v2
1 result
Select Git revision
Show changes
Commits on Source (40)
Showing
with 3204 additions and 187 deletions
......@@ -2,6 +2,17 @@
All notable changes to this project will be documented in this file.
## [0.36] - 2023-09-14
- Fixed bug with sorting in user management page
## [0.35] - 2023-09-12
- Added a publisher for the 2023 survey data
- Added report pages for the traffic volume and institution urls
- Added sorting functionality for the survey users admin page
- Added a new 'observer' role in addition to 'admin' and 'user'
- Added a missing policy field in the reporting datamodel
- Changed the 'country' question in the survey from a dropdown to a text field
## [0.34] - 2023-08-27
- Fixed bug with "To" address for single email recipient
......
......@@ -13,6 +13,7 @@ import ParentOrganisation from "./pages/ParentOrganisation";
import ECProjects from "./pages/ECProjects";
import Providers from "./Providers";
import PolicyPage from "./pages/Policy";
import TrafficVolumePage from "./pages/TrafficVolumePerNren";
import ConnectedInstitutionsURLs from "./pages/ConnectedInstitutionsURLs";
......@@ -26,6 +27,7 @@ const router = createBrowserRouter([
{ path: "/parentorganisation", element: <ParentOrganisation />},
{ path: "/ec-projects", element: <ECProjects />},
{ path: "/policy", element: <PolicyPage />},
{ path: "/traffic-volume", element: <TrafficVolumePage />},
{ path: "/data", element: <CompendiumData />},
{ path: "/institutions-urls", element: <ConnectedInstitutionsURLs />},
{ path: "*", element: <Landing />},
......
......@@ -18,6 +18,13 @@ export interface Budget extends NrenAndYearDatapoint {
budget: string
}
export interface TrafficVolume extends NrenAndYearDatapoint {
from_customers: number,
to_customers: number,
from_external: number,
to_external: number
}
export interface FundingSource extends NrenAndYearDatapoint {
client_institutions: number,
commercial: number,
......
......@@ -9,6 +9,7 @@ import PolicySidebar from "./PolicySidebar";
import { Chart as ChartJS } from 'chart.js';
import { usePreview } from "../helpers/usePreview";
import NetworkSidebar from "./NetworkSidebar";
import ConnectedUsersSidebar from "./ConnectedUsersSidebar";
ChartJS.defaults.font.size = 16;
......@@ -31,6 +32,7 @@ function DataPage({ title, description, filter, children, category }: inputProps
<>
{category === Sections.Organisation && <OrganizationSidebar />}
{category === Sections.Policy && <PolicySidebar />}
{category === Sections.Network && <NetworkSidebar />}
{category === Sections.ConnectedUsers && <ConnectedUsersSidebar />}
<PageHeader type={'data'} />
{ preview && <Row className="preview-banner">
......
import React from 'react';
import { Link } from 'react-router-dom';
import { Row } from 'react-bootstrap';
import Sidebar from './SideBar';
const PolicySidebar = () => {
return (
<Sidebar>
<h5>Network</h5>
<Row>
<Link to="/traffic-volume" className="link-text-underline">
<span>Traffic volume</span>
</Link>
</Row>
</Sidebar>
)
}
export default PolicySidebar
\ No newline at end of file
......@@ -32,10 +32,9 @@ const SectionNavigation = ({ activeCategory }: inputProps) => {
<span>{Sections.ConnectedUsers}</span>
</Button>
<Button
onClick={() => navigate(activeCategory === Sections.Network ? '.' : '.')}
onClick={() => navigate(activeCategory === Sections.Network ? '.' : '/traffic-volume')}
variant={'nav-box'}
active={activeCategory === Sections.Network}
disabled={true}>
active={activeCategory === Sections.Network}>
<span>{Sections.Network}</span>
</Button>
<Button
......
import { cartesianProduct } from 'cartesian-product-multiple-arrays';
import {
FundingSource, FundingSourceDataset, ChargingStructure,
Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, ConnectedInstitutionURLs
FundingSource, FundingSourceDataset, ChargingStructure, ConnectedInstitutionURLs,
Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, TrafficVolume
} from "../Schema";
// create a color from a string, credits https://stackoverflow.com/a/16348977
......@@ -54,6 +54,50 @@ function CreateDataLookup(data: FundingSource[]) {
return dataLookup
}
export const createTrafficVolumeDataset = (fundingSourcesData: TrafficVolume[]) => {
const data = fundingSourcesData;
const dataLookup = new Map<string, number>();
data.forEach((item: TrafficVolume) => {
const lookupKey = `${item.nren}/${item.year}`;
console.log(lookupKey );
dataLookup.set(lookupKey, item.from_customers); // we ignore the rest of the data for now..
})
const labelsYear = [...new Set(data.map((item: TrafficVolume) => item.year))];
const labelsNREN = [...new Set(data.map((item: TrafficVolume) => item.nren))];
const sets = labelsYear.map(year => {
return {
backgroundColor: 'rgba(40, 40, 250, 0.8)',
data: labelsNREN.map((nren) => dataLookup.get(`${nren}/${year}`) ?? null),
label: year.toString(),
borderSkipped: true,
barPercentage: 0.8,
borderWidth: 0.5,
categoryPercentage: 0.8,
hidden: false,
datalabels: {
display: true,
color: 'grey',
formatter: function(value, context) {
return context.dataset.label;
},
anchor: 'start',
align: 'end',
offset: function(context) {
return context.chart.chartArea.width;
}
}
}
});
const dataResponse: BasicDataset = {
datasets: sets,
labels: labelsNREN.map(l => l.toString())
}
return dataResponse;
}
export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) => {
const data = fundingSourcesData;
const dataLookup = CreateDataLookup(data)
......
......@@ -98,7 +98,11 @@ function CompendiumData(): ReactElement {
</CollapsibleBox>
<CollapsibleBox title={Sections.Network} startCollapsed>
<div className="collapsible-column">
<h5>Coming Soon</h5>
<Row>
<Link to="/traffic-volume" className="link-text-underline">
<span>Total yearly traffic volume per NREN</span>
</Link>
</Row>
</div>
</CollapsibleBox>
<CollapsibleBox title={Sections.Services} startCollapsed>
......
......@@ -16,6 +16,7 @@ function getJSXFromMap(data: Map<string, Map<number, Policy>>) {
['data_protection', 'Data Protection Policy'],
['environmental', 'Environmental Policy'],
['equal_opportunity', 'Equal Opportunity Policy'],
['gender_equality', 'Gender Equality Policy'],
['privacy_notice', 'Privacy Notice'],
['strategic_plan', 'Strategic Plan']
];
......
import React, { useContext } from 'react';
import { Bar } from 'react-chartjs-2';
import { Col, Row } from "react-bootstrap";
import { Chart as ChartJS } from 'chart.js';
import ChartDataLabels from 'chartjs-plugin-datalabels';
import { TrafficVolume } from "../Schema";
import { createTrafficVolumeDataset } from "../helpers/dataconversion";
import DataPage from '../components/DataPage';
import Filter from "../components/graphing/Filter"
import { ExportType, Sections } from '../helpers/constants';
import DownloadDataButton from "../components/DownloadDataButton";
import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
import DownloadImageChartButton from "../components/DownloadImageChartButton";
import ChartContainer from "../components/graphing/ChartContainer";
import { useData } from '../helpers/useData';
export const chartOptions = {
maintainAspectRatio: false,
layout: {
padding: {
right: 60
}
},
animation: {
duration: 0,
},
plugins: {
legend: {
display: false
}
},
scales: {
x: {
position: "top" as const
},
xBottom: {
grid: {
drawOnChartArea: false
},
afterDataLimits: function (axis) {
const indices = Object.keys(ChartJS.instances)
// initial values should be far outside possible range
let max = -99999999
let min = 99999999
for (const index of indices) {
if (ChartJS.instances[index] && axis.chart.scales.xBottom) {
min = Math.min(ChartJS.instances[index].scales.x.min, min);
max = Math.max(ChartJS.instances[index].scales.x.max, max);
}
}
axis.chart.scales.xBottom.options.min = min;
axis.chart.scales.xBottom.options.max = max;
axis.chart.scales.xBottom.min = min;
axis.chart.scales.xBottom.max = max;
},
},
y: {
ticks: {
autoSkip: false
},
}
},
indexAxis: "y" as const,
};
function TrafficVolumePage() {
const { filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
const { data: trafficVolumeData, years, nrens } = useData<TrafficVolume>('/api/traffic/', setFilterSelection);
const trafficVolumeDataset = createTrafficVolumeDataset(trafficVolumeData);
trafficVolumeDataset.datasets.forEach(dataset => {
dataset.hidden = !filterSelection.selectedYears.includes(parseInt(dataset.label));
});
// remove the datapoints and labels for the nrens that aren't selected
// unfortunately we cannot just hide them because chart.js doesn't want
// to create a stack from a single dataset
trafficVolumeDataset.datasets.forEach(dataset => {
dataset.data = dataset.data.filter((e, i) => {
return filterSelection.selectedNrens.includes(trafficVolumeDataset.labels[i]);
});
});
trafficVolumeDataset.labels = trafficVolumeDataset.labels.filter((e) => filterSelection.selectedNrens.includes(e));
const filterNode = <Filter
filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
filterSelection={filterSelection}
setFilterSelection={setFilterSelection}
/>
const numNrens = filterSelection.selectedNrens.length;
const numYears = filterSelection.selectedYears.length;
const heightPerBar = 2; // every added bar should give this much additional height
console.log(trafficVolumeDataset)
// set a minimum height of 20rem, additional years need some more space
const height = numNrens * numYears * heightPerBar + 5;
return (
<DataPage title="Traffic Volume Of NRENs per Year"
description='Total yearly traffic volume in terabyte per NREN'
category={Sections.Network} filter={filterNode}>
<>
<Row>
<DownloadDataButton data={trafficVolumeData} filename="traffic_volume_of_nren_per_year.csv" exportType={ExportType.CSV} />
<DownloadDataButton data={trafficVolumeData} filename="traffic_volume_of_nren_per_year.xlsx" exportType={ExportType.EXCEL} />
</Row>
<DownloadImageChartButton filename="traffic_volume_of_nren_per_year" />
<ChartContainer>
<div className="chart-container" style={{ 'height': `${height}rem` }}>
<Bar
plugins={[ChartDataLabels]}
data={trafficVolumeDataset}
options={chartOptions}
/>
</div>
</ChartContainer>
</>
</DataPage>
);
}
export default TrafficVolumePage;
......@@ -10,6 +10,7 @@ setup_logging()
logger = logging.getLogger(__name__)
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
def fetch_budget_excel_data():
......@@ -347,3 +348,47 @@ def fetch_organization_excel_data():
if parent_org not in [None, 'NA', 'N/A']:
yield nren.upper(), 2021, parent_org
def fetch_traffic_excel_data():
# load the xlsx file
wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Estimated_Traffic TByte"
ws = wb[sheet_name]
rows = list(ws.rows)
def convert_number(value, nren, year, description):
if value is None or value == '--' or value == 'No data':
return 0
try:
return float(value)
except (TypeError, ValueError):
logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
return 0
def create_points_for_year(year, start_column):
for i in range(6, 49):
nren_name = rows[i][start_column].value
if not nren_name:
continue
nren_name = nren_name.upper()
from_external = convert_number(rows[i][start_column + 1].value, nren_name, year, 'from_external')
to_external = convert_number(rows[i][start_column + 2].value, nren_name, year, 'to_external')
from_customer = convert_number(rows[i][start_column + 3].value, nren_name, year, 'from_customer')
to_customer = convert_number(rows[i][start_column + 4].value, nren_name, year, 'to_customer')
if from_external == 0 and to_external == 0 and from_customer == 0 and to_customer == 0:
continue
yield nren_name, year, from_external, to_external, from_customer, to_customer
yield from create_points_for_year(2016, 38)
yield from create_points_for_year(2017, 32)
yield from create_points_for_year(2018, 26)
yield from create_points_for_year(2019, 20)
yield from create_points_for_year(2020, 14)
yield from create_points_for_year(2021, 8)
yield from create_points_for_year(2022, 2)
......@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
class ROLES(Enum):
admin = "admin"
user = "user"
observer = "observer"
uuid_pk = Annotated[UUID, mapped_column(primary_key=True, default=lambda _: uuid4())]
......@@ -74,6 +75,10 @@ class User(UserMixin, db.Model):
def is_admin(self):
return self.roles == ROLES.admin
@property
def is_observer(self):
return self.roles == ROLES.observer
@property
def nren(self):
if len(self.nrens) == 0:
......
......@@ -4,7 +4,7 @@ from __future__ import annotations
import logging
from decimal import Decimal
from enum import Enum
from typing import Optional
from typing import List, Optional
from typing_extensions import Annotated
from sqlalchemy import String, JSON
......@@ -124,6 +124,18 @@ class Policy(db.Model):
acceptable_use: Mapped[str]
privacy_notice: Mapped[str]
data_protection: Mapped[str]
gender_equality: Mapped[str]
class TrafficVolume(db.Model):
__tablename__ = 'traffic_volume'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
to_customers: Mapped[Decimal]
from_customers: Mapped[Decimal]
to_external: Mapped[Decimal]
from_external: Mapped[Decimal]
class InstitutionURLs(db.Model):
......@@ -132,4 +144,4 @@ class InstitutionURLs(db.Model):
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
urls: Mapped[list[str]] = mapped_column(JSON)
urls: Mapped[List[str]] = mapped_column(JSON)
import smtplib
import threading
import logging
from threading import Thread
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Sequence, Union
......@@ -68,7 +69,7 @@ def send_mail(
# spin off a thread since this can take some time..
logger.debug('Sending email')
thread = threading.Thread(target=_send_mail, args=(smtp_server, port, recipients, message))
thread = Thread(target=_send_mail, args=(smtp_server, port, recipients, message))
thread.start()
......
import logging
import alembic_postgresql_enum # type: ignore # noqa: F401
from flask import current_app
from alembic import context
# this is the Alembic Config object, which provides
......
......@@ -312,182 +312,9 @@
"title": "Postal Code:"
},
{
"type": "dropdown",
"type": "text",
"name": "country",
"title": "Country:",
"choices": [
"Afghanistan",
"Albania",
"Algeria",
"Angola",
"Argentina",
"Armenia",
"Australia",
"Austria",
"Azerbaijan",
"Bahamas",
"Bahrain",
"Bangladesh",
"Barbados",
"Belarus",
"Belgium",
"Belize",
"Benin",
"Bermuda",
"Bhutan",
"Bolivia",
"Bosnia And Herzegowina",
"Botswana",
"Brazil",
"Brunei Darussalam",
"Bulgaria",
"Burkina Faso",
"Burundi",
"Cambodia",
"Cameroon",
"Canada",
"Cape Verde",
"Chad",
"Chile",
"China",
"Colombia",
"Congo",
"Congo, The Democratic Republic Of The",
"Costa Rica",
"Cote D'ivoire",
"Croatia",
"Cuba",
"Cyprus",
"Czech Republic",
"Denmark",
"Dominican Republic",
"Ecuador",
"Egypt",
"El Salvador",
"Estonia",
"Ethiopia",
"Fiji",
"Finland",
"France",
"Gabon",
"Gambia",
"Georgia",
"Germany",
"Ghana",
"Greece",
"Grenada",
"Guatemala",
"Guinea",
"Guyana",
"Haiti",
"Honduras",
"Hong Kong",
"Hungary",
"Iceland",
"India",
"Indonesia",
"Iran (islamic Republic Of)",
"Iraq",
"Ireland",
"Israel",
"Italy",
"Jamaica",
"Japan",
"Jordan",
"Kazakhstan",
"Kenya",
"Korea, Republic Of",
"Kuwait",
"Kyrgyzstan",
"Lao People's Democratic Republic",
"Latvia",
"Lebanon",
"Lesotho",
"Liberia",
"Libyan Arab Jamahiriya",
"Lithuania",
"Luxembourg",
"Macedonia, North",
"Madagascar",
"Malawi",
"Malaysia",
"Mali",
"Malta",
"Mauritania",
"Mauritius",
"Mexico",
"Micronesia, Federated States Of",
"Moldova, Republic Of",
"Mongolia",
"Montenegro",
"Morocco",
"Mozambique",
"Myanmar",
"Namibia",
"Nepal",
"Netherlands",
"New Zealand",
"Nicaragua",
"Niger",
"Nigeria",
"Norway",
"Oman",
"Pakistan",
"Palestine",
"Panama",
"Papua New Guinea",
"Paraguay",
"Peru",
"Philippines",
"Poland",
"Portugal",
"Puerto Rico",
"Qatar",
"Romania",
"Russian Federation",
"Rwanda",
"Samoa",
"Saudi Arabia",
"Senegal",
"Serbia",
"Serbia",
"Singapore",
"Slovakia (Slovak Republic)",
"Slovenia",
"Somalia",
"South Africa",
"South Sudan",
"Spain",
"Sri Lanka",
"Sudan",
"Suriname",
"Sweden",
"Switzerland",
"Syrian Arab Republic",
"Taiwan",
"Tajikistan",
"Tanzania, United Republic Of",
"Thailand",
"Togo",
"Trinidad And Tobago",
"Tunisia",
"Turkey",
"Turkmenistan",
"Uganda",
"Ukraine",
"United Arab Emirates",
"United Kingdom",
"United Kingdom",
"United States",
"Uruguay",
"Uzbekistan",
"Venezuela",
"Vietnam",
"Virgin Islands (british)",
"Yemen",
"Zambia",
"Zimbabwe"
]
"title": "Country (in English):"
},
{
"type": "text",
......
"""add traffic volume table
Revision ID: 3730c7f1ea1b
Revises: d6f581374e8f
Create Date: 2023-09-01 10:26:29.089050
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3730c7f1ea1b'
down_revision = 'f2879a6b15c8'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
'traffic_volume',
sa.Column('nren_id', sa.Integer(), nullable=False),
sa.Column('year', sa.Integer(), nullable=False),
sa.Column('to_customers', sa.Numeric(), nullable=False),
sa.Column('from_customers', sa.Numeric(), nullable=False),
sa.Column('to_external', sa.Numeric(), nullable=False),
sa.Column('from_external', sa.Numeric(), nullable=False),
sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_volume_nren_id_nren')),
sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_volume'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('traffic_volume')
# ### end Alembic commands ###
"""add gender equality policy field
Revision ID: 3cef2a22ebf1
Revises: 51a29df6148c
Create Date: 2023-09-07 14:29:11.149798
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3cef2a22ebf1'
down_revision = '51a29df6148c'
branch_labels = None
depends_on = None
def upgrade():
op.execute("DELETE FROM policy") # data is incomplete so force people to republish
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('policy', schema=None) as batch_op:
batch_op.add_column(sa.Column('gender_equality', sa.String(), nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('policy', schema=None) as batch_op:
batch_op.drop_column('gender_equality')
# ### end Alembic commands ###
"""Add observer role
Revision ID: 51a29df6148c
Revises: 3730c7f1ea1b
Create Date: 2023-09-07 10:48:30.087825
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '51a29df6148c'
down_revision = '3730c7f1ea1b'
branch_labels = None
depends_on = None
def upgrade():
op.sync_enum_values('public', 'roles', ['admin', 'user', 'observer'],
[('user', 'roles')],
enum_values_to_rename=[])
def downgrade():
op.sync_enum_values('public', 'roles', ['admin', 'user'],
[('user', 'roles')],
enum_values_to_rename=[])