Skip to content
Snippets Groups Projects
Commit 998db182 authored by Remco Tukker's avatar Remco Tukker
Browse files

backend part of the new traffic volume page

parent 1e47b8ce
Branches
Tags
1 merge request!74Feature/comp 274 traffic volume page
...@@ -10,6 +10,7 @@ setup_logging() ...@@ -10,6 +10,7 @@ setup_logging()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx") EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx")
NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx")
def fetch_budget_excel_data(): def fetch_budget_excel_data():
...@@ -347,3 +348,47 @@ def fetch_organization_excel_data(): ...@@ -347,3 +348,47 @@ def fetch_organization_excel_data():
if parent_org not in [None, 'NA', 'N/A']: if parent_org not in [None, 'NA', 'N/A']:
yield nren.upper(), 2021, parent_org yield nren.upper(), 2021, parent_org
def fetch_traffic_excel_data():
# load the xlsx file
wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "Estimated_Traffic TByte"
ws = wb[sheet_name]
rows = list(ws.rows)
def convert_number(value, nren, year, description):
if value is None or value == '--' or value == 'No data':
return 0
try:
return float(value)
except (TypeError, ValueError):
logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.')
return 0
def create_points_for_year(year, start_column):
for i in range(6, 49):
nren_name = rows[i][start_column].value
if not nren_name:
continue
nren_name = nren_name.upper()
from_external = convert_number(rows[i][start_column + 1].value, nren_name, year, 'from_external')
to_external = convert_number(rows[i][start_column + 2].value, nren_name, year, 'to_external')
from_customer = convert_number(rows[i][start_column + 3].value, nren_name, year, 'from_customer')
to_customer = convert_number(rows[i][start_column + 4].value, nren_name, year, 'to_customer')
if from_external == 0 and to_external == 0 and from_customer == 0 and to_customer == 0:
continue
yield nren_name, year, from_external, to_external, from_customer, to_customer
yield from create_points_for_year(2016, 38)
yield from create_points_for_year(2017, 32)
yield from create_points_for_year(2018, 26)
yield from create_points_for_year(2019, 20)
yield from create_points_for_year(2020, 14)
yield from create_points_for_year(2021, 8)
yield from create_points_for_year(2022, 2)
...@@ -124,3 +124,14 @@ class Policy(db.Model): ...@@ -124,3 +124,14 @@ class Policy(db.Model):
acceptable_use: Mapped[str] acceptable_use: Mapped[str]
privacy_notice: Mapped[str] privacy_notice: Mapped[str]
data_protection: Mapped[str] data_protection: Mapped[str]
class TrafficVolume(db.Model):
__tablename__ = 'traffic_volume'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
to_customers: Mapped[Decimal]
from_customers: Mapped[Decimal]
to_external: Mapped[Decimal]
from_external: Mapped[Decimal]
"""add traffic volume table
Revision ID: 3730c7f1ea1b
Revises: d6f581374e8f
Create Date: 2023-09-01 10:26:29.089050
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3730c7f1ea1b'
down_revision = 'd6f581374e8f'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
'traffic_volume',
sa.Column('nren_id', sa.Integer(), nullable=False),
sa.Column('year', sa.Integer(), nullable=False),
sa.Column('to_customers', sa.Numeric(), nullable=False),
sa.Column('from_customers', sa.Numeric(), nullable=False),
sa.Column('to_external', sa.Numeric(), nullable=False),
sa.Column('from_external', sa.Numeric(), nullable=False),
sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_volume_nren_id_nren')),
sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_volume'))
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('traffic_volume')
# ### end Alembic commands ###
...@@ -204,6 +204,27 @@ def db_organizations_migration(nren_dict): ...@@ -204,6 +204,27 @@ def db_organizations_migration(nren_dict):
db.session.commit() db.session.commit()
def db_traffic_volume_migration(nren_dict):
traffic_data = parse_excel_data.fetch_traffic_excel_data()
for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data:
if abbrev not in nren_dict:
logger.warning(f'{abbrev} unknown. Skipping.')
continue
nren = nren_dict[abbrev]
traffic_entry = model.TrafficVolume(
nren=nren,
nren_id=nren.id,
year=year,
from_customers=from_customers,
to_customers=to_customers,
from_external=from_external,
to_external=to_external
)
db.session.merge(traffic_entry)
db.session.commit()
def _cli(config, app): def _cli(config, app):
with app.app_context(): with app.app_context():
nren_dict = helpers.get_uppercase_nren_dict() nren_dict = helpers.get_uppercase_nren_dict()
...@@ -213,6 +234,7 @@ def _cli(config, app): ...@@ -213,6 +234,7 @@ def _cli(config, app):
db_staffing_migration(nren_dict) db_staffing_migration(nren_dict)
db_ecprojects_migration(nren_dict) db_ecprojects_migration(nren_dict)
db_organizations_migration(nren_dict) db_organizations_migration(nren_dict)
db_traffic_volume_migration(nren_dict)
@click.command() @click.command()
......
...@@ -14,6 +14,7 @@ from compendium_v2.routes.survey import routes as survey ...@@ -14,6 +14,7 @@ from compendium_v2.routes.survey import routes as survey
from compendium_v2.routes.user import routes as user_routes from compendium_v2.routes.user import routes as user_routes
from compendium_v2.routes.nren import routes as nren_routes from compendium_v2.routes.nren import routes as nren_routes
from compendium_v2.routes.response import routes as response_routes from compendium_v2.routes.response import routes as response_routes
from compendium_v2.routes.traffic import routes as traffic_routes
routes = Blueprint('compendium-v2-api', __name__) routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget') routes.register_blueprint(budget_routes, url_prefix='/budget')
...@@ -27,6 +28,7 @@ routes.register_blueprint(survey, url_prefix='/survey') ...@@ -27,6 +28,7 @@ routes.register_blueprint(survey, url_prefix='/survey')
routes.register_blueprint(user_routes, url_prefix='/user') routes.register_blueprint(user_routes, url_prefix='/user')
routes.register_blueprint(nren_routes, url_prefix='/nren') routes.register_blueprint(nren_routes, url_prefix='/nren')
routes.register_blueprint(response_routes, url_prefix='/response') routes.register_blueprint(response_routes, url_prefix='/response')
routes.register_blueprint(traffic_routes, url_prefix='/traffic')
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
import logging
from flask import Blueprint, jsonify
from compendium_v2.routes import common
from compendium_v2.db.model import TrafficVolume
from typing import Any
routes = Blueprint('traffic', __name__)
logger = logging.getLogger(__name__)
TRAFFIC_RESPONSE_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'traffic': {
'type': 'object',
'properties': {
'nren': {'type': 'string'},
'nren_country': {'type': 'string'},
'year': {'type': 'integer'},
'to_customers': {'type': 'number'},
'from_customers': {'type': 'number'},
'to_external': {'type': 'number'},
'from_external': {'type': 'number'}
},
'required': ['nren', 'nren_country', 'year', 'to_customers',
'from_customers', 'to_external', 'from_external'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/traffic'}
}
@routes.route('/', methods=['GET'])
@common.require_accepts_json
def traffic_volume_view() -> Any:
"""
handler for /api/traffic/ requests
response will be formatted as:
.. asjson::
compendium_v2.routes.traffic.TRAFFIC_RESPONSE_SCHEMA
:return:
"""
def _extract_data(entry: TrafficVolume):
return {
'nren': entry.nren.name,
'nren_country': entry.nren.country,
'year': entry.year,
'to_customers': float(entry.to_customers),
'from_customers': float(entry.from_customers),
'to_external': float(entry.to_external),
'from_external': float(entry.from_external)
}
entries = sorted(
[_extract_data(entry) for entry in common.get_data(TrafficVolume)],
key=lambda d: (d['nren'], d['year'])
)
return jsonify(entries)
...@@ -331,3 +331,27 @@ def test_policy_data(app): ...@@ -331,3 +331,27 @@ def test_policy_data(app):
)) ))
db.session.commit() db.session.commit()
@pytest.fixture
def test_traffic_data(app):
with app.app_context():
nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)]
nren_names = set(ny[0] for ny in nrens_and_years)
nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names}
db.session.add_all(nren_dict.values())
for (nren_name, year) in nrens_and_years:
nren = nren_dict[nren_name]
db.session.add(
model.TrafficVolume(
nren=nren,
year=year,
from_customers=2.23,
to_customers=5.2,
from_external=0,
to_external=1000
)
)
db.session.commit()
import json
import jsonschema
from compendium_v2.routes.traffic import TRAFFIC_RESPONSE_SCHEMA
def test_staff_response(client, test_traffic_data):
rv = client.get(
'/api/traffic/',
headers={'Accept': ['application/json']})
assert rv.status_code == 200
result = json.loads(rv.data.decode('utf-8'))
jsonschema.validate(result, TRAFFIC_RESPONSE_SCHEMA)
assert result
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment