diff --git a/compendium_v2/background_task/parse_excel_data.py b/compendium_v2/background_task/parse_excel_data.py index 162ec80d78eadb50fdc08664ba5ade43a2c13968..aba3ef6fb09ce2ec5f7b523788dfe22c2a7ff00c 100644 --- a/compendium_v2/background_task/parse_excel_data.py +++ b/compendium_v2/background_task/parse_excel_data.py @@ -10,6 +10,7 @@ setup_logging() logger = logging.getLogger(__name__) EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2021_Organisation_DataSeries.xlsx") +NETWORK_EXCEL_FILE = os.path.join(os.path.dirname(__file__), "xlsx", "2022_Networks_DataSeries.xlsx") def fetch_budget_excel_data(): @@ -347,3 +348,47 @@ def fetch_organization_excel_data(): if parent_org not in [None, 'NA', 'N/A']: yield nren.upper(), 2021, parent_org + + +def fetch_traffic_excel_data(): + # load the xlsx file + wb = openpyxl.load_workbook(NETWORK_EXCEL_FILE, data_only=True, read_only=True) + + # select the active worksheet + sheet_name = "Estimated_Traffic TByte" + ws = wb[sheet_name] + + rows = list(ws.rows) + + def convert_number(value, nren, year, description): + if value is None or value == '--' or value == 'No data': + return 0 + try: + return float(value) + except (TypeError, ValueError): + logger.info(f'NREN: {nren} year: {year} has {value} for {description}; set to 0.') + return 0 + + def create_points_for_year(year, start_column): + for i in range(6, 49): + nren_name = rows[i][start_column].value + if not nren_name: + continue + nren_name = nren_name.upper() + + from_external = convert_number(rows[i][start_column + 1].value, nren_name, year, 'from_external') + to_external = convert_number(rows[i][start_column + 2].value, nren_name, year, 'to_external') + from_customer = convert_number(rows[i][start_column + 3].value, nren_name, year, 'from_customer') + to_customer = convert_number(rows[i][start_column + 4].value, nren_name, year, 'to_customer') + if from_external == 0 and to_external == 0 and from_customer == 0 and to_customer == 0: + continue + + yield nren_name, year, from_external, to_external, from_customer, to_customer + + yield from create_points_for_year(2016, 38) + yield from create_points_for_year(2017, 32) + yield from create_points_for_year(2018, 26) + yield from create_points_for_year(2019, 20) + yield from create_points_for_year(2020, 14) + yield from create_points_for_year(2021, 8) + yield from create_points_for_year(2022, 2) diff --git a/compendium_v2/db/model.py b/compendium_v2/db/model.py index 3fb13df26688092ef9b875bfa8d311797d743fa1..7e594d01518c9f71d4ebbf47eb1f3a5a089ff534 100644 --- a/compendium_v2/db/model.py +++ b/compendium_v2/db/model.py @@ -124,3 +124,14 @@ class Policy(db.Model): acceptable_use: Mapped[str] privacy_notice: Mapped[str] data_protection: Mapped[str] + + +class TrafficVolume(db.Model): + __tablename__ = 'traffic_volume' + nren_id: Mapped[int_pk_fkNREN] + nren: Mapped[NREN] = relationship(lazy='joined') + year: Mapped[int_pk] + to_customers: Mapped[Decimal] + from_customers: Mapped[Decimal] + to_external: Mapped[Decimal] + from_external: Mapped[Decimal] diff --git a/compendium_v2/migrations/versions/3730c7f1ea1b_add_traffic_volume_table.py b/compendium_v2/migrations/versions/3730c7f1ea1b_add_traffic_volume_table.py new file mode 100644 index 0000000000000000000000000000000000000000..21ae1384ab82bc7fb7584b30b88c877e96e4e85b --- /dev/null +++ b/compendium_v2/migrations/versions/3730c7f1ea1b_add_traffic_volume_table.py @@ -0,0 +1,38 @@ +"""add traffic volume table + +Revision ID: 3730c7f1ea1b +Revises: d6f581374e8f +Create Date: 2023-09-01 10:26:29.089050 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '3730c7f1ea1b' +down_revision = 'd6f581374e8f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'traffic_volume', + sa.Column('nren_id', sa.Integer(), nullable=False), + sa.Column('year', sa.Integer(), nullable=False), + sa.Column('to_customers', sa.Numeric(), nullable=False), + sa.Column('from_customers', sa.Numeric(), nullable=False), + sa.Column('to_external', sa.Numeric(), nullable=False), + sa.Column('from_external', sa.Numeric(), nullable=False), + sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_traffic_volume_nren_id_nren')), + sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_traffic_volume')) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('traffic_volume') + # ### end Alembic commands ### diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_v1.py index 0731fe5b1a2752b060d4242aee6c54f16b9c3a65..a6e2376f5f1f12571e6f8fefa8b89c04fb405a3e 100644 --- a/compendium_v2/publishers/survey_publisher_v1.py +++ b/compendium_v2/publishers/survey_publisher_v1.py @@ -204,6 +204,27 @@ def db_organizations_migration(nren_dict): db.session.commit() +def db_traffic_volume_migration(nren_dict): + traffic_data = parse_excel_data.fetch_traffic_excel_data() + for (abbrev, year, from_external, to_external, from_customers, to_customers) in traffic_data: + if abbrev not in nren_dict: + logger.warning(f'{abbrev} unknown. Skipping.') + continue + + nren = nren_dict[abbrev] + traffic_entry = model.TrafficVolume( + nren=nren, + nren_id=nren.id, + year=year, + from_customers=from_customers, + to_customers=to_customers, + from_external=from_external, + to_external=to_external + ) + db.session.merge(traffic_entry) + db.session.commit() + + def _cli(config, app): with app.app_context(): nren_dict = helpers.get_uppercase_nren_dict() @@ -213,6 +234,7 @@ def _cli(config, app): db_staffing_migration(nren_dict) db_ecprojects_migration(nren_dict) db_organizations_migration(nren_dict) + db_traffic_volume_migration(nren_dict) @click.command() diff --git a/compendium_v2/routes/api.py b/compendium_v2/routes/api.py index 74c28a9def2437f7cec8241e196e1e09c3762383..15d6cd6238c32160d9420876ac1ca3e5e288707d 100644 --- a/compendium_v2/routes/api.py +++ b/compendium_v2/routes/api.py @@ -14,6 +14,7 @@ from compendium_v2.routes.survey import routes as survey from compendium_v2.routes.user import routes as user_routes from compendium_v2.routes.nren import routes as nren_routes from compendium_v2.routes.response import routes as response_routes +from compendium_v2.routes.traffic import routes as traffic_routes routes = Blueprint('compendium-v2-api', __name__) routes.register_blueprint(budget_routes, url_prefix='/budget') @@ -27,6 +28,7 @@ routes.register_blueprint(survey, url_prefix='/survey') routes.register_blueprint(user_routes, url_prefix='/user') routes.register_blueprint(nren_routes, url_prefix='/nren') routes.register_blueprint(response_routes, url_prefix='/response') +routes.register_blueprint(traffic_routes, url_prefix='/traffic') logger = logging.getLogger(__name__) diff --git a/compendium_v2/routes/traffic.py b/compendium_v2/routes/traffic.py new file mode 100644 index 0000000000000000000000000000000000000000..70255d0f847be12d54b62308cdc4763e6b263dde --- /dev/null +++ b/compendium_v2/routes/traffic.py @@ -0,0 +1,68 @@ +import logging + +from flask import Blueprint, jsonify + +from compendium_v2.routes import common +from compendium_v2.db.model import TrafficVolume +from typing import Any + + +routes = Blueprint('traffic', __name__) +logger = logging.getLogger(__name__) + +TRAFFIC_RESPONSE_SCHEMA = { + '$schema': 'http://json-schema.org/draft-07/schema#', + + 'definitions': { + 'traffic': { + 'type': 'object', + 'properties': { + 'nren': {'type': 'string'}, + 'nren_country': {'type': 'string'}, + 'year': {'type': 'integer'}, + 'to_customers': {'type': 'number'}, + 'from_customers': {'type': 'number'}, + 'to_external': {'type': 'number'}, + 'from_external': {'type': 'number'} + }, + 'required': ['nren', 'nren_country', 'year', 'to_customers', + 'from_customers', 'to_external', 'from_external'], + 'additionalProperties': False + } + }, + + 'type': 'array', + 'items': {'$ref': '#/definitions/traffic'} +} + + +@routes.route('/', methods=['GET']) +@common.require_accepts_json +def traffic_volume_view() -> Any: + """ + handler for /api/traffic/ requests + + response will be formatted as: + + .. asjson:: + compendium_v2.routes.traffic.TRAFFIC_RESPONSE_SCHEMA + + :return: + """ + + def _extract_data(entry: TrafficVolume): + return { + 'nren': entry.nren.name, + 'nren_country': entry.nren.country, + 'year': entry.year, + 'to_customers': float(entry.to_customers), + 'from_customers': float(entry.from_customers), + 'to_external': float(entry.to_external), + 'from_external': float(entry.from_external) + } + + entries = sorted( + [_extract_data(entry) for entry in common.get_data(TrafficVolume)], + key=lambda d: (d['nren'], d['year']) + ) + return jsonify(entries) diff --git a/test/conftest.py b/test/conftest.py index 429802eadcfed73e0d2ae77e9cef8198bea10aae..c87d613c212933a93e51cb395dd70fbbeebfc309 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -331,3 +331,27 @@ def test_policy_data(app): )) db.session.commit() + + +@pytest.fixture +def test_traffic_data(app): + with app.app_context(): + nrens_and_years = [('nren1', 2019), ('nren1', 2020), ('nren1', 2021), ('nren2', 2019), ('nren2', 2021)] + nren_names = set(ny[0] for ny in nrens_and_years) + nren_dict = {nren_name: model.NREN(name=nren_name, country='country') for nren_name in nren_names} + db.session.add_all(nren_dict.values()) + + for (nren_name, year) in nrens_and_years: + nren = nren_dict[nren_name] + + db.session.add( + model.TrafficVolume( + nren=nren, + year=year, + from_customers=2.23, + to_customers=5.2, + from_external=0, + to_external=1000 + ) + ) + db.session.commit() diff --git a/test/test_traffic.py b/test/test_traffic.py new file mode 100644 index 0000000000000000000000000000000000000000..d1427f5b5ec23aae6f5f2a38e3c918e5d79732d5 --- /dev/null +++ b/test/test_traffic.py @@ -0,0 +1,13 @@ +import json +import jsonschema +from compendium_v2.routes.traffic import TRAFFIC_RESPONSE_SCHEMA + + +def test_staff_response(client, test_traffic_data): + rv = client.get( + '/api/traffic/', + headers={'Accept': ['application/json']}) + assert rv.status_code == 200 + result = json.loads(rv.data.decode('utf-8')) + jsonschema.validate(result, TRAFFIC_RESPONSE_SCHEMA) + assert result