Skip to content
Snippets Groups Projects
Commit e958883a authored by Saket Agrahari's avatar Saket Agrahari
Browse files

COMP-119: charging structure be fe migration test

parent 2853d7d9
Branches
Tags
No related merge requests found
Showing
with 586 additions and 5 deletions
...@@ -2,6 +2,7 @@ import openpyxl ...@@ -2,6 +2,7 @@ import openpyxl
import os import os
import logging import logging
from compendium_v2.db.model import FeeType
from compendium_v2.environment import setup_logging from compendium_v2.environment import setup_logging
setup_logging() setup_logging()
...@@ -14,7 +15,6 @@ EXCEL_FILE = os.path.join( ...@@ -14,7 +15,6 @@ EXCEL_FILE = os.path.join(
def fetch_budget_excel_data(): def fetch_budget_excel_data():
# load the xlsx file # load the xlsx file
sheet_name = "1. Budget" sheet_name = "1. Budget"
wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True) wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
...@@ -43,7 +43,6 @@ def fetch_budget_excel_data(): ...@@ -43,7 +43,6 @@ def fetch_budget_excel_data():
def fetch_funding_excel_data(): def fetch_funding_excel_data():
# load the xlsx file # load the xlsx file
wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True) wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
...@@ -112,3 +111,47 @@ def fetch_funding_excel_data(): ...@@ -112,3 +111,47 @@ def fetch_funding_excel_data():
# For 2020 # For 2020
yield from create_points_for_year(8, 50, 2020, 3) yield from create_points_for_year(8, 50, 2020, 3)
def fetch_charging_structure_excel_data():
# load the xlsx file
wb = openpyxl.load_workbook(EXCEL_FILE, data_only=True, read_only=True)
# select the active worksheet
sheet_name = "3. Charging mechanism"
ws = wb[sheet_name]
# iterate over the rows in the worksheet
def create_points_for_year(start_row, end_row, year, col_start):
for row in range(start_row, end_row):
# extract the data from the row
nren = ws.cell(row=row, column=col_start).value
charging_structure = ws.cell(row=row, column=col_start + 1).value
logger.info(
f'NREN: {nren}, Charging Structure: {charging_structure},'
f' Year: {year}')
if charging_structure is not None:
if "do not charge" in charging_structure:
charging_structure = FeeType.no_charge.value
elif "combination" in charging_structure:
charging_structure = FeeType.combination.value
elif "flat" in charging_structure:
charging_structure = FeeType.flat_fee.value
elif "usage-based" in charging_structure:
charging_structure = FeeType.usage_based_fee.value
elif "Other" in charging_structure:
charging_structure = FeeType.other.value
else:
charging_structure = None
logger.info(
f'NREN: {nren}, Charging Structure: {charging_structure},'
f' Year: {year}')
yield nren, year, charging_structure
# For 2021
yield from create_points_for_year(3, 45, 2021, 2)
# For 2019
yield from create_points_for_year(3, 45, 2019, 6)
import logging import logging
import sqlalchemy as sa import sqlalchemy as sa
from enum import Enum
from typing import Any from typing import Any
...@@ -30,3 +31,21 @@ class FundingSource(base_schema): ...@@ -30,3 +31,21 @@ class FundingSource(base_schema):
gov_public_bodies = sa.Column(sa.Numeric(asdecimal=False), nullable=False) gov_public_bodies = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
commercial = sa.Column(sa.Numeric(asdecimal=False), nullable=False) commercial = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
other = sa.Column(sa.Numeric(asdecimal=False), nullable=False) other = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
class FeeType(Enum):
flat_fee = "flat_fee"
usage_based_fee = "usage_based_fee"
combination = "combination"
no_charge = "no_charge"
other = "other"
class ChargingStructure(base_schema):
__tablename__ = 'charging_structure'
nren = sa.Column(sa.String(128), primary_key=True)
year = sa.Column(sa.Integer, primary_key=True)
fee_type = sa.Column('fee_type', sa.Enum("flat_fee", "usage_based_fee",
"combination", "no_charge",
"other",
name="fee_type"), nullable=True)
"""adding charging structure schema
Revision ID: b123f21a8f4c
Revises: b70ada054046
Create Date: 2023-04-05 22:02:02.248236
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'b123f21a8f4c'
down_revision = 'b70ada054046'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'charging_structure',
sa.Column('nren', sa.String(length=128), nullable=False),
sa.Column('year', sa.Integer(), nullable=False),
sa.Column('fee_type', sa.Enum("flat_fee", "usage_based_fee",
"combination", "no_charge",
"other",
name="fee_type"), nullable=True),
sa.PrimaryKeyConstraint('nren', 'year')
)
def downgrade():
op.drop_table('charging_structure')
...@@ -82,10 +82,23 @@ def db_funding_migration(): ...@@ -82,10 +82,23 @@ def db_funding_migration():
session.commit() session.commit()
def db_charging_structure_migration():
with db.session_scope() as session:
# Import the data to database
data = parse_excel_data.fetch_charging_structure_excel_data()
for (nren, year, charging_structure) in data:
charging_structure_entry = model.ChargingStructure(
nren=nren, year=year, fee_type=charging_structure)
session.merge(charging_structure_entry)
session.commit()
def _cli(config): def _cli(config):
init_db(config) init_db(config)
db_budget_migration() db_budget_migration()
db_funding_migration() db_funding_migration()
db_charging_structure_migration()
@click.command() @click.command()
......
...@@ -17,10 +17,12 @@ from flask import Blueprint ...@@ -17,10 +17,12 @@ from flask import Blueprint
from compendium_v2.routes import common from compendium_v2.routes import common
from compendium_v2.routes.budget import routes as budget_routes from compendium_v2.routes.budget import routes as budget_routes
from compendium_v2.routes.funding import routes as funding_routes from compendium_v2.routes.funding import routes as funding_routes
from compendium_v2.routes.charging import routes as charging_routes
routes = Blueprint('compendium-v2-api', __name__) routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget') routes.register_blueprint(budget_routes, url_prefix='/budget')
routes.register_blueprint(funding_routes, url_prefix='/funding') routes.register_blueprint(funding_routes, url_prefix='/funding')
routes.register_blueprint(charging_routes, url_prefix='/charging')
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
......
import logging
from flask import Blueprint, jsonify, current_app
from compendium_v2 import db
from compendium_v2.routes import common
from compendium_v2.db import model
from typing import Any
routes = Blueprint('charging', __name__)
@routes.before_request
def before_request():
config = current_app.config['CONFIG_PARAMS']
dsn_prn = config['SQLALCHEMY_DATABASE_URI']
db.init_db_model(dsn_prn)
logger = logging.getLogger(__name__)
CHARGING_STRUCTURE_RESPONSE_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'charging': {
'type': 'object',
'properties': {
'NREN': {'type': 'string'},
'YEAR': {'type': 'integer'},
'FEE_TYPE': {'type': ["string", "null"]},
},
'required': ['NREN', 'YEAR'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/charging'}
}
@routes.route('/', methods=['GET'])
@common.require_accepts_json
def charging_structure_view() -> Any:
"""
handler for /api/charging/ requests
response will be formatted as:
.. asjson::
compendium_v2.routes.charging.CHARGING_STRUCTURE_RESPONSE_SCHEMA
:return:
"""
def _extract_data(entry: model.ChargingStructure):
return {
'NREN': entry.nren,
'YEAR': int(entry.year),
'FEE_TYPE': entry.fee_type,
}
with db.session_scope() as session:
entries = sorted([_extract_data(entry)
for entry in session.query(model.ChargingStructure)
.all()],
key=lambda d: (d['NREN'], d['YEAR']))
return jsonify(entries)
...@@ -149,3 +149,21 @@ def client(data_config_filename, mocked_db, mocked_survey_db): ...@@ -149,3 +149,21 @@ def client(data_config_filename, mocked_db, mocked_survey_db):
os.environ['SETTINGS_FILENAME'] = data_config_filename os.environ['SETTINGS_FILENAME'] = data_config_filename
with compendium_v2.create_app().test_client() as c: with compendium_v2.create_app().test_client() as c:
yield c yield c
@pytest.fixture
def test_charging_structure_data():
with db.session_scope() as session:
data = _test_data_csv("ChargingStructureTestData.csv")
for row in data:
nren = row["nren"]
year = row["year"]
fee_type = row["fee_type"]
if fee_type == "null":
fee_type = None
session.add(
model.ChargingStructure(
nren=nren, year=year,
fee_type=fee_type)
)
nren,year,fee_type
ACOnet,2021,usage_based_fee
AMRES,2021,no_charge
ANA,2021,combination
ARNES,2021,no_charge
ASNET-AM,2021,combination
BASNET,2021,combination
Belnet,2021,no_charge
BREN,2021,no_charge
CARNET,2021,no_charge
CESNET,2021,flat_fee
CYNET,2021,flat_fee
DeIC,2021,combination
DFN,2021,flat_fee
EENet,2021,no_charge
FCCN,2021,usage_based_fee
GARR,2021,no_charge
GRENA,2021,flat_fee
GRNET S.A.,2021,no_charge
HEAnet,2021,no_charge
Jisc,2021,usage_based_fee
LITNET,2021,no_charge
MARnet,2021,no_charge
MREN,2021,no_charge
PIONIER,2021,flat_fee
RedIRIS,2021,no_charge
RENAM,2021,combination
RENATER,2021,flat_fee
RESTENA,2021,flat_fee
RoEduNet,2021,no_charge
SANET,2021,flat_fee
SWITCH,2021,combination
ULAKBIM,2021,no_charge
ACOnet,2019,flat_fee
AMRES,2019,no_charge
ANA,2019,combination
ARNES,2019,no_charge
AzScienceNet,2019,no_charge
BASNET,2019,combination
BELNET,2019,flat_fee
BREN,2019,no_charge
CARNet,2019,no_charge
CESNET,2019,no_charge
DFN,2019,no_charge
EENet,2019,no_charge
FCCN,2019,no_charge
Funet,2019,combination
GARR,2019,no_charge
GRENA,2019,flat_fee
GRNET S.A.,2019,no_charge
HEAnet,2019,combination
Jisc,2019,flat_fee
LITNET,2019,no_charge
MARNET,2019,usage_based_fee
MREN,2019,no_charge
PIONIER,2019,flat_fee
RedIRIS,2019,no_charge
RENAM,2019,combination
RENATER,2019,no_charge
RESTENA,2019,no_charge
RoEduNet,2019,no_charge
SANET,2019,flat_fee
SURFnet,2019,combination
SWITCH,2019,combination
ULAKBIM,2019,no_charge
AzScienceNet,2021,null
LAT,2021,null
RHnet,2021,null
SURFnet,2021,null
Uninett,2021,null
UoM/RicerkaNet,2021,null
ASNET,2019,null
CYNET,2019,null
DeIC,2019,null
KIFU (NIIF),2019,null
LANET,2019,null
RhNET,2019,null
UNINETT,2019,null
UoM,2019,null
Funet,2021,other
IUCC,2021,other
KIFU,2021,other
SUNET,2021,other
IUCC,2019,other
SUNET,2019,other
import json
import jsonschema
from compendium_v2.routes.charging import CHARGING_STRUCTURE_RESPONSE_SCHEMA
def test_charging_structure_response(client, test_charging_structure_data):
rv = client.get(
'/api/charging/',
headers={'Accept': ['application/json']})
assert rv.status_code == 200
result = json.loads(rv.data.decode('utf-8'))
jsonschema.validate(result, CHARGING_STRUCTURE_RESPONSE_SCHEMA)
assert result
...@@ -20,3 +20,6 @@ def test_publisher(client, mocker, dummy_config): ...@@ -20,3 +20,6 @@ def test_publisher(client, mocker, dummy_config):
assert budget_count assert budget_count
funding_source_count = session.query(model.FundingSource.year).count() funding_source_count = session.query(model.FundingSource.year).count()
assert funding_source_count assert funding_source_count
charging_structure_count = session.query(model.ChargingStructure.year)\
.count()
assert charging_structure_count
...@@ -7,6 +7,8 @@ import DataAnalysis from "./pages/DataAnalysis"; ...@@ -7,6 +7,8 @@ import DataAnalysis from "./pages/DataAnalysis";
import AnnualReport from "./pages/AnnualReport"; import AnnualReport from "./pages/AnnualReport";
import CompendiumData from "./pages/CompendiumData"; import CompendiumData from "./pages/CompendiumData";
import FundingSourcePage from "./pages/FundingSource"; import FundingSourcePage from "./pages/FundingSource";
import ChargingStructurePage from "./pages/ChargingStructure";
function App(): ReactElement { function App(): ReactElement {
return ( return (
...@@ -19,6 +21,7 @@ function App(): ReactElement { ...@@ -19,6 +21,7 @@ function App(): ReactElement {
<Route path="/analysis" element={<DataAnalysis />} /> <Route path="/analysis" element={<DataAnalysis />} />
<Route path="/report" element={<AnnualReport />} /> <Route path="/report" element={<AnnualReport />} />
<Route path="/funding" element={<FundingSourcePage />} /> <Route path="/funding" element={<FundingSourcePage />} />
<Route path="/charging" element={<ChargingStructurePage />} />
<Route path="*" element={<Landing />} /> <Route path="*" element={<Landing />} />
</Routes> </Routes>
</Router> </Router>
......
...@@ -53,6 +53,26 @@ export interface FundingSourceDataset { ...@@ -53,6 +53,26 @@ export interface FundingSourceDataset {
}[] }[]
} }
export interface ChargingStructure{
NREN: string,
YEAR: number,
FEE_TYPE: (string | null),
}
export interface ChargingStructureDataset {
labels: string[],
datasets: {
label: string,
data: { x: number | null; y: number | null; }[],
backgroundColor: string
borderRadius: number,
borderSkipped: boolean,
barPercentage: number,
borderWidth: number,
categoryPercentage: number,
}[]
}
export interface DataEntrySection { export interface DataEntrySection {
name: string, name: string,
description: string, description: string,
......
...@@ -2,7 +2,9 @@ import { cartesianProduct } from 'cartesian-product-multiple-arrays'; ...@@ -2,7 +2,9 @@ import { cartesianProduct } from 'cartesian-product-multiple-arrays';
import { import {
FundingSource, FundingSource,
FundingSourceDataset FundingSourceDataset,
ChargingStructure,
ChargingStructureDataset
} from "../Schema"; } from "../Schema";
const DEFAULT_FUNDING_SOURCE_DATA = [ const DEFAULT_FUNDING_SOURCE_DATA = [
...@@ -16,6 +18,13 @@ const DEFAULT_FUNDING_SOURCE_DATA = [ ...@@ -16,6 +18,13 @@ const DEFAULT_FUNDING_SOURCE_DATA = [
"YEAR": 0, "YEAR": 0,
"id": 0 "id": 0
}] }]
const DEFAULT_CHARGING_STRUCTURE_DATA = [
{
"NREN": "",
"YEAR": 0,
"FEE_TYPE": "",
}
]
function getColorMap() { function getColorMap() {
const rgbToHex = (r: number, g: number, b: number) => '#' + [r, g, b].map(x => { const rgbToHex = (r: number, g: number, b: number) => '#' + [r, g, b].map(x => {
...@@ -99,4 +108,84 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[]) ...@@ -99,4 +108,84 @@ export const createFundingSourceDataset = (fundingSourcesData: FundingSource[])
labels: labelsNREN.map(l => l.toString()) labels: labelsNREN.map(l => l.toString())
} }
return dataResponse; return dataResponse;
} }
\ No newline at end of file
function createChargingStructureDataLookup(data: ChargingStructure[]) {
let dataLookup = new Map<string, (string|null)>();
data.forEach((item: ChargingStructure) => {
const lookupKey = `${item.NREN}/${item.YEAR}`
dataLookup.set(lookupKey, item.FEE_TYPE)
})
return dataLookup;
}
export function coordinateLookupForFeeStructure(feeType:string,nren:string,
yearIndex:number,nrenIndex:number){
switch(feeType){
case "flat_fee":
return {x:(0+((yearIndex+1)*2)),y:(nrenIndex+1),feeType:feeType}
case "usage_based_fee":
return {x:(5+((yearIndex+1)*2)),y:(nrenIndex+1),feeType:feeType}
case "combination":
return {x:(10+((yearIndex+1)*2)),y:(nrenIndex+1),feeType:feeType}
case "no_charge":
return {x:(15+((yearIndex+1)*2)),y:(nrenIndex+1),feeType:feeType}
case "other":
return {x:(20+((yearIndex+1)*2)),y:(nrenIndex+1),feeType:feeType}
default:
return {x:0,y:0}
}
}
export function lookupColorForYearIndex(yearIndex:number){
switch(yearIndex){
case 0:
return "rgba(244, 144, 28, 1)"
case 1:
return "rgba(140, 168, 128, 1)"
default:
return "rgba(0, 0, 0, 0)"
}
}
export function createChargingStructureDataset(chargingStructureData: ChargingStructure[]) {
const data = chargingStructureData ?? DEFAULT_CHARGING_STRUCTURE_DATA;
const dataLookup = createChargingStructureDataLookup(data)
const labelsNREN = [...new Set(data.map((item: ChargingStructure) => item.NREN))];
const labelsYear = [...new Set(data.map((item: ChargingStructure) => item.YEAR))];
const sizeOfYear = labelsYear.length;
const chargingStructureDataset = labelsYear.map(function (year,yearIndex) {
return {
label: "(" + year + ")",
data: labelsNREN.map((nren,nrenIndex) => {
// ensure that the data is in the same order as the labels
const lookupKey = `${nren}/${year}`
const fee_type =dataLookup.get(lookupKey) ?? ""
return coordinateLookupForFeeStructure(fee_type,nren,yearIndex,nrenIndex)
}),
backgroundColor: lookupColorForYearIndex(yearIndex),
borderColor: lookupColorForYearIndex(yearIndex),
pointRadius:20,
usePointStyle: true,
pointStyle: 'rectRounded',
pointBackgroundColor: lookupColorForYearIndex(yearIndex),
borderRadius: 0,
borderSkipped: false,
barPercentage: 0.5,
borderWidth: 0.5,
categoryPercentage: 0.8
}
}
)
const dataResponse: ChargingStructureDataset = {
datasets: chargingStructureDataset,
labels: labelsNREN.map(l => l.toString())
}
return dataResponse;
}
import React, { useEffect, useState } from "react";
import { Container, Row, Col } from "react-bootstrap";
import Chart, { ChartOptions, Plugin, ChartTypeRegistry, ScatterDataPoint } from 'chart.js';
import { Bubble, Scatter } from "react-chartjs-2";
import { createChargingStructureDataset } from "../helpers/dataconversion";
import { ChargingStructure, ChargingStructureDataset } from "../Schema";
import {
Chart as ChartJS,
CategoryScale,
LinearScale,
BarElement,
Title,
Tooltip,
Legend,
scales,
} from 'chart.js';
ChartJS.register(
CategoryScale,
LinearScale,
BarElement,
Title,
Tooltip,
Legend
);
const EMPTY_DATASET = { datasets: [], labels: [] };
const DATA_COUNT = 7;
const NUMBER_CFG = {count: DATA_COUNT, rmin: 1, rmax: 1, min: 0, max: 100};
const test_data = {
labels: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'],
datasets: [{
label: 'Weekly Sales',
data: [
{ x: 1, y: 5 },
{ x: 3, y: 0 },
{ x: 3, y: 2 },
{ x: 5, y: 3 },
{ x: 7, y: 4 }
],
backgroundColor: 'rgba(244, 144, 28, 1)',
borderColor: 'rgba(244, 144, 28, 1)',
borderWidth: 0,
pointRadius: 40,
usePointStyle: true,
pointStyle: 'rectRounded',
},
{
label: 'Monthly Sales',
data: [
{ x: 2, y: 1 },
{ x: 1, y: 2 },
{ x: 6, y: 3 },
{ x: 8, y: 4 },
{ x: 9, y: 5 }
],
backgroundColor: 'rgba(140, 168, 128, 1)',
borderColor: 'rgba(140, 168, 128, 1)',
borderWidth: 0,
pointRadius: 40,
usePointStyle: true,
pointStyle: 'rectRounded',
}]
};
const plugin = {
id: 'customCanvasBackgroundColor',
beforeDraw: (chart:any, args:any, options:any) => {
const {ctx} = chart;
ctx.save();
ctx.globalCompositeOperation = 'destination-over';
ctx.fillStyle = options.color || '#99ffff';
ctx.fillRect(0, 0, chart.width, chart.height);
ctx.restore();
}
};
const options = {
maintainAspectRatio: false,
layout: {
padding: {
left: 100,
top: 100,
}
},
plugins: {
legend: {
display: false,
},
customCanvasBackgroundColor: {
color: 'lightGreen',
}
},
scales: {
x: {
position: "top" as const,
ticks: {
display: false,
stepSize: 5,
},
},
y: {
grid: {
display: false
},
ticks: {
display: false,
stepSize: 5,
},
},
}
}
async function getData(): Promise<ChargingStructure[]> {
try {
const response = await fetch('/api/charging/');
return response.json();
} catch (error) {
console.error(`Failed to load data: ${error}`);
throw error;
}
}
function ChargingStructurePage (): React.ReactElement {
const [chargingStructureDataSet,setDataset]= useState<ChargingStructureDataset>(EMPTY_DATASET);
const [chargingStructureData, setChargingStructureData] = useState<ChargingStructure[]>([]);
console.log("ChargingStructurePage");
React.useEffect(() => {
const loadData = async () => {
const _chargingStructureData = await getData();
setChargingStructureData(_chargingStructureData);
}
loadData();
}, []);
useEffect(() => {
if(chargingStructureDataSet !== undefined) {
setDataset(createChargingStructureDataset(chargingStructureData));
console.log(chargingStructureDataSet);
}
}, [chargingStructureData]);
return (
<Container>
<Row>
<Col>
<h1>Charging Structure</h1>
</Col>
</Row>
<Row>
<Col>
<div className="chart-container" style={{ 'minHeight': '300vh', 'width': '80vw', }}>
<Bubble data={chargingStructureDataSet} options={options} />
</div>
</Col>
</Row>
</Container>
);
};
export default ChargingStructurePage;
...@@ -38,6 +38,12 @@ function CompendiumData(): ReactElement { ...@@ -38,6 +38,12 @@ function CompendiumData(): ReactElement {
<span>Income Source of NRENs per Year</span> <span>Income Source of NRENs per Year</span>
</Link> </Link>
</Row> </Row>
<Row>
<Link to="/charging" state={{ graph: 'charging_structure' }}>
<span>Charging Mechanism of NRENs per Year</span>
</Link>
</Row>
</div> </div>
</CollapsibleBox> </CollapsibleBox>
</div> </div>
......
...@@ -12,7 +12,9 @@ ...@@ -12,7 +12,9 @@
"resolveJsonModule": true, "resolveJsonModule": true,
"isolatedModules": true, "isolatedModules": true,
"noEmit": true, "noEmit": true,
"jsx": "react" "jsx": "react",
"declaration": true,
"declarationDir": "dist/types"
}, },
"include": ["src"] "include": ["src"]
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment