Skip to content
Snippets Groups Projects
Commit ef047f17 authored by Saket Agrahari's avatar Saket Agrahari
Browse files

logging and style correction

parent bc89db14
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
+ 41
− 20
View file @ ef047f17
......@@ -3,6 +3,12 @@ import csv
import os
from compendium_v2 import db
from compendium_v2.db import model
from compendium_v2.environment import setup_logging
import logging
setup_logging()
logger = logging.getLogger('xlsx_to_csv_sheet_parsing_task')
# Import the data to database
......@@ -20,7 +26,8 @@ def import_countries():
def parse_budget_xlsx_file():
try:
# load the xlsx file
filename = "compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx"
filename = "compendium_v2/background_task/xlsx" \
"/2021_Organisation_DataSeries.xlsx "
csv_out_file = "compendium_v2/background_task/csv/BudgetCsvFile.csv"
sheet_name = "1. Budget"
wb = openpyxl.load_workbook(
......@@ -41,6 +48,11 @@ def parse_budget_xlsx_file():
if budget is not None:
budget = round(budget / 1000000, 2)
if budget > 200:
logger.info(
f'{nren} has budget set to '
f'>200M EUR for {year}. ({budget})')
# process the data (e.g. save to database)
# print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
output_csv_file = csv.writer(
......@@ -57,8 +69,10 @@ def parse_budget_xlsx_file():
def parse_income_source_xlsx_file():
try:
# load the xlsx file
filename = "compendium_v2/background_task/xlsx/2021_Organisation_DataSeries.xlsx"
csv_out_file = "compendium_v2/background_task/csv/FundingSourceCsvFile.csv"
filename = "compendium_v2/background_task/xlsx" \
"/2021_Organisation_DataSeries.xlsx "
csv_out_file = "compendium_v2/background_task/csv" \
"/FundingSourceCsvFile.csv "
sheet_name = "2. Income Sources"
wb = openpyxl.load_workbook(
filename, data_only=True, read_only=True)
......@@ -67,20 +81,26 @@ def parse_income_source_xlsx_file():
ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file)
print("file deleted "+ csv_out_file)
print("file deleted " + csv_out_file)
def hard_number_convert(s):
def hard_number_convert(s, source_name, nren, year):
if s is None:
logger.info(
f'Invalid Value :{nren} has empty value for {source_name}.'
+ f'for year ({year})')
return float(0)
""" Returns True if string is a number. """
try:
return float(s)
except ValueError:
logger.info(
f'Invalid Value :{nren} has empty value for {source_name}.'
+ f'for year ({year}) with value ({s})')
return float(0)
# iterate over the rows in the worksheet
def create_csv_per_uear(start_row, end_row, yearI, col_start):
def create_csv_per_year(start_row, end_row, yearI, col_start):
for row in range(start_row, end_row):
# extract the data from the row
nren = ws.cell(row=row, column=col_start).value
......@@ -93,17 +113,18 @@ def parse_income_source_xlsx_file():
other = ws.cell(row=row, column=col_start + 7).value
year = yearI
client_institution = hard_number_convert(client_institution)
european_funding = hard_number_convert(european_funding)
gov_public_bodies = hard_number_convert(gov_public_bodies)
commercial = hard_number_convert(commercial)
other = hard_number_convert(other)
client_institution = hard_number_convert(
client_institution, "client institution", nren, year)
european_funding = hard_number_convert(
european_funding, "european funding", nren, year)
gov_public_bodies = hard_number_convert(
gov_public_bodies, "gov/public_bodies", nren, year)
commercial = hard_number_convert(
commercial, "commercial", nren, year)
other = hard_number_convert(
other, "other", nren, year)
# client_institution = is_number(client_institution) ? client_institution :'';
# budget = round(budget / 1000000, 2)
# process the data (e.g. save to database)
# print(f"NREN: {nren}, client_institution:"
# f" {client_institution}, Year: {year}")
if nren is not None:
output_csv_file = csv.writer(
open(csv_out_file, 'a'),
......@@ -114,15 +135,15 @@ def parse_income_source_xlsx_file():
commercial, other])
# For 2020
create_csv_per_uear(8,50, 2020, 3)
create_csv_per_year(8, 50, 2020, 3)
# # For 2019
create_csv_per_uear(8,50, 2019,12)
create_csv_per_year(8, 50, 2019, 12)
# # For 2018
create_csv_per_uear(8,50, 2018,21)
create_csv_per_year(8, 50, 2018, 21)
# # For 2017
create_csv_per_uear(8,50,2017,32)
create_csv_per_year(8, 50, 2017, 32)
# # For 2016
create_csv_per_uear(8,50,2016,43)
create_csv_per_year(8, 50, 2016, 43)
except Exception as e:
print(e)
compendium_v2/publishers/survey_publisher_v1.py
+ 26
− 4
View file @ ef047f17
import logging
import math
from collections import defaultdict
import click
......@@ -7,13 +8,17 @@ import csv
from flask import current_app
import compendium_v2
from compendium_v2.environment import setup_logging
from compendium_v2 import db, survey_db
from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
from compendium_v2.config import load
from compendium_v2.db import model
from compendium_v2.routes.api import logger
from compendium_v2.survey_db import model as survey_model
setup_logging()
logger = logging.getLogger('survey_publisher_v1')
def before_request(config):
dsn_prn = config['SQLALCHEMY_DATABASE_URI']
db.init_db_model(dsn_prn)
......@@ -40,9 +45,15 @@ def db_budget_migration():
abbrev = nren.abbreviation
year = budget.year
if float(budget.budget) > 200:
logger.info(
f'Incorrect Data: '
f'{abbrev} has budget set to '
f'>200M EUR for {year}. ({budget.budget})')
entry = session.query(
model.BudgetEntry).filter_by(nren=abbrev,
year=year)
model.BudgetEntry).filter_by(nren=abbrev,
year=year)
dup_entry: model.BudgetEntry = entry.first()
if dup_entry:
entry.update({"budget": budget.budget})
......@@ -92,10 +103,21 @@ def db_funding_migration():
for row in reader:
if row is not None:
_data = [float(row[2]), float(row[3]), float(row[4]),
float(row[5]), float(row[6])]
total = sum(_data)
if not math.isclose(total, 100, abs_tol=0.01):
logger.info(
f'Incorrect Data: '
f'{row[0]} funding sources for year ({row[1]})'
f'do not sum to 100%. ({total})')
entry = session.query(
model.FundingSource).filter_by(nren=row[0],
year=row[1])
model.FundingSource).filter_by(nren=row[0],
year=row[1])
dup_entry: model.FundingSource = entry.first()
if dup_entry:
entry.update({"client_institutions": row[2],
"european_funding": row[3],
......
compendium_v2/routes/budget.py
+ 2
− 1
View file @ ef047f17
......@@ -78,7 +78,8 @@ def budget_view() -> Any:
entries = sorted([_extract_data(entry)
for entry in session.query(model.BudgetEntry)],
key=lambda d: (d['BUDGET_YEAR'], d['NREN']))
return jsonify(entries)
dict_obj = {"data": entries}
return jsonify(dict_obj)
@routes.route('/migration', methods=['GET'])
......
webapp/src/pages/DataAnalysis.tsx
+ 5
− 1
View file @ ef047f17
......@@ -65,15 +65,18 @@ function DataAnalysis(): ReactElement {
useEffect(() => {
const loadData = async () => {
console.log("budgetResponse "+ budgetResponse)
if (budgetResponse == undefined) {
api<BudgetAPI>('/api/budget/', {})
.then((budget: BudgetAPI) => {
console.log('budget:', budget)
console.log('budget.data :', budget.data)
console.log('budget :', budget)
const entry = dataEntrySection?.items.find(i => i.id == selectedDataEntry)
console.log(selectedDataEntry, dataEntrySection, entry)
if (entry)
options.plugins.title.text = entry.title;
setBudget(budget.data)
console.log("budgetResponse after api "+ budgetResponse)
convertToBudgetPerYearDataResponse(budget.data)
})
.catch(error => {
......@@ -113,6 +116,7 @@ function DataAnalysis(): ReactElement {
const convertToBudgetPerYearDataResponse = (budgetResponse: Budget[]) => {
const barResponse = budgetResponse != undefined ? budgetResponse : empty_budget_response;
console.log("barResponse "+barResponse);
console.log(barResponse.map((item) => item.BUDGET_YEAR));
const labelsYear = [...new Set(barResponse.map((item) => item.BUDGET_YEAR))];
const labelsNREN = [...new Set(barResponse.map((item) => item.NREN))];
......
webapp/src/pages/FundingSource.tsx
+ 19
− 3
View file @ ef047f17
......@@ -19,6 +19,8 @@ import {
FundingSource,
FS, FundingGraphMatrix
} from "../Schema";
// import _default from "chart.js/dist/plugins/plugin.tooltip";
// import numbers = _default.defaults.animations.numbers;
ChartJS.register(
......@@ -244,13 +246,26 @@ function FundingSourcePage(): ReactElement {
return `#${red}${green}${blue}`;
}
const rgbToHex = (r:number, g:number, b:number) => '#' + [r, g, b].map(x => {
const hex = x.toString(16)
return hex.length === 1 ? '0' + hex : hex
}).join('')
let colorMap = new Map<string, string>();
colorMap.set("CLIENT INSTITUTIONS", rgbToHex(157,40,114))
colorMap.set("COMMERCIAL", rgbToHex(241,224,79))
colorMap.set("EUROPEAN FUNDING", rgbToHex(219,42,76))
colorMap.set("GOV/PUBLIC_BODIES", rgbToHex(237,141,24))
colorMap.set("OTHER", rgbToHex(137,166,121))
const datasetFunding = dataSetKey.map(function (entry) {
const randomColor = getRandomColor();
// const randomColor = getRandomColor();
const color:string = colorMap.get(entry[0])!;
console.log(color)
return {
backgroundColor: randomColor,
backgroundColor: color,
label: entry[0]+"("+entry[1]+")",//composition+year
data: labelsNREN.map(nren => dataPerCompositionPerYear(entry[1], nren,entry[0])),
stack:entry[1],
......@@ -309,7 +324,7 @@ function FundingSourcePage(): ReactElement {
const fundingAPIResponse: FundingGraphMatrix = fundingMatrixResponse !== undefined
? fundingMatrixResponse : empty_bar_response;
return (
<div>
<div id="canvas_container" >
<h1>Income Source</h1>
<Bar data={fundingAPIResponse}
width={80}
......@@ -317,6 +332,7 @@ function FundingSourcePage(): ReactElement {
options={{
plugins:{
legend:{
display:false,
labels:{
boxWidth:20,
boxHeight:30,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment