Skip to content
Snippets Groups Projects
Commit bc89db14 authored by Saket Agrahari's avatar Saket Agrahari
Browse files

correcting cconsole script for v1

parent 1f620e15
No related branches found
No related tags found
No related merge requests found
......@@ -30,7 +30,7 @@ def parse_budget_xlsx_file():
ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file)
print("file deleted")
print("file deleted " + csv_out_file)
# iterate over the rows in the worksheet
for row in range(14, 57):
for col in range(3, 8):
......@@ -42,7 +42,7 @@ def parse_budget_xlsx_file():
if budget is not None:
budget = round(budget / 1000000, 2)
# process the data (e.g. save to database)
print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
# print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
output_csv_file = csv.writer(
open(csv_out_file, 'a'),
delimiter=",")
......@@ -67,7 +67,7 @@ def parse_income_source_xlsx_file():
ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file)
print("file deleted")
print("file deleted "+ csv_out_file)
def hard_number_convert(s):
if s is None:
......@@ -102,8 +102,8 @@ def parse_income_source_xlsx_file():
# client_institution = is_number(client_institution) ? client_institution :'';
# budget = round(budget / 1000000, 2)
# process the data (e.g. save to database)
print(f"NREN: {nren}, client_institution:"
f" {client_institution}, Year: {year}")
# print(f"NREN: {nren}, client_institution:"
# f" {client_institution}, Year: {year}")
if nren is not None:
output_csv_file = csv.writer(
open(csv_out_file, 'a'),
......
......@@ -9,12 +9,12 @@ from flask import current_app
import compendium_v2
from compendium_v2 import db, survey_db
from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
from compendium_v2.config import load
from compendium_v2.db import model
from compendium_v2.survey_db import model as survey_model
def before_request():
config = current_app.config['CONFIG_PARAMS']
def before_request(config):
dsn_prn = config['SQLALCHEMY_DATABASE_URI']
db.init_db_model(dsn_prn)
dsn_survey = config['SURVEY_DATABASE_URI']
......@@ -27,10 +27,6 @@ def before_request():
def db_budget_migration():
with survey_db.session_scope() as survey_session, \
db.session_scope() as session:
# with app.app_context():
# xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
_entries = session.query(model.BudgetEntry)
inserted = defaultdict(dict)
......@@ -44,48 +40,90 @@ def db_budget_migration():
abbrev = nren.abbreviation
year = budget.year
if inserted.get(abbrev, {}).get(year):
continue
entry = session.query(
model.BudgetEntry).filter_by(nren=abbrev,
year=year)
dup_entry: model.BudgetEntry = entry.first()
if dup_entry:
entry.update({"budget": budget.budget})
else:
inserted[abbrev][year] = True
entry = model.BudgetEntry(
nren=abbrev, budget=budget.budget, year=year)
session.add(entry)
budget_entry = model.BudgetEntry(
nren=abbrev, budget=budget.budget, year=year)
session.add(budget_entry)
# if inserted.get(abbrev, {}).get(year):
# continue
# else:
# inserted[abbrev][year] = True
# entry = model.BudgetEntry(
# nren=abbrev, budget=budget.budget, year=year)
# session.add(entry)
# Import the data to database
xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if row is not None:
entry = session.query(
model.BudgetEntry).filter_by(nren=row[0],
year=row[
2])
dup_entry: model.BudgetEntry = entry.first()
if dup_entry:
dup_entry.budget = row[1]
entry.update({"budget": row[1]})
else:
budget_entry = model.BudgetEntry(
nren=row[0], budget=row[1], year=row[2])
session.add(budget_entry)
session.commit()
def db_funding_migration():
with db.session_scope() as session:
# Import the data to database
try:
with open('survey_excel/csv/BudgetCsvFile.csv', "rU",
newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if row is not None:
budget_entry: model.BudgetEntry = session.query(
model.BudgetEntry).filter_by(nren=row[0],
year=row[2])
if budget_entry is None:
budget_entry = model.BudgetEntry(
nren=row[0], budget=row[1], year=row[2])
session.add(budget_entry)
else:
budget_entry.budget = row[1]
print(row)
session.commit()
except Exception as error:
print(f"Unable to read :, {error}")
xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if row is not None:
entry = session.query(
model.FundingSource).filter_by(nren=row[0],
year=row[1])
dup_entry: model.FundingSource = entry.first()
if dup_entry:
entry.update({"client_institutions": row[2],
"european_funding": row[3],
"gov_public_bodies": row[4],
"commercial": row[5],
"other": row[6]})
else:
budget_entry = model.FundingSource(
nren=row[0], year=row[1],
client_institutions=row[2],
european_funding=row[3],
gov_public_bodies=row[4],
commercial=row[5],
other=row[6])
session.add(budget_entry)
session.commit()
@click.command()
# @click.option('--config', type=click.STRING, default='config.json')
@click.option(
"--migration",
multiple=True,
default=[],
help="Budget Migration from old compendium db to new presentation db")
def cli(migration):
@click.option('--config', type=click.STRING, default='config.json')
def cli(config):
app_config = load(open(config, 'r'))
print("survery-publisher-v1 starting")
print(" migration: %r" % str(migration))
before_request()
print(" config: %r" % str(config))
before_request(app_config)
db_budget_migration()
db_funding_migration()
if __name__ == "__main__":
......
......@@ -83,7 +83,7 @@ def budget_view() -> Any:
@routes.route('/migration', methods=['GET'])
@common.require_accepts_json
def db_budget_migration():
def db_funding_migration():
with db.session_scope() as session:
# Import the data to database
......@@ -99,11 +99,11 @@ def db_budget_migration():
year=row[1])
dup_entry: model.FundingSource = entry.first()
if dup_entry:
entry.update({"client_institutions": row[2]},
{"european_funding": row[3]},
{"gov_public_bodies": row[4]},
{"commercial": row[5]},
{"other": row[6]})
entry.update({"client_institutions": row[2],
"european_funding": row[3],
"gov_public_bodies": row[4],
"commercial": row[5],
"other": row[6]})
else:
print("add new")
......
......@@ -60,9 +60,10 @@ export interface FundingGraphMatrix {
data: string[],
backgroundColor: string
borderRadius:number,
borderSkipped:boolean,
borderSkipped: boolean ,
barPercentage:number,
borderWidth: number,
categoryPercentage:number
stack: string
}[]
}
......
......@@ -257,8 +257,10 @@ function FundingSourcePage(): ReactElement {
borderRadius:10,
borderSkipped:false,
barPercentage:0.5,
borderWidth: 1,
}
borderWidth: 0.5,
categoryPercentage:0.8
}
})
function dataPerCompositionPerYear(year: number, nren: string, composition:string) {
......@@ -298,8 +300,9 @@ function FundingSourcePage(): ReactElement {
borderRadius:0,
borderSkipped:false,
barPercentage:0,
borderWidth: 1,
borderWidth: 0,
stack: '0',
categoryPercentage:0.5
}],
labels: []
}
......@@ -309,6 +312,8 @@ function FundingSourcePage(): ReactElement {
<div>
<h1>Income Source</h1>
<Bar data={fundingAPIResponse}
width={80}
height={300}
options={{
plugins:{
legend:{
......@@ -318,6 +323,7 @@ function FundingSourcePage(): ReactElement {
pointStyle:"rectRounded",
borderRadius:6,
useBorderRadius:true,
},
},
},
......@@ -338,6 +344,7 @@ function FundingSourcePage(): ReactElement {
},
},
indexAxis: "y",
// maintainAspectRatio: false
}}
></Bar>
</div>
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment