Skip to content
Snippets Groups Projects
Commit bc89db14 authored by Saket Agrahari's avatar Saket Agrahari
Browse files

correcting cconsole script for v1

parent 1f620e15
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
+ 5
− 5
View file @ bc89db14
...@@ -30,7 +30,7 @@ def parse_budget_xlsx_file(): ...@@ -30,7 +30,7 @@ def parse_budget_xlsx_file():
ws = wb[sheet_name] ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file): if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file) os.remove(csv_out_file)
print("file deleted") print("file deleted " + csv_out_file)
# iterate over the rows in the worksheet # iterate over the rows in the worksheet
for row in range(14, 57): for row in range(14, 57):
for col in range(3, 8): for col in range(3, 8):
...@@ -42,7 +42,7 @@ def parse_budget_xlsx_file(): ...@@ -42,7 +42,7 @@ def parse_budget_xlsx_file():
if budget is not None: if budget is not None:
budget = round(budget / 1000000, 2) budget = round(budget / 1000000, 2)
# process the data (e.g. save to database) # process the data (e.g. save to database)
print(f"NREN: {nren}, Budget: {budget}, Year: {year}") # print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
output_csv_file = csv.writer( output_csv_file = csv.writer(
open(csv_out_file, 'a'), open(csv_out_file, 'a'),
delimiter=",") delimiter=",")
...@@ -67,7 +67,7 @@ def parse_income_source_xlsx_file(): ...@@ -67,7 +67,7 @@ def parse_income_source_xlsx_file():
ws = wb[sheet_name] ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file): if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file) os.remove(csv_out_file)
print("file deleted") print("file deleted "+ csv_out_file)
def hard_number_convert(s): def hard_number_convert(s):
if s is None: if s is None:
...@@ -102,8 +102,8 @@ def parse_income_source_xlsx_file(): ...@@ -102,8 +102,8 @@ def parse_income_source_xlsx_file():
# client_institution = is_number(client_institution) ? client_institution :''; # client_institution = is_number(client_institution) ? client_institution :'';
# budget = round(budget / 1000000, 2) # budget = round(budget / 1000000, 2)
# process the data (e.g. save to database) # process the data (e.g. save to database)
print(f"NREN: {nren}, client_institution:" # print(f"NREN: {nren}, client_institution:"
f" {client_institution}, Year: {year}") # f" {client_institution}, Year: {year}")
if nren is not None: if nren is not None:
output_csv_file = csv.writer( output_csv_file = csv.writer(
open(csv_out_file, 'a'), open(csv_out_file, 'a'),
......
compendium_v2/publishers/survey_publisher_v1.py
+ 78
− 40
View file @ bc89db14
...@@ -9,12 +9,12 @@ from flask import current_app ...@@ -9,12 +9,12 @@ from flask import current_app
import compendium_v2 import compendium_v2
from compendium_v2 import db, survey_db from compendium_v2 import db, survey_db
from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
from compendium_v2.config import load
from compendium_v2.db import model from compendium_v2.db import model
from compendium_v2.survey_db import model as survey_model from compendium_v2.survey_db import model as survey_model
def before_request(): def before_request(config):
config = current_app.config['CONFIG_PARAMS']
dsn_prn = config['SQLALCHEMY_DATABASE_URI'] dsn_prn = config['SQLALCHEMY_DATABASE_URI']
db.init_db_model(dsn_prn) db.init_db_model(dsn_prn)
dsn_survey = config['SURVEY_DATABASE_URI'] dsn_survey = config['SURVEY_DATABASE_URI']
...@@ -27,10 +27,6 @@ def before_request(): ...@@ -27,10 +27,6 @@ def before_request():
def db_budget_migration(): def db_budget_migration():
with survey_db.session_scope() as survey_session, \ with survey_db.session_scope() as survey_session, \
db.session_scope() as session: db.session_scope() as session:
# with app.app_context():
# xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
_entries = session.query(model.BudgetEntry) _entries = session.query(model.BudgetEntry)
inserted = defaultdict(dict) inserted = defaultdict(dict)
...@@ -44,48 +40,90 @@ def db_budget_migration(): ...@@ -44,48 +40,90 @@ def db_budget_migration():
abbrev = nren.abbreviation abbrev = nren.abbreviation
year = budget.year year = budget.year
if inserted.get(abbrev, {}).get(year): entry = session.query(
continue model.BudgetEntry).filter_by(nren=abbrev,
year=year)
dup_entry: model.BudgetEntry = entry.first()
if dup_entry:
entry.update({"budget": budget.budget})
else: else:
inserted[abbrev][year] = True budget_entry = model.BudgetEntry(
entry = model.BudgetEntry( nren=abbrev, budget=budget.budget, year=year)
nren=abbrev, budget=budget.budget, year=year) session.add(budget_entry)
session.add(entry) # if inserted.get(abbrev, {}).get(year):
# continue
# else:
# inserted[abbrev][year] = True
# entry = model.BudgetEntry(
# nren=abbrev, budget=budget.budget, year=year)
# session.add(entry)
# Import the data to database
xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
newline='') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if row is not None:
entry = session.query(
model.BudgetEntry).filter_by(nren=row[0],
year=row[
2])
dup_entry: model.BudgetEntry = entry.first()
if dup_entry:
dup_entry.budget = row[1]
entry.update({"budget": row[1]})
else:
budget_entry = model.BudgetEntry(
nren=row[0], budget=row[1], year=row[2])
session.add(budget_entry)
session.commit()
def db_funding_migration():
with db.session_scope() as session:
# Import the data to database # Import the data to database
try: xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
with open('survey_excel/csv/BudgetCsvFile.csv', "rU", with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
newline='') as csvfile: newline='') as csvfile:
reader = csv.reader(csvfile) reader = csv.reader(csvfile)
for row in reader:
if row is not None: for row in reader:
budget_entry: model.BudgetEntry = session.query( if row is not None:
model.BudgetEntry).filter_by(nren=row[0], entry = session.query(
year=row[2]) model.FundingSource).filter_by(nren=row[0],
if budget_entry is None: year=row[1])
budget_entry = model.BudgetEntry( dup_entry: model.FundingSource = entry.first()
nren=row[0], budget=row[1], year=row[2]) if dup_entry:
session.add(budget_entry) entry.update({"client_institutions": row[2],
else: "european_funding": row[3],
budget_entry.budget = row[1] "gov_public_bodies": row[4],
print(row) "commercial": row[5],
session.commit() "other": row[6]})
except Exception as error:
print(f"Unable to read :, {error}") else:
budget_entry = model.FundingSource(
nren=row[0], year=row[1],
client_institutions=row[2],
european_funding=row[3],
gov_public_bodies=row[4],
commercial=row[5],
other=row[6])
session.add(budget_entry)
session.commit()
@click.command() @click.command()
# @click.option('--config', type=click.STRING, default='config.json') @click.option('--config', type=click.STRING, default='config.json')
@click.option( def cli(config):
"--migration", app_config = load(open(config, 'r'))
multiple=True,
default=[],
help="Budget Migration from old compendium db to new presentation db")
def cli(migration):
print("survery-publisher-v1 starting") print("survery-publisher-v1 starting")
print(" migration: %r" % str(migration)) print(" config: %r" % str(config))
before_request() before_request(app_config)
db_budget_migration() db_budget_migration()
db_funding_migration()
if __name__ == "__main__": if __name__ == "__main__":
......
compendium_v2/routes/funding.py
+ 6
− 6
View file @ bc89db14
...@@ -83,7 +83,7 @@ def budget_view() -> Any: ...@@ -83,7 +83,7 @@ def budget_view() -> Any:
@routes.route('/migration', methods=['GET']) @routes.route('/migration', methods=['GET'])
@common.require_accepts_json @common.require_accepts_json
def db_budget_migration(): def db_funding_migration():
with db.session_scope() as session: with db.session_scope() as session:
# Import the data to database # Import the data to database
...@@ -99,11 +99,11 @@ def db_budget_migration(): ...@@ -99,11 +99,11 @@ def db_budget_migration():
year=row[1]) year=row[1])
dup_entry: model.FundingSource = entry.first() dup_entry: model.FundingSource = entry.first()
if dup_entry: if dup_entry:
entry.update({"client_institutions": row[2]}, entry.update({"client_institutions": row[2],
{"european_funding": row[3]}, "european_funding": row[3],
{"gov_public_bodies": row[4]}, "gov_public_bodies": row[4],
{"commercial": row[5]}, "commercial": row[5],
{"other": row[6]}) "other": row[6]})
else: else:
print("add new") print("add new")
......
webapp/src/Schema.tsx
+ 2
− 1
View file @ bc89db14
...@@ -60,9 +60,10 @@ export interface FundingGraphMatrix { ...@@ -60,9 +60,10 @@ export interface FundingGraphMatrix {
data: string[], data: string[],
backgroundColor: string backgroundColor: string
borderRadius:number, borderRadius:number,
borderSkipped:boolean, borderSkipped: boolean ,
barPercentage:number, barPercentage:number,
borderWidth: number, borderWidth: number,
categoryPercentage:number
stack: string stack: string
}[] }[]
} }
......
webapp/src/pages/FundingSource.tsx
+ 10
− 3
View file @ bc89db14
...@@ -257,8 +257,10 @@ function FundingSourcePage(): ReactElement { ...@@ -257,8 +257,10 @@ function FundingSourcePage(): ReactElement {
borderRadius:10, borderRadius:10,
borderSkipped:false, borderSkipped:false,
barPercentage:0.5, barPercentage:0.5,
borderWidth: 1, borderWidth: 0.5,
} categoryPercentage:0.8
}
}) })
function dataPerCompositionPerYear(year: number, nren: string, composition:string) { function dataPerCompositionPerYear(year: number, nren: string, composition:string) {
...@@ -298,8 +300,9 @@ function FundingSourcePage(): ReactElement { ...@@ -298,8 +300,9 @@ function FundingSourcePage(): ReactElement {
borderRadius:0, borderRadius:0,
borderSkipped:false, borderSkipped:false,
barPercentage:0, barPercentage:0,
borderWidth: 1, borderWidth: 0,
stack: '0', stack: '0',
categoryPercentage:0.5
}], }],
labels: [] labels: []
} }
...@@ -309,6 +312,8 @@ function FundingSourcePage(): ReactElement { ...@@ -309,6 +312,8 @@ function FundingSourcePage(): ReactElement {
<div> <div>
<h1>Income Source</h1> <h1>Income Source</h1>
<Bar data={fundingAPIResponse} <Bar data={fundingAPIResponse}
width={80}
height={300}
options={{ options={{
plugins:{ plugins:{
legend:{ legend:{
...@@ -318,6 +323,7 @@ function FundingSourcePage(): ReactElement { ...@@ -318,6 +323,7 @@ function FundingSourcePage(): ReactElement {
pointStyle:"rectRounded", pointStyle:"rectRounded",
borderRadius:6, borderRadius:6,
useBorderRadius:true, useBorderRadius:true,
}, },
}, },
}, },
...@@ -338,6 +344,7 @@ function FundingSourcePage(): ReactElement { ...@@ -338,6 +344,7 @@ function FundingSourcePage(): ReactElement {
}, },
}, },
indexAxis: "y", indexAxis: "y",
// maintainAspectRatio: false
}} }}
></Bar> ></Bar>
</div> </div>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment