diff --git a/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py b/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
index 1b834d38975cb12c1708ebd3ff9a96673316895b..c833535d3777149da1ef82d21749509c84b60fc6 100644
--- a/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
+++ b/compendium_v2/background_task/xlsx_to_csv_sheet_parsing_task.py
@@ -30,7 +30,7 @@ def parse_budget_xlsx_file():
ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file)
- print("file deleted")
+ print("file deleted " + csv_out_file)
# iterate over the rows in the worksheet
for row in range(14, 57):
for col in range(3, 8):
@@ -42,7 +42,7 @@ def parse_budget_xlsx_file():
if budget is not None:
budget = round(budget / 1000000, 2)
# process the data (e.g. save to database)
- print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
+ # print(f"NREN: {nren}, Budget: {budget}, Year: {year}")
output_csv_file = csv.writer(
open(csv_out_file, 'a'),
delimiter=",")
@@ -67,7 +67,7 @@ def parse_income_source_xlsx_file():
ws = wb[sheet_name]
if os.path.exists(csv_out_file) and os.path.isfile(csv_out_file):
os.remove(csv_out_file)
- print("file deleted")
+ print("file deleted "+ csv_out_file)
def hard_number_convert(s):
if s is None:
@@ -102,8 +102,8 @@ def parse_income_source_xlsx_file():
# client_institution = is_number(client_institution) ? client_institution :'';
# budget = round(budget / 1000000, 2)
# process the data (e.g. save to database)
- print(f"NREN: {nren}, client_institution:"
- f" {client_institution}, Year: {year}")
+ # print(f"NREN: {nren}, client_institution:"
+ # f" {client_institution}, Year: {year}")
if nren is not None:
output_csv_file = csv.writer(
open(csv_out_file, 'a'),
diff --git a/compendium_v2/publishers/survey_publisher_v1.py b/compendium_v2/publishers/survey_publisher_v1.py
index 3b425a612c3060aaafb7d9da20f88c37c513e638..1e020c7a662472208f38ba80f24ee31d6aa13f66 100644
--- a/compendium_v2/publishers/survey_publisher_v1.py
+++ b/compendium_v2/publishers/survey_publisher_v1.py
@@ -9,12 +9,12 @@ from flask import current_app
import compendium_v2
from compendium_v2 import db, survey_db
from compendium_v2.background_task import xlsx_to_csv_sheet_parsing_task
+from compendium_v2.config import load
from compendium_v2.db import model
from compendium_v2.survey_db import model as survey_model
-def before_request():
- config = current_app.config['CONFIG_PARAMS']
+def before_request(config):
dsn_prn = config['SQLALCHEMY_DATABASE_URI']
db.init_db_model(dsn_prn)
dsn_survey = config['SURVEY_DATABASE_URI']
@@ -27,10 +27,6 @@ def before_request():
def db_budget_migration():
with survey_db.session_scope() as survey_session, \
db.session_scope() as session:
-
- # with app.app_context():
- # xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
-
_entries = session.query(model.BudgetEntry)
inserted = defaultdict(dict)
@@ -44,48 +40,90 @@ def db_budget_migration():
abbrev = nren.abbreviation
year = budget.year
- if inserted.get(abbrev, {}).get(year):
- continue
+ entry = session.query(
+ model.BudgetEntry).filter_by(nren=abbrev,
+ year=year)
+ dup_entry: model.BudgetEntry = entry.first()
+ if dup_entry:
+ entry.update({"budget": budget.budget})
else:
- inserted[abbrev][year] = True
- entry = model.BudgetEntry(
- nren=abbrev, budget=budget.budget, year=year)
- session.add(entry)
+ budget_entry = model.BudgetEntry(
+ nren=abbrev, budget=budget.budget, year=year)
+ session.add(budget_entry)
+ # if inserted.get(abbrev, {}).get(year):
+ # continue
+ # else:
+ # inserted[abbrev][year] = True
+ # entry = model.BudgetEntry(
+ # nren=abbrev, budget=budget.budget, year=year)
+ # session.add(entry)
+ # Import the data to database
+ xlsx_to_csv_sheet_parsing_task.parse_budget_xlsx_file()
+ with open('compendium_v2/background_task/csv/BudgetCsvFile.csv',
+ newline='') as csvfile:
+ reader = csv.reader(csvfile)
+
+ for row in reader:
+ if row is not None:
+ entry = session.query(
+ model.BudgetEntry).filter_by(nren=row[0],
+ year=row[
+ 2])
+ dup_entry: model.BudgetEntry = entry.first()
+ if dup_entry:
+ dup_entry.budget = row[1]
+ entry.update({"budget": row[1]})
+
+ else:
+ budget_entry = model.BudgetEntry(
+ nren=row[0], budget=row[1], year=row[2])
+ session.add(budget_entry)
+ session.commit()
+
+
+def db_funding_migration():
+ with db.session_scope() as session:
# Import the data to database
- try:
- with open('survey_excel/csv/BudgetCsvFile.csv', "rU",
- newline='') as csvfile:
- reader = csv.reader(csvfile)
- for row in reader:
- if row is not None:
- budget_entry: model.BudgetEntry = session.query(
- model.BudgetEntry).filter_by(nren=row[0],
- year=row[2])
- if budget_entry is None:
- budget_entry = model.BudgetEntry(
- nren=row[0], budget=row[1], year=row[2])
- session.add(budget_entry)
- else:
- budget_entry.budget = row[1]
- print(row)
- session.commit()
- except Exception as error:
- print(f"Unable to read :, {error}")
+ xlsx_to_csv_sheet_parsing_task.parse_income_source_xlsx_file()
+ with open('compendium_v2/background_task/csv/FundingSourceCsvFile.csv',
+ newline='') as csvfile:
+ reader = csv.reader(csvfile)
+
+ for row in reader:
+ if row is not None:
+ entry = session.query(
+ model.FundingSource).filter_by(nren=row[0],
+ year=row[1])
+ dup_entry: model.FundingSource = entry.first()
+ if dup_entry:
+ entry.update({"client_institutions": row[2],
+ "european_funding": row[3],
+ "gov_public_bodies": row[4],
+ "commercial": row[5],
+ "other": row[6]})
+
+ else:
+ budget_entry = model.FundingSource(
+ nren=row[0], year=row[1],
+ client_institutions=row[2],
+ european_funding=row[3],
+ gov_public_bodies=row[4],
+ commercial=row[5],
+ other=row[6])
+ session.add(budget_entry)
+ session.commit()
@click.command()
-# @click.option('--config', type=click.STRING, default='config.json')
-@click.option(
- "--migration",
- multiple=True,
- default=[],
- help="Budget Migration from old compendium db to new presentation db")
-def cli(migration):
+@click.option('--config', type=click.STRING, default='config.json')
+def cli(config):
+ app_config = load(open(config, 'r'))
print("survery-publisher-v1 starting")
- print(" migration: %r" % str(migration))
- before_request()
+ print(" config: %r" % str(config))
+ before_request(app_config)
db_budget_migration()
+ db_funding_migration()
if __name__ == "__main__":
diff --git a/compendium_v2/routes/funding.py b/compendium_v2/routes/funding.py
index 2d32886a9d21b2f71d09b62d19b355214da65c6e..4a1d9ad431d8369b3bd162dd2035613a55dfcb8c 100644
--- a/compendium_v2/routes/funding.py
+++ b/compendium_v2/routes/funding.py
@@ -83,7 +83,7 @@ def budget_view() -> Any:
@routes.route('/migration', methods=['GET'])
@common.require_accepts_json
-def db_budget_migration():
+def db_funding_migration():
with db.session_scope() as session:
# Import the data to database
@@ -99,11 +99,11 @@ def db_budget_migration():
year=row[1])
dup_entry: model.FundingSource = entry.first()
if dup_entry:
- entry.update({"client_institutions": row[2]},
- {"european_funding": row[3]},
- {"gov_public_bodies": row[4]},
- {"commercial": row[5]},
- {"other": row[6]})
+ entry.update({"client_institutions": row[2],
+ "european_funding": row[3],
+ "gov_public_bodies": row[4],
+ "commercial": row[5],
+ "other": row[6]})
else:
print("add new")
diff --git a/webapp/src/Schema.tsx b/webapp/src/Schema.tsx
index b3a5828f006c67a234e38346e08a691f6d3c5632..873d308389a04bbbea17cfd22de2d7f8cb4556e4 100644
--- a/webapp/src/Schema.tsx
+++ b/webapp/src/Schema.tsx
@@ -60,9 +60,10 @@ export interface FundingGraphMatrix {
data: string[],
backgroundColor: string
borderRadius:number,
- borderSkipped:boolean,
+ borderSkipped: boolean ,
barPercentage:number,
borderWidth: number,
+ categoryPercentage:number
stack: string
}[]
}
diff --git a/webapp/src/pages/FundingSource.tsx b/webapp/src/pages/FundingSource.tsx
index b352877667ca4074b7eb8176a108cdb8dc44d6d9..7519cdd45b1492b242e5e0302316c7ad1374f8ae 100644
--- a/webapp/src/pages/FundingSource.tsx
+++ b/webapp/src/pages/FundingSource.tsx
@@ -257,8 +257,10 @@ function FundingSourcePage(): ReactElement {
borderRadius:10,
borderSkipped:false,
barPercentage:0.5,
- borderWidth: 1,
- }
+ borderWidth: 0.5,
+ categoryPercentage:0.8
+
+ }
})
function dataPerCompositionPerYear(year: number, nren: string, composition:string) {
@@ -298,8 +300,9 @@ function FundingSourcePage(): ReactElement {
borderRadius:0,
borderSkipped:false,
barPercentage:0,
- borderWidth: 1,
+ borderWidth: 0,
stack: '0',
+ categoryPercentage:0.5
}],
labels: []
}
@@ -309,6 +312,8 @@ function FundingSourcePage(): ReactElement {
<div>
<h1>Income Source</h1>
<Bar data={fundingAPIResponse}
+ width={80}
+ height={300}
options={{
plugins:{
legend:{
@@ -318,6 +323,7 @@ function FundingSourcePage(): ReactElement {
pointStyle:"rectRounded",
borderRadius:6,
useBorderRadius:true,
+
},
},
},
@@ -338,6 +344,7 @@ function FundingSourcePage(): ReactElement {
},
},
indexAxis: "y",
+ // maintainAspectRatio: false
}}
></Bar>
</div>