Skip to content
Snippets Groups Projects
Commit 1d23bf05 authored by Remco Tukker's avatar Remco Tukker
Browse files

linelength to 80 and updates to the correct NREN names

parent cf3cf17c
No related branches found
No related tags found
1 merge request!1normalization of nrens including datamigration
Hide whitespace changes
Inline Side-by-side
Showing
with 221 additions and 158 deletions
compendium_v2/db/model.py
+ 6
− 3
View file @ 1d23bf05
......@@ -32,7 +32,8 @@ class NREN(base_schema):
class BudgetEntry(base_schema):
__tablename__ = 'budgets'
nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren_id = sa.Column(
sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren = relationship(NREN, lazy='joined')
year = sa.Column(sa.Integer, primary_key=True)
budget = sa.Column(sa.Numeric(asdecimal=False), nullable=False)
......@@ -40,7 +41,8 @@ class BudgetEntry(base_schema):
class FundingSource(base_schema):
__tablename__ = 'funding_source'
nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren_id = sa.Column(
sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren = relationship(NREN, lazy='joined')
year = sa.Column(sa.Integer, primary_key=True)
client_institutions = sa.Column(
......@@ -61,7 +63,8 @@ class FeeType(Enum):
class ChargingStructure(base_schema):
__tablename__ = 'charging_structure'
nren_id = sa.Column(sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren_id = sa.Column(
sa.Integer, sa.schema.ForeignKey(NREN.id), primary_key=True)
nren = relationship(NREN, lazy='joined')
year = sa.Column(sa.Integer, primary_key=True)
fee_type = sa.Column('fee_type', sa.Enum("flat_fee", "usage_based_fee",
......
compendium_v2/migrations/versions/2b698bb45c09_normalize_nrens.py
+ 177
− 139
View file @ 1d23bf05
......@@ -25,172 +25,167 @@ def upgrade():
sa.PrimaryKeyConstraint('id', name=op.f('pk_nren'))
)
# TODO before merge: check if this list is ok, its from COMP-118 with UNINETT and LANET added because otherwise
# I wouldn't know how to map them
# TODO also check if the mapping of names in the 3 data migrations below are OK
op.execute(
"INSERT INTO nren (name) VALUES "
"('AAF - Australian Access Federation'),"
"('AARNet'),"
# "('AAF - Australian Access Federation'),"
# "('AARNet'),"
"('ACOnet'),"
"('AfgREN'),"
# "('AfgREN'),"
"('AMRES'),"
"('ANA'),"
"('Ankabut'),"
"('Arandu'),"
# "('Ankabut'),"
# "('Arandu'),"
"('ARENA'),"
"('ARN'),"
# "('ARN'),"
"('ARNES'),"
"('ASGC'),"
# "('ASGC'),"
"('ASNET-AM'),"
"('AzRena'),"
# "('AzRena'),"
"('AzScienceNet'),"
"('BARNet'),"
# "('BARNet'),"
"('BASNET'),"
"('BdREN'),"
# "('BdREN'),"
"('Belnet'),"
"('BIHARNET'),"
"('BOLNET'),"
# "('BOLNET'),"
"('BREN'),"
"('Brunet'),"
"('CameroonianNREN'),"
"('CANARIE'),"
# "('Brunet'),"
# "('CameroonianNREN'),"
# "('CANARIE'),"
"('CARNET'),"
"('CEDIA'),"
"('CERNET'),"
# "('CEDIA'),"
# "('CERNET'),"
"('CESNET'),"
"('CNRS'),"
"('CNTI'),"
"('CRNet'),"
"('CSTNet'),"
"('CUDI'),"
# "('CNRS'),"
# "('CNTI'),"
# "('CRNet'),"
# "('CSTNet'),"
# "('CUDI'),"
"('CYNET'),"
"('DeIC'),"
"('DFN'),"
"('e-ARENA'),"
"('eb@le'),"
# "('e-ARENA'),"
# "('eb@le'),"
"('EENet'),"
"('ErdemNet'),"
"('ERNET'),"
"('EtherNet'),"
"('EUN'),"
# "('ErdemNet'),"
# "('ERNET'),"
# "('EtherNet'),"
# "('EUN'),"
"('FCCN'),"
"('FREEnet'),"
# "('FREEnet'),"
"('Funet'),"
"('GabonREN'),"
"('GARNET'),"
# "('GabonREN'),"
# "('GARNET'),"
"('GARR'),"
"('GCC'),"
# "('GCC'),"
"('GRENA'),"
"('GRNET S.A.'),"
"('HARNET'),"
# "('HARNET'),"
"('HEAnet'),"
"('INNOVA|RED'),"
"('Internet2'),"
"('IRANET'),"
"('IRANET/IPM'),"
"('iRENALA'),"
"('ITB'),"
"('ITC'),"
# "('INNOVA|RED'),"
# "('Internet2'),"
# "('IRANET'),"
# "('IRANET/IPM'),"
# "('iRENALA'),"
# "('ITB'),"
# "('ITC'),"
"('IUCC'),"
"('Jisc'),"
"('JREN'),"
"('JUNet'),"
"('KAUST'),"
"('KAZRENA'),"
"('KENET'),"
# "('JREN'),"
# "('JUNet'),"
# "('KAUST'),"
# "('KAZRENA'),"
# "('KENET'),"
"('KIFU'),"
"('KOREN'),"
"('KREN'),"
"('KRENA-AKNET'),"
"('KREONET'),"
"('LANET'),"
# "('KOREN'),"
# "('KREN'),"
# "('KRENA-AKNET'),"
# "('KREONET'),"
"('LAT'),"
"('LEARN'),"
"('LERNET'),"
# "('LEARN'),"
# "('LERNET'),"
"('LITNET'),"
"('MaliREN'),"
"('MAREN'),"
# "('MaliREN'),"
# "('MAREN'),"
"('MARnet'),"
"('MARWAN'),"
"('mmREN'),"
"('MoRENet'),"
# "('MARWAN'),"
# "('mmREN'),"
# "('MoRENet'),"
"('MREN'),"
"('MYREN'),"
"('NCHC'),"
"('ngNER'),"
"('ngREN'),"
"('NiCT'),"
"('NigerREN'),"
"('NII'),"
"('NITC'),"
"('NREN'),"
"('OMREN'),"
"('PADI2'),"
"('PERN'),"
# "('MYREN'),"
# "('NCHC'),"
# "('ngNER'),"
# "('ngREN'),"
# "('NiCT'),"
# "('NigerREN'),"
# "('NII'),"
# "('NITC'),"
# "('NREN'),"
# "('OMREN'),"
# "('PADI2'),"
# "('PERN'),"
"('PIONIER'),"
"('PNGARNet'),"
"('PREGINET'),"
"('Qatar Foundation'),"
"('RAAP'),"
"('RADEI'),"
"('RAGIE'),"
"('RAICES'),"
"('RAU'),"
"('REACCIUN'),"
"('REANNZ'),"
"('RedCyT'),"
# "('PNGARNet'),"
# "('PREGINET'),"
# "('Qatar Foundation'),"
# "('RAAP'),"
# "('RADEI'),"
# "('RAGIE'),"
# "('RAICES'),"
"('RASH'),"
# "('RAU'),"
# "('REACCIUN'),"
# "('REANNZ'),"
# "('RedCyT'),"
"('RedIRIS'),"
"('RedUNIV'),"
# "('RedUNIV'),"
"('RENAM'),"
"('RENATA'),"
# "('RENATA'),"
"('RENATER'),"
"('RENER'),"
"('RENIA'),"
"('RENU'),"
"('RerBenin'),"
# "('RENER'),"
# "('RENIA'),"
# "('RENU'),"
# "('RerBenin'),"
"('RESTENA'),"
"('REUNA'),"
# "('REUNA'),"
"('RHnet'),"
"('RITER'),"
"('RNERT'),"
"('RNP'),"
"('RNRT'),"
"('RNU'),"
# "('RITER'),"
# "('RNERT'),"
# "('RNP'),"
# "('RNRT'),"
# "('RNU'),"
"('RoEduNet'),"
"('RUB'),"
"('RwEdNet'),"
# "('RUB'),"
# "('RwEdNet'),"
"('SANET'),"
"('SANReN'),"
"('SAREN'),"
# "('SANReN'),"
# "('SAREN'),"
"('SARNET'),"
"('SHERN'),"
# "('SHERN'),"
"('SigmaNet'),"
"('SIKT'),"
"('SingAREN'),"
"('snRER'),"
"('Somaliren'),"
"('SudREN'),"
# "('SingAREN'),"
# "('snRER'),"
# "('Somaliren'),"
# "('SudREN'),"
"('SUNET'),"
"('SUREN'),"
# "('SUREN'),"
"('SURF'),"
"('SWITCH'),"
"('TARENA'),"
"('TENET'),"
"('TERNET'),"
"('ThaiREN'),"
"('TTRENT'),"
"('TuRENA'),"
# "('TARENA'),"
# "('TENET'),"
# "('TERNET'),"
# "('ThaiREN'),"
# "('TTRENT'),"
# "('TuRENA'),"
"('UARNet'),"
"('ULAKBIM'),"
"('UNINETT'),"
"('UNITEC'),"
# "('UNITEC'),"
"('UNREN'),"
"('UoM/RicerkaNet'),"
"('URAN'),"
"('UzSciNet'),"
"('VinaREN'),"
"('ZAMREN')"
"('UoM'),"
"('URAN')"
# "('UzSciNet'),"
# "('VinaREN'),"
# "('ZAMREN')"
)
op.add_column('budgets', sa.Column('nren_id', sa.Integer()))
......@@ -200,58 +195,101 @@ def upgrade():
" OR (budgets.nren = 'ASNET' AND nren.name = 'ASNET-AM')"
" OR (budgets.nren = 'KIFU (NIIF)' AND nren.name = 'KIFU')"
" OR (budgets.nren = 'SURFnet' AND nren.name = 'SURF')"
" OR (budgets.nren = 'UoM' AND nren.name = 'UoM/RicerkaNet')"
" OR (budgets.nren = 'UoM/RicerkaNet' AND nren.name = 'UoM')"
" OR (budgets.nren = 'UNINETT' AND nren.name = 'SIKT')"
" OR (budgets.nren = 'LANET' AND nren.name = 'LAT')"
" OR (budgets.nren = 'ANA' AND nren.name = 'RASH')"
)
op.alter_column('budgets', 'nren_id', nullable=False)
op.create_foreign_key(op.f('fk_budgets_nren_id_nren'), 'budgets', 'nren', ['nren_id'], ['id'])
op.create_foreign_key(
op.f('fk_budgets_nren_id_nren'),
'budgets', 'nren', ['nren_id'], ['id'])
op.drop_column('budgets', 'nren')
op.add_column('funding_source', sa.Column('nren_id', sa.Integer()))
op.execute(
"UPDATE funding_source SET nren_id = nren.id FROM nren "
"WHERE (UPPER(funding_source.nren) = UPPER(nren.name))"
" OR (funding_source.nren = 'ASNET' AND nren.name = 'ASNET-AM')"
" OR (funding_source.nren = 'KIFU (NIIF)' AND nren.name = 'KIFU')"
" OR (funding_source.nren = 'SURFnet' AND nren.name = 'SURF')"
" OR (funding_source.nren = 'UoM' AND nren.name = 'UoM/RicerkaNet')"
"OR (funding_source.nren = 'ASNET' AND nren.name = 'ASNET-AM')"
"OR (funding_source.nren = 'KIFU (NIIF)' AND nren.name = 'KIFU')"
"OR (funding_source.nren = 'SURFnet' AND nren.name = 'SURF')"
"OR (funding_source.nren = 'UoM/RicerkaNet' AND nren.name = 'UoM')"
"OR (funding_source.nren = 'UNINETT' AND nren.name = 'SIKT')"
"OR (funding_source.nren = 'LANET' AND nren.name = 'LAT')"
"OR (funding_source.nren = 'ANA' AND nren.name = 'RASH')"
)
op.alter_column('funding_source', 'nren_id', nullable=False)
op.create_foreign_key(op.f('fk_funding_source_nren_id_nren'), 'funding_source', 'nren', ['nren_id'], ['id'])
op.create_foreign_key(
op.f('fk_funding_source_nren_id_nren'),
'funding_source', 'nren', ['nren_id'], ['id']
)
op.drop_column('funding_source', 'nren')
op.add_column('charging_structure', sa.Column('nren_id', sa.Integer()))
op.execute(
"UPDATE charging_structure SET nren_id = nren.id FROM nren "
"WHERE (UPPER(charging_structure.nren) = UPPER(nren.name))"
" OR (charging_structure.nren = 'ASNET' AND nren.name = 'ASNET-AM')"
" OR (charging_structure.nren = 'KIFU (NIIF)' AND nren.name = 'KIFU')"
" OR (charging_structure.nren = 'SURFnet' AND nren.name = 'SURF')"
" OR (charging_structure.nren = 'UoM' AND nren.name = 'UoM/RicerkaNet')"
"OR (charging_structure.nren = 'ASNET' AND nren.name = 'ASNET-AM')"
"OR (charging_structure.nren = 'KIFU (NIIF)' AND nren.name = 'KIFU')"
"OR (charging_structure.nren = 'SURFnet' AND nren.name = 'SURF')"
"OR (charging_structure.nren = 'UoM/RicerkaNet' AND nren.name = 'UoM')"
"OR (charging_structure.nren = 'UNINETT' AND nren.name = 'SIKT')"
"OR (charging_structure.nren = 'LANET' AND nren.name = 'LAT')"
"OR (charging_structure.nren = 'ANA' AND nren.name = 'RASH')"
)
op.alter_column('charging_structure', 'nren_id', nullable=False)
op.create_foreign_key(op.f('fk_charging_structure_nren_id_nren'), 'charging_structure', 'nren', ['nren_id'], ['id'])
op.create_foreign_key(
op.f('fk_charging_structure_nren_id_nren'),
'charging_structure', 'nren', ['nren_id'], ['id'])
op.drop_column('charging_structure', 'nren')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('charging_structure', sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False))
op.execute("UPDATE charging_structure SET nren = nren.name FROM nren WHERE charging_structure.nren_id = nren.id")
op.add_column(
'charging_structure',
sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False)
)
op.execute(
"UPDATE charging_structure SET nren = nren.name FROM nren"
" WHERE charging_structure.nren_id = nren.id"
)
op.alter_column('charging_structure', 'nren', nullable=False)
op.drop_constraint(op.f('fk_charging_structure_nren_id_nren'), 'charging_structure', type_='foreignkey')
op.drop_constraint(
op.f('fk_charging_structure_nren_id_nren'),
'charging_structure',
type_='foreignkey'
)
op.drop_column('charging_structure', 'nren_id')
op.add_column('funding_source', sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False))
op.execute("UPDATE funding_source SET nren = nren.name FROM nren WHERE funding_source.nren_id = nren.id")
op.add_column(
'funding_source',
sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False)
)
op.execute(
"UPDATE funding_source SET nren = nren.name FROM nren"
" WHERE funding_source.nren_id = nren.id"
)
op.alter_column('funding_source', 'nren', nullable=False)
op.drop_constraint(op.f('fk_funding_source_nren_id_nren'), 'funding_source', type_='foreignkey')
op.drop_constraint(
op.f('fk_funding_source_nren_id_nren'),
'funding_source',
type_='foreignkey'
)
op.drop_column('funding_source', 'nren_id')
op.add_column('budgets', sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False))
op.execute("UPDATE budgets SET nren = nren.name FROM nren WHERE budgets.nren_id = nren.id")
op.add_column(
'budgets',
sa.Column('nren', sa.VARCHAR(length=128), autoincrement=False)
)
op.execute(
"UPDATE budgets SET nren = nren.name FROM nren"
" WHERE budgets.nren_id = nren.id"
)
op.alter_column('budgets', 'nren', nullable=False)
op.drop_constraint(op.f('fk_budgets_nren_id_nren'), 'budgets', type_='foreignkey')
op.drop_constraint(
op.f('fk_budgets_nren_id_nren'), 'budgets', type_='foreignkey')
op.drop_column('budgets', 'nren_id')
op.drop_table('nren')
......
compendium_v2/publishers/helpers.py
+ 6
− 2
View file @ 1d23bf05
......@@ -12,7 +12,8 @@ def init_db(config):
def get_uppercase_nren_dict(session):
"""
:param session: db session that is used to query the known NRENs
:return: a dictionary of all known NRENs db entities keyed on the uppercased name
:return: a dictionary of all known NRENs db entities keyed on the
uppercased name
"""
current_nrens = session.query(model.NREN).all()
nren_dict = {nren.name.upper(): nren for nren in current_nrens}
......@@ -20,5 +21,8 @@ def get_uppercase_nren_dict(session):
nren_dict['ASNET'] = nren_dict['ASNET-AM']
nren_dict['KIFU (NIIF)'] = nren_dict['KIFU']
nren_dict['SURFNET'] = nren_dict['SURF']
nren_dict['UOM'] = nren_dict['UOM/RICERKANET']
nren_dict['UOM/RICERKANET'] = nren_dict['UOM']
nren_dict['UNINETT'] = nren_dict['SIKT']
nren_dict['LANET'] = nren_dict['LAT']
nren_dict['ANA'] = nren_dict['RASH']
return nren_dict
compendium_v2/publishers/survey_publisher_2022.py
+ 12
− 5
View file @ 1d23bf05
......@@ -77,12 +77,14 @@ def transfer_budget():
budget = float(_budget.replace('"', '').replace(',', ''))
except ValueError:
logger.info(
f'{nren_name} has no budget for 2022. Skipping. ({_budget}))')
f'{nren_name} has no budget for 2022. Skipping.'
f' ({_budget}))')
continue
if budget > 200:
logger.info(
f'{nren_name} has budget set to >200M EUR for 2022. ({budget})')
f'{nren_name} has budget set to >200M EUR for 2022.'
f' ({budget})')
if nren_name not in nren_dict:
logger.info(f'{nren_name} unknown. Skipping.')
......@@ -115,7 +117,10 @@ def transfer_funding_sources():
+ f' ({_value}))')
value = 0
nren_info = sourcedata.setdefault(nren_name, {source_type: 0 for source_type in FundingSource})
nren_info = sourcedata.setdefault(
nren_name,
{source_type: 0 for source_type in FundingSource}
)
nren_info[source] = value
for nren_name, nren_info in sourcedata.items():
......@@ -123,7 +128,8 @@ def transfer_funding_sources():
if not math.isclose(total, 100, abs_tol=0.01):
logger.info(
f'{nren_name} funding sources do not sum to 100%. ({total})')
f'{nren_name} funding sources do not sum to 100%.'
f' ({total})')
if nren_name not in nren_dict:
logger.info(f'{nren_name} unknown. Skipping.')
......@@ -132,7 +138,8 @@ def transfer_funding_sources():
funding_source = model.FundingSource(
nren=nren_dict[nren_name],
year=2022,
client_institutions=nren_info[FundingSource.CLIENT_INSTITUTIONS],
client_institutions=nren_info[
FundingSource.CLIENT_INSTITUTIONS],
european_funding=nren_info[FundingSource.EUROPEAN_FUNDING],
gov_public_bodies=nren_info[FundingSource.GOV_PUBLIC_BODIES],
commercial=nren_info[FundingSource.COMMERCIAL],
......
compendium_v2/publishers/survey_publisher_v1.py
+ 4
− 1
View file @ 1d23bf05
......@@ -39,7 +39,10 @@ def db_budget_migration():
continue
budget_entry = model.BudgetEntry(
nren=nren_dict[abbrev], budget=float(budget.budget), year=year)
nren=nren_dict[abbrev],
budget=float(budget.budget),
year=year
)
session.merge(budget_entry)
# Import the data from excel sheet to database
......
test/conftest.py
+ 6
− 3
View file @ 1d23bf05
......@@ -73,7 +73,8 @@ def test_budget_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("BudgetTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......@@ -121,7 +122,8 @@ def test_funding_source_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("FundingSourceTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......@@ -164,7 +166,8 @@ def test_charging_structure_data():
with db.session_scope() as session:
data = [row for row in _test_data_csv("ChargingStructureTestData.csv")]
nren_names = set([row["nren"] for row in data])
nren_dict = {nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
nren_dict = {
nren_name: model.NREN(name=nren_name) for nren_name in nren_names}
session.add_all(nren_dict.values())
for row in data:
......
test/test_survey_publisher_2022.py
+ 6
− 2
View file @ 1d23bf05
......@@ -37,8 +37,12 @@ def test_publisher(client, mocker, dummy_config):
funding_source_data)
with db.session_scope() as session:
nren_names = ['Nren1', 'Nren2', 'Nren3', 'Nren4', 'SURF', 'KIFU', 'UoM/RicerkaNet', 'ASNET-AM']
session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
nren_names = [
'Nren1', 'Nren2', 'Nren3', 'Nren4',
'SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH'
]
session.add_all(
[model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
......
test/test_survey_publisher_v1.py
+ 3
− 2
View file @ 1d23bf05
......@@ -14,8 +14,9 @@ def test_publisher(client, mocker, dummy_config):
EXCEL_FILE)
with db.session_scope() as session:
nren_names = ['SURF', 'KIFU', 'UoM/RicerkaNet', 'ASNET-AM']
session.add_all([model.NREN(name=nren_name) for nren_name in nren_names])
nren_names = ['SURF', 'KIFU', 'UoM', 'ASNET-AM', 'SIKT', 'LAT', 'RASH']
session.add_all(
[model.NREN(name=nren_name) for nren_name in nren_names])
_cli(dummy_config)
......
tox.ini
+ 1
− 1
View file @ 1d23bf05
......@@ -3,7 +3,7 @@ envlist = py39
[flake8]
exclude = venv,.tox,webapp
max-line-length = 120
max-line-length = 80
[testenv]
deps =
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment