Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • develop
  • feature/frontend-tests
  • master
  • 0.1
  • 0.10
  • 0.11
  • 0.12
  • 0.13
  • 0.14
  • 0.15
  • 0.16
  • 0.17
  • 0.18
  • 0.19
  • 0.2
  • 0.20
  • 0.21
  • 0.22
  • 0.23
  • 0.24
  • 0.25
  • 0.26
  • 0.27
  • 0.28
  • 0.29
  • 0.3
  • 0.30
  • 0.31
  • 0.32
  • 0.33
  • 0.34
  • 0.35
  • 0.36
  • 0.37
  • 0.38
  • 0.39
  • 0.4
  • 0.40
  • 0.41
  • 0.42
  • 0.43
  • 0.44
  • 0.45
  • 0.46
  • 0.47
  • 0.48
  • 0.49
  • 0.5
  • 0.50
  • 0.51
  • 0.52
  • 0.53
  • 0.54
  • 0.55
  • 0.56
  • 0.57
  • 0.58
  • 0.59
  • 0.6
  • 0.60
  • 0.61
  • 0.62
  • 0.63
  • 0.64
  • 0.65
  • 0.66
  • 0.67
  • 0.68
  • 0.69
  • 0.7
  • 0.70
  • 0.71
  • 0.72
  • 0.73
  • 0.74
  • 0.75
  • 0.76
  • 0.77
  • 0.78
  • 0.79
  • 0.8
  • 0.80
  • 0.81
  • 0.82
  • 0.83
  • 0.84
  • 0.85
  • 0.86
  • 0.87
  • 0.88
  • 0.89
  • 0.9
  • 0.90
  • 0.91
  • 0.92
  • 0.93
  • 0.94
  • 0.95
  • 0.96
  • 0.97
  • 0.98
101 results

Target

Select target project
  • geant-swd/compendium-v2
1 result
Select Git revision
  • develop
  • feature/frontend-tests
  • master
  • 0.1
  • 0.10
  • 0.11
  • 0.12
  • 0.13
  • 0.14
  • 0.15
  • 0.16
  • 0.17
  • 0.18
  • 0.19
  • 0.2
  • 0.20
  • 0.21
  • 0.22
  • 0.23
  • 0.24
  • 0.25
  • 0.26
  • 0.27
  • 0.28
  • 0.29
  • 0.3
  • 0.30
  • 0.31
  • 0.32
  • 0.33
  • 0.34
  • 0.35
  • 0.36
  • 0.37
  • 0.38
  • 0.39
  • 0.4
  • 0.40
  • 0.41
  • 0.42
  • 0.43
  • 0.44
  • 0.45
  • 0.46
  • 0.47
  • 0.48
  • 0.49
  • 0.5
  • 0.50
  • 0.51
  • 0.52
  • 0.53
  • 0.54
  • 0.55
  • 0.56
  • 0.57
  • 0.58
  • 0.59
  • 0.6
  • 0.60
  • 0.61
  • 0.62
  • 0.63
  • 0.64
  • 0.65
  • 0.66
  • 0.67
  • 0.68
  • 0.69
  • 0.7
  • 0.70
  • 0.71
  • 0.72
  • 0.73
  • 0.74
  • 0.75
  • 0.76
  • 0.77
  • 0.78
  • 0.79
  • 0.8
  • 0.80
  • 0.81
  • 0.82
  • 0.83
  • 0.84
  • 0.85
  • 0.86
  • 0.87
  • 0.88
  • 0.89
  • 0.9
  • 0.90
  • 0.91
  • 0.92
  • 0.93
  • 0.94
  • 0.95
  • 0.96
  • 0.97
  • 0.98
101 results
Show changes
Commits on Source (2)
Showing
with 407 additions and 10 deletions
......@@ -13,6 +13,7 @@ import ParentOrganisation from "./pages/ParentOrganisation";
import ECProjects from "./pages/ECProjects";
import Providers from "./Providers";
import PolicyPage from "./pages/Policy";
import ConnectedInstitutionsURLs from "./pages/ConnectedInstitutionsURLs";
const router = createBrowserRouter([
......@@ -26,6 +27,7 @@ const router = createBrowserRouter([
{ path: "/ec-projects", element: <ECProjects />},
{ path: "/policy", element: <PolicyPage />},
{ path: "/data", element: <CompendiumData />},
{ path: "/institutions-urls", element: <ConnectedInstitutionsURLs />},
{ path: "*", element: <Landing />},
]);
......
......@@ -56,6 +56,10 @@ export interface Policy extends NrenAndYearDatapoint {
strategic_plan: string
}
export interface ConnectedInstitutionURLs extends NrenAndYearDatapoint {
urls: string[]
}
export interface BasicDataset {
labels: string[];
datasets: {
......
import React from 'react';
import { Link } from 'react-router-dom';
import { Row } from 'react-bootstrap';
import Sidebar from './SideBar';
const ConnectedUsersSidebar = () => {
return (
<Sidebar>
<h5>Connected Users</h5>
<Row>
<Link to="/institutions-urls" className="link-text-underline">
<span>Connected Institutions URLs</span>
</Link>
</Row>
</Sidebar>
)
}
export default ConnectedUsersSidebar
\ No newline at end of file
......@@ -9,6 +9,7 @@ import PolicySidebar from "./PolicySidebar";
import { Chart as ChartJS } from 'chart.js';
import { usePreview } from "../helpers/usePreview";
import ConnectedUsersSidebar from "./ConnectedUsersSidebar";
ChartJS.defaults.font.size = 16;
ChartJS.defaults.font.family = 'Open Sans';
......@@ -30,6 +31,7 @@ function DataPage({ title, description, filter, children, category }: inputProps
<>
{category === Sections.Organisation && <OrganizationSidebar />}
{category === Sections.Policy && <PolicySidebar />}
{category === Sections.ConnectedUsers && <ConnectedUsersSidebar />}
<PageHeader type={'data'} />
{ preview && <Row className="preview-banner">
<span>You are viewing a preview of the website which includes pre-published survey data. <a href={locationWithoutPreview}>Click here</a> to deactivate preview mode.</span>
......
......@@ -26,10 +26,9 @@ const SectionNavigation = ({ activeCategory }: inputProps) => {
<span>{Sections.Policy}</span>
</Button>
<Button
onClick={() => navigate(activeCategory === Sections.ConnectedUsers ? '.' : '.')}
onClick={() => navigate(activeCategory === Sections.ConnectedUsers ? '.' : '/institutions-urls')}
variant={'nav-box'}
active={activeCategory === Sections.ConnectedUsers}
disabled={true}>
active={activeCategory === Sections.ConnectedUsers}>
<span>{Sections.ConnectedUsers}</span>
</Button>
<Button
......
import { cartesianProduct } from 'cartesian-product-multiple-arrays';
import {
FundingSource, FundingSourceDataset, ChargingStructure,
Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy
Budget, BasicDataset, NrenStaff, NrenStaffDataset, Organisation, ECProject, Policy, ConnectedInstitutionURLs
} from "../Schema";
// create a color from a string, credits https://stackoverflow.com/a/16348977
......@@ -216,6 +216,20 @@ export function createPolicyDataLookup(policyEntries: Policy[]) {
return policyMap;
}
export function createConnectedInstitutionsURLsDataLookup(institutionEntries: ConnectedInstitutionURLs[]) {
const institutionMap = new Map<string, Map<number, ConnectedInstitutionURLs>>();
institutionEntries.forEach(entry => {
let nrenEntry = institutionMap.get(entry.nren);
if (!nrenEntry) {
nrenEntry = new Map<number, ConnectedInstitutionURLs>();
}
nrenEntry.set(entry.year, entry);
institutionMap.set(entry.nren, nrenEntry);
});
return institutionMap;
}
export const createNRENStaffDataset = (data: NrenStaff[], roles: boolean, selectedYear: number) => {
let categories;
......
......@@ -90,8 +90,11 @@ function CompendiumData(): ReactElement {
<CollapsibleBox title={Sections.ConnectedUsers} startCollapsed>
<div className="collapsible-column">
<h5>Coming Soon</h5>
</div>
<Row>
<Link to="/institutions-urls" className="link-text-underline">
<span>Connected Institutions URLs</span>
</Link>
</Row> </div>
</CollapsibleBox>
<CollapsibleBox title={Sections.Network} startCollapsed>
<div className="collapsible-column">
......
import React, { useContext} from 'react';
import { Table } from "react-bootstrap";
import {ConnectedInstitutionURLs} from "../Schema";
import { createConnectedInstitutionsURLsDataLookup } from '../helpers/dataconversion';
import DataPage from '../components/DataPage';
import Filter from "../components/graphing/Filter";
import { Sections } from '../helpers/constants';
import { FilterSelectionContext } from '../helpers/FilterSelectionProvider';
import {useData} from "../helpers/useData";
function getJSXFromMap(data: Map<string, Map<number, ConnectedInstitutionURLs>>) {
return Array.from(data.entries()).map(([nren, nrenMap]) => {
return Array.from(nrenMap.entries()).map(([year, institution], yearIndex) => (
<tr key={nren + year}>
<td className='pt-3 nren-column'>{yearIndex === 0 && nren}</td>
<td className='pt-3 year-column'>{year}</td>
<td className='pt-3 blue-column'>
<ul>
{/* Display URLs */}
{institution.urls.map((url, index) => (
<li key={index}>
<a href={url} target="_blank" rel="noopener noreferrer">{url}</a>
</li>
))}
</ul>
</td>
</tr>
));
});
}
function ConnectedInstitutionsURLsPage() {
const {filterSelection, setFilterSelection } = useContext(FilterSelectionContext);
const {data: institutionData, years, nrens} = useData<ConnectedInstitutionURLs>('/api/institutions-urls/', setFilterSelection);
const selectedData = institutionData.filter(institution =>
filterSelection.selectedYears.includes(institution.year) &&
filterSelection.selectedNrens.includes(institution.nren)
);
const institutionDataByYear = createConnectedInstitutionsURLsDataLookup(selectedData);
const filterNode = <Filter
filterOptions={{ availableYears: [...years], availableNrens: [...nrens.values()] }}
filterSelection={filterSelection}
setFilterSelection={setFilterSelection}
/>;
return (
<DataPage title="Connected Institutions URLs"
description='The table shows URLs of the institutions connected to NRENs. You can filter the data by years and by NRENs.'
category={Sections.ConnectedUsers} filter={filterNode}>
<Table borderless className='compendium-table'>
<thead>
<tr>
<th className='nren-column'>NREN</th>
<th className='year-column'>Year</th>
<th className='blue-column'>URLs</th>
</tr>
</thead>
<tbody>
{getJSXFromMap(institutionDataByYear)}
</tbody>
</Table>
</DataPage>
);
}
export default ConnectedInstitutionsURLsPage;
......@@ -7,7 +7,7 @@ from enum import Enum
from typing import Optional
from typing_extensions import Annotated
from sqlalchemy import String
from sqlalchemy import String, JSON
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.schema import ForeignKey
......@@ -124,3 +124,12 @@ class Policy(db.Model):
acceptable_use: Mapped[str]
privacy_notice: Mapped[str]
data_protection: Mapped[str]
class InstitutionURLs(db.Model):
__tablename__ = 'institution_urls'
nren_id: Mapped[int_pk_fkNREN]
nren: Mapped[NREN] = relationship(lazy='joined')
year: Mapped[int_pk]
urls: Mapped[list[str]] = mapped_column(JSON)
"""added InstitutionURL
Revision ID: f2879a6b15c8
Revises: 96f26bf37f6c
Create Date: 2023-08-30 14:39:37.133633
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'f2879a6b15c8'
down_revision = 'd6f581374e8f'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'institution_urls',
sa.Column('nren_id', sa.Integer(), nullable=False),
sa.Column('year', sa.Integer(), nullable=False),
sa.Column('urls', sa.JSON(), nullable=False),
sa.ForeignKeyConstraint(['nren_id'], ['nren.id'], name=op.f('fk_institution_urls_nren_id_nren')),
sa.PrimaryKeyConstraint('nren_id', 'year', name=op.f('pk_institution_urls'))
)
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('institution_urls')
# ### end Alembic commands ###
import re
from sqlalchemy import select
from compendium_v2.db import db, model
URL_PATTERN = re.compile(
r'\b(https?://[^\s<>";,(){}\[\]!\\]+|www\.[^\s<>";,(){}\[\]!\\]+|[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4})\b(?=\s|\b|[,!?.;:\\])'
)
def get_uppercase_nren_dict():
"""
......@@ -22,3 +28,7 @@ def get_uppercase_nren_dict():
nren_dict['GRNET S.A.'] = nren_dict['GRNET']
nren_dict['FUNET'] = nren_dict['CSC']
return nren_dict
def extract_urls(text: str) -> list[str]:
return re.findall(URL_PATTERN, text)
......@@ -20,6 +20,7 @@ import compendium_v2
from compendium_v2.db.model import FeeType
from compendium_v2.environment import setup_logging
from compendium_v2.config import load
from compendium_v2.publishers.helpers import extract_urls
from compendium_v2.survey_db import model as survey_model
from compendium_v2.db import db, model
from compendium_v2.publishers import helpers
......@@ -59,6 +60,35 @@ WHERE
ORDER BY n.id, a.question_id, a.updated_at DESC
"""
INSTITUTIONS_URLS_QUERY_UNTIL_2022 = """
WITH RECURSIVE parent_questions AS (
-- Base case
SELECT q.id, q.equivalent_question_id, c.year, q.title
FROM questions q
JOIN sections s ON q.section_id = s.id
JOIN compendia c ON s.compendium_id = c.id
WHERE q.id = 16507
UNION ALL
-- Recursive case
SELECT q.id, q.equivalent_question_id, c.year, q.title
FROM questions q
INNER JOIN parent_questions pq ON q.id = pq.equivalent_question_id
JOIN sections s ON q.section_id = s.id
JOIN compendia c ON s.compendium_id = c.id)
SELECT DISTINCT ON (n.id, answers.question_id) answers.id,
UPPER(n.abbreviation) AS nren,
parent_questions.year,
answers.value as answer
FROM answers
JOIN parent_questions ON answers.question_id = parent_questions.id
JOIN nrens n on answers.nren_id = n.id
WHERE UPPER(answers.value) NOT IN ('"NA"', '"N/A"', '[""]', '["-"]', '["/"]', '/', '["NA"]', '""', '[]', '[n/a]')
ORDER BY n.id, answers.question_id, answers.updated_at DESC;
"""
class FundingSource(enum.Enum):
CLIENT_INSTITUTIONS = 16405
......@@ -120,6 +150,11 @@ def query_budget():
return db.session.execute(text(BUDGET_QUERY), bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
def query_institutions_urls():
return db.session.execute(text(INSTITUTIONS_URLS_QUERY_UNTIL_2022),
bind_arguments={'bind': db.engines[survey_model.SURVEY_DB_BIND]})
def query_funding_sources():
for source in FundingSource:
query = QUESTION_TEMPLATE_QUERY.format(source.value, 2022)
......@@ -167,6 +202,45 @@ def transfer_budget(nren_dict):
db.session.commit()
def transfer_institutions_urls(nren_dict):
def _parse_json(value):
if value and not value.startswith('['):
value = f'[{value}]'
try:
return [url.strip() for url in json.loads(value) if url.strip()]
except json.decoder.JSONDecodeError:
logger.info(f'JSON decode error for institution urls for {nren_name}.')
return []
rows = query_institutions_urls()
for row in rows:
answer_id, nren_name, year, answer = row
if nren_name not in nren_dict:
logger.info(f'{nren_name} unknown. Skipping.')
continue
urls = extract_urls(text=answer)
urls_json = _parse_json(answer)
if urls != urls_json:
logger.info(f'Institution URLs for {nren_name} do not match between json and regex. {urls} != {urls_json}')
if not urls:
logger.info(f'{nren_name} has no urls for {year}. Skipping.')
continue
institution_urls = model.InstitutionURLs(
nren=nren_dict[nren_name],
nren_id=nren_dict[nren_name].id,
urls=urls,
year=year,
)
db.session.merge(institution_urls)
db.session.commit()
def transfer_funding_sources(nren_dict):
sourcedata = {}
for source, data in query_funding_sources():
......@@ -491,6 +565,7 @@ def _cli(config, app):
transfer_charging_structure(nren_dict)
transfer_ec_projects(nren_dict)
transfer_policies(nren_dict)
transfer_institutions_urls(nren_dict)
@click.command()
......
......@@ -14,6 +14,7 @@ from compendium_v2.routes.survey import routes as survey
from compendium_v2.routes.user import routes as user_routes
from compendium_v2.routes.nren import routes as nren_routes
from compendium_v2.routes.response import routes as response_routes
from compendium_v2.routes.institutions_urls import routes as institutions_urls_routes
routes = Blueprint('compendium-v2-api', __name__)
routes.register_blueprint(budget_routes, url_prefix='/budget')
......@@ -27,6 +28,7 @@ routes.register_blueprint(survey, url_prefix='/survey')
routes.register_blueprint(user_routes, url_prefix='/user')
routes.register_blueprint(nren_routes, url_prefix='/nren')
routes.register_blueprint(response_routes, url_prefix='/response')
routes.register_blueprint(institutions_urls_routes, url_prefix='/institutions-urls')
logger = logging.getLogger(__name__)
......
from typing import Any
from flask import Blueprint, jsonify
from sqlalchemy import select
from compendium_v2.db import db
from compendium_v2.db.model import NREN, InstitutionURLs
from compendium_v2.routes import common
routes = Blueprint('institutions-urls', __name__)
INSTITUTION_URLS_RESPONSE_SCHEMA = {
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'institution_urls': {
'type': 'object',
'properties': {
'nren': {'type': 'string'},
'nren_country': {'type': 'string'},
'year': {'type': 'integer'},
'urls': {'type': 'array', 'items': {'type': 'string'}}
},
'required': ['nren', 'nren_country', 'year', 'urls'],
'additionalProperties': False
}
},
'type': 'array',
'items': {'$ref': '#/definitions/institution_urls'}
}
@routes.route('/', methods=['GET'])
@common.require_accepts_json
def institutions_urls_view() -> Any:
"""
handler for /api/institutions-urls/ requests
Endpoint for getting the URLs of institutions connected to the NREN.
This endpoint retrieves the URLs of webpages that list the institutions
or organizations connected to the NREN.
Many NRENs have one or more pages on their website listing such user institutions.
response will be formatted as per INSTITUTION_URLS_RESPONSE_SCHEMA
:return:
"""
def _extract_data(institution: InstitutionURLs) -> dict:
return {
'nren': institution.nren.name,
'nren_country': institution.nren.country,
'year': institution.year,
'urls': institution.urls
}
entries = []
records = db.session.scalars(
select(InstitutionURLs).join(NREN).order_by(NREN.name.asc(), InstitutionURLs.year.desc()))
for entry in records:
entries.append(_extract_data(entry))
return jsonify(entries)
This diff is collapsed.
......@@ -331,3 +331,41 @@ def test_policy_data(app):
))
db.session.commit()
@pytest.fixture
def test_institution_urls_data(app):
def _create_and_save_nrens(nren_names):
nrens = {}
for nren_name in nren_names:
nren_instance = model.NREN(name=nren_name, country='country')
nrens[nren_name] = nren_instance
db.session.add(nren_instance)
return nrens
def _create_and_save_institution_urls(nrens_and_years, nrens):
for nren_name, year in nrens_and_years:
nren_instance = nrens[nren_name]
urls = ['https://example.com', 'http://example.org']
institution_urls_model = model.InstitutionURLs(
nren=nren_instance,
year=year,
urls=urls
)
db.session.add(institution_urls_model)
with app.app_context():
predefined_nrens_and_years = [
('nren1', 2019),
('nren1', 2020),
('nren1', 2021),
('nren2', 2019),
('nren2', 2021)
]
unique_nren_names = {nren for nren, _ in predefined_nrens_and_years}
created_nrens = _create_and_save_nrens(unique_nren_names)
_create_and_save_institution_urls(predefined_nrens_and_years, created_nrens)
db.session.commit()
import pytest
from compendium_v2.publishers.helpers import extract_urls
@pytest.mark.parametrize(
"text, expected",
[
("Check this: https://example.com and http://example.com\\", ["https://example.com", "http://example.com"]),
("No URL here", []),
("Incomplete URL: http://", []),
("Here is a www link: www.example.com", ["www.example.com"]),
("Another format: example.com", ["example.com"]),
("Mixed: https://example.com, http://site.org, www.site.net",
["https://example.com", "http://site.org", "www.site.net"]),
("With Parentheses: (https://example.com)", ["https://example.com"]),
("With Brackets: [https://example.com]", ["https://example.com"]),
("With Braces: {https://example.com}", ["https://example.com"]),
("With Semicolon: https://example.com;", ["https://example.com"]),
("With Dot at the end: https://example.com.", ["https://example.com"]),
("With Comma at the end: https://example.com,", ["https://example.com"]),
("With Exclamation at the end: https://example.com!", ["https://example.com"]),
("With Multiple special characters: https://example.com)!..d!;www.m.asd!)",
["https://example.com", "www.m.asd"]),
("URL with path: https://example.com/path/to/page", ["https://example.com/path/to/page"]),
("URL with path and query: https://example.com/path?query=value", ["https://example.com/path?query=value"]),
("URL with hash: https://example.com#section", ["https://example.com#section"]),
("URL with path, query and hash: https://example.com/path?query=value#section",
["https://example.com/path?query=value#section"]),
("URL with port: http://example.com:8080", ["http://example.com:8080"]),
("Multiple Complex: https://example.com/path, http://example.com:8080/page?query=value#section",
["https://example.com/path", "http://example.com:8080/page?query=value#section"]),
("URL with encoded characters: https://example.com/path%20to%20page", ["https://example.com/path%20to%20page"]),
("URL with encoded query: https://example.com/path?query=value%20with%20space",
["https://example.com/path?query=value%20with%20space"]),
("[URL with encoded hash: https://example.com/path#section%20two;]", ["https://example.com/path#section%20two"]),
]
)
def test_extract_urls_from_a_text(text, expected):
result = extract_urls(text)
assert result == expected
import json
import jsonschema
from compendium_v2.routes.institutions_urls import INSTITUTION_URLS_RESPONSE_SCHEMA
def test_institutions_urls_response(client, test_institution_urls_data):
rv = client.get(
'/api/institutions-urls/',
headers={'Accept': ['application/json']})
assert rv.status_code == 200
result = json.loads(rv.data.decode('utf-8'))
jsonschema.validate(result, INSTITUTION_URLS_RESPONSE_SCHEMA)
assert result
......@@ -3,7 +3,7 @@ import jsonschema
from compendium_v2.routes.policy import POLICY_RESPONSE_SCHEMA
def test_ec_project_response(client, test_policy_data):
def test_policy_response(client, test_policy_data):
rv = client.get(
'/api/policy/',
headers={'Accept': ['application/json']})
......