Skip to content
Snippets Groups Projects
Commit 54e9b910 authored by Erik Reid's avatar Erik Reid
Browse files

updated storage, added preliminary http api

parent c9ffdee8
No related branches found
No related tags found
No related merge requests found
......@@ -4,6 +4,7 @@ import json
import logging
import queue
import random
import re
import threading
from distutils.util import strtobool
......@@ -266,3 +267,10 @@ def load_snmp_indexes(hostname=None):
result[router] = {e['name']: e for e in doc['value']}
return result
def ims_equipment_to_hostname(equipment):
hostname = equipment.lower()
if not re.match(r'.*\.geant\.(net|org)$', hostname):
hostname = f'{hostname}.geant.net'
return hostname
......@@ -12,7 +12,6 @@ These endpoints are intended for use by LG.
"""
import json
import logging
import re
from flask import Blueprint, Response, request
......@@ -119,14 +118,12 @@ def routers(access):
for k in redis.scan_iter('ims:lg:*', count=1000):
rtr = redis.get(k.decode('utf-8')).decode('utf-8')
rtr = json.loads(rtr)
hostname = rtr['equipment name'].lower()
if ' ' in hostname:
if ' ' in rtr['equipment name']:
logger.warning(
'skipping LG router with ws in hostname: {hostname}')
continue
if not re.match(r'.*\.geant\.(net|org)$', hostname):
hostname = f'{hostname}.geant.net'
rtr['equipment name'] = hostname
rtr['equipment name'] = common.ims_equipment_to_hostname(
rtr['equipment name'])
yield rtr
cache_key = f'classifier-cache:ims-lg:{access}'
......
......@@ -12,6 +12,12 @@ These endpoints are intended for use by MSR.
.. autofunction:: inventory_provider.routes.msr.access_services
/msr/gws-indirect
---------------------------------
.. autofunction:: inventory_provider.routes.msr.gws_indirect
/msr/bgp/logical-systems
-------------------------------------
......@@ -63,74 +69,70 @@ from flask import Blueprint, Response, request
from inventory_provider.routes import common
from inventory_provider.routes.common import _ignore_cache_or_retrieve
routes = Blueprint("msr-query-routes", __name__)
routes = Blueprint('msr-query-routes', __name__)
ACCESS_SERVICES_LIST_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"service": {
"type": "object",
"properties": {
"id": {"type": "integer"},
"name": {"type": "string"},
"equipment": {"type": "string"},
"pop_name": {"type": "string"},
"other_end_equipment": {"type": "string"},
"other_end_pop_name": {"type": "string"},
"speed_value": {"type": "integer"},
"speed_unit": {"type": "string"}
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'service': {
'type': 'object',
'properties': {
'id': {'type': 'integer'},
'name': {'type': 'string'},
'customer': {'type': 'string'},
'speed': {'type': 'integer'},
'pop': {'type': 'string'},
'hostname': {'type': 'string'},
'interface': {'type': 'string'},
},
"required": [
"id", "name",
"pop_name", "equipment",
"other_end_pop_name", "other_end_equipment",
"speed_value", "speed_unit"
],
"additionalProperties": False
'required': [
'id', 'name', 'customer', 'speed',
'pop', 'hostname', 'interface'],
'additionalProperties': False
}
},
"type": "array",
"items": {"$ref": "#/definitions/service"}
'type': 'array',
'items': {'$ref': '#/definitions/service'}
}
PEERING_GROUP_LIST_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "array",
"items": {"type": "string"}
'$schema': 'http://json-schema.org/draft-07/schema#',
'type': 'array',
'items': {'type': 'string'}
}
PEERING_LIST_SCHEMA = {
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"peering-instance": {
"type": "object",
"properties": {
"address": {"type": "string"},
"description": {"type": "string"},
"logical-system": {"type": "string"},
"group": {"type": "string"},
"hostname": {"type": "string"},
"remote-asn": {"type": "integer"},
"local-asn": {"type": "integer"},
"instance": {"type": "string"}
'$schema': 'http://json-schema.org/draft-07/schema#',
'definitions': {
'peering-instance': {
'type': 'object',
'properties': {
'address': {'type': 'string'},
'description': {'type': 'string'},
'logical-system': {'type': 'string'},
'group': {'type': 'string'},
'hostname': {'type': 'string'},
'remote-asn': {'type': 'integer'},
'local-asn': {'type': 'integer'},
'instance': {'type': 'string'}
},
# only vrr peerings have remote-asn
# only group peerings have local-asn or instance
# not all group peerings have 'description'
# and only vrr or vpn-proxy peerings are within a logical system
"required": [
"address",
"group",
"hostname"],
"additionalProperties": False
'required': [
'address',
'group',
'hostname'],
'additionalProperties': False
}
},
"type": "array",
"items": {"$ref": "#/definitions/peering-instance"}
'type': 'array',
'items': {'$ref': '#/definitions/peering-instance'}
}
......@@ -139,7 +141,7 @@ def after_request(resp):
return common.after_request(resp)
@routes.route("/access-services", methods=['GET', 'POST'])
@routes.route('/access-services', methods=['GET', 'POST'])
@common.require_accepts_json
def access_services():
"""
......@@ -159,7 +161,18 @@ def access_services():
def _services():
for k in redis.scan_iter('ims:access_services:*'):
service = redis.get(k.decode('utf-8')).decode('utf-8')
yield json.loads(service)
service = json.loads(service)
yield {
'id': service['id'],
'name': service['name'],
'customer': service['project'],
'speed': service['speed_value'],
'pop': service['here']['pop']['name'],
'hostname': common.ims_equipment_to_hostname(
service['here']['equipment']),
'interface': service['here']['port']
}
cache_key = 'classifier-cache:msr:access-services'
......@@ -172,13 +185,65 @@ def access_services():
return Response(
response='no access services found',
status=404,
mimetype="text/html")
mimetype='text/html')
# cache this data for the next call
result = json.dumps(result)
redis.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype='application/json')
@routes.route('/gws-indirect', methods=['GET', 'POST'])
@common.require_accepts_json
def gws_indirect():
"""
Handler for `/msr/gws-indirect`.
This method is in development, not yet used.
The response will be formatted according to the following schema:
.. asjson::
inventory_provider.routes.msr.ACCESS_SERVICES_LIST_SCHEMA
:return:
"""
redis = common.get_current_redis()
def _services():
for k in redis.scan_iter('ims:gws_indirect:*'):
service = redis.get(k.decode('utf-8')).decode('utf-8')
service = json.loads(service)
yield {
'id': service['id'],
'name': service['name'],
'customer': service['project'],
'speed': service['speed_value'],
'pop': service['here']['pop']['name'],
'hostname': common.ims_equipment_to_hostname(
service['here']['equipment']),
'interface': service['here']['port']
}
cache_key = 'classifier-cache:msr:gws-indirect'
result = _ignore_cache_or_retrieve(request, cache_key, redis)
if not result:
result = list(_services())
if not result:
return Response(
response='no gws indirect services found',
status=404,
mimetype='text/html')
# cache this data for the next call
result = json.dumps(result)
redis.set(cache_key, result.encode('utf-8'))
return Response(result, mimetype="application/json")
return Response(result, mimetype='application/json')
def _handle_peering_group_request(name, cache_key, group_key_base):
......@@ -239,8 +304,8 @@ def _handle_peering_group_request(name, cache_key, group_key_base):
return Response(items, mimetype="application/json")
@routes.route("/bgp/logical-system-peerings", methods=['GET', 'POST'])
@routes.route("/bgp/logical-system-peerings/<name>", methods=['GET', 'POST'])
@routes.route('/bgp/logical-system-peerings', methods=['GET', 'POST'])
@routes.route('/bgp/logical-system-peerings/<name>', methods=['GET', 'POST'])
@common.require_accepts_json
def logical_system_peerings(name=None):
"""
......@@ -258,8 +323,8 @@ def logical_system_peerings(name=None):
group_key_base='juniper-peerings:logical-system')
@routes.route("/bgp/group-peerings", methods=['GET', 'POST'])
@routes.route("/bgp/group-peerings/<name>", methods=['GET', 'POST'])
@routes.route('/bgp/group-peerings', methods=['GET', 'POST'])
@routes.route('/bgp/group-peerings/<name>', methods=['GET', 'POST'])
@common.require_accepts_json
def bgp_group_peerings(name=None):
"""
......@@ -277,8 +342,8 @@ def bgp_group_peerings(name=None):
group_key_base='juniper-peerings:group')
@routes.route("/bgp/routing-instance-peerings", methods=['GET', 'POST'])
@routes.route("/bgp/routing-instance-peerings/<name>", methods=['GET', 'POST'])
@routes.route('/bgp/routing-instance-peerings', methods=['GET', 'POST'])
@routes.route('/bgp/routing-instance-peerings/<name>', methods=['GET', 'POST'])
@common.require_accepts_json
def bgp_routing_instance_peerings(name=None):
"""
......@@ -338,7 +403,7 @@ def _handle_peering_group_list_request(cache_key, group_key_base):
return Response(names, mimetype="application/json")
@routes.route("/bgp/logical-systems", methods=['GET', 'POST'])
@routes.route('/bgp/logical-systems', methods=['GET', 'POST'])
@common.require_accepts_json
def get_logical_systems():
"""
......@@ -354,7 +419,7 @@ def get_logical_systems():
group_key_base='juniper-peerings:logical-system')
@routes.route("/bgp/groups", methods=['GET', 'POST'])
@routes.route('/bgp/groups', methods=['GET', 'POST'])
@common.require_accepts_json
def get_peering_groups():
"""
......@@ -370,7 +435,7 @@ def get_peering_groups():
group_key_base='juniper-peerings:group')
@routes.route("/bgp/routing-instances", methods=['GET', 'POST'])
@routes.route('/bgp/routing-instances', methods=['GET', 'POST'])
@common.require_accepts_json
def get_peering_routing_instances():
"""
......
......@@ -578,8 +578,7 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
port_id_details = defaultdict(list)
port_id_services = defaultdict(list)
interface_services = defaultdict(list)
access_services = {}
gws_indirect_services = {}
services_by_type = {}
def _convert_to_bits(value, unit):
unit = unit.lower()
......@@ -763,12 +762,12 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
circ['calculated-speed'] = _get_speed(circ['id'])
_format_service(circ)
if circ['service_type'].lower() == 'geant ip':
access_services[circ['id']] = circ
service_type_key = re.sub(
r'[^a-zA-Z]+', '_', circ['service_type'].lower())
if circ['service_type'].lower() == 'gws - indirect' \
and circ['status'].lower() == 'operational':
gws_indirect_services[circ['id']] = circ
type_services = services_by_type.setdefault(
service_type_key, dict())
type_services[circ['id']] = circ
interface_services[k].extend(circuits)
......@@ -804,31 +803,34 @@ def update_circuit_hierarchy_and_port_id_services(self, use_current=False):
populate_poller_cache(interface_services, r)
for v in access_services.values():
rp.set(
f'ims:access_services:{v["name"]}',
json.dumps({
'id': v['id'],
'name': v['name'],
'pop_name': v['pop_name'],
'other_end_pop_name': v['other_end_pop_name'],
'equipment': v['equipment'],
'other_end_equipment': v['other_end_equipment'],
'speed_value': v['calculated-speed'],
'speed_unit': 'n/a'
}))
for v in gws_indirect_services.values():
rp.set(
f'ims:gws_indirect:{v["name"]}',
json.dumps({
'id': v['id'],
'name': v['name'],
'customer': v['project'],
'hostname': v['equipment'],
'interface_name': v['port'],
'speed_value': v['calculated-speed']
}))
for service_type, services in services_by_type.items():
for v in services.values():
rp.set(
f'ims:{service_type}:{v["name"]}',
json.dumps({
'id': v['id'],
'name': v['name'],
'project': v['project'],
'here': {
'pop': {
'name': v['pop_name'],
'abbreviation': v['pop_abbreviation']
},
'equipment': v['equipment'],
'port': v['port'],
},
'there': {
'pop': {
'name': v['other_end_pop_name'],
'abbreviation': v['other_end_pop_abbreviation']
},
'equipment': v['other_end_equipment'],
'port': v['other_end_port'],
},
'speed_value': v['calculated-speed'],
'speed_unit': 'n/a'
}))
rp.execute()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment