Skip to content
Snippets Groups Projects
Commit b996e00c authored by Pelle Koster's avatar Pelle Koster
Browse files

more refactor and test things

parent d0ee269e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 536 additions and 500 deletions
brian_polling_manager/interface_stats/brian.py
+ 0
− 68
View file @ b996e00c
BRIAN_COUNTER_DICT_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema',
'type': 'object',
'properties': {
'hostname': {'type': 'string'},
'interface_name': {'type': 'string'},
'egressOctets': {'type': 'integer'},
'egressPackets': {'type': 'integer'},
'ingressOctets': {'type': 'integer'},
'ingressPackets': {'type': 'integer'},
'egressOctetsv6': {'type': 'integer'},
'egressPacketsv6': {'type': 'integer'},
'ingressOctetsv6': {'type': 'integer'},
'ingressPacketsv6': {'type': 'integer'},
'egressErrors': {'type': 'integer'},
'ingressDiscards': {'type': 'integer'},
'ingressErrors': {'type': 'integer'},
},
'required': [
'hostname',
'interface_name',
'egressOctets',
'egressPackets',
'ingressOctets',
'ingressPackets',
],
'additionalProperties': False
}
def counters(router_fqdn, interface_counters):
"""
:param router_fqdn: hostname
:param interface_counters: either PHYSICAL_INTERFACE_COUNTER_SCHEMA
or LOGICAL_INTERFACE_COUNTER_SCHEMA
:return: iterable of BRIAN_POINT_SCHEMA objects
"""
def _counters2point(interface):
brian_counters = interface['brian']
point = {
'hostname': router_fqdn,
'interface_name': interface['name'],
'egressOctets': brian_counters['egress']['bytes'],
'egressPackets': brian_counters['egress']['packets'],
'ingressOctets': brian_counters['ingress']['bytes'],
'ingressPackets': brian_counters['ingress']['packets'],
}
if 'v6' in brian_counters['egress']:
point['egressOctetsv6'] = brian_counters['egress']['v6']['bytes']
point['egressPacketsv6'] = brian_counters['egress']['v6']['packets']
if 'v6' in brian_counters['ingress']:
point['ingressOctetsv6'] = brian_counters['ingress']['v6']['bytes']
point['ingressPacketsv6'] = brian_counters['ingress']['v6']['packets']
if 'errors' in brian_counters['egress']:
point['egressErrors'] = brian_counters['egress']['errors']['errors']
if 'errors' in brian_counters['ingress']:
point['ingressDiscards'] = brian_counters['ingress']['errors']['discards']
point['ingressErrors'] = brian_counters['ingress']['errors']['errors']
return point
return map(_counters2point, interface_counters)
brian_polling_manager/interface_stats/cli.py
+ 30
− 69
View file @ b996e00c
...@@ -8,7 +8,6 @@ from typing import Callable, Iterable, List ...@@ -8,7 +8,6 @@ from typing import Callable, Iterable, List
import click import click
from brian_polling_manager.interface_stats import brian, errors
from brian_polling_manager.interface_stats.click_helpers import ( from brian_polling_manager.interface_stats.click_helpers import (
validate_config, validate_config,
validate_hostname, validate_hostname,
...@@ -22,6 +21,7 @@ from brian_polling_manager.interface_stats.services.writers import ( ...@@ -22,6 +21,7 @@ from brian_polling_manager.interface_stats.services.writers import (
get_point_writer, get_point_writer,
) )
from brian_polling_manager.interface_stats.vendors import Vendor, juniper from brian_polling_manager.interface_stats.vendors import Vendor, juniper
from brian_polling_manager.interface_stats import vendors
logger = logging.getLogger(__file__) logger = logging.getLogger(__file__)
...@@ -75,72 +75,6 @@ def setup_logging(): ...@@ -75,72 +75,6 @@ def setup_logging():
logging.config.dictConfig(logging_config) logging.config.dictConfig(logging_config)
def counters_to_point(measurement, timestamp, counters):
"""
:param measurement: the measurement where the point will be written
:param counters: either a BRIAN_COUNTER_DICT_SCHEMA or
ERROR_COUNTER_DICT_SCHEMA object
:return: a brian_polling_manager.influx.INFLUX_POINT object
"""
def _is_tag(key_name):
return key_name == "hostname" or key_name == "interface_name"
fields = dict([_k, _v] for _k, _v in counters.items() if not _is_tag(_k))
tags = dict([_k, _v] for _k, _v in counters.items() if _is_tag(_k))
return {
"time": timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
"measurement": measurement,
"tags": tags,
"fields": fields,
}
def _brian_points(router_fqdn, netconf_doc, timestamp, measurement_name):
"""
returns an interable of points that can be written to influxdb
:param router_fqdn: 'hostname' tag to be set in the generated points
:param netconf_doc: a valid loaded netconf doc
:param timestamp: timestamp to put in the generated points
:param measurement_name: measurement name to put in the generated points
:return:
"""
_ctr2point = partial(counters_to_point, measurement_name, timestamp)
interfaces = juniper.physical_interface_counters(netconf_doc)
counters = brian.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
interfaces = juniper.logical_interface_counters(netconf_doc)
counters = brian.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
def _error_points(router_fqdn, netconf_doc, timestamp, measurement_name):
"""
returns an interable of points that can be written to influxdb
:param router_fqdn: 'hostname' tag to be set in the generated points
:param netconf_doc: a valid loaded netconf doc
:param timestamp: timestamp to put in the generated points
:param measurement_name: measurement name to put in the generated points
:return:
"""
_ctr2point = partial(counters_to_point, measurement_name, timestamp)
interfaces = juniper.physical_interface_counters(netconf_doc)
counters = errors.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
# [2024-03-21] We currently have no definition for error points on logical
# interfaces. This operation is essentially a no-op. Perhaps in the future we will
# get a definition for errors on logical interfaces
interfaces = juniper.logical_interface_counters(netconf_doc)
counters = errors.counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
def process_juniper_router( def process_juniper_router(
router_fqdn: str, router_fqdn: str,
all_influx_params: dict, all_influx_params: dict,
...@@ -153,7 +87,7 @@ def process_juniper_router( ...@@ -153,7 +87,7 @@ def process_juniper_router(
timestamp = datetime.now() timestamp = datetime.now()
influx_params = all_influx_params["brian-counters"] influx_params = all_influx_params["brian-counters"]
points = _brian_points( points = _juniper_brian_points(
router_fqdn=router_fqdn, router_fqdn=router_fqdn,
netconf_doc=document, netconf_doc=document,
timestamp=timestamp, timestamp=timestamp,
...@@ -164,7 +98,7 @@ def process_juniper_router( ...@@ -164,7 +98,7 @@ def process_juniper_router(
writer.write_points(points) writer.write_points(points)
influx_params = all_influx_params["error-counters"] influx_params = all_influx_params["error-counters"]
points = _error_points( points = _juniper_error_points(
router_fqdn=router_fqdn, router_fqdn=router_fqdn,
netconf_doc=document, netconf_doc=document,
timestamp=timestamp, timestamp=timestamp,
...@@ -175,6 +109,33 @@ def process_juniper_router( ...@@ -175,6 +109,33 @@ def process_juniper_router(
writer.write_points(points) writer.write_points(points)
def _juniper_brian_points(router_fqdn, netconf_doc, timestamp, measurement_name):
interfaces = juniper.physical_interface_counters(netconf_doc)
yield from vendors.brian_points(
router_fqdn, interfaces, timestamp, measurement_name
)
interfaces = juniper.logical_interface_counters(netconf_doc)
yield from vendors.brian_points(
router_fqdn, interfaces, timestamp, measurement_name
)
def _juniper_error_points(router_fqdn, netconf_doc, timestamp, measurement_name):
interfaces = juniper.physical_interface_counters(netconf_doc)
yield from vendors.error_points(
router_fqdn, interfaces, timestamp, measurement_name
)
# [2024-03-21] We currently have no definition for error points on logical
# interfaces. This operation is essentially a no-op. Perhaps in the future we will
# get a definition for errors on logical interfaces
interfaces = juniper.logical_interface_counters(netconf_doc)
yield from vendors.error_points(
router_fqdn, interfaces, timestamp, measurement_name
)
def process_nokia_router( def process_nokia_router(
router_fqdn: str, router_fqdn: str,
all_influx_params: dict, all_influx_params: dict,
......
brian_polling_manager/interface_stats/config.py
+ 57
− 57
View file @ b996e00c
...@@ -12,65 +12,63 @@ DEFAULT_NETCONF_PORT = 830 ...@@ -12,65 +12,63 @@ DEFAULT_NETCONF_PORT = 830
DEFAULT_HOSTKEY_VERIFY = False DEFAULT_HOSTKEY_VERIFY = False
CONFIG_SCHEMA = { CONFIG_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema', "$schema": "https://json-schema.org/draft/2020-12/schema",
"definitions": {
'definitions': { "ssh-params": {
'ssh-params': { "type": "object",
'type': 'object', "properties": {
'properties': { "username": {"type": "string"},
'username': {'type': 'string'}, "password": {"type": "string"},
'password': {'type': 'string'}, "port": {"type": "integer"},
'port': {'type': 'integer'}, "key_filename": {"type": "string"},
'key_filename': {'type': 'string'}, "ssh_config": {"type": "string"},
'ssh_config': {'type': 'string'}, "hostkey_verify": {"type": "boolean"},
'hostkey_verify': {'type': 'boolean'}
}, },
'additionalProperties': False "additionalProperties": False,
}, },
'influx-db-measurement': { "influx-db-measurement": {
'type': 'object', "type": "object",
'properties': { "properties": {
'ssl': {'type': 'boolean'}, "ssl": {"type": "boolean"},
'hostname': {'type': 'string'}, "hostname": {"type": "string"},
'port': {'type': 'integer'}, "port": {"type": "integer"},
'username': {'type': 'string'}, "username": {"type": "string"},
'password': {'type': 'string'}, "password": {"type": "string"},
'database': {'type': 'string'}, "database": {"type": "string"},
'measurement': {'type': 'string'} "measurement": {"type": "string"},
}, },
'required': [ "required": [
# ssl, port are optional # ssl, port are optional
'hostname', "hostname",
'username', "username",
'password', "password",
'database', "database",
'measurement' "measurement",
], ],
'additionalProperties': False "additionalProperties": False,
} },
}, },
"type": "object",
'type': 'object', "properties": {
'properties': { "juniper": {"$ref": "#/definitions/ssh-params"},
'juniper': {'$ref': '#/definitions/ssh-params'}, "nokia": {"$ref": "#/definitions/ssh-params"},
'nokia': {'$ref': '#/definitions/ssh-params'}, "inventory": {
'inventory': { "type": "array",
'type': 'array', "items": {"type": "string", "format": "uri"},
'items': {'type': 'string', 'format': 'uri'}, "minItems": 1,
'minItems': 1
}, },
'influx': { "influx": {
'type': 'object', "type": "object",
'properties': { "properties": {
'brian-counters': {'$ref': '#/definitions/influx-db-measurement'}, "brian-counters": {"$ref": "#/definitions/influx-db-measurement"},
'error-counters': {'$ref': '#/definitions/influx-db-measurement'}, "error-counters": {"$ref": "#/definitions/influx-db-measurement"},
}, },
'required': ['brian-counters', 'error-counters'], "required": ["brian-counters", "error-counters"],
'additionalProperties': False "additionalProperties": False,
}, },
}, },
'required': ['influx'], "required": ["influx"],
'additionalProperties': False "additionalProperties": False,
} }
...@@ -88,23 +86,25 @@ def load(config_file): ...@@ -88,23 +86,25 @@ def load(config_file):
# set some defaults # set some defaults
for router_vendor in 'juniper', 'nokia': for router_vendor in "juniper", "nokia":
ssh_params = config.setdefault(router_vendor, {}) ssh_params = config.setdefault(router_vendor, {})
ssh_params.setdefault('port', DEFAULT_NETCONF_PORT) ssh_params.setdefault("port", DEFAULT_NETCONF_PORT)
ssh_params.setdefault('hostkey_verify', DEFAULT_HOSTKEY_VERIFY) ssh_params.setdefault("hostkey_verify", DEFAULT_HOSTKEY_VERIFY)
# verify any declared files are present # verify any declared files are present
for _k in 'key_filename', 'ssh_config': for _k in "key_filename", "ssh_config":
if _k in ssh_params: if _k in ssh_params:
# TODO: why is this necessary? # TODO: why is this necessary?
# (user is automatically expanded for me when running from pycharm, but not from cli???)] # (user is automatically expanded for me when running from pycharm, but not from cli???)]
ssh_params[_k] = os.path.expanduser(ssh_params[_k]) ssh_params[_k] = os.path.expanduser(ssh_params[_k])
assert os.path.exists(ssh_params[_k]), f'{ssh_params[_k]} does not exist' assert os.path.exists(
ssh_params[_k]
), f"{ssh_params[_k]} does not exist"
for db in config['influx'].values(): for db in config["influx"].values():
db.setdefault('ssl', DEFAULT_INFLUX_SSL) db.setdefault("ssl", DEFAULT_INFLUX_SSL)
db.setdefault('port', DEFAULT_INFLUX_PORT) db.setdefault("port", DEFAULT_INFLUX_PORT)
return config return config
brian_polling_manager/interface_stats/errors.py
+ 0
− 102
View file @ b996e00c
ERROR_COUNTER_DICT_SCHEMA = {
'$schema': 'https://json-schema.org/draft/2020-12/schema',
'type': 'object',
'properties': {
'hostname': {'type': 'string'},
'interface_name': {'type': 'string'},
'bit_error_seconds': {'type': 'integer'},
'errored_blocks_seconds': {'type': 'integer'},
'input_crc_errors': {'type': 'integer'},
'input_fifo_errors': {'type': 'integer'},
'input_total_errors': {'type': 'integer'},
'input_discards': {'type': 'integer'},
'input_drops': {'type': 'integer'},
'output_drops': {'type': 'integer'},
'output_crc_errors': {'type': 'integer'},
'output_fifo_errors': {'type': 'integer'},
'output_total_errors': {'type': 'integer'},
'input_framing_errors': {'type': 'integer'},
'output_resource_errors': {'type': 'integer'},
'input_resource_errors': {'type': 'integer'},
'output_collisions': {'type': 'integer'},
},
'required': ['hostname', 'interface_name'],
'additionalProperties': False
}
def counters(router_fqdn, interface_counters):
"""
:param router_fqdn: hostname
:param interface_counters: PHYSICAL_INTERFACE_COUNTER_SCHEMA
:return: iterable of BRIAN_POINT_SCHEMA objects
"""
def _counters2point(interface):
brian_counters = interface['brian']
point = {
'hostname': router_fqdn,
'interface_name': interface['name']
}
ingress_errors = brian_counters['ingress'].get('errors', {})
if 'framing' in ingress_errors:
point['input_framing_errors'] = ingress_errors['framing']
if 'discards' in ingress_errors:
point['input_discards'] = ingress_errors['discards']
if 'drops' in ingress_errors:
point['input_drops'] = ingress_errors['drops']
if 'fifo' in ingress_errors:
point['input_fifo_errors'] = ingress_errors['fifo']
if 'drops' in ingress_errors:
point['input_drops'] = ingress_errors['drops']
if 'resource' in ingress_errors:
point['input_resource_errors'] = ingress_errors['resource']
egress_errors = brian_counters['egress'].get('errors', {})
if 'drops' in egress_errors:
point['output_drops'] = egress_errors['drops']
if 'fifo' in egress_errors:
point['output_fifo_errors'] = egress_errors['fifo']
if 'collisions' in egress_errors:
point['output_collisions'] = egress_errors['collisions']
if 'resource' in egress_errors:
point['output_resource_errors'] = egress_errors['resource']
if 'l2' in interface:
l2_counter = interface['l2']
ingress_errors = l2_counter['ingress'].get('errors', {})
if 'errors' in l2_counter['ingress']:
if 'crc' in ingress_errors:
point['input_crc_errors'] = ingress_errors['crc']
# use the value from brian/l3
# if 'fifo' in ingress_errors:
# _p['input_fifo_errors'] = ingress_errors['fifo']
if 'total' in ingress_errors:
point['input_total_errors'] = ingress_errors['total']
egress_errors = l2_counter['egress'].get('errors', {})
if 'errors' in l2_counter['egress']:
if 'crc' in egress_errors:
point['output_crc_errors'] = egress_errors['crc']
# use the value from brian/l3
# if 'fifo' in egress_errors:
# _p['output_fifo_errors'] = egress_errors['fifo']
if 'total' in egress_errors:
point['output_total_errors'] = egress_errors['total']
pcs_stats = l2_counter.get('pcs', {})
if 'bit-error-seconds' in pcs_stats:
point['bit_error_seconds'] = pcs_stats['bit-error-seconds']
if 'errored-blocks-seconds' in pcs_stats:
point['errored_blocks_seconds'] = pcs_stats['errored-blocks-seconds']
return point
def _has_data(ctrs):
return len(set(ctrs.keys()) - {'hostname', 'interface_name'}) > 0
return filter(_has_data, map(_counters2point, interface_counters))
brian_polling_manager/interface_stats/vendors/__init__.py
+ 29
− 164
View file @ b996e00c
import enum import enum
from .common import (
brian_counters,
error_counters,
brian_points,
error_points,
BRIAN_COUNTER_DICT_SCHEMA,
ERROR_COUNTER_DICT_SCHEMA,
PHYSICAL_INTERFACE_COUNTER_SCHEMA,
LOGICAL_INTERFACE_COUNTER_SCHEMA,
)
PHYSICAL_INTERFACE_COUNTER_SCHEMA = { from .juniper import (
"$schema": "https://json-schema.org/draft/2020-12/schema", physical_interface_counters,
"definitions": { logical_interface_counters,
"brian-v6-counters": { )
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-ingress-errors": {
"type": "object",
"properties": {
"errors": {"type": "integer"},
"drops": {"type": "integer"},
"resource": {"type": "integer"},
"discards": {"type": "integer"},
"fifo": {"type": "integer"},
"framing": {"type": "integer"},
},
"required": ["errors", "drops", "resource", "discards"],
"additionalProperties": False,
},
"brian-egress-errors": {
"type": "object",
"properties": {
"errors": {"type": "integer"},
"drops": {"type": "integer"},
"resource": {"type": "integer"},
"fifo": {"type": "integer"},
"collisions": {"type": "integer"},
},
"required": ["errors", "drops", "resource"],
"additionalProperties": False,
},
"brian-ingress-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
"errors": {"$ref": "#/definitions/brian-ingress-errors"},
},
# srx's don't have v6 counters
"required": ["bytes", "packets"],
# 'required': ['bytes', 'packets', 'v6'],
"additionalProperties": False,
},
"brian-egress-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
"errors": {"$ref": "#/definitions/brian-egress-errors"},
},
# srx's don't have v6 counters
"required": ["bytes", "packets"],
# 'required': ['bytes', 'packets', 'v6'],
"additionalProperties": False,
},
"brian-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/brian-ingress-counters"},
"egress": {"$ref": "#/definitions/brian-egress-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
"l2-dir-counters": {
"type": "object",
"properties": {
"broadcast": {"type": "integer"},
"multicast": {"type": "integer"},
"unicast": {"type": "integer"},
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"errors": {
"type": "object",
"properties": {
"crc": {"type": "integer"},
"fifo": {"type": "integer"},
"total": {"type": "integer"},
},
"required": ["crc", "fifo"],
"additionalProperties": False,
},
},
"required": ["broadcast", "multicast"],
"additionalProperties": False,
},
"l2-pcs-counters": {
"type": "object",
"properties": {
"bit-error-seconds": {"type": "integer"},
"errored-blocks-seconds": {"type": "integer"},
},
"required": ["bit-error-seconds", "errored-blocks-seconds"],
"additionalProperties": False,
},
"l2-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/l2-dir-counters"},
"egress": {"$ref": "#/definitions/l2-dir-counters"},
"pcs": {"$ref": "#/definitions/l2-pcs-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
},
"type": "object",
"properties": {
"name": {"type": "string"},
"brian": {"$ref": "#/definitions/brian-counters"},
"l2": {"$ref": "#/definitions/l2-counters"},
},
"required": ["name", "brian"],
"additionalProperties": False,
}
LOGICAL_INTERFACE_COUNTER_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"definitions": {
"brian-v6-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-dir-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/brian-dir-counters"},
"egress": {"$ref": "#/definitions/brian-dir-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
},
"type": "object",
"properties": {
"name": {"type": "string"},
"brian": {"$ref": "#/definitions/brian-counters"},
},
"required": ["name", "brian"],
"additionalProperties": False,
}
class Vendor(enum.Enum): class Vendor(enum.Enum):
JUNIPER = "juniper" JUNIPER = "juniper"
NOKIA = "nokia" NOKIA = "nokia"
__all__ = [
"brian_counters",
"error_counters",
"BRIAN_COUNTER_DICT_SCHEMA",
"ERROR_COUNTER_DICT_SCHEMA",
"PHYSICAL_INTERFACE_COUNTER_SCHEMA",
"LOGICAL_INTERFACE_COUNTER_SCHEMA",
"physical_interface_counters",
"logical_interface_counters",
"brian_points",
"error_points",
"Vendor",
]
brian_polling_manager/interface_stats/vendors/common.py 0 → 100644
+ 389
− 0
View file @ b996e00c
from functools import partial
PHYSICAL_INTERFACE_COUNTER_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"definitions": {
"brian-v6-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-ingress-errors": {
"type": "object",
"properties": {
"errors": {"type": "integer"},
"drops": {"type": "integer"},
"resource": {"type": "integer"},
"discards": {"type": "integer"},
"fifo": {"type": "integer"},
"framing": {"type": "integer"},
},
"required": ["errors", "drops", "resource", "discards"],
"additionalProperties": False,
},
"brian-egress-errors": {
"type": "object",
"properties": {
"errors": {"type": "integer"},
"drops": {"type": "integer"},
"resource": {"type": "integer"},
"fifo": {"type": "integer"},
"collisions": {"type": "integer"},
},
"required": ["errors", "drops", "resource"],
"additionalProperties": False,
},
"brian-ingress-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
"errors": {"$ref": "#/definitions/brian-ingress-errors"},
},
# srx's don't have v6 counters
"required": ["bytes", "packets"],
# 'required': ['bytes', 'packets', 'v6'],
"additionalProperties": False,
},
"brian-egress-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
"errors": {"$ref": "#/definitions/brian-egress-errors"},
},
# srx's don't have v6 counters
"required": ["bytes", "packets"],
# 'required': ['bytes', 'packets', 'v6'],
"additionalProperties": False,
},
"brian-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/brian-ingress-counters"},
"egress": {"$ref": "#/definitions/brian-egress-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
"l2-dir-counters": {
"type": "object",
"properties": {
"broadcast": {"type": "integer"},
"multicast": {"type": "integer"},
"unicast": {"type": "integer"},
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"errors": {
"type": "object",
"properties": {
"crc": {"type": "integer"},
"fifo": {"type": "integer"},
"total": {"type": "integer"},
},
"required": ["crc", "fifo"],
"additionalProperties": False,
},
},
"required": ["broadcast", "multicast"],
"additionalProperties": False,
},
"l2-pcs-counters": {
"type": "object",
"properties": {
"bit-error-seconds": {"type": "integer"},
"errored-blocks-seconds": {"type": "integer"},
},
"required": ["bit-error-seconds", "errored-blocks-seconds"],
"additionalProperties": False,
},
"l2-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/l2-dir-counters"},
"egress": {"$ref": "#/definitions/l2-dir-counters"},
"pcs": {"$ref": "#/definitions/l2-pcs-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
},
"type": "object",
"properties": {
"name": {"type": "string"},
"brian": {"$ref": "#/definitions/brian-counters"},
"l2": {"$ref": "#/definitions/l2-counters"},
},
"required": ["name", "brian"],
"additionalProperties": False,
}
LOGICAL_INTERFACE_COUNTER_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"definitions": {
"brian-v6-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-dir-counters": {
"type": "object",
"properties": {
"bytes": {"type": "integer"},
"packets": {"type": "integer"},
"v6": {"$ref": "#/definitions/brian-v6-counters"},
},
"required": ["bytes", "packets"],
"additionalProperties": False,
},
"brian-counters": {
"type": "object",
"properties": {
"ingress": {"$ref": "#/definitions/brian-dir-counters"},
"egress": {"$ref": "#/definitions/brian-dir-counters"},
},
"required": ["ingress", "egress"],
"additionalProperties": False,
},
},
"type": "object",
"properties": {
"name": {"type": "string"},
"brian": {"$ref": "#/definitions/brian-counters"},
},
"required": ["name", "brian"],
"additionalProperties": False,
}
BRIAN_COUNTER_DICT_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"hostname": {"type": "string"},
"interface_name": {"type": "string"},
"egressOctets": {"type": "integer"},
"egressPackets": {"type": "integer"},
"ingressOctets": {"type": "integer"},
"ingressPackets": {"type": "integer"},
"egressOctetsv6": {"type": "integer"},
"egressPacketsv6": {"type": "integer"},
"ingressOctetsv6": {"type": "integer"},
"ingressPacketsv6": {"type": "integer"},
"egressErrors": {"type": "integer"},
"ingressDiscards": {"type": "integer"},
"ingressErrors": {"type": "integer"},
},
"required": [
"hostname",
"interface_name",
"egressOctets",
"egressPackets",
"ingressOctets",
"ingressPackets",
],
"additionalProperties": False,
}
ERROR_COUNTER_DICT_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"hostname": {"type": "string"},
"interface_name": {"type": "string"},
"bit_error_seconds": {"type": "integer"},
"errored_blocks_seconds": {"type": "integer"},
"input_crc_errors": {"type": "integer"},
"input_fifo_errors": {"type": "integer"},
"input_total_errors": {"type": "integer"},
"input_discards": {"type": "integer"},
"input_drops": {"type": "integer"},
"output_drops": {"type": "integer"},
"output_crc_errors": {"type": "integer"},
"output_fifo_errors": {"type": "integer"},
"output_total_errors": {"type": "integer"},
"input_framing_errors": {"type": "integer"},
"output_resource_errors": {"type": "integer"},
"input_resource_errors": {"type": "integer"},
"output_collisions": {"type": "integer"},
},
"required": ["hostname", "interface_name"],
"additionalProperties": False,
}
def counters_to_point(measurement, timestamp, counters):
"""
:param measurement: the measurement where the point will be written
:param counters: either a BRIAN_COUNTER_DICT_SCHEMA or
ERROR_COUNTER_DICT_SCHEMA object
:return: a brian_polling_manager.influx.INFLUX_POINT object
"""
def _is_tag(key_name):
return key_name == "hostname" or key_name == "interface_name"
fields = dict([_k, _v] for _k, _v in counters.items() if not _is_tag(_k))
tags = dict([_k, _v] for _k, _v in counters.items() if _is_tag(_k))
return {
"time": timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
"measurement": measurement,
"tags": tags,
"fields": fields,
}
def brian_counters(router_fqdn, interface_counters):
"""
:param router_fqdn: hostname
:param interface_counters: either PHYSICAL_INTERFACE_COUNTER_SCHEMA
or LOGICAL_INTERFACE_COUNTER_SCHEMA
:return: iterable of BRIAN_POINT_SCHEMA objects
"""
def _counters2point(interface):
brian_counters = interface["brian"]
point = {
"hostname": router_fqdn,
"interface_name": interface["name"],
"egressOctets": brian_counters["egress"]["bytes"],
"egressPackets": brian_counters["egress"]["packets"],
"ingressOctets": brian_counters["ingress"]["bytes"],
"ingressPackets": brian_counters["ingress"]["packets"],
}
if "v6" in brian_counters["egress"]:
point["egressOctetsv6"] = brian_counters["egress"]["v6"]["bytes"]
point["egressPacketsv6"] = brian_counters["egress"]["v6"]["packets"]
if "v6" in brian_counters["ingress"]:
point["ingressOctetsv6"] = brian_counters["ingress"]["v6"]["bytes"]
point["ingressPacketsv6"] = brian_counters["ingress"]["v6"]["packets"]
if "errors" in brian_counters["egress"]:
point["egressErrors"] = brian_counters["egress"]["errors"]["errors"]
if "errors" in brian_counters["ingress"]:
point["ingressDiscards"] = brian_counters["ingress"]["errors"]["discards"]
point["ingressErrors"] = brian_counters["ingress"]["errors"]["errors"]
return point
return map(_counters2point, interface_counters)
def error_counters(router_fqdn, interface_counters):
"""
:param router_fqdn: hostname
:param interface_counters: PHYSICAL_INTERFACE_COUNTER_SCHEMA
:return: iterable of BRIAN_POINT_SCHEMA objects
"""
def _counters2point(interface):
brian_counters = interface["brian"]
point = {"hostname": router_fqdn, "interface_name": interface["name"]}
ingress_errors = brian_counters["ingress"].get("errors", {})
if "framing" in ingress_errors:
point["input_framing_errors"] = ingress_errors["framing"]
if "discards" in ingress_errors:
point["input_discards"] = ingress_errors["discards"]
if "drops" in ingress_errors:
point["input_drops"] = ingress_errors["drops"]
if "fifo" in ingress_errors:
point["input_fifo_errors"] = ingress_errors["fifo"]
if "drops" in ingress_errors:
point["input_drops"] = ingress_errors["drops"]
if "resource" in ingress_errors:
point["input_resource_errors"] = ingress_errors["resource"]
egress_errors = brian_counters["egress"].get("errors", {})
if "drops" in egress_errors:
point["output_drops"] = egress_errors["drops"]
if "fifo" in egress_errors:
point["output_fifo_errors"] = egress_errors["fifo"]
if "collisions" in egress_errors:
point["output_collisions"] = egress_errors["collisions"]
if "resource" in egress_errors:
point["output_resource_errors"] = egress_errors["resource"]
if "l2" in interface:
l2_counter = interface["l2"]
ingress_errors = l2_counter["ingress"].get("errors", {})
if "errors" in l2_counter["ingress"]:
if "crc" in ingress_errors:
point["input_crc_errors"] = ingress_errors["crc"]
# use the value from brian/l3
# if 'fifo' in ingress_errors:
# _p['input_fifo_errors'] = ingress_errors['fifo']
if "total" in ingress_errors:
point["input_total_errors"] = ingress_errors["total"]
egress_errors = l2_counter["egress"].get("errors", {})
if "errors" in l2_counter["egress"]:
if "crc" in egress_errors:
point["output_crc_errors"] = egress_errors["crc"]
# use the value from brian/l3
# if 'fifo' in egress_errors:
# _p['output_fifo_errors'] = egress_errors['fifo']
if "total" in egress_errors:
point["output_total_errors"] = egress_errors["total"]
pcs_stats = l2_counter.get("pcs", {})
if "bit-error-seconds" in pcs_stats:
point["bit_error_seconds"] = pcs_stats["bit-error-seconds"]
if "errored-blocks-seconds" in pcs_stats:
point["errored_blocks_seconds"] = pcs_stats["errored-blocks-seconds"]
return point
def _has_data(ctrs):
return len(set(ctrs.keys()) - {"hostname", "interface_name"}) > 0
return filter(_has_data, map(_counters2point, interface_counters))
def brian_points(router_fqdn, interfaces, timestamp, measurement_name):
"""
returns an interable of points that can be written to influxdb
:param router_fqdn: 'hostname' tag to be set in the generated points
:param interfaces: an iterable of PHYSICAL_INTERFACE_COUNTER_SCHEMA or
LOGICAL_INTERFACE_COUNTER_SCHEMA
:param timestamp: timestamp to put in the generated points
:param measurement_name: measurement name to put in the generated points
:return:
"""
_ctr2point = partial(counters_to_point, measurement_name, timestamp)
counters = brian_counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
def error_points(router_fqdn, interfaces, timestamp, measurement_name):
"""
returns an interable of points that can be written to influxdb
:param router_fqdn: 'hostname' tag to be set in the generated points
:param interfaces: an iterable of PHYSICAL_INTERFACE_COUNTER_SCHEMA or
LOGICAL_INTERFACE_COUNTER_SCHEMA
:param timestamp: timestamp to put in the generated points
:param measurement_name: measurement name to put in the generated points
:return:
"""
_ctr2point = partial(counters_to_point, measurement_name, timestamp)
counters = error_counters(router_fqdn=router_fqdn, interface_counters=interfaces)
yield from map(_ctr2point, counters)
brian_polling_manager/interface_stats/vendors/juniper.py
+ 0
− 13
View file @ b996e00c
"""
# rpc: 'get-interface-information' (child elem: <extensive />)
# cli: 'show interfaces extensive'
"""
import logging import logging
from brian_polling_manager.interface_stats.services import netconf
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def get_interface_info_ncrpc(router_name, ssh_params):
return netconf.JuniperNetconfProvider(ssh_params).get(router_name)
def _counter_dict(node, paths_and_keys): def _counter_dict(node, paths_and_keys):
""" """
:param node: a node element :param node: a node element
...@@ -38,7 +27,6 @@ def _ifc_name(ifc_node): ...@@ -38,7 +27,6 @@ def _ifc_name(ifc_node):
def physical_interface_counters(ifc_doc): def physical_interface_counters(ifc_doc):
def _brian_counters(ifc_node): def _brian_counters(ifc_node):
ingress_fields = [ ingress_fields = [
{ {
...@@ -218,7 +206,6 @@ def physical_interface_counters(ifc_doc): ...@@ -218,7 +206,6 @@ def physical_interface_counters(ifc_doc):
def logical_interface_counters(ifc_doc): def logical_interface_counters(ifc_doc):
def _brian_counters(ifc_node): def _brian_counters(ifc_node):
traffic = ( traffic = (
"traffic-statistics" "traffic-statistics"
......
brian_polling_manager/interface_stats/vendors/nokia.py
+ 14
− 13
View file @ b996e00c
# from lxml import etree def physical_interface_counters(ifc_doc):
# ...
# from brian_polling_manager.interface_stats.vendors import netconf
def get_interface_info_ncrpc(router_name, ssh_config, host_verify=False): def logical_interface_counters(ifc_doc):
# request = etree.Element('get-interface-information') ...
# request.append(etree.Element('extensive'))
# return netconf.netconf_rpc(
# router_name=router_name, def brian_points(router_fqdn, netconf_doc, timestamp, measurement_name):
# ssh_config=ssh_config, ...
# command=request,
# host_verify=host_verify)
assert False, 'to do' def error_points(router_fqdn, netconf_doc, timestamp, measurement_name):
...
test/interface_stats/test_interface_stats.py
+ 9
− 7
View file @ b996e00c
...@@ -12,12 +12,12 @@ from brian_polling_manager.interface_stats.services.writers import ( ...@@ -12,12 +12,12 @@ from brian_polling_manager.interface_stats.services.writers import (
import jsonschema import jsonschema
import pytest import pytest
from brian_polling_manager import influx from brian_polling_manager import influx
from brian_polling_manager.interface_stats import brian, cli, errors, vendors from brian_polling_manager.interface_stats import cli, vendors
from brian_polling_manager.interface_stats.services.netconf import ( from brian_polling_manager.interface_stats.services.netconf import (
JuniperNetconfProvider, JuniperNetconfProvider,
get_netconf_provider, get_netconf_provider,
) )
from brian_polling_manager.interface_stats.vendors import juniper from brian_polling_manager.interface_stats.vendors import common, juniper
from ncclient.operations.rpc import RPCReply from ncclient.operations.rpc import RPCReply
...@@ -102,8 +102,8 @@ def test_interface_counters(router_fqdn, interface_counter_calculator, get_netco ...@@ -102,8 +102,8 @@ def test_interface_counters(router_fqdn, interface_counter_calculator, get_netco
@pytest.fixture( @pytest.fixture(
params=[ params=[
(brian.counters, brian.BRIAN_COUNTER_DICT_SCHEMA), (common.brian_counters, common.BRIAN_COUNTER_DICT_SCHEMA),
(errors.counters, errors.ERROR_COUNTER_DICT_SCHEMA), (common.error_counters, common.ERROR_COUNTER_DICT_SCHEMA),
] ]
) )
def point_counter_calculator(request): def point_counter_calculator(request):
...@@ -120,7 +120,7 @@ def test_validate_point_counters( ...@@ -120,7 +120,7 @@ def test_validate_point_counters(
point_fun, schema = point_counter_calculator point_fun, schema = point_counter_calculator
if ( if (
interface_fun is juniper.logical_interface_counters interface_fun is juniper.logical_interface_counters
and point_fun is errors.counters and point_fun is common.error_counters
): ):
# We should not have any error counters for logical interfaces # We should not have any error counters for logical interfaces
pytest.skip() pytest.skip()
...@@ -149,7 +149,7 @@ def test_point_counters( ...@@ -149,7 +149,7 @@ def test_point_counters(
counters = list(point_fun(router_fqdn=router_fqdn, interface_counters=interfaces)) counters = list(point_fun(router_fqdn=router_fqdn, interface_counters=interfaces))
if ( if (
interface_fun is juniper.logical_interface_counters interface_fun is juniper.logical_interface_counters
and point_fun is errors.counters and point_fun is common.error_counters
): ):
# We should not have any error counters for logical interfaces # We should not have any error counters for logical interfaces
assert not counters assert not counters
...@@ -157,7 +157,7 @@ def test_point_counters( ...@@ -157,7 +157,7 @@ def test_point_counters(
assert counters assert counters
@pytest.fixture(params=[cli._brian_points, cli._error_points]) @pytest.fixture(params=[cli._juniper_brian_points, cli._juniper_error_points])
def influx_point_calculator(request): def influx_point_calculator(request):
return request.param return request.param
...@@ -191,6 +191,8 @@ def test_influx_points(router_fqdn, influx_point_calculator, get_netconf): ...@@ -191,6 +191,8 @@ def test_influx_points(router_fqdn, influx_point_calculator, get_netconf):
assert len(points) assert len(points)
for point in points: for point in points:
assert point["fields"] # any trivial points should already be filtered assert point["fields"] # any trivial points should already be filtered
assert point["measurement"] == "blah"
assert set(point["tags"]) == {"hostname", "interface_name"}
def test_main_for_all_juniper_routers(all_juniper_routers, data_dir): def test_main_for_all_juniper_routers(all_juniper_routers, data_dir):
......
test/interface_stats/test_interface_stats_e2e.py
+ 8
− 7
View file @ b996e00c
...@@ -8,10 +8,11 @@ import subprocess ...@@ -8,10 +8,11 @@ import subprocess
import tempfile import tempfile
import time import time
from unittest.mock import patch from unittest.mock import patch
from brian_polling_manager.interface_stats.vendors import common
from click.testing import CliRunner from click.testing import CliRunner
from typing import Any, Dict from typing import Any, Dict
from brian_polling_manager import influx from brian_polling_manager import influx
from brian_polling_manager.interface_stats import brian, cli, click_helpers, errors from brian_polling_manager.interface_stats import cli, click_helpers
import concurrent.futures import concurrent.futures
import pytest import pytest
import yaml import yaml
...@@ -262,7 +263,7 @@ def app_config_params(free_host_port): ...@@ -262,7 +263,7 @@ def app_config_params(free_host_port):
@pytest.mark.skipif( @pytest.mark.skipif(
not _use_docker_compose(), reason="docker compose not found or disabled" not _use_docker_compose(), reason="docker compose not found or disabled"
) )
@patch.object(cli, 'setup_logging') @patch.object(cli, "setup_logging")
def test_e2e( def test_e2e(
unused_setup_logging, unused_setup_logging,
app_config_params: Dict[str, Any], app_config_params: Dict[str, Any],
...@@ -295,22 +296,22 @@ def test_e2e( ...@@ -295,22 +296,22 @@ def test_e2e(
expected_count_fields = {f"count_{n}" for n in expected_fields} expected_count_fields = {f"count_{n}" for n in expected_fields}
assert expected_count_fields <= set(counts.keys()) assert expected_count_fields.issubset(counts.keys())
assert all(counts[_f] > 0 for _f in expected_count_fields) assert all(counts[_f] > 0 for _f in expected_count_fields)
def _expected_fields_from_schema(schema): def _expected_fields_from_schema(schema):
assert ( assert (
schema == errors.ERROR_COUNTER_DICT_SCHEMA schema == common.ERROR_COUNTER_DICT_SCHEMA
or brian.BRIAN_COUNTER_DICT_SCHEMA or schema == common.BRIAN_COUNTER_DICT_SCHEMA
) )
return schema["properties"].keys() - {"hostname", "interface_name"} return schema["properties"].keys() - {"hostname", "interface_name"}
verify_influx_content( verify_influx_content(
influx_config=app_config_params["influx"]["brian-counters"], influx_config=app_config_params["influx"]["brian-counters"],
expected_fields=_expected_fields_from_schema(brian.BRIAN_COUNTER_DICT_SCHEMA), expected_fields=_expected_fields_from_schema(common.BRIAN_COUNTER_DICT_SCHEMA),
) )
verify_influx_content( verify_influx_content(
influx_config=app_config_params["influx"]["error-counters"], influx_config=app_config_params["influx"]["error-counters"],
expected_fields=_expected_fields_from_schema(errors.ERROR_COUNTER_DICT_SCHEMA), expected_fields=_expected_fields_from_schema(common.ERROR_COUNTER_DICT_SCHEMA),
) )
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment