small improvement in complexity

brian_polling_manager/interface_stats/__init__.py

@@ -88,11 +88,21 @@ PHYSICAL_INTERFACE_COUNTER_SCHEMA = {
             'required': ['broadcast', 'multicast'],
             'additionalProperties': False
         },
+        'l2-pcs-counters': {
+            'type': 'object',
+            'properties': {
+                'bit-error-seconds': {'type': 'integer'},
+                'errored-blocks-seconds': {'type': 'integer'}
+            },
+            'required': ['bit-error-seconds', 'errored-blocks-seconds'],
+            'additionalProperties': False
+        },
         'l2-counters': {
             'type': 'object',
             'properties': {
                 'ingress': {'$ref': '#/definitions/l2-dir-counters'},
                 'egress': {'$ref': '#/definitions/l2-dir-counters'},
+                'pcs': {'$ref': '#/definitions/l2-pcs-counters'},
             },
             'required': ['ingress', 'egress'],
             'additionalProperties': False

brian_polling_manager/interface_stats/errors.py

+
+
+
+
+ERROR_POINT_SCHEMA = {
+    '$schema': 'https://json-schema.org/draft/2020-12/schema',
+
+    'type': 'object',
+    'properties': {
+        'hostname': {'type': 'string'},
+        'interface_name': {'type': 'string'},
+        'framing-errors': {'type': 'integer'},
+        'bit-error-seconds': {'type': 'integer'},
+        'input-crc-errors': {'type': 'integer'},
+        'input-total-errors': {'type': 'integer'},
+        'input-discards': {'type': 'integer'},
+        'input-drops': {'type': 'integer'},
+        'output-drops': {'type': 'integer'},
+    },
+    'required': ['hostname', 'interface_name', 'egressOctets', 'ingressOctets'],
+    'additionalProperties': False
+}
+
+def points(router_fqdn, interface_counters):
+    """
+    :param router_fqdn: hostname
+    :param interface_counters: either PHYSICAL_INTERFACE_COUNTER_SCHEMA
+            or LOGICAL_INTERFACE_COUNTER_SCHEMA
+    :return: iterable of BRIAN_POINT_SCHEMA objects
+    """
+
+    def _counters2point(_ifc):
+        _brct = _ifc['brian']
+        _p = {
+            'hostname': router_fqdn,
+            'interface_name': _ifc['name'],
+            'egressOctets': _brct['egress']['bytes'],
+            'ingressOctets': _brct['ingress']['bytes']
+        }
+
+        if 'v6' in _brct['egress']:
+            _p['egressOctetsv6'] = _brct['egress']['v6']['bytes']
+
+        if 'v6' in _brct['ingress']:
+            _p['ingressOctetsv6'] = _brct['ingress']['v6']['bytes']
+
+        if 'errors' in _brct['egress']:
+            _p['egressErrors'] = _brct['egress']['errors']['errors']
+
+        if 'errors' in _brct['ingress']:
+            _p['ingressDiscards'] = _brct['ingress']['errors']['discards']
+            _p['ingressErrors'] = _brct['ingress']['errors']['errors']
+
+        return _p
+
+    return map(_counters2point, interface_counters)

brian_polling_manager/interface_stats/juniper.py

@@ -65,6 +65,25 @@ def _elem_int(node, name):
     return int(_subelems[0])
 
 
+def _counter_dict(node, paths_and_keys):
+    """
+    :param node: a node element
+    :param paths_and_keys: list of dicts {'path': str, 'key': str, 'required': boolean (optional)}
+    :return:
+    """
+    result = {}
+    for pk in paths_and_keys:
+        _elems = node.xpath(pk['path'] + '/text()')
+        if _elems:
+            if len(_elems) > 1:
+                logger.warning(f'found more than one {pk["path"]} in {node}')
+            result[pk['key']] = int(_elems[0])
+        else:
+            if pk.get('required', False):
+                assert False, f'path {pk["path"]}  not found in {node}'
+    return result
+
+
 def _ifc_name(ifc_node):
     return ifc_node.xpath('./name/text()')[0].strip()
 
@@ -77,88 +96,109 @@ def physical_interface_counters(ifc_doc):
         return any(map(name.startswith, log_prefixes))
 
     def _brian_counters(ifc_node):
-        tr_stats = ifc_node.xpath('./traffic-statistics')[0]
-        v6_stats = ifc_node.xpath('./traffic-statistics/ipv6-transit-statistics')[0]
+        ingress_fields = [
+            {'path': './traffic-statistics/input-bytes', 'key': 'bytes', 'required': True},
+            {'path': './traffic-statistics/input-packets', 'key': 'packets', 'required': True},
+        ]
+        egress_fields = [
+            {'path': './traffic-statistics/output-bytes', 'key': 'bytes', 'required': True},
+            {'path': './traffic-statistics/output-packets', 'key': 'packets', 'required': True},
+        ]
 
         baseline_stats = {
-            'ingress': {
-                'bytes': _elem_int(tr_stats, 'input-bytes'),
-                'packets': _elem_int(tr_stats, 'input-packets'),
-                'v6': {
-                    'bytes': _elem_int(v6_stats, 'input-bytes'),
-                    'packets': _elem_int(v6_stats, 'input-packets'),
-                }
-            },
-            'egress': {
-                'bytes': _elem_int(tr_stats, 'output-bytes'),
-                'packets': _elem_int(tr_stats, 'output-packets'),
-                'v6': {
-                    'bytes': _elem_int(v6_stats, 'output-bytes'),
-                    'packets': _elem_int(v6_stats, 'output-packets'),
-                }
-            }
+            'ingress': _counter_dict(ifc_node, ingress_fields),
+            'egress': _counter_dict(ifc_node, egress_fields)
         }
 
-        in_errors = ifc_node.xpath('./input-error-list')
-        out_errors = ifc_node.xpath('./output-error-list')
+        v6_ingress_fields = [
+            {'path': './traffic-statistics/ipv6-transit-statistics/input-bytes', 'key': 'bytes'},
+            {'path': './traffic-statistics/ipv6-transit-statistics/input-packets', 'key': 'packets'},
+        ]
+        v6_egress_fields = [
+            {'path': './traffic-statistics/ipv6-transit-statistics/output-bytes', 'key': 'bytes'},
+            {'path': './traffic-statistics/ipv6-transit-statistics/output-packets', 'key': 'packets'},
+        ]
+
+        v6 = _counter_dict(ifc_node, v6_ingress_fields)
+        if v6:
+            baseline_stats['ingress']['v6'] = v6
+        v6 = _counter_dict(ifc_node, v6_egress_fields)
+        if v6:
+            baseline_stats['egress']['v6'] = v6
+
+        ingress_error_fields = [
+            {'path': './input-error-list/input-errors', 'key': 'errors'},
+            {'path': './input-error-list/input-drops', 'key': 'drops'},
+            {'path': './input-error-list/input-resource-errors', 'key': 'resource'},
+            {'path': './input-error-list/input-discards', 'key': 'discards'},
+            {'path': './input-error-list/input-fifo-errors', 'key': 'fifo'},
+        ]
+        egress_error_fields = [
+            {'path': './output-error-list/output-errors', 'key': 'errors'},
+            {'path': './output-error-list/output-drops', 'key': 'drops'},
+            {'path': './output-error-list/output-resource-errors', 'key': 'resource'},
+            {'path': './output-error-list/output-fifo-errors', 'key': 'fifo'},
+            {'path': './output-error-list/output-collisions', 'key': 'collisions'},
+        ]
+        errors = _counter_dict(ifc_node, ingress_error_fields)
+        if errors:
+            baseline_stats['ingress']['errors'] = errors
+        errors = _counter_dict(ifc_node, egress_error_fields)
+        if errors:
+            baseline_stats['egress']['errors'] = errors
 
-        if in_errors and out_errors:
-            in_errors = in_errors[0]
-            out_errors = out_errors[0]
+        return baseline_stats
 
-            baseline_stats['ingress']['errors'] = {
-                'errors': _elem_int(in_errors, 'input-errors'),
-                'drops': _elem_int(in_errors, 'input-drops'),
-                'resource': _elem_int(in_errors, 'input-resource-errors'),
-                'discards': _elem_int(in_errors, 'input-discards'),
-            }
-            baseline_stats['egress']['errors'] = {
-                'errors': _elem_int(out_errors, 'output-errors'),
-                'drops': _elem_int(out_errors, 'output-drops'),
-                'resource': _elem_int(out_errors, 'output-resource-errors')
-            }
+    def _l2_counters(ifc_node):
 
-            if not _is_logical(ifc_node):
-                baseline_stats['ingress']['errors']['fifo'] = _elem_int(in_errors, 'input-fifo-errors')
-                baseline_stats['egress']['errors']['fifo'] = _elem_int(out_errors, 'output-fifo-errors')
-                baseline_stats['egress']['errors']['collisions'] = _elem_int(out_errors, 'output-collisions')
+        ingress_fields = [
+            {'path': '././ethernet-mac-statistics/input-broadcasts', 'key': 'broadcast', 'required': True},
+            {'path': '././ethernet-mac-statistics/input-multicasts', 'key': 'multicast', 'required': True},
+            {'path': '././ethernet-mac-statistics/input-bytes', 'key': 'bytes'},
+            {'path': '././ethernet-mac-statistics/input-packets', 'key': 'packets'},
+            {'path': '././ethernet-mac-statistics/input-unicasts', 'key': 'unicast'},
+        ]
 
-        return baseline_stats
+        egress_fields = [
+            {'path': '././ethernet-mac-statistics/output-broadcasts', 'key': 'broadcast', 'required': True},
+            {'path': '././ethernet-mac-statistics/output-multicasts', 'key': 'multicast', 'required': True},
+            {'path': '././ethernet-mac-statistics/output-bytes', 'key': 'bytes'},
+            {'path': '././ethernet-mac-statistics/output-packets', 'key': 'packets'},
+            {'path': '././ethernet-mac-statistics/output-unicasts', 'key': 'unicast'},
 
-    def _l2_counters(ifc_node):
-        mac_stats = ifc_node.xpath('./ethernet-mac-statistics')[0]
+        ]
 
         baseline_stats = {
-            'ingress': {
-                'broadcast': _elem_int(mac_stats, 'input-broadcasts'),
-                'multicast': _elem_int(mac_stats, 'input-multicasts')
-            },
-            'egress': {
-                'broadcast': _elem_int(mac_stats, 'output-broadcasts'),
-                'multicast': _elem_int(mac_stats, 'output-multicasts')
-            }
+            'ingress': _counter_dict(ifc_node, ingress_fields),
+            'egress': _counter_dict(ifc_node, egress_fields)
         }
 
-        if not _is_logical(ifc_node):
-            baseline_stats['ingress']['bytes'] = _elem_int(mac_stats, 'input-bytes')
-            baseline_stats['ingress']['packets'] = _elem_int(mac_stats, 'input-packets')
-            baseline_stats['ingress']['unicast'] = _elem_int(mac_stats, 'input-unicasts')
-            baseline_stats['ingress']['errors'] = {
+        pcs_stats_fields = [
+            {'path': '././ethernet-pcs-statistics/bit-error-seconds', 'key': 'bit-error-seconds'},
+            {'path': '././ethernet-pcs-statistics/errored-blocks-seconds', 'key': 'errored-blocks-seconds'},
+        ]
+        pcs = _counter_dict(ifc_node, pcs_stats_fields)
+        if pcs:
+            baseline_stats['pcs'] = pcs
+
+        ingress_error_fields = [
             # this one isn't available on qfx's
             # 'total': _elem_int(mac_stats, 'input-total-errors'),
-                'crc': _elem_int(mac_stats, 'input-crc-errors'),
-                'fifo': _elem_int(mac_stats, 'input-fifo-errors'),
-            }
+            {'path': '././ethernet-mac-statistics/input-crc-errors', 'key': 'crc'},
+            {'path': '././ethernet-mac-statistics/input-fifo-errors', 'key': 'fifo'},
+        ]
+        egress_error_fields = [
+            # 'total': _elem_int(mac_stats, 'input-total-errors'),
+            {'path': '././ethernet-mac-statistics/output-crc-errors', 'key': 'crc'},
+            {'path': '././ethernet-mac-statistics/output-fifo-errors', 'key': 'fifo'},
+        ]
 
-            baseline_stats['egress']['bytes'] = _elem_int(mac_stats, 'output-bytes')
-            baseline_stats['egress']['packets'] = _elem_int(mac_stats, 'output-packets')
-            baseline_stats['egress']['unicast'] = _elem_int(mac_stats, 'output-unicasts')
-            baseline_stats['egress']['errors'] = {
-                # this one isn't available on qfx's
-                # 'total': _elem_int(mac_stats, 'output-total-errors'),
-                'crc': _elem_int(mac_stats, 'output-crc-errors'),
-                'fifo': _elem_int(mac_stats, 'output-fifo-errors'),
-            }
+        errors = _counter_dict(ifc_node, ingress_error_fields)
+        if errors:
+            baseline_stats['ingress']['errors'] = errors
+        errors = _counter_dict(ifc_node, egress_error_fields)
+        if errors:
+            baseline_stats['egress']['errors'] = errors
 
         return baseline_stats
 
@@ -187,32 +227,37 @@ def physical_interface_counters(ifc_doc):
 def logical_interface_counters(ifc_doc):
 
     def _brian_counters(ifc_node):
-        tr_stats = ifc_node.xpath('./lag-traffic-statistics/lag-bundle') or ifc_node.xpath('./traffic-statistics')
-        if not tr_stats:
-            assert False, f'no traffic-statistics for {ifc_node}'
-        tr_stats = tr_stats[0]
+        traffic = 'traffic-statistics' if ifc_node.xpath('./traffic-statistics') else 'lag-traffic-statistics/lag-bundle'
+
+        ingress_fields = [
+            {'path': f'./{traffic}/input-bytes', 'key': 'bytes', 'required': True},
+            {'path': f'./{traffic}/input-packets', 'key': 'packets', 'required': True},
+        ]
+        egress_fields = [
+            {'path': f'./{traffic}/output-bytes', 'key': 'bytes', 'required': True},
+            {'path': f'./{traffic}/output-packets', 'key': 'packets', 'required': True},
+        ]
 
         baseline_stats = {
-            'ingress': {
-                'bytes': _elem_int(tr_stats, 'input-bytes'),
-                'packets': _elem_int(tr_stats, 'input-packets'),
-            },
-            'egress': {
-                'bytes': _elem_int(tr_stats, 'output-bytes'),
-                'packets': _elem_int(tr_stats, 'output-packets')
-            }
+            'ingress': _counter_dict(ifc_node, ingress_fields),
+            'egress': _counter_dict(ifc_node, egress_fields)
         }
 
-        v6_stats = ifc_node.xpath('./traffic-statistics/ipv6-transit-statistics')
-        if v6_stats:
-            baseline_stats['ingress']['v6'] = {
-                'bytes': _elem_int(v6_stats[0], 'input-bytes'),
-                'packets': _elem_int(v6_stats[0], 'input-packets'),
-            }
-            baseline_stats['egress']['v6'] = {
-                'bytes': _elem_int(v6_stats[0], 'output-bytes'),
-                'packets': _elem_int(v6_stats[0], 'output-packets'),
-            }
+        v6_ingress_fields = [
+            {'path': './traffic-statistics/ipv6-transit-statistics/input-bytes', 'key': 'bytes'},
+            {'path': './traffic-statistics/ipv6-transit-statistics/input-packets', 'key': 'packets'},
+        ]
+        v6_egress_fields = [
+            {'path': './traffic-statistics/ipv6-transit-statistics/output-bytes', 'key': 'bytes'},
+            {'path': './traffic-statistics/ipv6-transit-statistics/output-packets', 'key': 'packets'},
+        ]
+
+        v6 = _counter_dict(ifc_node, v6_ingress_fields)
+        if v6:
+            baseline_stats['ingress']['v6'] = v6
+        v6 = _counter_dict(ifc_node, v6_egress_fields)
+        if v6:
+            baseline_stats['egress']['v6'] = v6
 
         return baseline_stats
 

test/test_interface_stats.py

@@ -48,7 +48,7 @@ def test_validate_physical_counter_schema(router_fqdn, ifc_netconf_rpc):
 
 
 @pytest.mark.parametrize('router_fqdn', ROUTERS)
-def test_logical_counters(router_fqdn, ifc_netconf_rpc):
+def test_validate_logical_counters_schema(router_fqdn, ifc_netconf_rpc):
     doc = juniper.get_interface_info_ncrpc(router_fqdn)
     for ifc in juniper.logical_interface_counters(doc):
         jsonschema.validate(ifc, LOGICAL_INTERFACE_COUNTER_SCHEMA)
@@ -123,6 +123,7 @@ def test_physical_brian_points(router_fqdn, ifc_netconf_rpc):
     doc = juniper.get_interface_info_ncrpc(router_fqdn)
     interfaces = juniper.physical_interface_counters(doc)
     for point in brian.points(router_fqdn=router_fqdn, interface_counters=interfaces):
+        print(point)
         jsonschema.validate(point, brian.BRIAN_POINT_SCHEMA)