Finished release 0.55.

Changelog.md

@@ -2,6 +2,9 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.55] - 2020-12-08
+- LGR-73: updated business logic for classifying LG routers public/internal
+
 ## [0.54] - 2020-10-07
 - DBOARD3-334: peer-info classifier performance improvement
 

inventory_provider/db/opsdb.py

@@ -400,27 +400,40 @@ SELECT
     g.longitude AS pop_longitude,
     p.country_code AS pop_country_code,
     g.country AS pop_country,
-    g.city AS pop_city
+    e.model AS equipment_model
 FROM
     opsdb.equipment e
     LEFT JOIN opsdb.pop p ON p.absid = e.PTR_pop
     LEFT JOIN opsdb.geocoding g ON g.absid = p.PTR_geocoding
     LEFT JOIN opsdb.organisation o ON o.absid = e.PTR_owner
 WHERE
-    e.model LIKE 'mx%'
+    e.manufacturer = 'Juniper'
     AND e.status = 'Operational'
     AND o.name = 'DANTE / GEANT'
-    AND NOT (e.name REGEXP 'vpn-proxy|vrr|taas')
+    AND NOT (e.name REGEXP 'vpn-proxy|vrr|taas|junosspace')
 """
 
+    def _public(row):
+        # cf. LGR-73
+
+        if row['pop_name'] in INTERNAL_POP_NAMES:
+            return False
+
+        router_name_lower = row['router_name'].lower()
+        if any(router_name_lower.startswith(prefix)
+                for prefix in ['srx', 'gts', 'sw', 'qfx']):
+            return False
+
+        if row['pop_city'].lower() == 'slough' \
+                and not row['equipment_model'].upper().startswith('MX'):
+            return False
+
+        return True
+
     def _row2rsp(row):
-        print(row)
         return {
             'equipment name': row['router_name'],
-            'type':
-                'INTERNAL'
-                if row['pop_name'] in INTERNAL_POP_NAMES
-                else 'CORE',
+            'type': 'CORE' if _public(row) else 'INTERNAL',
             'pop': {
                 'name': row['pop_name'],
                 'city': row['pop_city'],

inventory_provider/routes/lg.py

 import json
 
-from flask import Blueprint, jsonify, Response
+from flask import Blueprint, jsonify, Response, current_app
 
 from inventory_provider.routes import common
 
@@ -25,29 +25,25 @@ def routers(access):
             status=404,
             mimetype='text/html')
 
-    redis = common.get_current_redis()
-
     def _visible(router):
         if access == ACCESS_INTERNAL:
             return True
         return router['type'] == 'CORE'
 
     def _routers():
-        i = 0
-        for k in redis.scan_iter('opsdb:lg:*'):
-            rtr = redis.get(k.decode('utf-8')).decode('utf-8')
-            rtr = json.loads(rtr)
-            i += 1
-            if _visible(rtr):
-                yield rtr
+        for doc in common.load_json_docs(
+                config_params=current_app.config['INVENTORY_PROVIDER_CONFIG'],
+                key_pattern='opsdb:lg:*'):
+            yield doc['value']
 
+    redis = common.get_current_redis()
     cache_key = f'classifier-cache:lg:{access}'
     result = redis.get(cache_key)
 
     if result:
         result = json.loads(result.decode('utf-8'))
     else:
-        result = list(_routers())
+        result = list(filter(_visible, _routers()))
         # cache this data for the next call
         redis.set(cache_key, json.dumps(result).encode('utf-8'))
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='inventory-provider',
-    version="0.54",
+    version="0.55",
     author='GEANT',
     author_email='swd@geant.org',
     description='Dashboard inventory provider',

test/test_opsdb_queries.py

@@ -281,6 +281,26 @@ def test_lookup_lg_routers(connection, cached_test_data):
     jsonschema.validate(routers, LG_ROUTERS_SCHEMA)
     assert routers  # shouldn't be empty
 
+    # LGR-73
+    assert any(
+        r['equipment name'].startswith('sw2.am.office') for r in routers)
+    assert any(
+        r['equipment name'].startswith('sw3.am.office') for r in routers)
+
+    switches = filter(
+        lambda r: r['equipment name'].startswith('sw'), routers)
+    assert all(s['type'] == 'INTERNAL' for s in switches)
+
+    assert not any(
+        r['equipment name'].lower().startswith('junosspace') for r in routers)
+
+    internal_prefix_patterns = ['srx', 'gts', 'qfx']
+    for prefix in internal_prefix_patterns:
+        filtered = filter(
+            lambda r: r['equipment name'].lower().startswith(prefix), routers)
+        assert all(r['type'] == 'INTERNAL' for r in filtered), \
+            f'not all {prefix}* routers are INTERNAL'
+
 
 CIRCUIT_INFO_SCHEMA = {
     "$schema": "http://json-schema.org/draft-07/schema#",

tox.ini

@@ -11,7 +11,7 @@ deps =
 
 commands =
     coverage erase
-    coverage run --source inventory_provider -m py.test {posargs}
+    coverage run --source inventory_provider --omit='inventory_provider/routes/ims*,inventory_provider/db/ims*,inventory_provider/tasks/ims*' -m py.test {posargs}
     coverage xml
     coverage html
     coverage report --fail-under 75