diff --git a/requirements.txt b/requirements.txt index 719a1d2fc226e3c9b9b41a5e606c192edf14bfeb..f2f76e1e9eb2ab7fc19646125e129eefff059a7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,16 @@ jsonschema requests sqlalchemy alembic + +httpx +fastapi +pydantic +uvicorn[standard] + mysqlclient -click junos-eznc pytest +responses sphinx sphinx-rtd-theme diff --git a/resource_management/__init__.py b/resource_management/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1589417bb4e9d55b64017dcb53a4a653fe75d8cf 100644 --- a/resource_management/__init__.py +++ b/resource_management/__init__.py @@ -0,0 +1,58 @@ +""" +automatically invoked app factory +""" +import logging + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from resource_management import environment +from resource_management import config + +from resource_management.routes import default +from resource_management.routes import interfaces + + +def create_app(): + """ + overrides default settings with those found + in the file read from env var SETTINGS_FILENAME + + :return: a new flask app instance + """ + + app = FastAPI() + # app = FastAPI(dependencies=[Depends(get_query_token)]) + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + app.include_router( + default.router, + prefix='/api', + # tags="default"], + # dependencies=[Depends(get_token_header)], + # responses={418: {"description": "I'm a teapot"}}, + ) + + app.include_router( + interfaces.router, + prefix='/api/interfaces', + # tags=["trunk"], + # dependencies=[Depends(get_token_header)], + # responses={418: {"description": "I'm a teapot"}}, + ) + + # test that config params are available and can be loaded + config.load() + + logging.info('FastAPI app initialized') + + environment.setup_logging() + + return app diff --git a/resource_management/cli.py b/resource_management/cli.py deleted file mode 100644 index 9936f42f77a25058ba6fab427fca934e220c034a..0000000000000000000000000000000000000000 --- a/resource_management/cli.py +++ /dev/null @@ -1,127 +0,0 @@ -""" - -script name [TODO] -==================== - -TODO: add a console script for this - -.. code-block:: bash - - Usage: script name [OPTIONS] - - notes: - first call init_db - (dns=mysql_dsn(params from config)) - then create - a session (as in unit test) - ... and perform the db queries - - Options: - --config FILENAME config filename [required] - --fqdn TEXT config filename [required] - --help Show this message and exit. - -The configuration filename must be formatted according -to this schema: - - .. asjson:: - resource_management.config.CONFIG_SCHEMA - - -""" -import json -import click -import logging -from jsonschema import validate, ValidationError - -from resource_management.hardware.juniper \ - import load_router_ports, LINE_CARDS_LIST_SCHEMA -from resource_management import db, config, environment -from resource_management.db import model - -environment.setup_logging() -logger = logging.getLogger(__name__) - - -def _save_router_info(fqdn, fpcs): - - with db.session_scope() as session: - - node = model.Node(fqdn=fqdn) - session.add(node) - - for _line_card in fpcs: - - line_card_record = model.LineCard( - model=_line_card['model'], - slot=_line_card['slot'], - serial=_line_card['serial'], - description=_line_card['description'], - node=node) - session.add(line_card_record) - - for port in _line_card['ports']: - - port_record = model.Port( - pic=port['pic'], - position=port['position'], - interface=port.get('interface', None), - cable=port.get('cable', None), - line_card=line_card_record) - session.add(port_record) - - for speed in port.get('speeds', []): - session.add( - model.PortSpeedCapability( - speed=speed, - port=port_record)) - - logger.info(f'saved router information: {fqdn}') - - -def _validate_config(_unused_ctx, _unused_param, file): - try: - params = config.load(file) - logger.info("Loaded params from config file: {0}".format(params)) - return params - except json.JSONDecodeError: - raise click.BadParameter('config file is not valid json') - except ValidationError as e: - raise click.BadParameter(e.message) - - -@click.command() -@click.option( - '--config', - required=True, - type=click.File('r'), - help='config filename', - callback=_validate_config) -@click.option( - '--fqdn', - required=True, - type=click.STRING, - help='config filename') -def cli(config, fqdn): - """ - notes: - - first call init_db(dns=mysql_dsn(params from config)) - - then create a session (as in unit test test_db_model.py) - - ... and perform the db queries - """ - - line_cards = list(load_router_ports(fqdn, config['ssh'])) - # sanity check, shouldn't fail ... otherwise die badly - validate(line_cards, LINE_CARDS_LIST_SCHEMA) - - mysql_config = config["mysql"] - dsn = db.mysql_dsn( - username=mysql_config["username"], - password=mysql_config["password"], - hostname=mysql_config["hostname"], - db_name=mysql_config["dbname"]) - - db.init_db_model(dsn) - - _save_router_info(fqdn, line_cards) - - -if __name__ == '__main__': - cli() diff --git a/resource_management/config.py b/resource_management/config.py index 041b26c9dc2e4e1ba66e5ddf391603ff72d114cb..67f9df25153fa00fc0597fa65fb3d5a321856ebd 100644 --- a/resource_management/config.py +++ b/resource_management/config.py @@ -1,21 +1,11 @@ import json import jsonschema +import os CONFIG_SCHEMA = { '$schema': 'http://json-schema.org/draft-07/schema#', 'definitions': { - 'database-credentials': { - 'type': 'object', - 'properties': { - 'hostname': {'type': 'string'}, - 'dbname': {'type': 'string'}, - 'username': {'type': 'string'}, - 'password': {'type': 'string'} - }, - 'required': ['hostname', 'dbname', 'username', 'password'], - 'additionalProperties': False - }, 'ssh-credentials': { 'type': 'object', 'properties': { @@ -41,15 +31,15 @@ CONFIG_SCHEMA = { 'type': 'object', 'properties': { - 'mysql': {'$ref': '#/definitions/database-credentials'}, + 'db': {'type': 'string'}, 'ssh': {'$ref': '#/definitions/ssh-credentials'} }, - 'required': ['mysql', 'ssh'], + 'required': ['db', 'ssh'], 'additionalProperties': False } -def load(f): +def load_from_file(f): """ Loads, validates and returns configuration parameters. @@ -63,3 +53,9 @@ def load(f): config = json.loads(f.read()) jsonschema.validate(config, CONFIG_SCHEMA) return config + + +def load(): + assert 'SETTINGS_FILENAME' in os.environ + with open(os.environ['SETTINGS_FILENAME']) as f: + return load_from_file(f) diff --git a/resource_management/db/__init__.py b/resource_management/db/__init__.py index 6b3d5f7d10462f13c5eaa4842e20972853b1e155..661cc76cc4d2782824633e99f7657199429a790b 100644 --- a/resource_management/db/__init__.py +++ b/resource_management/db/__init__.py @@ -39,6 +39,8 @@ def init_db_model(dsn): # cf. https://docs.sqlalchemy.org/en # /latest/orm/extensions/automap.html engine = create_engine( - dsn, pool_size=10, max_overflow=0, pool_recycle=3600) + dsn, pool_size=10, pool_recycle=3600) + # engine = create_engine( + # dsn, pool_size=10, max_overflow=0, pool_recycle=3600) _SESSION_MAKER = sessionmaker(bind=engine) diff --git a/resource_management/db/model.py b/resource_management/db/model.py index 75599c45513e97d75ea906764f7f2f41abdad345..4203d874debb734dc58298affb2d93046354c59e 100644 --- a/resource_management/db/model.py +++ b/resource_management/db/model.py @@ -18,52 +18,72 @@ logger = logging.getLogger(__name__) base_schema: Any = declarative_base() -class Node(base_schema): - __tablename__ = 'nodes' +class Router(base_schema): + __tablename__ = 'routers' id = Column('id', Integer, primary_key=True) fqdn = Column('fqdn', String, nullable=False) - line_cards = relationship( - "LineCard", - back_populates="node", + + physical = relationship( + "PhysicalInterface", + back_populates="router", + cascade="all, delete, delete-orphan") + + lags = relationship( + "LAG", + back_populates="router", cascade="all, delete, delete-orphan") -class LineCard(base_schema): - __tablename__ = 'line_cards' +class LAG(base_schema): + __tablename__ = 'lags' id = Column('id', Integer, primary_key=True) - slot = Column('slot', Integer, nullable=False) - model = Column('model', String, nullable=False) - serial = Column('serial', String, nullable=False) - description = Column('description', String, nullable=False) - node_id = Column('node_id', Integer, ForeignKey('nodes.id')) - node = relationship('Node', back_populates='line_cards') - ports = relationship( - 'Port', - back_populates='line_card', + name = Column('name', String, nullable=False) + + router_id = Column( + 'router_id', + Integer, + ForeignKey('routers.id'), + nullable=False) + router = relationship('Router', back_populates='lags') + + physical = relationship( + "PhysicalInterface", + back_populates="lag", cascade="all, delete, delete-orphan") -class Port(base_schema): - __tablename__ = 'ports' +class PhysicalInterface(base_schema): + __tablename__ = 'physical_interfaces' id = Column('id', Integer, primary_key=True) - pic = Column('pic', Integer, nullable=False) - position = Column('position', Integer, nullable=False) - interface = Column('interface', Integer, nullable=True) - cable = Column('cable', Integer, nullable=True) - line_card_id = Column( - 'line_card_id', Integer, ForeignKey('line_cards.id')) - line_card = relationship('LineCard', back_populates='ports') - - speed_capabilities = relationship( - 'PortSpeedCapability', - back_populates='port', + name = Column('name', String, nullable=False) + + router_id = Column( + 'router_id', + Integer, + ForeignKey('routers.id'), + nullable=False) + router = relationship('Router', back_populates='physical') + + lag_id = Column( + 'lag_id', + Integer, + ForeignKey('lags.id'), + nullable=True) + lag = relationship('LAG', back_populates='physical') + + logical = relationship( + 'LogicalInterface', + back_populates='physical', cascade="all, delete, delete-orphan") -class PortSpeedCapability(base_schema): - __tablename__ = 'port_speed_capabilities' +class LogicalInterface(base_schema): + __tablename__ = 'logical_interfaces' id = Column('id', Integer, primary_key=True) - speed = Column('speed', String, nullable=False) - port_id = Column( - 'port_id', Integer, ForeignKey('ports.id')) - port = relationship('Port', back_populates='speed_capabilities') + name = Column('name', String, nullable=False) + physical_id = Column( + 'physical_id', + Integer, + ForeignKey('physical_interfaces.id'), + nullable=False) + physical = relationship('PhysicalInterface', back_populates='logical') diff --git a/resource_management/hardware/juniper.py b/resource_management/hardware/juniper.py index 460f4e4c0de169048265fecb69e439bb80959049..4c3eb4dc5f0e277b303d6b382b6d153cdb33a511 100644 --- a/resource_management/hardware/juniper.py +++ b/resource_management/hardware/juniper.py @@ -383,13 +383,14 @@ def load_installed_ethernet_ports(router: 'jnpr.junos.Device'): some routers in our network don't return int for mtu from that command, and the pyez view has some validation that fails - for each physical port on the router, yields a dict formatted like: - - { - 'name': str, # interface name + returns a dict formatted as: + { + physical interface name: { 'admin': bool, # admin status ('up' == True) 'oper': bool, # operational status ('up' == True) - } + } + } + """ catalog_spec = { @@ -413,12 +414,13 @@ def load_installed_ethernet_ports(router: 'jnpr.junos.Device'): catalog = loader.load(catalog_spec) PhysicalInterfacesTable = catalog['PhysicalInterfacesTable'] + interfaces = {} for ifc in PhysicalInterfacesTable(router).get(): - yield { - 'name': ifc.interface, + interfaces[ifc.interface] = { 'admin': ifc.admin == 'up', 'oper': ifc.admin == 'up' } + return interfaces def load_router_ports(hostname, ssh_config, port=830): diff --git a/resource_management/router_interfaces.py b/resource_management/router_interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..33fcc0761249212ccfc3e148685856b8caae4b81 --- /dev/null +++ b/resource_management/router_interfaces.py @@ -0,0 +1,71 @@ +import logging + +from resource_management.hardware import juniper +from resource_management import db, config +from resource_management.db import model + +logger = logging.getLogger(__name__) + + +def load_router_interfaces(fqdn: str): + """ + load all interface info from the router and + update the db with the current information + + TODO: merge, not just create new + """ + + params = config.load() + db.init_db_model(params['db']) + + with juniper.router( + hostname=fqdn, + port=830, # TODO: make this configurable? + ssh_config=params['ssh']) as dev: + + aggregates = juniper.load_aggregates(dev) + physical = juniper.load_installed_ethernet_ports(dev) + logical = juniper.load_logical_interfaces(dev) + + def _find_lag_name(interface_name): + for name, interfaces in aggregates.items(): + if interface_name in interfaces: + return name + return None + + with db.session_scope() as session: + + router_record = model.Router(fqdn=fqdn) + session.add(router_record) + + # add all the lag records + # keep a map for referencing below when adding physical interfaces + agg_records = {} + for a in aggregates.keys(): + record = model.LAG( + name=a, + router=router_record) + session.add(record) + agg_records[a] = record + + # add all the physical interface records + # keep a map for referencing below when adding logical interfaces + physical_records = {} + for physical_name, _ in physical.items(): + record = model.PhysicalInterface( + name=physical_name, + router=router_record) + lag_name = _find_lag_name(physical_name) + if lag_name: + record.lag = agg_records[lag_name] + physical_records[physical_name] = record + session.add(record) + + for physical_name, logical_names in logical.items(): + for ln in logical_names: + record = model.LogicalInterface( + name=ln, + physical=physical_records[physical_name]) + session.add(record) + + session.commit() diff --git a/resource_management/routes/__init__.py b/resource_management/routes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/resource_management/routes/default.py b/resource_management/routes/default.py new file mode 100644 index 0000000000000000000000000000000000000000..5231ffe5af71071993d5fa7d4af392f484fade57 --- /dev/null +++ b/resource_management/routes/default.py @@ -0,0 +1,23 @@ +import pkg_resources + +from fastapi import APIRouter +import pydantic + +API_VERSION = '0.1' +VERSION_STRING = pydantic.constr(regex=r'\d+\.\d+') + +router = APIRouter() + + +class Version(pydantic.BaseModel): + api: VERSION_STRING + module: VERSION_STRING + + +@router.get('/version') +def version() -> Version: + return { + 'api': API_VERSION, + 'module': + pkg_resources.get_distribution('resource-management').version + } diff --git a/resource_management/routes/interfaces.py b/resource_management/routes/interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..771bc2a0047238e701e8a8cad9a96c0bf75e20c6 --- /dev/null +++ b/resource_management/routes/interfaces.py @@ -0,0 +1,40 @@ +from fastapi import APIRouter, HTTPException +import pydantic + +from resource_management import db +from resource_management.db import model +from resource_management import router_interfaces + +router = APIRouter() + + +class InterfaceCounts(pydantic.BaseModel): + total: int + available: int + + +class InterfacesSummary(pydantic.BaseModel): + fqdn: str + lag: int # number of defined lag interfaces + physical: InterfaceCounts + + +@router.post('/import/{fqdn}') +async def load_router_interfaces(fqdn: str) -> InterfacesSummary: + + router_interfaces.load_router_interfaces(fqdn) + + with db.session_scope() as session: + router = session.query(model.Router).filter_by(fqdn=fqdn).one() + num_lags = len(router.lags) + num_physical = len(router.physical) + num_available_physical = sum(1 for p in router.physical if p.lag) + + return { + 'fqdn': fqdn, + 'lag': num_lags, + 'physical': { + 'total': num_physical, + 'available': num_available_physical + } + } diff --git a/test/conftest.py b/test/conftest.py index f768395899ac0096c64c1e16eda57a4dbbedc284..5090be14184b7080fbea743b8f7edbf5a605d338 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -12,7 +12,11 @@ from ncclient.manager import Manager, make_device_handler from ncclient.transport import SSHSession from ncclient.xml_ import NCElement +import httpx +from fastapi.testclient import TestClient + from resource_management.db import model +import resource_management RPC_DATA_DIR = os.path.join(os.path.dirname(__file__), 'rpc-data') HOSTNAMES = [ @@ -72,10 +76,17 @@ HOSTNAMES = [ ] + + @pytest.fixture def resources_db(): # cf. https://stackoverflow.com/a/33057675 + # engine = create_engine( + # 'sqlite://', + # connect_args={'check_same_thread': False}, + # poolclass=StaticPool, + # echo=False) engine = create_engine( 'sqlite://', connect_args={'check_same_thread': False}, @@ -86,8 +97,9 @@ def resources_db(): with patch( 'resource_management.db._SESSION_MAKER', - sessionmaker(bind=engine)): - yield # wait until caller context ends + new=sessionmaker(bind=engine)): + with patch('resource_management.db.init_db_model'): + yield # wait until caller context ends class _namespace(object): @@ -97,16 +109,10 @@ class _namespace(object): def pytest_generate_tests(metafunc): metafunc.parametrize("router_name", HOSTNAMES) - @pytest.fixture def config_data(router_name): return { - 'mysql': { - 'username': 'bogus-user', - 'password': 'bogus-password', - 'hostname': 'bogus hostname', - 'dbname': 'bogus database name' - }, + 'db': 'sqlite://', 'ssh': { 'username': 'another-bogus-user', 'private-key': 'bogus private key filename' @@ -115,19 +121,12 @@ def config_data(router_name): @pytest.fixture -def config_filename(config_data, router_name): - # for os indepence see: - # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file - # Generate a random temporary file name - file_name = os.path.join(tempfile.gettempdir(), os.urandom(24).hex()) - # Ensure the file is created - open(file_name, "x").close() - # Open the file in the given mode - tempFile = open(file_name, 'w') - - tempFile .write(json.dumps(config_data)) - tempFile .flush() - yield tempFile .name +def config_file(config_data, router_name): + with tempfile.NamedTemporaryFile(mode='w') as f: + f.write(json.dumps(config_data)) + f.flush() + os.environ['SETTINGS_FILENAME'] = f.name + yield f.name @pytest.fixture @@ -195,3 +194,10 @@ def mocked_router(netconf_rpc_replies): # it needs to be here ... with patch('jnpr.junos.device.Device.close'): yield # wait here until parent context ends + + + +@pytest.fixture +def client(config_file): + app = resource_management.create_app() + yield TestClient(app) # wait here until calling context ends diff --git a/test/test_cli.py b/test/test_cli.py index d3f8ca9b247bcc3e2b5752f29a5d64fffbd14c31..52d6ce80cf6e6943ddb47717c82632546d92921c 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -1,6 +1,6 @@ from unittest.mock import patch from click.testing import CliRunner -from resource_management.cli import cli +from resource_management.router_interfaces import cli @patch('resource_management.db.init_db_model') diff --git a/test/test_config.py b/test/test_config.py index 72ac321ed2878404399e53a90756fe7fd1f79a1f..2c89593260c32adfb0d5b84eaf4b0e44bc5af65c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -4,15 +4,13 @@ import json from resource_management import config -def test_config_with_file(config_filename): - # trivial sanity check on the config fixture - with open(config_filename) as f: - params = config.load(f) - assert params +def test_config_with_file(config_file): + params = config.load() + assert params def test_config(config_data): with io.StringIO(json.dumps(config_data)) as f: f.seek(0) # rewind file position to the beginning - params = config.load(f) + params = config.load_from_file(f) assert params diff --git a/test/test_db_model.py b/test/test_db_model.py index 7f9138b880423d435510e79b5df0b38872381ff1..23d4943e473f1aac537be0ef4cef8b792d11c7a1 100644 --- a/test/test_db_model.py +++ b/test/test_db_model.py @@ -1,6 +1,6 @@ from resource_management.db import model from resource_management.db import session_scope -from resource_management.cli import _save_router_info +from resource_management.router_interfaces import _save_router_info from resource_management.hardware.juniper import load_router_ports # diff --git a/test/test_interfaces_routes.py b/test/test_interfaces_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..09e1f933e8a4632ee47b5672d15cb5a57fb4c92e --- /dev/null +++ b/test/test_interfaces_routes.py @@ -0,0 +1,22 @@ +import jsonschema + +from resource_management.routes.default import Version +from resource_management.routes import interfaces + +def test_bad_method(client): + rv = client.post('/api/version') + assert rv.status_code == 405 + + +def test_version_request(client): + rv = client.get('/api/version') + assert rv.status_code == 200 + jsonschema.validate(rv.json(), Version.schema()) + + +def test_update_router_interfaces(client, resources_db, mocked_router, router_name): + rv = client.post(f'/api/interfaces/import/{router_name}') + assert rv.status_code == 200 + jsonschema.validate(rv.json(), interfaces.InterfacesSummary.schema()) + + diff --git a/test/test_parse_router.py b/test/test_parse_router.py index ca4ae63cc27e592b82b3de86855eb5247bdf14b9..6b79bc8bc90de1a90b3832b03a534054806bedca 100644 --- a/test/test_parse_router.py +++ b/test/test_parse_router.py @@ -4,7 +4,9 @@ from jsonschema import validate from lxml import etree from resource_management.hardware import juniper - +from resource_management import router_interfaces +from resource_management import db +from resource_management.db import model def _remove_ns(xml): # cf. https://stackoverflow.com/a/51972010 @@ -47,9 +49,9 @@ def test_router_ports(mocked_router, netconf_rpc_replies): == {f['slot'] for f in fpcs} -def test_new_load_interfaces(mocked_router, netconf_rpc_replies): +def test_no_db_load_interfaces(mocked_router): """ - renewed design for this app: no chassis info for now + new design for this app: no chassis info for now """ ssh = { @@ -67,9 +69,25 @@ def test_new_load_interfaces(mocked_router, netconf_rpc_replies): # print(len(config)) # just sanity check there are non-empty responses - physical = list(juniper.load_installed_ethernet_ports(dev)) + physical = juniper.load_installed_ethernet_ports(dev) assert physical logical = juniper.load_logical_interfaces(dev) assert logical aggregates = juniper.load_aggregates(dev) assert aggregates + + +def test_load_interfaces(mocked_router, resources_db, config_file, router_name): + """ + new design for this app: no chassis info for now + """ + + router_interfaces.load_router_interfaces(router_name) + + with db.session_scope() as session: + router = session.query(model.Router).filter_by(fqdn=router_name).one() + assert router + assert router.physical + assert router.lags + assert all(l.physical for l in router.lags) + assert any(p.logical for p in router.physical)