Skip to content
Snippets Groups Projects
Commit 6c918887 authored by Pelle Koster's avatar Pelle Koster
Browse files

implement email notification and finalize testing

parent 14ddc675
Branches
Tags
No related merge requests found
"""
Report Interface Errors
=======================
CLI tool for generating an interface error report and sending it by email to the OC.
Every day an email may be sent that summarizes interface errors for all (*) GEANT
routers and other network devices. The error report is sent as an html attachment to
that email. First all relevant routers and interfaces are requested from Inventory
Provider. Then InfluxDB is queried for the latest, and yesterday's, error measurement
points. For every interface, the latest error counts are compared against yesterday's
error count to determine whether it has suffered new errors. Currently the following
errors are checked:
* ``framing-errors``
* ``bit-error-seconds``
* ``errored-blocks-seconds``
* ``input-crc-errors``
* ``input-total-errors``
* ``input-discards``
* ``input-drops``
* ``output-drops``
For every interface with new errors is added to a summary report. This report is then
sent to the OC.
.. code-block:: bash
Usage: report-interface-errors [OPTIONS]
Options:
--config PATH Path to a config file for this tool. The schema this config
file must adhere to can be found in
``brian_polling_manager.error_report.config.ERROR_REPORT_CONFIG_SCHEMA``
[2024-04-09] This tool is the successor of a bash-script that was used before. That
script has some peculiarities in it's output and as of this new version mimics the
output of the earlier tool as much as possible.
*) There are some rules which routers/interfaces to include and exclude. See the
`get_relevant_interfaces`_ function for more details.
"""
from datetime import datetime from datetime import datetime
import json import json
import logging import logging
...@@ -6,16 +49,17 @@ import pathlib ...@@ -6,16 +49,17 @@ import pathlib
from typing import Sequence from typing import Sequence
from brian_polling_manager.interface_stats.services import influx_client from brian_polling_manager.interface_stats.services import influx_client
from brian_polling_manager.inventory import load_interfaces from brian_polling_manager.inventory import load_interfaces
import click
from influxdb import InfluxDBClient from influxdb import InfluxDBClient
from brian_polling_manager.error_report.config import load from brian_polling_manager.error_report.config import load
from brian_polling_manager.error_report.mailer import render_html from brian_polling_manager.error_report.report import (
render_email,
render_html,
send_email,
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
CONFIG_FILE = "/home/pellek/develop/klanten/geant/error_report-config.json"
INFLUX_TIME_WINDOW_TODAY = "time > now() - 1d"
INFLUX_TIME_WINDOW_YESTERDAY = "time < now() - 1d and time > now() - 2d"
# The error field names in the influx query vs their reporting name # The error field names in the influx query vs their reporting name
ERROR_FIELDS = { ERROR_FIELDS = {
...@@ -29,6 +73,8 @@ ERROR_FIELDS = { ...@@ -29,6 +73,8 @@ ERROR_FIELDS = {
"last_output_drops": "output-drops", "last_output_drops": "output-drops",
} }
INFLUX_TIME_WINDOW_TODAY = "time > now() - 1d"
INFLUX_TIME_WINDOW_YESTERDAY = "time < now() - 1d and time > now() - 2d"
PROCESSED_ERROR_COUNTERS_SCHEMA = { PROCESSED_ERROR_COUNTERS_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
...@@ -106,7 +152,7 @@ LOGGING_DEFAULT_CONFIG = { ...@@ -106,7 +152,7 @@ LOGGING_DEFAULT_CONFIG = {
"class": "logging.StreamHandler", "class": "logging.StreamHandler",
"level": "INFO", "level": "INFO",
"formatter": "simple", "formatter": "simple",
"stream": "ext://sys.stderr", "stream": "ext://sys.stdout",
}, },
}, },
"loggers": { "loggers": {
...@@ -276,27 +322,53 @@ def _filter_and_convert_interfaces(interfaces): ...@@ -276,27 +322,53 @@ def _filter_and_convert_interfaces(interfaces):
) )
def main(): def main(config: dict):
setup_logging() """Main function for the error reporting script
config = load(config_file=pathlib.Path(CONFIG_FILE))
:param config: An instance of `ERROR_REPORT_CONFIG_SCHEMA`
"""
logger.info(f"Retrieving interfaces from inventory provider: {config['inventory']}")
all_interfaces = get_relevant_interfaces(config["inventory"]) all_interfaces = get_relevant_interfaces(config["inventory"])
client = influx_client(config["influx"]) client = influx_client(config["influx"])
with client: with client:
logger.info("Retrieving error points from influxdb...")
all_error_counters = interface_errors( all_error_counters = interface_errors(
client, client,
interface_info=all_interfaces, interface_info=all_interfaces,
errors=ERROR_FIELDS, errors=ERROR_FIELDS,
exclusions=config["exclude-interfaces"], exclusions=config["exclude-interfaces"],
) )
body = render_html( logger.info("Generating report...")
all_error_counters,
date=datetime.utcnow().strftime("%a %d %b %H:%M:%S UTC %Y"),
)
# TODO: ensure data is from the day that we're interested in (today or yesterday) body = render_html(
# TODO: send script failures to admin email all_error_counters,
date=datetime.utcnow().strftime("%a %d %b %H:%M:%S UTC %Y"),
)
email = render_email(config["email"], html=body)
logger.info("Sending email...")
send_email(email, config=config["email"])
logger.info("Done!")
@click.command()
@click.option(
"-c",
"--config",
type=click.Path(
exists=True,
file_okay=True,
dir_okay=False,
readable=True,
path_type=pathlib.Path,
),
help="path to a config file",
)
def cli(config):
setup_logging()
config = load(config_file=config)
main(config)
if __name__ == "__main__": if __name__ == "__main__":
main() cli()
...@@ -4,7 +4,9 @@ ...@@ -4,7 +4,9 @@
"reply_to": "noreply@geant.org", "reply_to": "noreply@geant.org",
"to": "some-bogus-email", "to": "some-bogus-email",
"cc": "some-cc", "cc": "some-cc",
"contact": "someone@geant.org / NE team" "hostname": "some.smtp.server",
"username": "smtp-user",
"password": "smtp-password"
}, },
"inventory": ["blah"], "inventory": ["blah"],
"influx": { "influx": {
......
...@@ -6,7 +6,7 @@ import jsonschema ...@@ -6,7 +6,7 @@ import jsonschema
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
CONFIG_SCHEMA = { ERROR_REPORT_CONFIG_SCHEMA = {
"$schema": "https://json-schema.org/draft/2020-12/schema", "$schema": "https://json-schema.org/draft/2020-12/schema",
"definitions": { "definitions": {
"email-params": { "email-params": {
...@@ -14,12 +14,26 @@ CONFIG_SCHEMA = { ...@@ -14,12 +14,26 @@ CONFIG_SCHEMA = {
"properties": { "properties": {
"from": {"type": "string"}, "from": {"type": "string"},
"reply_to": {"type": "string"}, "reply_to": {"type": "string"},
"to": {"type": "string"}, "to": {"$ref": "#/definitions/string-or-array"},
"cc": {"type": "string"}, "cc": {"$ref": "#/definitions/string-or-array"},
"contact": {"type": "string"}, "hostname": {"type": "string"},
"username": {"type": "string"},
"password": {"type": "string"},
"starttls": {"type": "boolean"},
}, },
"required": [
"from",
"to",
"hostname",
],
"additionalProperties": False, "additionalProperties": False,
}, },
"string-or-array": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}, "minItems": 1},
]
},
"influx-db-measurement": { "influx-db-measurement": {
"type": "object", "type": "object",
"properties": { "properties": {
...@@ -71,6 +85,11 @@ def load(config_file: pathlib.Path): ...@@ -71,6 +85,11 @@ def load(config_file: pathlib.Path):
""" """
config = json.loads(config_file.read_text()) config = json.loads(config_file.read_text())
jsonschema.validate(config, CONFIG_SCHEMA) jsonschema.validate(config, ERROR_REPORT_CONFIG_SCHEMA)
# convert str addresses into list of str
if isinstance(config["email"]["to"], str):
config["email"]["to"] = [config["email"]["to"]]
if isinstance(config["email"].get("cc"), str):
config["email"]["cc"] = [config["email"]["cc"]]
return config return config
{#-
We have a lot of hardcoded stuff in here. Ideally we don't want to use jinja here
at all but use pythons EmailMessage or MIMEBase classes to construct the email.
However, there are a bunch of peculiarities in the email message that make it hard to
do this, unless we can simplify the email. But for now we stick to the original email
message format
-#}
From: {{ config.from }}
Sender: {{ config.from }}
{%- if config.reply_to is defined %}
Reply-To: {{ config.reply_to }}
{%- endif %}
To: {{ config.to | join('; ') }}
{%- if config.cc is defined %}
Cc: {{ config.cc | join('; ') }}
{%- endif %}
Mime-Version: 1.0
Subject: {{ subject }}
X-Mailer: /home/neteam/code/juniper_errors_report/email.sh
Content-Type: multipart/mixed; boundary="-"
---
Content-Type: text/plain; format=flowed; charset=ISO-8859-1
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
Hi,
The latest errors report is attached.
Regards,
neteam@neteam-server01.geant.org:/home/neteam/code/juniper_errors_report
---
Content-Type: text/plain; name="errors_report.html"
Content-Transfer-Encoding: base64
Content-Disposition: inline; filename="errors_report.html"
Content-MD5: {{ md5_hash }}
{{ b64_report | wordwrap(76) }}
\ No newline at end of file
import pathlib
import jinja2
THIS_DIR = pathlib.Path(__file__).parent
def send_email(errors, config):
pass
def render_html(errors, date):
"""
Render error struct to email body
:param errors: an instance of `PROCESSED_ERROR_COUNTERS_SCHEMA`
:param template_file: a jinja2 templat to rende
"""
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(THIS_DIR), newline_sequence='\r\n'
)
template = env.get_template("error_report.html.jinja2")
return template.render(**errors, date=date)
if __name__ == "__main__":
pass
import logging
import pathlib
import smtplib
import jinja2
import hashlib
import base64
THIS_DIR = pathlib.Path(__file__).parent
logger = logging.getLogger(__name__)
SMTP_TIMEOUT = 5 # s
def render_html(errors, date):
"""
Render error struct to email body
:param errors: an instance of `PROCESSED_ERROR_COUNTERS_SCHEMA`
:param template_file: a jinja2 template to render
"""
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(THIS_DIR), newline_sequence="\r\n"
)
template = env.get_template("error_report.html.jinja2")
return template.render(**errors, date=date)
def render_email(
email_config: dict, html: str, subject="GEANT Juniper Interface Errors Report"
):
env = jinja2.Environment(loader=jinja2.FileSystemLoader(THIS_DIR))
template = env.get_template("email.jinja2")
html_as_bytes = html.encode()
md5_hash = hashlib.md5(html_as_bytes).hexdigest()
b64_report = base64.b64encode(html_as_bytes).decode()
return template.render(
config=email_config, subject=subject, md5_hash=md5_hash, b64_report=b64_report
)
def send_email(payload: str, config: dict):
with smtplib.SMTP(host=config["hostname"], port=25, timeout=SMTP_TIMEOUT) as server:
if config.get("starttls"):
server.starttls()
if config.get("password"):
username = config.get("username", config["from"])
try:
server.login(username, config["password"])
except smtplib.SMTPNotSupportedError:
logger.warning(
"Authentication not supported, continuing without authentication. "
"Unset 'email.password' in the config to suppress this message"
)
recipients = [*config["to"], *config.get("cc", [])]
server.sendmail(config["from"], recipients, payload)
import base64
import hashlib
import json import json
import pathlib import pathlib
import smtplib
from unittest.mock import Mock, patch, call from unittest.mock import Mock, patch, call
import brian_polling_manager.error_report.report
from brian_polling_manager.error_report.mailer import render_html from brian_polling_manager.error_report.report import (
SMTP_TIMEOUT,
render_email,
render_html,
send_email,
)
import jsonschema import jsonschema
import pytest import pytest
from brian_polling_manager.error_report import cli, config from brian_polling_manager.error_report import cli, config
...@@ -20,6 +28,8 @@ from brian_polling_manager.error_report.cli import ( ...@@ -20,6 +28,8 @@ from brian_polling_manager.error_report.cli import (
select_error_fields, select_error_fields,
) )
from click.testing import CliRunner
DATA_DIR = pathlib.Path(__file__).parent / "data" DATA_DIR = pathlib.Path(__file__).parent / "data"
...@@ -39,11 +49,6 @@ def small_inventory(): ...@@ -39,11 +49,6 @@ def small_inventory():
return json.loads(((DATA_DIR / "small-inventory.json").read_text())) return json.loads(((DATA_DIR / "small-inventory.json").read_text()))
@pytest.fixture(params=["full_inventory", "small_inventory"])
def inventory(request):
return request.getfixturevalue(request.param)
@pytest.fixture @pytest.fixture
def mock_influx_client(): def mock_influx_client():
class FakeInfluxClient: class FakeInfluxClient:
...@@ -56,6 +61,9 @@ def mock_influx_client(): ...@@ -56,6 +61,9 @@ def mock_influx_client():
def __enter__(self): def __enter__(self):
pass pass
def __exit__(self, *args, **kwargs):
pass
def add_point(self, hostname, interface, timestamp, payload): def add_point(self, hostname, interface, timestamp, payload):
converted_payload = { converted_payload = {
self.INFLUX_ERROR_FIELDS[k]: v for k, v in payload.items() self.INFLUX_ERROR_FIELDS[k]: v for k, v in payload.items()
...@@ -102,12 +110,6 @@ def create_error_point(mock_influx_client): ...@@ -102,12 +110,6 @@ def create_error_point(mock_influx_client):
return _create return _create
@pytest.fixture
def mocked_load_interfaces(inventory):
with patch.object(cli, "load_interfaces", return_value=inventory) as mock:
yield mock
@pytest.fixture @pytest.fixture
def get_interface_errors(small_inventory, mock_influx_client): def get_interface_errors(small_inventory, mock_influx_client):
interfaces = _filter_and_convert_interfaces(small_inventory) interfaces = _filter_and_convert_interfaces(small_inventory)
...@@ -133,7 +135,10 @@ def test_validate_config(tmp_path): ...@@ -133,7 +135,10 @@ def test_validate_config(tmp_path):
"from": "noreply@geant.org", "from": "noreply@geant.org",
"reply_to": "noreply@geant.org", "reply_to": "noreply@geant.org",
"to": "some-bogus-email", "to": "some-bogus-email",
"contact": "someone@geant.org / NE team", "cc": ["recipient01@geant.org", "recipient02@geant.org"],
"hostname": "some.smtp.server",
"username": "smtp-user",
"password": "smtp-password",
}, },
"inventory": ["blah"], "inventory": ["blah"],
"influx": { "influx": {
...@@ -146,7 +151,27 @@ def test_validate_config(tmp_path): ...@@ -146,7 +151,27 @@ def test_validate_config(tmp_path):
"exclude-interfaces": ["SOME DESCRIPTION PART"], "exclude-interfaces": ["SOME DESCRIPTION PART"],
} }
config_file.write_text(json.dumps(content)) config_file.write_text(json.dumps(content))
assert config.load(config_file) == content result = config.load(config_file)
assert result == {
"email": {
"from": "noreply@geant.org",
"reply_to": "noreply@geant.org",
"to": ["some-bogus-email"],
"cc": ["recipient01@geant.org", "recipient02@geant.org"],
"hostname": "some.smtp.server",
"username": "smtp-user",
"password": "smtp-password",
},
"inventory": ["blah"],
"influx": {
"hostname": "hostname",
"database": "dbname",
"measurement": "errors",
"username": "some-username",
"password": "user-password",
},
"exclude-interfaces": ["SOME DESCRIPTION PART"],
}
def test_get_relevant_interfaces(full_inventory): def test_get_relevant_interfaces(full_inventory):
...@@ -469,3 +494,165 @@ Generated <some date> ...@@ -469,3 +494,165 @@ Generated <some date>
</body> </body>
</html>""" </html>"""
assert result == expected.replace("\n", "\r\n") assert result == expected.replace("\n", "\r\n")
def test_render_email():
body = "<SOME_BODY>"
md5 = hashlib.md5(body.encode()).hexdigest()
b64 = base64.b64encode(body.encode()).decode()
config = {"from": "someone@geant.org", "to": ["someone.else@geant.org"]}
result = render_email(config, html=body, subject="<subject>")
assert "From: someone@geant.org" in result
assert "To: someone.else@geant.org" in result
assert md5 in result
assert b64 in result
def test_render_email_for_multiple_recipients():
body = "<SOME_BODY>"
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org", "to2@geant.org"],
"cc": ["cc1@geant.org", "cc2@geant.org"],
}
result = render_email(config, html=body, subject="<subject>")
assert "To: someone.else@geant.org; to2@geant.org" in result
assert "Cc: cc1@geant.org; cc2@geant.org" in result
@patch.object(smtplib, "SMTP")
def test_send_email(SMTP):
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org"],
"hostname": "smtp.some.host",
}
send_email("<payload>", config)
assert SMTP.call_args == call(
host=config["hostname"], port=25, timeout=SMTP_TIMEOUT
)
assert SMTP().__enter__().sendmail.call_args == call(
config["from"], ["someone.else@geant.org"], "<payload>"
)
@patch.object(smtplib, "SMTP")
def test_send_email_to_multiple_recipients(SMTP):
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org", "to2@geant.org"],
"cc": ["cc1@geant.org", "cc2@geant.org"],
"hostname": "smtp.some.host",
}
send_email("<payload>", config)
assert SMTP().__enter__().sendmail.call_args == call(
config["from"],
["someone.else@geant.org", "to2@geant.org", "cc1@geant.org", "cc2@geant.org"],
"<payload>",
)
@patch.object(smtplib, "SMTP")
def test_send_email_without_authentication(SMTP):
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org"],
"hostname": "smtp.some.host",
}
send_email("<payload>", config)
assert not SMTP().__enter__().starttls.called
assert not SMTP().__enter__().login.called
@patch.object(smtplib, "SMTP")
def test_send_email_with_password(SMTP):
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org", "to2@geant.org"],
"cc": ["cc1@geant.org", "cc2@geant.org"],
"hostname": "smtp.some.host",
"password": "some-password",
}
send_email("<payload>", config)
assert SMTP().__enter__().login.call_args == call(
config["from"], config["password"]
)
@patch.object(smtplib, "SMTP")
def test_send_email_with_starttls(SMTP):
config = {
"from": "someone@geant.org",
"to": ["someone.else@geant.org", "to2@geant.org"],
"cc": ["cc1@geant.org", "cc2@geant.org"],
"hostname": "smtp.some.host",
"starttls": True,
}
send_email("<payload>", config)
assert SMTP().__enter__().starttls.called
@pytest.fixture
def config_file(tmp_path):
config = {
"email": {
"from": "noreply@geant.org",
"reply_to": "noreply@geant.org",
"to": "some-bogus-email",
"hostname": "some.smtp.server",
},
"inventory": ["blah"],
"influx": {
"hostname": "hostname",
"database": "dbname",
"measurement": "errors",
"username": "some-username",
"password": "user-password",
},
"exclude-interfaces": ["FOO"],
}
path = tmp_path / "config.json"
path.write_text(json.dumps(config))
return path
@patch.object(cli, "setup_logging")
@patch.object(smtplib, "SMTP")
def test_e2e(
SMTP,
unused_setup_logging,
mock_influx_client,
small_inventory,
config_file,
create_error_point,
):
create_error_point(
"mx1.ams.nl.geant.net", "ae1", "today", input_drops=1
)
with patch.object(
cli, "load_interfaces", return_value=small_inventory
), patch.object(cli, "influx_client", return_value=mock_influx_client):
runner = CliRunner()
result = runner.invoke(cli.cli, ["--config", str(config_file)])
assert result.exit_code == 0, str(result)
sendmail_call_args = SMTP().__enter__().sendmail.call_args[0]
assert sendmail_call_args[0] == "noreply@geant.org"
assert sendmail_call_args[1] == ["some-bogus-email"]
payload = sendmail_call_args[2]
assert "The latest errors report is attached." in payload
report_b64 = payload.split("\n\n")[-1]
report = base64.b64decode(report_b64).decode()
assert "mx1.ams.nl.geant.net" in report
assert "ae1" in report
assert "input-drops\t1" in report
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment