diff --git a/wile_coyote/__init__.py b/wile_coyote/__init__.py index f8b0d8e2058102d262165b4421a7c28811c99360..1327bbddc1eaa41459eaf552e6176109f77d85bb 100644 --- a/wile_coyote/__init__.py +++ b/wile_coyote/__init__.py @@ -1,3 +1,5 @@ +"""Wile Coyote is a Python library for creating and managing virtual environments.""" + import pkgutil -__version__ = '0.7.6' +__version__ = "0.7.6" diff --git a/wile_coyote/acme/__init__.py b/wile_coyote/acme/__init__.py index 593af3ef7b1381e9cffa3a1867151ef60ee8e394..24633bc5ae2841faa941fbfcb11aae5ed9ade3a8 100644 --- a/wile_coyote/acme/__init__.py +++ b/wile_coyote/acme/__init__.py @@ -1,3 +1,5 @@ +""" This module is a package for the Acme Corporation. """ + import pkgutil __all__ = [] diff --git a/wile_coyote/acme/nomad_uploader.py b/wile_coyote/acme/nomad_uploader.py index 5ec8c65b0e7cf82e1da0440ae0b0b177d8be9eb1..b63ac71c6a6c8fe790cf4e78635aa5418babad7e 100644 --- a/wile_coyote/acme/nomad_uploader.py +++ b/wile_coyote/acme/nomad_uploader.py @@ -7,6 +7,7 @@ Options: nomad_env = staging nomad_env wildcard = Wildcard (Bool) """ + import os import tempfile import wile_coyote.tools @@ -16,92 +17,88 @@ from wile_coyote.common import sys_kit def redis_uploader(certpath, env, proj, uploaded_cert_name, suffix, leader, log_file): - """ upload key to Redis """ - with open(certpath, 'r', encoding="utf8") as certfile: + """upload key to Redis""" + with open(certpath, "r", encoding="utf8") as certfile: cert_value_local = certfile.read() - consul_key_path = f'nomad/{env}/{proj}/{uploaded_cert_name}_{suffix}' + consul_key_path = f"nomad/{env}/{proj}/{uploaded_cert_name}_{suffix}" certdata_upstream = wile_coyote.tools.consul_kit.get( - consul_key_path, log_file, leader) + consul_key_path, log_file, leader + ) if cert_value_local != certdata_upstream: - log.handler( - f'uploading to Consul: {consul_key_path}', log_file) + log.handler(f"uploading to Consul: {consul_key_path}", log_file) wile_coyote.tools.consul_kit.put( - consul_key_path, cert_value_local, log_file, leader) + consul_key_path, cert_value_local, log_file, leader + ) else: - log.handler( - f'consul key {consul_key_path} unchanged: skipping', log_file) + log.handler(f"consul key {consul_key_path} unchanged: skipping", log_file) def vault_v2_uploader(mount_point, keypath, env, proj, uploaded_cert_name, log_file): - """ upload key to Vault v2 """ - with open(keypath, 'r', encoding="utf8") as keyfile: + """upload key to Vault v2""" + with open(keypath, "r", encoding="utf8") as keyfile: key_value_local = keyfile.read() - vault_key_path = f'{env}/{proj}/{uploaded_cert_name}.key' + vault_key_path = f"{env}/{proj}/{uploaded_cert_name}.key" key_value_upstream = wile_coyote.tools.vault_kit.get_v2( - vault_key_path, mount_point) + vault_key_path, mount_point + ) if key_value_local != key_value_upstream: - log.handler( - f'uploading to Vault: {mount_point}/{vault_key_path}', log_file) + log.handler(f"uploading to Vault: {mount_point}/{vault_key_path}", log_file) wile_coyote.tools.vault_kit.put_v2( - vault_key_path, key_value_local, log_file, mount_point) + vault_key_path, key_value_local, log_file, mount_point + ) else: log.handler( - f'vault key {mount_point}/{vault_key_path} unchanged: skipping', log_file) + f"vault key {mount_point}/{vault_key_path} unchanged: skipping", + log_file, + ) def uploader(provider, project, domain, nomad_env, wildcard=None): - """ Upload wildcard certificate to Consul and Vault """ - - if type(project) is str: - project_list = [project] - else: - project_list = project - - if type(nomad_env) is str: - nomad_env_list = [nomad_env] - else: - nomad_env_list = nomad_env + """Upload wildcard certificate to Consul and Vault""" + project_list = [project] if isinstance(project, str) else project + nomad_env_list = [nomad_env] if isinstance(nomad_env, str) else nomad_env if wildcard: - uploaded_cert_name = f'{provider}_wildcard_{domain}' - uploaded_haproxy_cert_name = f'{provider}_wildcard_haproxy_{domain}' + uploaded_cert_name = f"{provider}_wildcard_{domain}" + uploaded_haproxy_cert_name = f"{provider}_wildcard_haproxy_{domain}" else: - uploaded_cert_name = f'{provider}_{domain}' - uploaded_haproxy_cert_name = f'{provider}_haproxy_{domain}' + uploaded_cert_name = f"{provider}_{domain}" + uploaded_haproxy_cert_name = f"{provider}_haproxy_{domain}" - basedir = f'/etc/{provider}/live' - keypath = os.path.join(basedir, domain, 'privkey.pem') - log_file = f'/var/log/acme_{provider}/acme.log' - suffixes_list = ['cert.pem', 'chain.pem', 'fullchain.pem'] + basedir = f"/etc/{provider}/live" + keypath = os.path.join(basedir, domain, "privkey.pem") + log_file = f"/var/log/acme_{provider}/acme.log" + suffixes_list = ["cert.pem", "chain.pem", "fullchain.pem"] leader, _, __ = wile_coyote.tools.consul_kit.get_leader(log_file) # Ensure that we upload consistent data: check certificates validity and accessibility for suffix in suffixes_list: certpath = os.path.join(basedir, domain, suffix) sys_kit.file_access(certpath, log_file) - sys_kit.check_validity(certpath, 'public', log_file) + sys_kit.check_validity(certpath, "public", log_file) # check key validity and accessibility sys_kit.file_access(keypath, log_file) - sys_kit.check_validity(keypath, 'private', log_file) + sys_kit.check_validity(keypath, "private", log_file) for env in nomad_env_list: for proj in project_list: # upload certificates to Consul for suffix in suffixes_list: certpath = os.path.join(basedir, domain, suffix) - redis_uploader(certpath, env, proj, - uploaded_cert_name, suffix, leader, log_file) + redis_uploader( + certpath, env, proj, uploaded_cert_name, suffix, leader, log_file + ) # upload key to Vault - vault_v2_uploader('nomad', keypath, env, proj, - uploaded_cert_name, log_file) + vault_v2_uploader("nomad", keypath, env, proj, uploaded_cert_name, log_file) # upload key for haproxy to Vault tmp_path = tempfile.NamedTemporaryFile().name - cert_path = os.path.join(basedir, domain, 'cert.pem') + cert_path = os.path.join(basedir, domain, "cert.pem") combine.keys(cert_path, provider, keypath, tmp_path) - vault_v2_uploader('nomad', tmp_path, env, proj, - uploaded_haproxy_cert_name, log_file) + vault_v2_uploader( + "nomad", tmp_path, env, proj, uploaded_haproxy_cert_name, log_file + ) os.unlink(tmp_path) diff --git a/wile_coyote/acme/uploader.py b/wile_coyote/acme/uploader.py index 8e93b870294d931492715af4d3f6c4ce7e0b2219..b30d7d702e87149d64cc63ab602c20d3abe4377f 100644 --- a/wile_coyote/acme/uploader.py +++ b/wile_coyote/acme/uploader.py @@ -7,6 +7,7 @@ Options: unit = Unit, entity or team wildcard = Wildcard (Bool) """ + import os import wile_coyote.tools import wile_coyote.common @@ -14,81 +15,92 @@ from wile_coyote.common import sys_kit from wile_coyote.common import log -def redis_uploader(certpath, domain, suffix, unit, new_client, provider_prefix, wildcard_prefix, log_file): - """ upload key to Redis """ - with open(certpath, 'r', encoding="utf8") as certfile: +def redis_uploader(certpath, domain, suffix, unit, new_client, prov_prefix, wcard_prefix, log_file): + """upload key to Redis""" + with open(certpath, "r", encoding="utf8") as certfile: cert_local = certfile.read() - domain_underscored = domain.replace('.', '_') - cert_renamed = suffix.replace('cert.pem', 'pem').replace('.', '_') - redis_key_path = f'{unit}:{new_client}:redis_{provider_prefix}' \ - + f'{wildcard_prefix}{domain_underscored}_{cert_renamed}' + domain_underscored = domain.replace(".", "_") + cert_renamed = suffix.replace("cert.pem", "pem").replace(".", "_") + redis_key_path = ( + f"{unit}:{new_client}:redis_{prov_prefix}" + + f"{wcard_prefix}{domain_underscored}_{cert_renamed}" + ) cert_upstream = wile_coyote.tools.redis_kit.get(redis_key_path) if cert_local != cert_upstream: - log.handler(f'uploading to Redis: {redis_key_path}', log_file) - wile_coyote.tools.redis_kit.put( - redis_key_path, cert_local, log_file) + log.handler(f"uploading to Redis: {redis_key_path}", log_file) + wile_coyote.tools.redis_kit.put(redis_key_path, cert_local, log_file) else: - log.handler(f'redis key {redis_key_path} unchanged: skipping', - log_file) + log.handler(f"redis key {redis_key_path} unchanged: skipping", log_file) -def vault_uploader(keypath, domain, unit, new_client, provider_prefix, wildcard_prefix, log_file): - """ upload key to Vault v1 """ - with open(keypath, 'r', encoding="utf8") as keyfile: +def vault_uploader(keypath, domain, unit, new_client, prov_prefix, wcard_prefix, log_file): + """upload key to Vault v1""" + with open(keypath, "r", encoding="utf8") as keyfile: key_local = keyfile.read() - domain_underscored = domain.replace('.', '_') - vault_key_path = f'{unit}/{new_client}/vault_{provider_prefix}' \ - + f'{wildcard_prefix}{domain_underscored}_key' + domain_underscored = domain.replace(".", "_") + vault_key_path = ( + f"{unit}/{new_client}/vault_{prov_prefix}" + + f"{wcard_prefix}{domain_underscored}_key" + ) key_upstream = wile_coyote.tools.vault_kit.get(vault_key_path, unit) if key_local != key_upstream: - log.handler(f'uploading to Vault: {vault_key_path}', log_file) - wile_coyote.tools.vault_kit.put( - vault_key_path, key_local, log_file, unit) + log.handler(f"uploading to Vault: {vault_key_path}", log_file) + wile_coyote.tools.vault_kit.put(vault_key_path, key_local, log_file, unit) else: - log.handler(f'vault key {vault_key_path} unchanged: skipping', - log_file) + log.handler(f"vault key {vault_key_path} unchanged: skipping", log_file) def uploader(domain, provider, unit, client=None, wildcard=None): - """ Upload keys to Redis and Vault """ + """Upload keys to Redis and Vault""" if wildcard: - wildcard_prefix = 'wildcard_' + wcard_prefix = "wildcard_" if client: new_client = client # not a TLD wildcard else: - new_client = ['common'] # this is TLD and everyone can get it + new_client = ["common"] # this is TLD and everyone can get it else: new_client = client - wildcard_prefix = '' + wcard_prefix = "" - provider_prefix = f'{provider}_' + prov_prefix = f"{provider}_" - if unit != 'puppet': # outside puppet we don't have client + if unit != "puppet": # outside puppet we don't have client new_client = domain - basedir = f'/etc/{provider}/live' - keypath = os.path.join(basedir, domain, 'privkey.pem') - log_file = f'/var/log/acme_{provider}/acme.log' - suffixes_list = ['cert.pem', 'chain.pem', 'fullchain.pem'] + basedir = f"/etc/{provider}/live" + keypath = os.path.join(basedir, domain, "privkey.pem") + log_file = f"/var/log/acme_{provider}/acme.log" + suffixes_list = ["cert.pem", "chain.pem", "fullchain.pem"] # check public key validity and accessibility for suffix in suffixes_list: certpath = os.path.join(basedir, domain, suffix) sys_kit.file_access(certpath, log_file) - sys_kit.check_validity(certpath, 'public', log_file) + sys_kit.check_validity(certpath, "public", log_file) # check private key validity and accessibility sys_kit.file_access(keypath, log_file) - sys_kit.check_validity(keypath, 'private', log_file) + sys_kit.check_validity(keypath, "private", log_file) # upload certificates to Redis for suffix in suffixes_list: certpath = os.path.join(basedir, domain, suffix) - redis_uploader(certpath, domain, suffix, unit, new_client, provider_prefix, wildcard_prefix, log_file) + redis_uploader( + certpath, + domain, + suffix, + unit, + new_client, + prov_prefix, + wcard_prefix, + log_file, + ) # upload key to Vault - vault_uploader(keypath, domain, unit, new_client, provider_prefix, wildcard_prefix, log_file) + vault_uploader( + keypath, domain, unit, new_client, prov_prefix, wcard_prefix, log_file + ) wile_coyote.tools.redis_kit.save(log_file) diff --git a/wile_coyote/acme/wildcard_uploader.py b/wile_coyote/acme/wildcard_uploader.py index 73030cba07e81c736076744d893aeb43c15b0d43..c87970690ea3a67b6850d730a66b3bc760e15aa6 100644 --- a/wile_coyote/acme/wildcard_uploader.py +++ b/wile_coyote/acme/wildcard_uploader.py @@ -4,6 +4,7 @@ Options: provider = ACME Provider (sectigo_ev, sectigo_ov, letsencrypt) domain = Certificate name """ + import os import tempfile import wile_coyote.tools @@ -13,102 +14,113 @@ from wile_coyote.common import log def vault_uploader(keypath, domain, provider, log_file): - """ upload key to Vault v1 """ - with open(keypath, 'r', encoding="utf8") as keyfile: + """upload key to Vault v1""" + with open(keypath, "r", encoding="utf8") as keyfile: keydata_local = keyfile.read() - domain_underscored = domain.replace('.', '_') + domain_underscored = domain.replace(".", "_") # upload key into Puppet stanza on Vault - unit = 'puppet' - puppet_vault_key_path = f'puppet/common/vault_{provider}_' \ - + f'wildcard_{domain_underscored}_key' + unit = "puppet" + puppet_vault_key_path = ( + f"puppet/common/vault_{provider}_" + f"wildcard_{domain_underscored}_key" + ) puppet_keydata_upstream = wile_coyote.tools.vault_kit.get( - puppet_vault_key_path, unit) + puppet_vault_key_path, unit + ) if keydata_local != puppet_keydata_upstream: - log.handler(f'uploading to Vault: {puppet_vault_key_path}', - log_file) - wile_coyote.tools.vault_kit.put(puppet_vault_key_path, - keydata_local, log_file, unit) + log.handler(f"uploading to Vault: {puppet_vault_key_path}", log_file) + wile_coyote.tools.vault_kit.put( + puppet_vault_key_path, keydata_local, log_file, unit + ) else: - log.handler(f'vault key {puppet_vault_key_path} unchanged: skipping', - log_file) + log.handler( + f"vault key {puppet_vault_key_path} unchanged: skipping", log_file + ) def vault_v2_uploader(keypath, uploaded_name, log_file): - """ upload key to Vault v1 """ - - with open(keypath, 'r', encoding="utf8") as keyfile: + """upload key to Vault v1""" + with open(keypath, "r", encoding="utf8") as keyfile: keydata_local = keyfile.read() - unit = 'nomad' - mount_point = 'nomad' - nomad_vault_key_path = f'common/{uploaded_name}.key' + unit = "nomad" + mount_point = "nomad" + nomad_vault_key_path = f"common/{uploaded_name}.key" nomad_keydata_upstream = wile_coyote.tools.vault_kit.get_v2( - nomad_vault_key_path, unit, mount_point) + nomad_vault_key_path, unit, mount_point + ) if keydata_local != nomad_keydata_upstream: - log.handler(f'uploading to Vault: {mount_point}/{nomad_vault_key_path}', - log_file) - wile_coyote.tools.vault_kit.put_v2(nomad_vault_key_path, - keydata_local, log_file, mount_point) + log.handler( + f"uploading to Vault: {mount_point}/{nomad_vault_key_path}", log_file + ) + wile_coyote.tools.vault_kit.put_v2( + nomad_vault_key_path, keydata_local, log_file, mount_point + ) else: - log.handler(f'vault key {mount_point}/{nomad_vault_key_path} unchanged: skipping', - log_file) + log.handler( + f"vault key {mount_point}/{nomad_vault_key_path} unchanged: skipping", + log_file, + ) def uploader(provider, domain): - """ Upload wildcard certificate to Redis, Consul and Vault """ - uploaded_cert_name = f'{provider}_wildcard_{domain}' - uploaded_haproxy_cert_name = f'{provider}_wildcard_haproxy_{domain}' - basedir = f'/etc/{provider}/live' - keypath = os.path.join(basedir, domain, 'privkey.pem') - log_file = f'/var/log/acme_{provider}/acme.log' - suffix_list = ['cert.pem', 'chain.pem', 'fullchain.pem'] + """Upload wildcard certificate to Redis, Consul and Vault""" + uploaded_cert_name = f"{provider}_wildcard_{domain}" + uploaded_haproxy_cert_name = f"{provider}_wildcard_haproxy_{domain}" + basedir = f"/etc/{provider}/live" + keypath = os.path.join(basedir, domain, "privkey.pem") + log_file = f"/var/log/acme_{provider}/acme.log" + suffix_list = ["cert.pem", "chain.pem", "fullchain.pem"] leader, _, __ = wile_coyote.tools.consul_kit.get_leader(log_file) # check certificates validity and accessibility for suffix in suffix_list: certpath = os.path.join(basedir, domain, suffix) sys_kit.file_access(certpath, log_file) - sys_kit.check_validity(certpath, 'public', log_file) + sys_kit.check_validity(certpath, "public", log_file) # check key validity and accessibility sys_kit.file_access(keypath, log_file) - sys_kit.check_validity(keypath, 'private', log_file) + sys_kit.check_validity(keypath, "private", log_file) for suffix in suffix_list: certpath = os.path.join(basedir, domain, suffix) - with open(certpath, 'r', encoding="utf8") as certfile: + with open(certpath, "r", encoding="utf8") as certfile: certdata_local = certfile.read() # upload certificates to Redis - domain_underscored = domain.replace('.', '_') - cert_renamed = suffix.replace(suffix_list[0], 'pem').replace('.', '_') - redis_key_path = f'puppet:common:redis_{provider}_' \ - + f'wildcard_{domain_underscored}_{cert_renamed}' + domain_underscored = domain.replace(".", "_") + cert_renamed = suffix.replace(suffix_list[0], "pem").replace(".", "_") + redis_key_path = ( + f"puppet:common:redis_{provider}_" + + f"wildcard_{domain_underscored}_{cert_renamed}" + ) cert_upstream = wile_coyote.tools.redis_kit.get(redis_key_path) if certdata_local != cert_upstream: - log.handler(f'uploading to Redis: {redis_key_path}', log_file) + log.handler(f"uploading to Redis: {redis_key_path}", log_file) wile_coyote.tools.redis_kit.put( - redis_key_path, certdata_local, log_file) + redis_key_path, certdata_local, log_file + ) else: - log.handler(f'redis key {redis_key_path} unchanged: skipping', - log_file) + log.handler(f"redis key {redis_key_path} unchanged: skipping", log_file) # upload certificates to Consul for Nomad - consul_key_path = f'nomad/common/{uploaded_cert_name}_{suffix}' + consul_key_path = f"nomad/common/{uploaded_cert_name}_{suffix}" certdata_upstream = wile_coyote.tools.consul_kit.get( - consul_key_path, log_file, leader) + consul_key_path, log_file, leader + ) if certdata_local != certdata_upstream: - log.handler( - f'uploading to Consul: {consul_key_path}', log_file) + log.handler(f"uploading to Consul: {consul_key_path}", log_file) wile_coyote.tools.consul_kit.put( - consul_key_path, certdata_local, log_file, leader) + consul_key_path, certdata_local, log_file, leader + ) else: - log.handler(f'consul key {consul_key_path} unchanged: skipping', - log_file) + log.handler( + f"consul key {consul_key_path} unchanged: skipping", log_file + ) # upload key to Vault vault_uploader(keypath, domain, provider, log_file) diff --git a/wile_coyote/bin/anvil b/wile_coyote/bin/anvil index 7c06ee3911a7bbda28bc8271d87926737342855b..a16b9425d69033ee7401a4f9fe97041e006d9863 100755 --- a/wile_coyote/bin/anvil +++ b/wile_coyote/bin/anvil @@ -30,99 +30,104 @@ from wile_coyote.common import constants from wile_coyote.common import log from wile_coyote.common import sys_kit +LOGFILE = "/dev/stdout" +START_TIMEDATE = datetime.datetime.now() + def certificates_delete(provider: str, certificates: list): - """ delete list of certificates from a given provider """ + """delete list of certificates from a given provider""" for cert in certificates: - cbot_cmd = '/usr/local/bin/certbot delete --non-interactive' \ - + f' -c /etc/{provider}/cli.ini --cert-name {cert}' + cbot_cmd = ( + "/usr/local/bin/certbot delete --non-interactive" + + f" -c /etc/{provider}/cli.ini --cert-name {cert}" + ) cbot_child = sp.Popen(cbot_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE) cbot_out, cbot_err = cbot_child.communicate() if cbot_child.returncode != 0: - err = cbot_err.decode("utf-8") - log.handler(f"error executing certbot: {err}", LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + childerr = cbot_err.decode("utf-8") + log.handler(f"error executing certbot: {childerr}", LOGFILE, True) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) else: decoded_msg = cbot_out.decode("utf-8") - msg = decoded_msg[:decoded_msg.rfind('\n')] + msg = decoded_msg[: decoded_msg.rfind("\n")] log.handler(msg, LOGFILE) def redis_keys(server, token): - """ download keys from Redis """ - log.handler('fetching keys from Redis...', LOGFILE) - r_client = redis.StrictRedis( - host=server, password=token, port=6379, db=0) + """download keys from Redis""" + log.handler("fetching keys from Redis...", LOGFILE) + r_client = redis.StrictRedis(host=server, password=token, port=6379, db=0) try: - r_keys = [n.decode("utf-8") for n in r_client.keys('*')] - except Exception as err: # pylint: disable=w0703 - r_keys = err + r_keys = [n.decode("utf-8") for n in r_client.keys("*")] + except Exception as redis_err: # pylint: disable=w0703 + r_keys = redis_err return sorted(r_keys) def redis_prune(server, token): - """ prune keys from Redis """ - log.handler('purging keys from Redis...', LOGFILE) + """prune keys from Redis""" + log.handler("purging keys from Redis...", LOGFILE) r_client = redis.StrictRedis(host=server, password=token, port=6379, db=0) - r_keys = [n.decode("utf-8") for n in r_client.keys('*')] + r_keys = [n.decode("utf-8") for n in r_client.keys("*")] for key in r_keys: r_client.delete(key) def vault_keys(server, token, mount_points, ver): - """ download key from vault """ - log.handler(f'fetching keys from Vault v{ver}...', LOGFILE) + """download key from vault""" + log.handler(f"fetching keys from Vault v{ver}...", LOGFILE) all_vault_keys = [] for mount in mount_points: - vault_cmd = f'/usr/local/bin/rvault --token {token}' \ - + f' --insecure --address https://{server} list {mount}/' - vault_child = sp.Popen( - vault_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE) + vault_cmd = ( + f"/usr/local/bin/rvault --token {token}" + + f" --insecure --address https://{server} list {mount}/" + ) + vault_child = sp.Popen(vault_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE) vault_out, vault_err = vault_child.communicate() if vault_child.returncode != 0: - err = vault_err.decode("utf-8") + vault_err = vault_err.decode("utf-8") log.handler( - f'error listing keys for the mount point {mount}: {err}', LOGFILE) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"error listing keys for the mount point {mount}: {vault_err}", LOGFILE + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) else: decoded_msg = vault_out.decode("utf-8") - _msg = [x for x in decoded_msg.split('\n') if x != ''] - msg = [f'/{mount}{x}' for x in _msg] + _msg = [x for x in decoded_msg.split("\n") if x != ""] + msg = [f"/{mount}{x}" for x in _msg] all_vault_keys.extend(msg) return sorted(all_vault_keys) def vault_prune(server, token, mount_points, ver): - """ prune key to vault """ - log.handler(f'purging keys from Vault v{ver}...', LOGFILE) - v_client = hvac.Client(url=f'https://{server}', token=token) + """prune key to vault""" + log.handler(f"purging keys from Vault v{ver}...", LOGFILE) + v_client = hvac.Client(url=f"https://{server}", token=token) for mount in mount_points: v_client.sys.disable_secrets_engine(mount) - v_client.sys.enable_secrets_engine( - 'kv', path=mount, options={'version': ver}) + v_client.sys.enable_secrets_engine("kv", path=mount, options={"version": ver}) def consul_keys(server, token): - """ download keys from consul """ - log.handler('fetching keys from Consul...', LOGFILE) - c_client = Consul(host=server, port='443', token=token, scheme='https') + """download keys from consul""" + log.handler("fetching keys from Consul...", LOGFILE) + c_client = Consul(host=server, port="443", token=token, scheme="https") try: - c_keys = sorted(c_client.kv.get('nomad', recurse=True, keys=True)[1]) - except Exception as err: # pylint: disable=W0703 - return err + c_keys = sorted(c_client.kv.get("nomad", recurse=True, keys=True)[1]) + except Exception as consul_err: # pylint: disable=W0703 + return consul_err return sorted(c_keys) def consul_prune(server, token): - """ prune keys from consul """ - log.handler('purging keys from Consul...', LOGFILE) - c_client = Consul(host=server, port='443', token=token, scheme='https') - for nomad_env in ['test', 'uat', 'prod']: - c_client.kv.delete(f'nomad/{nomad_env}', recurse=True) + """prune keys from consul""" + log.handler("purging keys from Consul...", LOGFILE) + c_client = Consul(host=server, port="443", token=token, scheme="https") + for nomad_env in ["test", "uat", "prod"]: + c_client.kv.delete(f"nomad/{nomad_env}", recurse=True) # Here we Go. @@ -130,18 +135,15 @@ if __name__ == "__main__": VERSION = __import__("wile_coyote").__version__ ARGS = docopt(__doc__, version=VERSION) - PRUNE = ARGS['--prune'] - LOGFILE = '/dev/stdout' - - START_TIMEDATE = datetime.datetime.now() - PLEASE_CHECK = 'Please check the logs' + PRUNE = ARGS["--prune"] + PLEASE_CHECK = "Please check the logs" # Anvil should not be used in production VAULT_ROOT_TOKEN = wile_coyote.tools.VAULT_ROOT_TOKEN if not VAULT_ROOT_TOKEN: - log.handler('you can use this tool ONLY in test', LOGFILE, True) + log.handler("you can use this tool ONLY in test", LOGFILE, True) log.handler(constants.GIVEUP, LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) ACME_PROVIDERS = wile_coyote.tools.ACME_PROVIDERS REDIS_HOST = wile_coyote.tools.REDIS_HOST @@ -160,22 +162,22 @@ if __name__ == "__main__": # prune certificates locally if PRUNE: - if 'all' in PRUNE: + if "all" in PRUNE: prune = ACME_PROVIDERS else: prune = PRUNE for prov in prune: if prov not in ACME_PROVIDERS: - log.handler(f'{prov} is not a valid provider', LOGFILE, True) + log.handler(f"{prov} is not a valid provider", LOGFILE, True) log.handler(constants.GIVEUP, LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) else: prune = [] for acme_provider in prune: - acme_certificates = glob(f'/etc/{acme_provider}/live/*') + acme_certificates = glob(f"/etc/{acme_provider}/live/*") try: - acme_certificates.remove(f'/etc/{acme_provider}/live/README') + acme_certificates.remove(f"/etc/{acme_provider}/live/README") except ValueError: pass cert_names = [os.path.basename(x) for x in acme_certificates] @@ -188,19 +190,17 @@ if __name__ == "__main__": vault_prune(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V2, 2) # run all scripts under /opt/acme/bin - if PRUNE: - params = '' - else: - params = ' --renew --reuse-key' - for script in glob('/opt/acme/bin/*'): - log.handler(f'running script {script}{params}', LOGFILE) + PARAMS = "" if PRUNE else " --renew --reuse-key" + for script in glob("/opt/acme/bin/*"): + log.handler(f"running script {script}{PARAMS}", LOGFILE) script_child = sp.Popen(script.split(), stdout=sp.PIPE, stderr=sp.PIPE) script_out, script_err = script_child.communicate() if script_child.returncode != 0: - err = script_err.decode("utf-8") + CHILD_ERR = script_err.decode("utf-8") log.handler( - f"{script} execution failed. {PLEASE_CHECK}", LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"{script} execution failed. {PLEASE_CHECK}: {CHILD_ERR}", LOGFILE, True + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) # upstream keys C_KEYS = consul_keys(CONSUL_LEADER, CONSUL_TOKEN) @@ -213,103 +213,124 @@ if __name__ == "__main__": C_DIFFS = [item for item in CONSUL_KEYS if item not in C_KEYS] R_DIFFS = [item for item in REDIS_KEYS if item not in R_KEYS] V_DIFFS = [item for item in VAULT_KEYS if item not in V_KEYS_ALL] - for key_store in ['Consul', 'Redis', 'Vault']: - if key_store == 'Consul': - key_list = C_DIFFS - elif key_store == 'Redis': - key_list = R_DIFFS - elif key_store == 'Vault': - key_list = V_DIFFS - if key_list: + for key_store in ["Consul", "Redis", "Vault"]: + KEY_LIST = "" # make the linter happy + if key_store == "Consul": + KEY_LIST = C_DIFFS + elif key_store == "Redis": + KEY_LIST = R_DIFFS + elif key_store == "Vault": + KEY_LIST = V_DIFFS + if KEY_LIST: log.handler( - f'the following keys are not available on {key_store}:', LOGFILE, True) - log.handler(' '.join(key_list), LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"the following keys are not available on {key_store}:", LOGFILE, True + ) + log.handler(" ".join(KEY_LIST), LOGFILE, True) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) else: log.handler( - f'{key_store} key names have been successfully checked', LOGFILE) + f"{key_store} key names have been successfully checked", LOGFILE + ) if REDIS_KEYS != R_KEYS: - log.handler( - 'downstream and upstream Redis keys are different', LOGFILE, True) - missing_upstream = list(set(REDIS_KEYS) - set(R_KEYS)) - missing_downstream = list(set(R_KEYS) - set(REDIS_KEYS)) - if missing_upstream: + log.handler("downstream and upstream Redis keys are different", LOGFILE, True) + gone_upstream = list(set(REDIS_KEYS) - set(R_KEYS)) + gone_downstream = list(set(R_KEYS) - set(REDIS_KEYS)) + if gone_upstream: log.handler( - f'the following keys are missing on Redis: {missing_upstream}', LOGFILE, True) - if missing_downstream: + f"the following keys are missing on Redis: {gone_upstream}", + LOGFILE, + True, + ) + if gone_downstream: log.handler( - f'the following Redis keys are missing in acme.ini: {missing_downstream}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"the following Redis keys are missing in acme.ini: {gone_downstream}", + LOGFILE, + True, + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) elif CONSUL_KEYS != C_KEYS: - log.handler( - 'downstream and upstream Consul keys are different', LOGFILE, True) - missing_upstream = list(set(CONSUL_KEYS) - set(C_KEYS)) - missing_downstream = list(set(C_KEYS) - set(CONSUL_KEYS)) - if missing_upstream: + log.handler("downstream and upstream Consul keys are different", LOGFILE, True) + gone_upstream = list(set(CONSUL_KEYS) - set(C_KEYS)) + gone_downstream = list(set(C_KEYS) - set(CONSUL_KEYS)) + if gone_upstream: log.handler( - f'the following keys are missing on Consul: {missing_upstream}', LOGFILE, True) - if missing_downstream: + f"the following keys are missing on Consul: {gone_upstream}", + LOGFILE, + True, + ) + if gone_downstream: log.handler( - f'the following Consul keys are missing in acme.ini: {missing_downstream}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"the following Consul keys are missing in acme.ini: {gone_downstream}", + LOGFILE, + True, + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) elif VAULT_KEYS != V_KEYS_ALL: - log.handler( - 'downstream and upstream Vault keys are different', LOGFILE, True) - missing_upstream = list(set(VAULT_KEYS) - set(V_KEYS_ALL)) - missing_downstream = list(set(V_KEYS_ALL) - set(VAULT_KEYS)) - if missing_upstream: + log.handler("downstream and upstream Vault keys are different", LOGFILE, True) + gone_upstream = list(set(VAULT_KEYS) - set(V_KEYS_ALL)) + gone_downstream = list(set(V_KEYS_ALL) - set(VAULT_KEYS)) + if gone_upstream: log.handler( - f'the following keys are missing on Vault: {missing_upstream}', LOGFILE, True) - if missing_downstream: + f"the following keys are missing on Vault: {gone_upstream}", + LOGFILE, + True, + ) + if gone_downstream: log.handler( - f'the following Vault keys are missing in acme.ini: {missing_downstream}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"the following Vault keys are missing in acme.ini: {gone_downstream}", + LOGFILE, + True, + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) # validate public keys - for r_key in [item for item in REDIS_KEYS if '_chain.pem' not in R_KEYS and item != 'do_not_delete']: - log.handler(f'checking certificate {r_key} on Redis', LOGFILE) + for r_key in [item for item in REDIS_KEYS if "_chain.pem" not in R_KEYS and item != "do_not_delete"]: + log.handler(f"checking certificate {r_key} on Redis", LOGFILE) pubkey = wile_coyote.tools.redis_kit.get(r_key) - key_test = wile_coyote.tools.key_kit.Check( - pubkey, LOGFILE, r_key, False).public() - if not key_test: + KEY_TEST = wile_coyote.tools.key_kit.Check( + pubkey, LOGFILE, r_key, False + ).public() + if not KEY_TEST: log.handler( - f'The certificate {r_key} is malformed. {PLEASE_CHECK}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) - - for c_key in [item for item in CONSUL_KEYS if '_chain.pem' not in C_KEYS]: - log.handler(f'checking certificate {c_key} on Consul', LOGFILE) - pubkey = wile_coyote.tools.consul_kit.get( - c_key, LOGFILE, CONSUL_LEADER) - key_test = wile_coyote.tools.key_kit.Check( - pubkey, LOGFILE, c_key, False).public() - if not key_test: + f"The certificate {r_key} is malformed. {PLEASE_CHECK}", LOGFILE, True + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) + + for c_key in [item for item in CONSUL_KEYS if "_chain.pem" not in C_KEYS]: + log.handler(f"checking certificate {c_key} on Consul", LOGFILE) + pubkey = wile_coyote.tools.consul_kit.get(c_key, LOGFILE, CONSUL_LEADER) + KEY_TEST = wile_coyote.tools.key_kit.Check( + pubkey, LOGFILE, c_key, False + ).public() + if not KEY_TEST: log.handler( - f'The certificate {c_key} is malformed. {PLEASE_CHECK}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + f"The certificate {c_key} is malformed. {PLEASE_CHECK}", LOGFILE, True + ) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) # validate private keys: we check only if it's not malformed - for v_key in [item for item in V_KEYS_V1 if item != '/puppet/do_not_delete']: - log.handler(f'checking private key {v_key} on Vault v1', LOGFILE) - privkey = wile_coyote.tools.vault_kit.get(v_key, 'root') - key_test = wile_coyote.tools.key_kit.Check( - privkey, LOGFILE, v_key, False).private() - if not key_test: - log.handler( - f'The key {v_key} is malformed. {PLEASE_CHECK}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + for v_key in [item for item in V_KEYS_V1 if item != "/puppet/do_not_delete"]: + log.handler(f"checking private key {v_key} on Vault v1", LOGFILE) + privkey = wile_coyote.tools.vault_kit.get(v_key, "root") + KEY_TEST = wile_coyote.tools.key_kit.Check( + privkey, LOGFILE, v_key, False + ).private() + if not KEY_TEST: + log.handler(f"The key {v_key} is malformed. {PLEASE_CHECK}", LOGFILE, True) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) for v2_key in V_KEYS_V2: - log.handler(f'checking private key {v2_key} on Vault v2', LOGFILE) - fixed_v2_key = re.sub(r'^/nomad/', '/', v2_key) - privkey = wile_coyote.tools.vault_kit.get_v2(fixed_v2_key, 'root') - key_test = wile_coyote.tools.key_kit.Check( - privkey, LOGFILE, v2_key, False).private() - if not key_test: - log.handler( - f'The key {v2_key} is malformed. {PLEASE_CHECK}', LOGFILE, True) - sys_kit.exit(LOGFILE, START_TIMEDATE, 1) + log.handler(f"checking private key {v2_key} on Vault v2", LOGFILE) + fixed_v2_key = re.sub(r"^/nomad/", "/", v2_key) + privkey = wile_coyote.tools.vault_kit.get_v2(fixed_v2_key, "root") + KEY_TEST = wile_coyote.tools.key_kit.Check( + privkey, LOGFILE, v2_key, False + ).private() + if not KEY_TEST: + log.handler(f"The key {v2_key} is malformed. {PLEASE_CHECK}", LOGFILE, True) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE, 1) # if we are here the test was successful - log.handler('The test was successfully executed!', LOGFILE) - sys_kit.exit(LOGFILE, START_TIMEDATE) + log.handler("The test was successfully executed!", LOGFILE) + sys_kit.coyote_exit(LOGFILE, START_TIMEDATE) diff --git a/wile_coyote/bin/coyote b/wile_coyote/bin/coyote index aa2405a497922bc775e951d02128f3c5422004d8..07764f230fb4446dabdd7c69df5ad05753084491 100755 --- a/wile_coyote/bin/coyote +++ b/wile_coyote/bin/coyote @@ -46,7 +46,7 @@ import wile_coyote.common.config def atomic_checks(log_file, leader): - """ test connectivity to Redis, Consul, Vault """ + """test connectivity to Redis, Consul, Vault""" wile_coyote.tools.consul_kit.test(log_file, leader) wile_coyote.tools.redis_kit.test(log_file) wile_coyote.tools.vault_kit.test(log_file) @@ -54,7 +54,7 @@ def atomic_checks(log_file, leader): def get_certificate_metadata(certificate): - """ retrieve certificate expiration and SAN """ + """retrieve certificate expiration and SAN""" st_cert = open(certificate, 'rt', encoding="utf8").read() ssl_crypto = OpenSSL.crypto cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert) @@ -63,31 +63,32 @@ def get_certificate_metadata(certificate): for i in range(ext_count): ext = cert.get_extension(i) if 'subjectAltName' in str(ext.get_short_name()): - san = ext.__str__() + san = str(ext) break return [san, not_after] -def run_certbot(start, cbot_domain, provider, action, reuse_key, force, dry, wild_card=None, extra_commands=None): - """ get certificate from ACME provider - a few parameters for certbot are defined in /etc/<provider>/cli.ini +def run_certbot(start, cbot_dom, provider, action, reuse_key, force, dry, wild_card=None, extra_cmds=None): + """ + get certificate from ACME provider + a few parameters for certbot are defined in /etc/<provider>/cli.ini """ if action == 'renew': domain_list = '' # renew does not support list of domains else: if wild_card: - domain_list = '-d *.{}'.format(' -d *'.join(list(cbot_domain))) + domain_list = f'-d *.{' -d *'.join(list(cbot_dom))}' else: - domain_list = '-d {}'.format(' -d '.join(list(cbot_domain))) + domain_list = f'-d {' -d '.join(list(cbot_dom))}' - if extra_commands: - extra_cmd = ' ' + ' '.join(extra_commands) + if extra_cmds: + extra_cmd = ' ' + ' '.join(extra_cmds) else: extra_cmd = '' cbot_cmd = f'/usr/local/bin/certbot {action} --non-interactive' \ - + f' -c /etc/{provider}/cli.ini --cert-name {cbot_domain[0]}' \ + + f' -c /etc/{provider}/cli.ini --cert-name {cbot_dom[0]}' \ + f' {domain_list}{dry}{force}{reuse_key}{extra_cmd}' log.handler(f'executing: {cbot_cmd}', LOG_FILE) @@ -97,7 +98,7 @@ def run_certbot(start, cbot_domain, provider, action, reuse_key, force, dry, wil if cbot_child.returncode != 0: err = cbot_err.decode("utf-8") log.handler(f"error executing certbot: {err}", LOG_FILE, True) - sys_kit.exit(LOG_FILE, start, 1) + sys_kit.coyote_exit(LOG_FILE, start, 1) else: decoded_msg = cbot_out.decode("utf-8") msg = decoded_msg[:decoded_msg.rfind('\n')] @@ -112,32 +113,17 @@ if __name__ == "__main__": VERSION = __import__("wile_coyote").__version__ ARGS = docopt(__doc__, version=VERSION) PROVIDER = ARGS['--provider'] - if ARGS['--renew']: - action = 'renew' - else: - action = 'certonly' - if ARGS['--reuse-key']: - reuse_key = ' --reuse-key' - else: - reuse_key = '' - if ARGS['--dry']: - dry = ' --dry-run' - else: - dry = '' - if ARGS['--force-renewal']: - force = ' --force-renewal' - else: - force = '' - if ARGS['--stdout']: - LOG_FILE = '/dev/stdout' - else: - LOG_FILE = f'/var/log/acme_{PROVIDER}/acme.log' + LOG_FILE = '/dev/stdout' if ARGS['--stdout'] else f'/var/log/acme_{PROVIDER}/acme.log' + ACTION = 'renew' if ARGS['--renew'] else 'certonly' + REUSE_KEY = '--reuse-key' if ARGS['--reuse-key'] else '' + DRY = '--dry-run' if ARGS['--dry'] else '' + FORCE = '--force-renewal' if ARGS['--force-renewal'] else '' LEADER, _, __ = wile_coyote.tools.consul_kit.get_leader(LOG_FILE) atomic_checks(LOG_FILE, LEADER) # start logging ASAP - START_TIMEDATE = datetime.datetime.now() + START_TIME = datetime.datetime.now() CMD_LINE = ' '.join(os.sys.argv) log.handler(80*'=', LOG_FILE) # since we use flock logs are tidy log.handler(f'JOB STARTED: {CMD_LINE}', LOG_FILE) @@ -168,13 +154,13 @@ if __name__ == "__main__": SORTED_DOMAIN = sorted(_DOMAIN) if os.path.islink(cert_path): cert_metadata = get_certificate_metadata(cert_path) - san_string = cert_metadata[0] + SAN_STR = cert_metadata[0] notafter = cert_metadata[1] - if ', DNS:' in san_string: - _SAN = sorted(san_string.replace('DNS:', '').split(', ')) + if ', DNS:' in SAN_STR: + _SAN = sorted(SAN_STR.replace('DNS:', '').split(', ')) SAN = [x.replace('*.', '') for x in _SAN] else: - SAN = [san_string.replace('DNS:', '').replace('*.', '')] + SAN = [SAN_STR.replace('DNS:', '').replace('*.', '')] expiration = datetime.datetime.strptime(notafter, '%Y%m%d%H%M%S%fZ') # either initial value and value 1 here will be set to ZERO DAYS_LEFT = (expiration - datetime.datetime.now()).days @@ -184,25 +170,22 @@ if __name__ == "__main__": PRINTABLE_SAN = ', '.join(SAN) PRINTABLE_SORTED_DOMAIN = ', '.join(SORTED_DOMAIN) if DAYS_LEFT == "BOFH": - days_left_msg = 'the certificate has NOT yet been created' + DAYS_LEFT_MSG = 'the certificate has NOT yet been created' elif DAYS_LEFT == 1: - days_left_msg = 'there are ZERO days left' + DAYS_LEFT_MSG = 'there are ZERO days left' else: - days_left_msg = f'there are {DAYS_LEFT} days left' + DAYS_LEFT_MSG = f'there are {DAYS_LEFT} days left' log.handler(f'the SAN on this server is ..... {PRINTABLE_SAN}', LOG_FILE) log.handler(f'the SAN defined in Puppet is .. {PRINTABLE_SORTED_DOMAIN}', LOG_FILE) - log.handler(days_left_msg, LOG_FILE) + log.handler(DAYS_LEFT_MSG, LOG_FILE) if SAN != SORTED_DOMAIN or DAYS_LEFT < DAYS: - run_certbot(START_TIMEDATE, DOMAIN, PROVIDER, action, reuse_key, force, dry, WILDCARD, EXTRA) + run_certbot(START_TIME, DOMAIN, PROVIDER, ACTION, REUSE_KEY, FORCE, DRY, WILDCARD, EXTRA) # if we are here, everything went fine and we can upload the certificates - if WILDCARD: - wilcard_string = ' Wildcard' - else: - wilcard_string = '' + WILDCARD_STRING = ' Wildcard' if WILDCARD else '' if ARGS['--tld']: log.handler(f'uploading {FIRST_NAME} {PROVIDER} wildcard', LOG_FILE) @@ -212,15 +195,15 @@ if __name__ == "__main__": if single_unit == 'puppet': for client in CLIENTS: log.handler( - f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for Puppet', LOG_FILE) + f'uploading {FIRST_NAME} {PROVIDER}{WILDCARD_STRING} for Puppet', LOG_FILE) uploader.uploader(FIRST_NAME, PROVIDER, single_unit, client, WILDCARD) elif single_unit == 'nomad': log.handler( - f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for Nomad', LOG_FILE) + f'uploading {FIRST_NAME} {PROVIDER}{WILDCARD_STRING} for Nomad', LOG_FILE) nomad_uploader.uploader(PROVIDER, PROJECT, FIRST_NAME, ENV, WILDCARD) else: log.handler( - f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for {UNIT}', LOG_FILE) + f'uploading {FIRST_NAME} {PROVIDER}{WILDCARD_STRING} for {UNIT}', LOG_FILE) uploader.uploader(FIRST_NAME, PROVIDER, single_unit, None, WILDCARD) - sys_kit.exit(LOG_FILE, START_TIMEDATE) + sys_kit.coyote_exit(LOG_FILE, START_TIME) diff --git a/wile_coyote/common/__init__.py b/wile_coyote/common/__init__.py index 92434cf1c536e7c9cbf00c8318190fdb43f84b9f..caeb0432c040ea939cf99da8f9daba459724aec8 100644 --- a/wile_coyote/common/__init__.py +++ b/wile_coyote/common/__init__.py @@ -1,3 +1,5 @@ +""" Common package for Wile Coyote project. """ + import pkgutil diff --git a/wile_coyote/common/combine.py b/wile_coyote/common/combine.py index a024120b08313301adb78da9b2f92aa7129cc913..ecf88860b9f3f1f5f6f886ce7a5947bd4a56eb91 100644 --- a/wile_coyote/common/combine.py +++ b/wile_coyote/common/combine.py @@ -1,21 +1,27 @@ +"""Module providing a function to combine certificate, CA and private key""" + import os import distro def keys(certpath, provider, keypath, outpath): - """ combine certificate, CA and private key """ + """combine certificate, CA and private key""" - if distro.os_release_info()['id_like'] in ['debian', 'arch']: + if distro.os_release_info()["id_like"] in ["debian", "arch"]: ssl_dir = "/etc/ssl/certs" - elif distro.os_release_info()['id_like'] == 'rhel fedora': + elif distro.os_release_info()["id_like"] == "rhel fedora": ssl_dir = "/etc/pki/tls" + else: + raise NotImplementedError("OS not supported") - if provider == 'sectigo_ov': + if provider == "sectigo_ov": capath = os.path.join(ssl_dir, "COMODO_OV.crt") - elif provider == 'sectigo_ev': + elif provider == "sectigo_ev": capath = os.path.join(ssl_dir, "COMODO_EV.crt") - elif provider == 'letsencrypt': + elif provider == "letsencrypt": capath = os.path.join(ssl_dir, "LE.crt") + else: + raise NotImplementedError(f"Provider {provider} not supported") filenames = [certpath, capath, keypath] @@ -24,8 +30,8 @@ def keys(certpath, provider, keypath, outpath): except FileNotFoundError: pass - with open(outpath, 'w') as outfile: + with open(outpath, "w", encoding="utf-8") as outfile: for fname in filenames: - with open(fname) as infile: + with open(fname, "r", encoding="utf-8") as infile: outfile.write(infile.read()) outfile.close() diff --git a/wile_coyote/common/config.py b/wile_coyote/common/config.py index 09a207de22d1c34f6780fd33189f51cd8a7adaaf..59c80baed0c050579b2c535e3b81990c308d3bc3 100644 --- a/wile_coyote/common/config.py +++ b/wile_coyote/common/config.py @@ -1,3 +1,5 @@ +""" Check if the config file exists and is writable """ + import os import wile_coyote.common.constants from wile_coyote.common import log @@ -28,14 +30,14 @@ vault_token_team2 = fake_vault_token_for_team2 def check(log_file): - """ Check config file """ + """Check config file""" cred_conf = wile_coyote.common.constants.CRED_CONF if not os.access(cred_conf, os.W_OK): - cred_file = open(cred_conf, 'w+') + cred_file = open(cred_conf, "w+", encoding="utf-8") cred_file.write(CRED_FILE_CONTENT) cred_file.close() print(f"\nthe following file has been created: {cred_conf}\n") print("Fill it with proper values and run the script again\n") log.handler(f"\nthe file {cred_conf} has been created\n", log_file) log.handler("Fill in the values and run the script again\n", log_file) - os.exit(1) + os.sys.exit(1) diff --git a/wile_coyote/common/constants.py b/wile_coyote/common/constants.py index 56475117237bcedf3724bd26f512b16422d160b9..6a14cdf876c6950787357d6b0be28cbdef974adf 100644 --- a/wile_coyote/common/constants.py +++ b/wile_coyote/common/constants.py @@ -1,3 +1,5 @@ +""" Constants for the Wile Coyote project. """ + import os from datetime import datetime diff --git a/wile_coyote/common/log.py b/wile_coyote/common/log.py index 585da7dc49e5273302bf5e604ed0eab16a1eb391..082a2fd187e5a97a592afafe6e66260593dda3e1 100644 --- a/wile_coyote/common/log.py +++ b/wile_coyote/common/log.py @@ -1,3 +1,5 @@ +""" Logging module """ + import os import re import inspect @@ -5,18 +7,22 @@ import logging def handler(log_message, log_file, error=None): - """ handle logging """ + """handle logging""" # https://www.calazan.com/how-to-retrieve-the-name-of-the-calling-module-in-python/ frame_records = inspect.stack()[1] # REMINDER: # calling_module = inspect.getmodulename(frame_records[1]) _calling_module = os.path.basename(frame_records.filename) - calling_module = re.sub(r'\.py$', '', _calling_module) + calling_module = re.sub(r"\.py$", "", _calling_module) - extra = {'app_name': calling_module} - log_format = '%(asctime)s [%(app_name)s] %(levelname)s: %(message)s' - logging.basicConfig(filename=log_file, level=logging.INFO, - format=log_format, datefmt='%Y-%m-%d %H:%M:%S') + extra = {"app_name": calling_module} + log_format = "%(asctime)s [%(app_name)s] %(levelname)s: %(message)s" + logging.basicConfig( + filename=log_file, + level=logging.INFO, + format=log_format, + datefmt="%Y-%m-%d %H:%M:%S", + ) if error: logging.error(log_message, extra=extra) diff --git a/wile_coyote/common/sys_kit.py b/wile_coyote/common/sys_kit.py index 4c120996403045526a8cf7f8747bb12fc30fc03e..655d3a17dfdba7a2d6989be11705be620111c253 100644 --- a/wile_coyote/common/sys_kit.py +++ b/wile_coyote/common/sys_kit.py @@ -1,3 +1,5 @@ +""" system kit """ + import os import time from datetime import datetime @@ -5,35 +7,36 @@ from wile_coyote.common import log import wile_coyote.common.constants -def exit(logfile, start=None, status=0): - """ just exit """ +def coyote_exit(logfile, start=None, status=0): + """just exit""" if start: end = datetime.now() # end logging seconds_spent = time.gmtime((end - start).seconds) time_spent = time.strftime( - "%H hour(s), %M minute(s), %S second(s)", seconds_spent) + "%H hour(s), %M minute(s), %S second(s)", seconds_spent + ) else: - time_spent = '' + time_spent = "" if status != 0: - log.handler(f'JOB COMPLETED IN: {time_spent}', logfile, True) + log.handler(f"JOB COMPLETED IN: {time_spent}", logfile, True) else: - log.handler(f'JOB COMPLETED IN: {time_spent}', logfile) - quit(status) + log.handler(f"JOB COMPLETED IN: {time_spent}", logfile) + os.sys.exit(status) def file_access(file_name, log_file): - """ check if file can be accessed """ + """check if file can be accessed""" giveup = wile_coyote.common.constants.GIVEUP if not os.access(file_name, os.R_OK): - log.handler(f'could not access {file_name}: {giveup}', log_file, True) - quit(1) + log.handler(f"could not access {file_name}: {giveup}", log_file, True) + os.sys.exit(1) -def check_validity(path, type, log_file): - """ check certificate availability and expiration """ - if type == 'public': +def check_validity(path, key_type, log_file): + """check certificate availability and expiration""" + if key_type == "public": key_test = wile_coyote.tools.key_kit.Check(path, log_file).public() else: key_test = wile_coyote.tools.key_kit.Check(path, log_file).private() if not key_test: - os._exit(1) + os.sys.exit(1) diff --git a/wile_coyote/tools/consul_kit.py b/wile_coyote/tools/consul_kit.py index 4ba44fc6c48af46951019c6d388c909a9ca2ee5c..11be9e235b80be8ed1ba92fa5207d3c612acd71c 100644 --- a/wile_coyote/tools/consul_kit.py +++ b/wile_coyote/tools/consul_kit.py @@ -1,3 +1,5 @@ +""" Consul toolkit """ + import os import ssl import string @@ -17,7 +19,7 @@ def delete(key, leader): try: c_client.kv.delete(key) - except Exception as _: + except Exception as _: # pylint: disable=w0703 pass @@ -28,7 +30,7 @@ def get(keyname, log_file, leader): try: _, data = c_client.kv.get(keyname) - except Exception as err: + except Exception as err: # pylint: disable=w0703 log.handler( f'could not connect to Consul {leader}', log_file, True) return err @@ -86,7 +88,7 @@ def test(log_file, leader): try: _ = c_client.kv.put('do_not_delete', rnd) - except Exception as err: + except Exception as err: # pylint: disable=w0703 log.handler(f'{atomic_msg} could not write test key to {leader}: {err}', log_file, True) log.handler(wile_coyote.common.constants.GIVEUP, log_file) diff --git a/wile_coyote/tools/key_kit.py b/wile_coyote/tools/key_kit.py index bc5fc0adf6f8797e17c53ecf7d49a92eb2a7e826..f74a3430bcddb6990eed4bf4eb80204f623275f6 100644 --- a/wile_coyote/tools/key_kit.py +++ b/wile_coyote/tools/key_kit.py @@ -1,72 +1,79 @@ -import os +""" Check key """ + import OpenSSL.crypto from wile_coyote.common import log import wile_coyote.common.constants class Check: - """ Check key """ - def __init__(self, key, log_file, key_name='unknown', from_file=True): + """Check key""" + + def __init__(self, key, log_file, key_name="unknown", from_file=True): self.key = key self.log_file = log_file self.key_name = key_name self.from_file = from_file def public(self): - """ check certificate expiration """ + """check certificate expiration""" if self.from_file: - key_path = self.key + keypath = self.key try: - st_cert = open(self.key, 'rt', encoding="utf8").read() + st_cert = open(self.key, "rt", encoding="utf8").read() except Exception as err: # pylint: disable=w0703 - log.handler(f'Could not open/find the certificate {key_path}: {err}', self.log_file, True) + log.handler( + f"Could not open/find the certificate {keypath}: {err}", + self.log_file, + True, + ) return False else: st_cert = self.key - key_path = self.key_name - + keypath = self.key_name + ssl_crypto = OpenSSL.crypto - + try: cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert) except Exception as err: # pylint: disable=w0703 - log.handler(f'The certificate {key_path} is malformed: {err}', self.log_file, True) + log.handler( + f"The certificate {keypath} is malformed: {err}", self.log_file, True + ) return False - + if cert.has_expired(): log.handler( - f'{key_path} expired and it will not be uploaded', self.log_file, True) - log.handler(wile_coyote.common.constants.giveup, self.log_file) + f"{keypath} expired and it will not be uploaded", self.log_file, True + ) + log.handler(wile_coyote.common.constants.GIVEUP, self.log_file) return False - - return True + return True def private(self): - """ check private key validity """ + """check private key validity""" if self.from_file: - key_path = self.key + keypath = self.key try: - st_key = open(self.key, 'rt', encoding="utf8").read() + st_key = open(self.key, "rt", encoding="utf8").read() except Exception as err: # pylint: disable=w0703 - log.handler(f'{key_path} is not a valid key: {err}', self.log_file, True) + log.handler(f"{keypath} is not a valid key: {err}", self.log_file, True) return False else: st_key = self.key - key_path = self.key_name - + keypath = self.key_name + ssl_crypto = OpenSSL.crypto try: key = ssl_crypto.load_privatekey(ssl_crypto.FILETYPE_PEM, st_key) except Exception as err: # pylint: disable=w0703 - log.handler(f'{key_path} is not a valid key: {err}', self.log_file, True) + log.handler(f"{keypath} is not a valid key: {err}", self.log_file, True) return False try: key.check() except Exception as err: # pylint: disable=w0703 - log.handler(f'{key_path} is not a valid key: {err}', self.log_file, True) + log.handler(f"{keypath} is not a valid key: {err}", self.log_file, True) return False - - return True + return True diff --git a/wile_coyote/tools/redis_kit.py b/wile_coyote/tools/redis_kit.py index 4c2bda0e91c88ab5e6a0dbe15887b947a92ca810..a45492a454c33041791d76781743f43f1ef2242e 100644 --- a/wile_coyote/tools/redis_kit.py +++ b/wile_coyote/tools/redis_kit.py @@ -1,3 +1,5 @@ +""" Redis tools """ + import os import string import random @@ -62,7 +64,7 @@ def test(log_file): try: _ = r_client.set('do_not_delete', rnd) - except Exception as err: + except Exception as err: # pylint: disable=W0703 log.handler(f'{atomic_msg} could not write test key to {host}: {err}', log_file, True) log.handler('giving up...', log_file) diff --git a/wile_coyote/tools/vault_kit.py b/wile_coyote/tools/vault_kit.py index f0dd1e15d6e30d134c66b5c0df5ff8da8402335d..695033dafac966c744140bd62ff925ac72aeb933 100644 --- a/wile_coyote/tools/vault_kit.py +++ b/wile_coyote/tools/vault_kit.py @@ -1,98 +1,101 @@ +""" Vault operations """ + import os import string import random import hvac -from requests.packages.urllib3 import disable_warnings # pylint: disable=E0401 -from requests.packages.urllib3.exceptions import InsecureRequestWarning # pylint: disable=E0401 +import urllib3 import wile_coyote.tools import wile_coyote.common.constants from wile_coyote.common import log def get(keyname, unit): - """ Download key from Vault """ - disable_warnings(InsecureRequestWarning) # pylint: disable=E1101 + """Download key from Vault""" + urllib3.disable_warnings() # pylint: disable=E1101 host = wile_coyote.tools.VAULT_HOST - token = wile_coyote.tools.config.get('acme', f'vault_token_{unit}') - client = hvac.Client(url=f'https://{host}', token=token) + token = wile_coyote.tools.config.get("acme", f"vault_token_{unit}") + client = hvac.Client(url=f"https://{host}", token=token) try: - vault_value = client.read(keyname)['data']['value'] + vault_value = client.read(keyname)["data"]["value"] except Exception as err: # pylint: disable=w0703 vault_value = err return vault_value -def get_v2(key, unit='nomad', mount_point='nomad'): - """ Download key to Vault v2 """ - disable_warnings(InsecureRequestWarning) # pylint: disable=E1101 +def get_v2(key, unit="nomad", mount_point="nomad"): + """Download key to Vault v2""" + urllib3.disable_warnings() # pylint: disable=E1101 host = wile_coyote.tools.VAULT_HOST - token = wile_coyote.tools.config.get('acme', f'vault_token_{unit}') - v_client = hvac.Client(url=f'https://{host}', token=token) + token = wile_coyote.tools.config.get("acme", f"vault_token_{unit}") + v_client = hvac.Client(url=f"https://{host}", token=token) try: data_obj = v_client.secrets.kv.read_secret_version( - mount_point=mount_point, path=key) + mount_point=mount_point, path=key + ) except Exception as err: # pylint: disable=w0703 return err if data_obj: - return data_obj['data']['data']['value'] + return data_obj["data"]["data"]["value"] - return 'BOFH' + return "BOFH" def put(key, value, log_file, unit): - """ Upload a key to Vault """ - disable_warnings(InsecureRequestWarning) # pylint: disable=E1101 + """Upload a key to Vault""" + urllib3.disable_warnings() # pylint: disable=E1101 host = wile_coyote.tools.VAULT_HOST - token = wile_coyote.tools.config.get('acme', f'vault_token_{unit}') - v_client = hvac.Client(url=f'https://{host}', token=token) + token = wile_coyote.tools.config.get("acme", f"vault_token_{unit}") + v_client = hvac.Client(url=f"https://{host}", token=token) try: _ = v_client.write(key, value=value) except Exception as err: # pylint: disable=w0703 - log.handler(f'could not write key {key} to Vault {host}: {err}', - log_file, True) + log.handler(f"could not write key {key} to Vault {host}: {err}", log_file, True) log.handler(wile_coyote.common.constants.GIVEUP, log_file) - os._exit(1) + os.sys.exit(1) -def put_v2(key, value, log_file, unit='nomad', mount_point='nomad'): - """ upload key to Vault v2 """ - disable_warnings(InsecureRequestWarning) # pylint: disable=E1101 +def put_v2(key, value, log_file, unit="nomad", mount_point="nomad"): + """upload key to Vault v2""" + urllib3.disable_warnings() # pylint: disable=E1101 host = wile_coyote.tools.VAULT_HOST - token = wile_coyote.tools.config.get('acme', f'vault_token_{unit}') - v_client = hvac.Client(url=f'https://{host}', token=token) + token = wile_coyote.tools.config.get("acme", f"vault_token_{unit}") + v_client = hvac.Client(url=f"https://{host}", token=token) try: _ = v_client.secrets.kv.v2.create_or_update_secret( - mount_point=mount_point, - path=key, - secret=dict(value=value) + mount_point=mount_point, path=key, secret=dict(value=value) ) except Exception as err: # pylint: disable=w0703 - log.loghandler( - f'could not write key {key} to {host}: {err}', log_file, True) - log.loghandler(wile_coyote.common.constants.GIVEUP, log_file) - os._exit(1) + log.handler(f"could not write key {key} to {host}: {err}", log_file, True) + log.handler(wile_coyote.common.constants.GIVEUP, log_file) + os.sys.exit(1) def test(log_file): - """ Check Vault connection """ + """Check Vault connection""" atomic_msg = wile_coyote.tools.ATOMIC_MSG - disable_warnings(InsecureRequestWarning) # pylint: disable=E1101 + urllib3.disable_warnings() # pylint: disable=E1101 host = wile_coyote.tools.VAULT_HOST - token = wile_coyote.tools.config.get('acme', f'vault_token_puppet') - key_name = 'puppet/do_not_delete' - rnd = ''.join([random.choice(string.ascii_letters) for _ in range(8)]) - v_client = hvac.Client(url=f'https://{host}', token=token, verify=False) + token = wile_coyote.tools.config.get("acme", "vault_token_puppet") + key_name = "puppet/do_not_delete" + rnd = "".join([random.choice(string.ascii_letters) for _ in range(8)]) + v_client = hvac.Client(url=f"https://{host}", token=token, verify=False) try: _ = v_client.write(key_name, value=rnd) except Exception as err: # pylint: disable=w0703 - log.handler(f'{atomic_msg} could not write key {key_name} to Vault {host}: {err}', - log_file, True) + log.handler( + f"{atomic_msg} could not write key {key_name} to Vault {host}: {err}", + log_file, + True, + ) log.handler(wile_coyote.common.constants.GIVEUP, log_file) - os._exit(1) + os.sys.exit(1) else: - log.handler(f'{atomic_msg} successfully wrote Vault test key to {host}', log_file) + log.handler( + f"{atomic_msg} successfully wrote Vault test key to {host}", log_file + )