diff --git a/.gitignore b/.gitignore
index af6ad96183e7aea0d454b4004df65f22ba4285df..36b9f0571cf6a4f336326e320765d28f07616145 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
.*.sw[op]
+wile_coyote-*.tar.gz
# Byte-compiled / optimized / DLL files
__pycache__/
diff --git a/Description.txt b/Description.txt
index 8da220a81fb455c351b06fc6ad96721c3e63bea6..555c396514df97c4372a580cc3e678129b674555 100644
--- a/Description.txt
+++ b/Description.txt
@@ -1,3 +1,3 @@
== A tool to manage certificate lifecycle on Vault, Redis, Consul
-this tool is used in conjunction with certbot to upload certificates to Vault, Redis and Consul
+this tool is used in conjunction with certbot to upload certificates to the key stores
diff --git a/README.md b/README.md
index 6c80339537f4633fa09ff3ed2ca474a5e48edc3e..74db8ad2257e88fc375fa3b37efa1396b71745f3 100644
--- a/README.md
+++ b/README.md
@@ -7,21 +7,21 @@
The version number is defined in `./version/__init__.py`\
In this README we assume that we are using version `0.5.0`
-### Use pip
+### Build pip
```sh
python3 setup.py bdist_wheel
pip install dist wile_coyote-0.5.0-py3-none-any.whl
```
-### Use RPM
+### UBuildse RPM
```sh
python3 setup.py bdist_rpm
sudo rpm -Uvh dist/wile_coyote-0.5.0-1.noarch.rpm
```
-### Use DEB
+### Build DEB
`DEB_BUILD_OPTIONS=nocheck` is required because `dh_auto_test` resets `$HOME` variable and the unit test fails because it won't find `acme.ini`
diff --git a/acme_kit/nomad_uploader.py b/acme_kit/nomad_uploader.py
index df3b37695b4e4a73268ca7d16a46281c9b6a9bf1..98f62da55a8fb3fce4fe4d6a04703a995bfb1c01 100644
--- a/acme_kit/nomad_uploader.py
+++ b/acme_kit/nomad_uploader.py
@@ -3,11 +3,11 @@
"""Nomad Uploader
Options:
- provider = ACME Provider (sectigo_ev, sectigo_ov, letsencrypt)
- domain = Certificate name
- project = Nomad Project
- environment = staging environment
- wildcard = Wildcard (Bool)
+ provider = ACME Provider (sectigo_ev, sectigo_ov, letsencrypt)
+ domain = Certificate name
+ project = Nomad Project
+ nomad_env = staging nomad_env
+ wildcard = Wildcard (Bool)
"""
import os
import wile_coyote.tools
@@ -15,7 +15,7 @@ import wile_coyote.common_kit
from wile_coyote.common_kit import log
-def uploader(provider, project, domain, environment, wildcard=None):
+def uploader(provider, project, domain, nomad_env, wildcard=None):
""" Upload wildcard certificate to Consul and Vault """
if type(provider) is str:
@@ -23,10 +23,10 @@ def uploader(provider, project, domain, environment, wildcard=None):
else:
project_list = project
- if type(environment) is str:
- environment_list = [environment]
+ if type(nomad_env) is str:
+ nomad_env_list = [nomad_env]
else:
- environment_list = environment
+ nomad_env_list = nomad_env
if wildcard:
wcard_flag = 'wildcard_'
@@ -52,7 +52,7 @@ def uploader(provider, project, domain, environment, wildcard=None):
wile_coyote.common_kit.private_key.check(keypath, log_file)
- for env in environment_list:
+ for env in nomad_env_list:
for proj in project_list:
# upload certificates to Consul
for suffix in suffixes_list:
diff --git a/bin/anvil.py b/bin/anvil.py
new file mode 100755
index 0000000000000000000000000000000000000000..e394e9931fadfcd29ce34f5d2350175281b9ba60
--- /dev/null
+++ b/bin/anvil.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Anvil
+ wile_coyote test tool, erases, uploads and check certificates
+ validity onto test instances of Vault, Redis and Consul
+
+Usage:
+ anvil.py [--prune]
+ anvil.py (-h | --help)
+
+Options:
+ -h --help Show this screen.
+ -p --prune Client
+"""
+import os
+import re
+from glob import glob
+import subprocess as sp
+from docopt import docopt
+import redis
+from consul import Consul
+import hvac
+import wile_coyote.tools
+import wile_coyote.common_kit
+from wile_coyote.common_kit import log
+
+
+def certificates_delete(provider: str, certificates: list):
+ """ delete list of certificates from a given provider """
+ for cert in certificates:
+ cbot_cmd = '/usr/local/bin/certbot delete -non-interactive' \
+ + f' -c /etc/${provider}/cli.ini --cert-name ${cert}'
+ cbot_child = sp.Popen(cbot_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
+ cbot_out, cbot_err = cbot_child.communicate()
+ if cbot_child.returncode != 0:
+ err = cbot_err.decode("utf-8")
+ log.handler(f"error executing certbot: {err}", LOGFILE, True)
+ os.sys.exit(1)
+ else:
+ decoded_msg = cbot_out.decode("utf-8")
+ msg = decoded_msg[:decoded_msg.rfind('\n')]
+ log.handler(msg, LOGFILE)
+
+
+def redis_keys(server, token):
+ """ download keys from Redis """
+ log.handler('fetching keys from Redis...', LOGFILE)
+ r_client = redis.StrictRedis(
+ host=server, password=token, port=6379, db=0)
+
+ try:
+ r_keys = [n.decode("utf-8") for n in r_client.keys('*')]
+ except Exception as err: # pylint: disable=w0703
+ r_keys = err
+
+ return sorted(r_keys)
+
+
+def redis_prune(server, token):
+ """ prune keys from Redis """
+ log.handler('purging keys from Redis...', LOGFILE)
+ r_client = redis.StrictRedis(host=server, password=token, port=6379, db=0)
+ r_keys = [n.decode("utf-8") for n in r_client.keys('*')]
+ for key in r_keys:
+ r_client.delete(key)
+
+
+def vault_keys(server, token, mount_points):
+ """ download key from vault """
+ log.handler('fetching keys from Vault...', LOGFILE)
+ all_vault_keys = []
+ for mount in mount_points:
+ vault_cmd = f'/usr/local/bin/rvault --token {token}' \
+ + f' --insecure --address https://{server} list {mount}/'
+ vault_child = sp.Popen(
+ vault_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
+ vault_out, vault_err = vault_child.communicate()
+ if vault_child.returncode != 0:
+ err = vault_err.decode("utf-8")
+ log.handler(f'error listing keys for the mount point {mount}: {err}', LOGFILE)
+ os.sys.exit(1)
+ else:
+ decoded_msg = vault_out.decode("utf-8")
+ _msg = [x for x in decoded_msg.split('\n') if x != '']
+ msg = [f'/{mount}{x}' for x in _msg]
+ all_vault_keys.extend(msg)
+
+ return sorted(all_vault_keys)
+
+
+def vault_prune(server, token, mount_points, ver):
+ """ prune key to vault """
+ log.handler('purging keys from Vault...')
+ v_client = hvac.Client(url=f'https://{server}', token=token)
+ for mount in mount_points:
+ v_client.sys.disable_secrets_engine(mount)
+ v_client.sys.enable_secrets_engine(
+ 'kv', path=mount, options={'version': ver})
+
+
+def consul_keys(server, token):
+ """ download keys from consul """
+ log.handler('fetching keys from Consul...', LOGFILE)
+ c_client = Consul(host=server, port='443', token=token, scheme='https')
+ try:
+ c_keys = sorted(c_client.kv.get('nomad', recurse=True, keys=True)[1])
+ except Exception as err: # pylint: disable=W0703
+ return err
+
+ return sorted(c_keys)
+
+
+def consul_prune(server, token):
+ """ prune keys from consul """
+ log.handler('purging keys from Consul...', LOGFILE)
+ c_client = Consul(host=server, port='443', token=token, scheme='https')
+ for nomad_env in ['test', 'uat', 'prod']:
+ c_client.kv.delete(f'nomad/{nomad_env}', recurse=True)
+
+
+# Here we Go.
+if __name__ == "__main__":
+
+ ARGS = docopt(__doc__)
+ PRUNE = ARGS['--prune']
+ LOGFILE = '/dev/stdout'
+
+ # Anvil should not be used in production
+ VAULT_ROOT_TOKEN = wile_coyote.tools.VAULT_ROOT_TOKEN
+ if not VAULT_ROOT_TOKEN:
+ log.handler('you can use this tool ONLY in test', LOGFILE, True)
+ log.handler('exiting ...', LOGFILE, True)
+ os.sys.exit()
+
+ REDIS_HOST = wile_coyote.tools.REDIS_HOST
+ REDIS_TOKEN = wile_coyote.tools.REDIS_TOKEN
+ VAULT_HOST = wile_coyote.tools.VAULT_HOST
+ CONSUL_SERVERS = wile_coyote.tools.CONSUL_SERVERS
+ CONSUL_TOKEN = wile_coyote.tools.CONSUL_TOKEN
+ CONSUL_LEADER, _, __ = wile_coyote.tools.consul_leader.get(LOGFILE)
+ MOUNT_POINTS_V1 = wile_coyote.tools.MOUNT_POINTS_V1
+ MOUNT_POINTS_V2 = wile_coyote.tools.MOUNT_POINTS_V2
+
+ # keys define in .acme.ini
+ REDIS_KEYS = wile_coyote.tools.REDIS_KEYS
+ VAULT_KEYS = wile_coyote.tools.VAULT_KEYS
+ CONSUL_KEYS = wile_coyote.tools.CONSUL_KEYS
+
+ # delete and upload keys again
+ consul_prune(CONSUL_LEADER, CONSUL_TOKEN)
+ redis_prune(REDIS_HOST, REDIS_TOKEN)
+ vault_prune(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V1, 1)
+ vault_prune(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V2, 2)
+
+ # prune certificates locally
+ ACME_PROVIDERS = ['sectigo_ev', 'sectigo_ev', 'letsencrypt']
+ for acme_provider in ACME_PROVIDERS:
+ acme_certificates = glob(f'/etc/{acme_provider}/live/*')
+ try:
+ acme_certificates.remove(f'/etc/{acme_provider}/live/README')
+ except ValueError:
+ pass
+ certificates_delete(acme_provider, acme_certificates)
+
+ # run all scripts under /opt/acme/bin
+ for script in glob('/opt/acme/bin/*'):
+ log.handler(f'running script {script}', LOGFILE)
+ os.system(script)
+
+ # upstream keys
+ C_KEYS = consul_keys(CONSUL_LEADER, CONSUL_TOKEN)
+ R_KEYS = redis_keys(REDIS_HOST, REDIS_TOKEN)
+ V_KEYS_V1 = vault_keys(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V1)
+ V_KEYS_V2 = vault_keys(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V2)
+ V_KEYS_ALL = sorted(V_KEYS_V1 + V_KEYS_V2)
+
+ # differences
+ C_DIFFS = [item for item in CONSUL_KEYS if item not in C_KEYS]
+ R_DIFFS = [item for item in REDIS_KEYS if item not in R_KEYS]
+ V_DIFFS = [item for item in VAULT_KEYS if item not in V_KEYS_ALL]
+ for key_store in ['Consul', 'Redis', 'Vault']:
+ if key_store == 'Consul':
+ key_list = C_DIFFS
+ elif key_store == 'Redis':
+ key_list = R_DIFFS
+ elif key_store == 'Vault':
+ key_list = V_DIFFS
+ if key_list:
+ log.handler(f'the following keys are not available on {key_store}:', LOGFILE, True)
+ log.handler(' '.join(key_list), LOGFILE, True)
+ os.sys.exit(1)
+ else:
+ log.handler(f'{key_store} keys names have been successfully checked')
+
+ if REDIS_KEYS != R_KEYS:
+ log.handler('downstream and upstream Redis keys are different', LOGFILE, True)
+ os.sys.exit(1)
+ elif CONSUL_KEYS != C_KEYS:
+ log.handler('downstream and upstream Consul keys are different', LOGFILE, True)
+ os.sys.exit(1)
+ elif VAULT_KEYS != V_KEYS_ALL:
+ log.handler('downstream and upstream Vault keys are different', LOGFILE, True)
+ os.sys.exit(1)
+
+ # validate public keys
+ for r_key in [item for item in REDIS_KEYS if '_chain.pem' not in R_KEYS]:
+ log.handler(f'checking certificate {r_key} on Redis', LOGFILE)
+ pubkey = wile_coyote.tools.redis_get.get(r_key)
+ wile_coyote.common_kit.public_key.check(pubkey, LOGFILE, r_key, False)
+
+ for c_key in [item for item in CONSUL_KEYS if '_chain.pem' not in C_KEYS]:
+ log.handler(f'checking certificate {c_key} on Consul', LOGFILE)
+ pubkey = wile_coyote.tools.consul_get.get(c_key)
+ wile_coyote.common_kit.public_key.check(pubkey, LOGFILE, c_key, False)
+
+ # validate private keys: we check only if it's not malformed
+ for v_key in V_KEYS_V1:
+ log.handler(f'checking private key {v_key} on Vault', LOGFILE)
+ privkey = wile_coyote.tools.vault_get.get(v_key, 'root')
+ wile_coyote.common_kit.public_key.check(privkey, LOGFILE, v_key, False)
+
+ for v_key in V_KEYS_V2:
+ log.handler(f'checking private key {v_key} on Vault v2', LOGFILE)
+ fixed_v_key = re.sub(r'^/nomad/', '/', v_key)
+ privkey = wile_coyote.tools.vault_get_v2.get(fixed_v_key, 'root')
+ wile_coyote.common_kit.public_key.check(privkey, LOGFILE, v_key, False)
diff --git a/bin/wile_coyote.py b/bin/wile_coyote.py
index b6395e993fb8fa145ca862f3f480d85f10dcc3f4..cff709aefddccf6d97827e759eee4e3b86d60552 100644
--- a/bin/wile_coyote.py
+++ b/bin/wile_coyote.py
@@ -1,25 +1,25 @@
#!/usr/bin/env python3
#
-"""Geant Acme
+"""Wile Coyote
Usage:
- acme.py (--domain <DOMAIN>...) (--provider <PROVIDER>) --days <DAYS> \
-[--project <PROJECT>...] [--environment <ENVIRONMENT>...] [--tld] \
-[--unit <UNIT>] [--client <CLIENT>...] [--wildcard] [--extra <EXTRA>...]
- acme.py (-h | --help)
+ wile_coyote.py (--domain <DOMAIN>...) (--provider <PROVIDER>) --days <DAYS> \
+[--project <PROJECT>...] [--nomad-env <ENV>...] [--tld] [--unit <UNIT>] \
+[--client <CLIENT>...] [--wildcard] [--extra <EXTRA>...]
+ wile_coyote.py (-h | --help)
Options:
- -h --help Show this screen.
- -c CLIENT --client=CLIENT Client
- -d DOMAIN --domain=DOMAIN Domain
- -p PROVIDER --provider=PROVIDER Provider
- -u UNIT --unit=UNIT Unit, entity or team
- --days DAYS CRIT days before expiration
- --project=PROJECT Project (Nomad only)
- -e ENVIRONMENT --environment=ENVIRONMENT Environment (Nomad only)
- -t --tld Top Level Domain
- -w --wildcard Use wildcard
- -x --extra=EXTRA Supply extra parameters (check certbot documentation)
+ -h --help Show this screen.
+ -c CLIENT --client=CLIENT Client
+ -d DOMAIN --domain=DOMAIN Domain
+ -p PROVIDER --provider=PROVIDER Provider
+ -u UNIT --unit=UNIT Unit, entity or team
+ --days DAYS CRIT days before expiration
+ --project=PROJECT Project (Nomad only)
+ --nomad-env=ENV Environment (Nomad only)
+ -t --tld Top Level Domain
+ -w --wildcard Use wildcard
+ -x --extra=EXTRA Supply extra parameters (check certbot documentation)
"""
import os
import time
@@ -124,7 +124,7 @@ if __name__ == "__main__":
DOMAIN = ARGS['--domain']
UNIT = ARGS['--unit']
PROJECT = ARGS['--project']
- ENVIRONMENT = ARGS['--environment']
+ ENV = ARGS['--nomad-env']
FIRST_NAME = DOMAIN[0]
WILDCARD = ARGS['--wildcard'] # True or None
EXTRA = ARGS['--extra']
@@ -192,7 +192,7 @@ if __name__ == "__main__":
log.handler(
f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for Nomad', LOG_FILE)
wile_coyote.acme_kit.nomad_uploader.uploader(
- PROVIDER, PROJECT, FIRST_NAME, ENVIRONMENT, WILDCARD)
+ PROVIDER, PROJECT, FIRST_NAME, ENV, WILDCARD)
else:
log.handler(
f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for {UNIT}', LOG_FILE)
diff --git a/common_kit/private_key.py b/common_kit/private_key.py
index 026f632e0fc3cc7e0f5918e012150ec2e8e2d444..029dcad592293609a7dfd1f7c7d7000ae79abd6b 100644
--- a/common_kit/private_key.py
+++ b/common_kit/private_key.py
@@ -7,14 +7,19 @@ from wile_coyote.common_kit import log
import wile_coyote.common_kit.constants
-def check(privkey, log_file):
+def check(privkey, log_file, key_name='unknown', from_file=True):
""" check private key validity """
giveup = wile_coyote.common_kit.constants.GIVEUP
is_broken = False
- try:
- st_key = open(privkey, 'rt', encoding="utf8").read()
- except Exception: # pylint: disable=w0703
- is_broken = True
+ if from_file:
+ key_path = privkey
+ try:
+ st_key = open(privkey, 'rt', encoding="utf8").read()
+ except Exception: # pylint: disable=w0703
+ is_broken = True
+ else:
+ st_key = privkey
+ key_path = key_name
ssl_crypto = OpenSSL.crypto
try:
@@ -28,7 +33,7 @@ def check(privkey, log_file):
is_broken = True
if is_broken:
- log.handler(f'{privkey} is not a valid key: {giveup}', log_file, True)
+ log.handler(f'{key_path} is not a valid key: {giveup}', log_file, True)
os.sys.exit(1)
return True
diff --git a/common_kit/public_key.py b/common_kit/public_key.py
index c2574dce763ea13844b69a1dd118004ec3924732..c0e8a1261ffdb3708060d6a269c32b215a6acfa1 100644
--- a/common_kit/public_key.py
+++ b/common_kit/public_key.py
@@ -7,14 +7,25 @@ from wile_coyote.common_kit import log
import wile_coyote.common_kit.constants
-def check(certificate, log_file):
+def check(certificate, log_file, key_name='unknown', from_file=True):
""" check certificate expiration """
- st_cert = open(certificate, 'rt', encoding="utf8").read()
+ if from_file:
+ st_cert = open(certificate, 'rt', encoding="utf8").read()
+ key_path = certificate
+ else:
+ st_cert = certificate
+ key_path = key_name
ssl_crypto = OpenSSL.crypto
- cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert)
+
+ try:
+ cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert)
+ except Exception as err: # pylint: disable=w0703
+ log.handler(f'Error. The certificate {key_path} is malformed: {err}')
+ os.sys.exit(1)
+
if cert.has_expired():
log.handler(
- f'{certificate} expired and it will not be uploaded', log_file, True)
+ f'{key_path} expired and it will not be uploaded', log_file, True)
log.handler(wile_coyote.common_kit.constants.giveup, log_file)
os.sys.exit(1)
diff --git a/tools/__init__.py b/tools/__init__.py
index 1f9615ea7edbe52ab1654f02e80d7d4ed4dff205..bbb7fde200097c9a5980ce3f72bea197acdc8f86 100644
--- a/tools/__init__.py
+++ b/tools/__init__.py
@@ -1,3 +1,4 @@
+from ast import literal_eval as l_eval
import pkgutil
import configparser
import wile_coyote.common_kit.constants
@@ -15,3 +16,35 @@ for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
REDIS_TOKEN = config.get('acme', 'redis_token')
CONSUL_LIST = config.get('acme', 'consul_servers')
CONSUL_TOKEN = config.get('acme', 'consul_token')
+
+ # these parameters only work in test
+ #
+ try:
+ VAULT_ROOT_TOKEN = config.get('unit-test', 'vault_token_root')
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ VAULT_ROOT_TOKEN = None
+
+ try:
+ MOUNT_POINTS_V1 = l_eval(config.get('unit-test', 'mount_points_v1'))
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ MOUNT_POINTS_V1 = None
+
+ try:
+ MOUNT_POINTS_V2 = l_eval(config.get('unit-test', 'mount_points_v2'))
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ MOUNT_POINTS_V2 = None
+
+ try:
+ REDIS_KEYS = sorted(l_eval(config.get('unit-test', 'redis_keys')))
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ REDIS_KEYS = None
+
+ try:
+ VAULT_KEYS = sorted(l_eval(config.get('unit-test', 'vault_keys')))
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ VAULT_KEYS = None
+
+ try:
+ CONSUL_KEYS = sorted(l_eval(config.get('unit-test', 'consul_keys')))
+ except (configparser.NoSectionError, configparser.NoOptionError):
+ CONSUL_KEYS = None