added anvil.py

.gitignore

 .*.sw[op]
+wile_coyote-*.tar.gz
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Description.txt

 == A tool to manage certificate lifecycle on Vault, Redis, Consul
 
-this tool is used in conjunction with certbot to upload certificates to Vault, Redis and Consul
+this tool is used in conjunction with certbot to upload certificates to the key stores

README.md

@@ -7,21 +7,21 @@
 The version number is defined in `./version/__init__.py`\
 In this README we assume that we are using version `0.5.0`
 
-### Use pip
+### Build pip
 
 ```sh
 python3 setup.py bdist_wheel
 pip install dist wile_coyote-0.5.0-py3-none-any.whl
 ```
 
-### Use RPM
+### UBuildse RPM
 
 ```sh
 python3 setup.py bdist_rpm
 sudo rpm -Uvh dist/wile_coyote-0.5.0-1.noarch.rpm
 ```
 
-### Use DEB
+### Build DEB
 
 `DEB_BUILD_OPTIONS=nocheck` is required because `dh_auto_test` resets `$HOME` variable and the unit test fails because it won't find `acme.ini`
 

acme_kit/nomad_uploader.py

@@ -3,11 +3,11 @@
 """Nomad Uploader
 
 Options:
-  provider    = ACME Provider (sectigo_ev, sectigo_ov, letsencrypt)
-  domain      = Certificate name
-  project     = Nomad Project
-  environment = staging environment
-  wildcard    = Wildcard (Bool)
+  provider  = ACME Provider (sectigo_ev, sectigo_ov, letsencrypt)
+  domain    = Certificate name
+  project   = Nomad Project
+  nomad_env = staging nomad_env
+  wildcard  = Wildcard (Bool)
 """
 import os
 import wile_coyote.tools
@@ -15,7 +15,7 @@ import wile_coyote.common_kit
 from wile_coyote.common_kit import log
 
 
-def uploader(provider, project, domain, environment, wildcard=None):
+def uploader(provider, project, domain, nomad_env, wildcard=None):
     """ Upload wildcard certificate to Consul and Vault """
 
     if type(provider) is str:
@@ -23,10 +23,10 @@ def uploader(provider, project, domain, environment, wildcard=None):
     else:
         project_list = project
 
-    if type(environment) is str:
-        environment_list = [environment]
+    if type(nomad_env) is str:
+        nomad_env_list = [nomad_env]
     else:
-        environment_list = environment
+        nomad_env_list = nomad_env
 
     if wildcard:
         wcard_flag = 'wildcard_'
@@ -52,7 +52,7 @@ def uploader(provider, project, domain, environment, wildcard=None):
     wile_coyote.common_kit.private_key.check(keypath, log_file)
 
 
-    for env in environment_list:
+    for env in nomad_env_list:
         for proj in project_list:
             # upload certificates to Consul
             for suffix in suffixes_list:

bin/anvil.py

+#!/usr/bin/env python3
+"""Anvil
+   wile_coyote test tool, erases, uploads and check certificates
+   validity onto test instances of Vault, Redis and Consul 
+
+Usage:
+  anvil.py [--prune]
+  anvil.py (-h | --help)
+
+Options:
+  -h --help    Show this screen.
+  -p --prune   Client
+"""
+import os
+import re
+from glob import glob
+import subprocess as sp
+from docopt import docopt
+import redis
+from consul import Consul
+import hvac
+import wile_coyote.tools
+import wile_coyote.common_kit
+from wile_coyote.common_kit import log
+
+
+def certificates_delete(provider: str, certificates: list):
+    """ delete list of certificates from a given provider """
+    for cert in certificates:
+        cbot_cmd = '/usr/local/bin/certbot delete -non-interactive' \
+            + f' -c /etc/${provider}/cli.ini --cert-name ${cert}'
+        cbot_child = sp.Popen(cbot_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
+        cbot_out, cbot_err = cbot_child.communicate()
+        if cbot_child.returncode != 0:
+            err = cbot_err.decode("utf-8")
+            log.handler(f"error executing certbot: {err}", LOGFILE, True)
+            os.sys.exit(1)
+        else:
+            decoded_msg = cbot_out.decode("utf-8")
+            msg = decoded_msg[:decoded_msg.rfind('\n')]
+            log.handler(msg, LOGFILE)
+
+
+def redis_keys(server, token):
+    """ download keys from Redis """
+    log.handler('fetching keys from Redis...', LOGFILE)
+    r_client = redis.StrictRedis(
+        host=server, password=token, port=6379, db=0)
+
+    try:
+        r_keys = [n.decode("utf-8") for n in r_client.keys('*')]
+    except Exception as err:  # pylint: disable=w0703
+        r_keys = err
+
+    return sorted(r_keys)
+
+
+def redis_prune(server, token):
+    """ prune keys from Redis """
+    log.handler('purging keys from Redis...', LOGFILE)
+    r_client = redis.StrictRedis(host=server, password=token, port=6379, db=0)
+    r_keys = [n.decode("utf-8") for n in r_client.keys('*')]
+    for key in r_keys:
+        r_client.delete(key)
+
+
+def vault_keys(server, token, mount_points):
+    """ download key from vault """
+    log.handler('fetching keys from Vault...', LOGFILE)
+    all_vault_keys = []
+    for mount in mount_points:
+        vault_cmd = f'/usr/local/bin/rvault --token {token}' \
+            + f' --insecure --address https://{server} list {mount}/'
+        vault_child = sp.Popen(
+            vault_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
+        vault_out, vault_err = vault_child.communicate()
+        if vault_child.returncode != 0:
+            err = vault_err.decode("utf-8")
+            log.handler(f'error listing keys for the mount point {mount}: {err}', LOGFILE)
+            os.sys.exit(1)
+        else:
+            decoded_msg = vault_out.decode("utf-8")
+            _msg = [x for x in decoded_msg.split('\n') if x != '']
+            msg = [f'/{mount}{x}' for x in _msg]
+        all_vault_keys.extend(msg)
+
+    return sorted(all_vault_keys)
+
+
+def vault_prune(server, token, mount_points, ver):
+    """ prune key to vault """
+    log.handler('purging keys from Vault...')
+    v_client = hvac.Client(url=f'https://{server}', token=token)
+    for mount in mount_points:
+        v_client.sys.disable_secrets_engine(mount)
+        v_client.sys.enable_secrets_engine(
+            'kv', path=mount, options={'version': ver})
+
+
+def consul_keys(server, token):
+    """ download keys from consul """
+    log.handler('fetching keys from Consul...', LOGFILE)
+    c_client = Consul(host=server, port='443', token=token, scheme='https')
+    try:
+        c_keys = sorted(c_client.kv.get('nomad', recurse=True, keys=True)[1])
+    except Exception as err:  # pylint: disable=W0703
+        return err
+
+    return sorted(c_keys)
+
+
+def consul_prune(server, token):
+    """ prune keys from consul """
+    log.handler('purging keys from Consul...', LOGFILE)
+    c_client = Consul(host=server, port='443', token=token, scheme='https')
+    for nomad_env in ['test', 'uat', 'prod']:
+        c_client.kv.delete(f'nomad/{nomad_env}', recurse=True)
+
+
+# Here we Go.
+if __name__ == "__main__":
+
+    ARGS = docopt(__doc__)
+    PRUNE = ARGS['--prune']
+    LOGFILE = '/dev/stdout'
+
+    # Anvil should not be used in production
+    VAULT_ROOT_TOKEN = wile_coyote.tools.VAULT_ROOT_TOKEN
+    if not VAULT_ROOT_TOKEN:
+        log.handler('you can use this tool ONLY in test', LOGFILE, True)
+        log.handler('exiting ...', LOGFILE, True)
+        os.sys.exit()
+
+    REDIS_HOST = wile_coyote.tools.REDIS_HOST
+    REDIS_TOKEN = wile_coyote.tools.REDIS_TOKEN
+    VAULT_HOST = wile_coyote.tools.VAULT_HOST
+    CONSUL_SERVERS = wile_coyote.tools.CONSUL_SERVERS
+    CONSUL_TOKEN = wile_coyote.tools.CONSUL_TOKEN
+    CONSUL_LEADER, _, __ = wile_coyote.tools.consul_leader.get(LOGFILE)
+    MOUNT_POINTS_V1 = wile_coyote.tools.MOUNT_POINTS_V1
+    MOUNT_POINTS_V2 = wile_coyote.tools.MOUNT_POINTS_V2
+
+    # keys define in .acme.ini
+    REDIS_KEYS = wile_coyote.tools.REDIS_KEYS
+    VAULT_KEYS = wile_coyote.tools.VAULT_KEYS
+    CONSUL_KEYS = wile_coyote.tools.CONSUL_KEYS
+
+    # delete and upload keys again
+    consul_prune(CONSUL_LEADER, CONSUL_TOKEN)
+    redis_prune(REDIS_HOST, REDIS_TOKEN)
+    vault_prune(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V1, 1)
+    vault_prune(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V2, 2)
+
+    # prune certificates locally
+    ACME_PROVIDERS = ['sectigo_ev', 'sectigo_ev', 'letsencrypt']
+    for acme_provider in ACME_PROVIDERS:
+        acme_certificates = glob(f'/etc/{acme_provider}/live/*')
+        try:
+            acme_certificates.remove(f'/etc/{acme_provider}/live/README')
+        except ValueError:
+            pass
+        certificates_delete(acme_provider, acme_certificates)
+
+    # run all scripts under /opt/acme/bin
+    for script in glob('/opt/acme/bin/*'):
+        log.handler(f'running script {script}', LOGFILE)
+        os.system(script)
+
+    # upstream keys
+    C_KEYS = consul_keys(CONSUL_LEADER, CONSUL_TOKEN)
+    R_KEYS = redis_keys(REDIS_HOST, REDIS_TOKEN)
+    V_KEYS_V1 = vault_keys(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V1)
+    V_KEYS_V2 = vault_keys(VAULT_HOST, VAULT_ROOT_TOKEN, MOUNT_POINTS_V2)
+    V_KEYS_ALL = sorted(V_KEYS_V1 + V_KEYS_V2)
+
+    # differences
+    C_DIFFS = [item for item in CONSUL_KEYS if item not in C_KEYS]
+    R_DIFFS = [item for item in REDIS_KEYS if item not in R_KEYS]
+    V_DIFFS = [item for item in VAULT_KEYS if item not in V_KEYS_ALL]
+    for key_store in ['Consul', 'Redis', 'Vault']:
+        if key_store == 'Consul':
+            key_list = C_DIFFS
+        elif key_store == 'Redis':
+            key_list = R_DIFFS
+        elif key_store == 'Vault':
+            key_list = V_DIFFS
+        if key_list:
+            log.handler(f'the following keys are not available on {key_store}:', LOGFILE, True)
+            log.handler(' '.join(key_list), LOGFILE, True)
+            os.sys.exit(1)
+        else:
+            log.handler(f'{key_store} keys names have been successfully checked')
+
+    if REDIS_KEYS != R_KEYS:
+        log.handler('downstream and upstream Redis keys are different', LOGFILE, True)
+        os.sys.exit(1)
+    elif CONSUL_KEYS != C_KEYS:
+        log.handler('downstream and upstream Consul keys are different', LOGFILE, True)
+        os.sys.exit(1)
+    elif VAULT_KEYS != V_KEYS_ALL:
+        log.handler('downstream and upstream Vault keys are different', LOGFILE, True)
+        os.sys.exit(1)
+
+    # validate public keys
+    for r_key in [item for item in REDIS_KEYS if '_chain.pem' not in R_KEYS]:
+        log.handler(f'checking certificate {r_key} on Redis', LOGFILE)
+        pubkey = wile_coyote.tools.redis_get.get(r_key)
+        wile_coyote.common_kit.public_key.check(pubkey, LOGFILE, r_key, False)
+
+    for c_key in [item for item in CONSUL_KEYS if '_chain.pem' not in C_KEYS]:
+        log.handler(f'checking certificate {c_key} on Consul', LOGFILE)
+        pubkey = wile_coyote.tools.consul_get.get(c_key)
+        wile_coyote.common_kit.public_key.check(pubkey, LOGFILE, c_key, False)
+
+    # validate private keys: we check only if it's not malformed
+    for v_key in V_KEYS_V1:
+        log.handler(f'checking private key {v_key} on Vault', LOGFILE)
+        privkey = wile_coyote.tools.vault_get.get(v_key, 'root')
+        wile_coyote.common_kit.public_key.check(privkey, LOGFILE, v_key, False)
+
+    for v_key in V_KEYS_V2:
+        log.handler(f'checking private key {v_key} on Vault v2', LOGFILE)
+        fixed_v_key = re.sub(r'^/nomad/', '/', v_key)
+        privkey = wile_coyote.tools.vault_get_v2.get(fixed_v_key, 'root')
+        wile_coyote.common_kit.public_key.check(privkey, LOGFILE, v_key, False)

bin/wile_coyote.py

 #!/usr/bin/env python3
 #
-"""Geant Acme
+"""Wile Coyote
 
 Usage:
-  acme.py (--domain <DOMAIN>...) (--provider <PROVIDER>) --days <DAYS> \
-[--project <PROJECT>...] [--environment <ENVIRONMENT>...] [--tld] \
-[--unit <UNIT>] [--client <CLIENT>...] [--wildcard] [--extra <EXTRA>...]
-  acme.py (-h | --help)
+  wile_coyote.py (--domain <DOMAIN>...) (--provider <PROVIDER>) --days <DAYS> \
+[--project <PROJECT>...] [--nomad-env <ENV>...] [--tld] [--unit <UNIT>] \
+[--client <CLIENT>...] [--wildcard] [--extra <EXTRA>...]
+  wile_coyote.py (-h | --help)
 
 Options:
-  -h --help                                 Show this screen.
-  -c CLIENT --client=CLIENT                 Client
-  -d DOMAIN --domain=DOMAIN                 Domain
-  -p PROVIDER --provider=PROVIDER           Provider
-  -u UNIT --unit=UNIT                       Unit, entity or team
-  --days DAYS                               CRIT days before expiration
-  --project=PROJECT                         Project      (Nomad only)
-  -e ENVIRONMENT --environment=ENVIRONMENT  Environment  (Nomad only)
-  -t --tld                                  Top Level Domain
-  -w --wildcard                             Use wildcard
-  -x --extra=EXTRA                          Supply extra parameters (check certbot documentation)
+  -h --help                        Show this screen.
+  -c CLIENT --client=CLIENT        Client
+  -d DOMAIN --domain=DOMAIN        Domain
+  -p PROVIDER --provider=PROVIDER  Provider
+  -u UNIT --unit=UNIT              Unit, entity or team
+  --days DAYS                      CRIT days before expiration
+  --project=PROJECT                Project      (Nomad only)
+  --nomad-env=ENV                  Environment  (Nomad only)
+  -t --tld                         Top Level Domain
+  -w --wildcard                    Use wildcard
+  -x --extra=EXTRA                 Supply extra parameters (check certbot documentation)
 """
 import os
 import time
@@ -124,7 +124,7 @@ if __name__ == "__main__":
     DOMAIN = ARGS['--domain']
     UNIT = ARGS['--unit']
     PROJECT = ARGS['--project']
-    ENVIRONMENT = ARGS['--environment']
+    ENV = ARGS['--nomad-env']
     FIRST_NAME = DOMAIN[0]
     WILDCARD = ARGS['--wildcard']  # True or None
     EXTRA = ARGS['--extra']
@@ -192,7 +192,7 @@ if __name__ == "__main__":
             log.handler(
                 f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for Nomad', LOG_FILE)
             wile_coyote.acme_kit.nomad_uploader.uploader(
-                PROVIDER, PROJECT, FIRST_NAME, ENVIRONMENT, WILDCARD)
+                PROVIDER, PROJECT, FIRST_NAME, ENV, WILDCARD)
         else:
             log.handler(
                 f'uploading {FIRST_NAME} {PROVIDER}{wilcard_string} for {UNIT}', LOG_FILE)

common_kit/private_key.py

@@ -7,14 +7,19 @@ from wile_coyote.common_kit import log
 import wile_coyote.common_kit.constants
 
 
-def check(privkey, log_file):
+def check(privkey, log_file, key_name='unknown', from_file=True):
     """ check private key validity """
     giveup = wile_coyote.common_kit.constants.GIVEUP
     is_broken = False
-    try:
-        st_key = open(privkey, 'rt', encoding="utf8").read()
-    except Exception:  # pylint: disable=w0703
-        is_broken = True
+    if from_file:
+        key_path = privkey
+        try:
+            st_key = open(privkey, 'rt', encoding="utf8").read()
+        except Exception:  # pylint: disable=w0703
+            is_broken = True
+    else:
+        st_key = privkey
+        key_path = key_name
 
     ssl_crypto = OpenSSL.crypto
     try:
@@ -28,7 +33,7 @@ def check(privkey, log_file):
         is_broken = True
 
     if is_broken:
-        log.handler(f'{privkey} is not a valid key: {giveup}', log_file, True)
+        log.handler(f'{key_path} is not a valid key: {giveup}', log_file, True)
         os.sys.exit(1)
 
     return True

common_kit/public_key.py

@@ -7,14 +7,25 @@ from wile_coyote.common_kit import log
 import wile_coyote.common_kit.constants
 
 
-def check(certificate, log_file):
+def check(certificate, log_file, key_name='unknown', from_file=True):
     """ check certificate expiration """
-    st_cert = open(certificate, 'rt', encoding="utf8").read()
+    if from_file:
+        st_cert = open(certificate, 'rt', encoding="utf8").read()
+        key_path = certificate
+    else:
+        st_cert = certificate
+        key_path = key_name
 
     ssl_crypto = OpenSSL.crypto
-    cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert)
+
+    try:
+        cert = ssl_crypto.load_certificate(ssl_crypto.FILETYPE_PEM, st_cert)
+    except Exception as err:  # pylint: disable=w0703
+        log.handler(f'Error. The certificate {key_path} is malformed: {err}')
+        os.sys.exit(1)
+
     if cert.has_expired():
         log.handler(
-            f'{certificate} expired and it will not be uploaded', log_file, True)
+            f'{key_path} expired and it will not be uploaded', log_file, True)
         log.handler(wile_coyote.common_kit.constants.giveup, log_file)
         os.sys.exit(1)

tools/__init__.py

+from ast import literal_eval as l_eval
 import pkgutil
 import configparser
 import wile_coyote.common_kit.constants
@@ -15,3 +16,35 @@ for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
     REDIS_TOKEN = config.get('acme', 'redis_token')
     CONSUL_LIST = config.get('acme', 'consul_servers')
     CONSUL_TOKEN = config.get('acme', 'consul_token')
+
+    # these parameters only work in test
+    #
+    try:
+        VAULT_ROOT_TOKEN = config.get('unit-test', 'vault_token_root')
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        VAULT_ROOT_TOKEN = None
+
+    try:
+        MOUNT_POINTS_V1 = l_eval(config.get('unit-test', 'mount_points_v1'))
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        MOUNT_POINTS_V1 = None
+
+    try:
+        MOUNT_POINTS_V2 = l_eval(config.get('unit-test', 'mount_points_v2'))
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        MOUNT_POINTS_V2 = None
+
+    try:
+        REDIS_KEYS = sorted(l_eval(config.get('unit-test', 'redis_keys')))
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        REDIS_KEYS = None
+
+    try:
+        VAULT_KEYS = sorted(l_eval(config.get('unit-test', 'vault_keys')))
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        VAULT_KEYS = None
+
+    try:
+        CONSUL_KEYS = sorted(l_eval(config.get('unit-test', 'consul_keys')))
+    except (configparser.NoSectionError, configparser.NoOptionError):
+        CONSUL_KEYS = None