Integration of The Hive

config.py

@@ -11,6 +11,7 @@ CA_CERT_FILE = os.path.join(SOCTOOLS_BASE, "secrets/CA/ca.crt")
 KEYCLOAK_ADMIN_PASSWORD_FILE = os.path.join(SOCTOOLS_BASE, "secrets/passwords/keykloak_admin") # Note: should be keycloak, not keykloak
 
 MISP_API_KEY_FILE = os.path.join(SOCTOOLS_BASE, "secrets/tokens/misp")
+THEHIVE_API_KEY_FILE = os.path.join(SOCTOOLS_BASE, "secrets/tokens/thehive_secret_key")
 
 # Credentials of the special user for account management
 # Cert and key should be in .pem format, unencrypted
@@ -31,3 +32,6 @@ KEYCLOAK_USERS_URL = None
 KEYCLOAK_ADMIN_PASSWORD = None
 
 MISP_API_KEY = None
+
+THEHIVE_API_KEY = None
+THEHIVE_ORG_NAME = None # set to "domain" from variables file

main.py

@@ -18,6 +18,7 @@ import config
 import certificates
 from nifi import *
 from misp import *
+from thehive import *
 
 app = Flask(__name__)
 app.secret_key = "ASDF1234 - CHANGE ME!"
@@ -40,10 +41,16 @@ def load_config():
     config.KEYCLOAK_ADMIN_PASSWORD = open(config.KEYCLOAK_ADMIN_PASSWORD_FILE, "r").read(100).strip() # read max 100 B, the key should never be so long
     # Load API key for MISP
     config.MISP_API_KEY = open(config.MISP_API_KEY_FILE, "r").read(100).strip() # read max 100 B, the key should never be so long
+    # Load API key for The Hive
+    config.THEHIVE_API_KEY = open(config.THEHIVE_API_KEY_FILE, "r").read(100).strip()  # read max 100 B, the key should never be so long
+    # Load organization name (for The Hive)
+    config.THEHIVE_ORG_NAME = variables["domain"]
 
     print(f"Config loaded:\nSOCTOOLSPROXY={config.SOCTOOLSPROXY}\nKEYCLOAK_BASE_URL={config.KEYCLOAK_BASE_URL}\n"
           f"KEYCLOAK_ADMIN_PASSWORD={config.KEYCLOAK_ADMIN_PASSWORD[:3]}...{config.KEYCLOAK_ADMIN_PASSWORD[-4:]}\n"
-          f"MISP_API_KEY={config.MISP_API_KEY[:3]}...{config.MISP_API_KEY[-4:]}\n")
+          f"MISP_API_KEY={config.MISP_API_KEY[:3]}...{config.MISP_API_KEY[-4:]}\n"
+          f"THEHIVE_API_KEY={config.THEHIVE_API_KEY[:3]}...{config.THEHIVE_API_KEY[-4:]}\n"
+          f"THEHIVE_ORG_NAME={config.THEHIVE_ORG_NAME}\n")
 
 
 # *** Custom Jinja filters ***
@@ -297,6 +304,20 @@ def main():
     # List of usernames only (for easier cross-check with Keycloak users)
     misp_emails = set(mu["email"] for mu in misp_users)
 
+    # ===================
+    # Load The Hive users
+    try:
+        thehive_users = thehive_get_users()
+    except TheHiveError as e:
+        flash(f"ERROR: {e}", "error")
+        thehive_users = []
+    # Mark "internal" users
+    for u in thehive_users:
+        if u["login"].startswith("admin@") or u["login"].startswith("kibana@"):
+            u["internal"] = True
+    # List of usernames only (for easier cross-check with Keycloak users)
+    thehive_usernames = set(u["login"] for u in thehive_users)
+
     return render_template("main.html", **locals())
 
 
@@ -350,9 +371,18 @@ def add_user():
         except Exception as e:
             flash(f'Error when creating user in MISP: {e}', "error")
 
+        # The Hive
+        try:
+            thehive_add_user(user)
+            flash(f'User "{user.email}" successfully created in The Hive.', "success")
+        except TheHiveUserExistsError:
+            flash(f'User with email "{user.email}" already exists in TheHive, nothing has changed.', "warning")
+        except Exception as e:
+            flash(f'Error when creating user in The Hive: {e}', "error")
+
         # Send email to the user
         if form_user.send_email.data:
-            ok, err = _send_token(user.username, user.email)
+            ok, err = _send_token(user.username, user.email) # TODO
             if ok:
                 flash(f"Email successfully sent to '{user.email}'", "success")
             else:
@@ -404,11 +434,26 @@ def edit_user(username: str):
                     flash(f'Error when updating user in MISP: User with email "{user.email}" not found', "error")
                 except MISPUserExistsError:
                     flash(f'Error when updating user in MISP: User with email "{new_user.email}" already exists. '
-                          'BEWARE: An inconsistency in user accounts in Keycloak and MISP was probably just introduced '
-                          'which needs to be fixed manually in the administration of the individual services!', "error")
+                          'BEWARE: An inconsistency in user accounts in Keycloak and MISP was probably just created. '
+                          'It needs to be fixed manually in the administration of the individual services!', "error")
                 except Exception as e:
                     flash(f'Error when updating user in MISP: {e}', "error")
 
+            # The Hive
+            # IMPORTANT: Email is used as user identifier, it can't be changed - we have to delete and re-create the account.
+            if user.email != new_user.email:
+                thehive_delete_user(user.email)
+                thehive_add_user(new_user)
+                flash(f'User {user.email} deleted and new user "{new_user.username}" created in The Hive.', "success")
+            else:
+                try:
+                    thehive_edit_user(user.email, new_user)
+                    flash(f'User "{new_user.username}" successfully updated in The Hive.', "success")
+                except TheHiveUserNotFoundError:
+                    flash(f'Error when updating user in The Hive: User with email "{user.email}" not found', "error")
+                except Exception as e:
+                    flash(f'Error when updating user in The Hive: {e}', "error")
+
             return redirect("/") # Success - go back to main page
 
         # data not valid - show form again
@@ -454,6 +499,15 @@ def delete_user(username: str):
     except MISPError as e:
         flash(f'Error when deleting user from MISP: {e}', "error")
 
+    # The Hive
+    try:
+        thehive_delete_user(user_spec.email)
+        flash(f'User "{user_spec.email}" successfully deleted from The Hive.', "success")
+    except TheHiveUserNotFoundError:
+        flash(f'Error when deleting user from The Hive: User with email "{user_spec.email}" not found', "error")
+    except TheHiveError as e:
+        flash(f'Error when deleting user from The Hive: {e}', "error")
+
     return redirect("/")
 
 
@@ -541,7 +595,7 @@ def _send_token(username: str, email: str) -> Tuple[bool, Optional[str]]:
 #   (re)send cert-access token for existing user - DONE (on click in table)
 #   automatically create certificate when creating new user (optionally automatically send email with token) - DONE
 #   revoke and delete certificate when user is deleted
-#   make CN=username (so cert filename also matches the username (it's stored by CN))
+#   make CN=username (so cert filename also matches the username (it's stored by CN)) - DONE
 
 
 # @app.route("/test_cert/<func>")

templates/main.html

@@ -14,7 +14,7 @@ document.getElementById('show-internal').addEventListener('change', function(eve
 </script>
 
 <table>
-<tr><th>Username</th><th>First name</th><th>Last name</th><th>Email</th><th>CN</th><th>DN</th><th>Time created (UTC)</th><th>NiFi</th><th>MISP</th><th>Actions</th>
+<tr><th>Username</th><th>First name</th><th>Last name</th><th>Email</th><th>CN</th><th>DN</th><th>Time created (UTC)</th><th>NiFi</th><th>MISP</th><th>The Hive</th><th>Actions</th>
 {% for user in users %}
 <tr{% if user.internal %} class="internal-user"{% endif %}>
 <td>{{ user.username }}</td>
@@ -26,13 +26,14 @@ document.getElementById('show-internal').addEventListener('change', function(eve
 <td>{{ user.ts_created|ts_to_str }}</td>
 <td>{{ icon('check' if user.username in nifi_usernames else 'close') }}</td>
 <td>{{ icon('check' if user.email in misp_emails else 'close') }}</td>
+<td>{{ icon('check' if user.login in thehive_usernames else 'close') }}</td>
 <td>
 {% if not user.internal -%}
 <a href="{{ url_for('edit_user', username=user.username) }}" title="Edit user">{{ icon('pencil') }}</a>
 <a href="{{ url_for('send_token', username=user.username) }}" title="Re-send email with token for certificate download"
  onclick="return confirm('Send an email to &quot;{{user.email}}&quot; containing a unique URL allowing to download the user\'s certificate and private key?')">{{ icon('envelope') }}</a>
 <a href="{{ url_for('delete_user', username=user.username) }}" title="Delete user"
- onclick="return confirm('Are you sure you want to permanently delete user account &quot;{{user.username}}&quot; ({{user.cn}}, {{user.email}})?')">{{ icon('trash') }}</a>
+ onclick="return confirm('Are you sure you want to permanently delete user account &quot;{{user.username}}&quot; ({{user.email}})?')">{{ icon('trash') }}</a>
 {%- endif %}
 </td>
 </tr>
@@ -72,4 +73,21 @@ document.getElementById('show-internal').addEventListener('change', function(eve
 {% endfor %}
 </table>
 
+
+<h3>The Hive</h3>
+<table>
+<tr><th>Login</th><th>Name</th><th>Organization</th><th>Role (profile)</th><th>ID</th><th>Created</th><th></th>
+
+{% for user in thehive_users %}
+<tr{% if user.internal %} class="internal-user"{% endif %}>
+<td>{{ user.login }}</td>
+<td>{{ user.name }}</td>
+<td>{{ user.org }}</td>
+<td>{{ user.role }}</td>
+<td>{{ user.id }}</td>
+<td>{{ user.created|ts_to_str }}</td>
+</tr>
+{% endfor %}
+</table>
+
 {% endblock %}
\ No newline at end of file

thehive.py

+"""Functions to manage user accounts in The Hive"""
+
+from typing import List, Dict, Optional
+import requests
+from datetime import datetime
+import re
+from operator import itemgetter
+import urllib.parse
+
+import config
+from main import UserAccount
+
+# Base URL to The Hive API endpoints
+THEHIVE_API_BASE_URL = "https://{soctools_proxy}:9000/api/v1"
+
+# The Hive API documentation: http://docs.thehive-project.org/thehive/api/
+# But a lot of information is missing there, it is easier to just intercept queries made by a browser.
+
+class TheHiveError(Exception):
+    pass
+
+class TheHiveUserNotFoundError(TheHiveError):
+    pass
+
+class TheHiveUserExistsError(TheHiveError):
+    pass
+
+class TheHiveUnexpectedReplyError(TheHiveError):
+    pass
+
+
+# =========================
+# Public interface
+
+def thehive_get_users() -> List[Dict]:
+    """
+    List users defined in The Hive in the configured organization
+
+    :return List of dicts with keys 'id', 'login', 'name', 'role', 'org', 'created' (datetime)
+    :raise TheHiveUnexpectedReplyError
+    """
+    url = THEHIVE_API_BASE_URL.format(soctools_proxy=config.SOCTOOLSPROXY) + "/query"
+    data = {
+        "query": [
+            {"_name": "getOrganisation", "idOrName": config.THEHIVE_ORG_NAME},
+            {"_name": "users"}
+        ]
+    }
+
+    resp = _send_request("post", url, data=data)
+    #print(url)
+    #print(resp.request.headers)
+    #print(resp.status_code)
+    if not resp.ok:
+        raise TheHiveUnexpectedReplyError(f"Can't get list of users from The Hive: Unexpected reply {resp.status_code}")
+    #print(resp.json())
+    users = []
+    try:
+        for user_entry in resp.json():
+            users.append({
+                "id": user_entry["_id"],
+                "login": user_entry["login"],
+                "name": user_entry["name"],
+                "role": user_entry["profile"],
+                "org": user_entry["organisation"],
+                "created": datetime.utcfromtimestamp(user_entry["_createdAt"]/1000), # time of account creation (timestamp in ms -> datetime)
+            })
+    except (ValueError, TypeError, KeyError) as e:
+        print(f"Can't get list of users from The Hive: Unexpected content received: {type(e).__name__}: {e})")
+        raise TheHiveUnexpectedReplyError(f"Can't get list of users from The Hive: Unexpected content received")
+
+    return users
+
+
+def thehive_add_user(user: UserAccount) -> None:
+    """Add a new user to TheHive
+
+    :raise TheHiveUnexpectedReplyError
+    """
+    url = THEHIVE_API_BASE_URL.format(soctools_proxy=config.SOCTOOLSPROXY) + "/user"
+    data = {
+        "login": user.email,
+        "name": f"{user.firstname} {user.lastname}" if (user.firstname or user.lastname) else user.username,
+        "organisation": config.THEHIVE_ORG_NAME,
+        "profile": "analyst", # TODO allow to set different roles?
+        #"email": user.email,
+        #"password": "",
+    }
+    resp = _send_request("post", url, data)
+    #print(resp.json())
+    if not resp.ok:
+        print(f"Can't add user to The Hive: Unexpected reply {resp.status_code}: {resp.text[:500]}")
+        raise TheHiveUnexpectedReplyError(f"Can't add user to The Hive: Unexpected reply {resp.status_code}")
+        # Note: There is no check for existing user, because it seems TheHive just do nothing if user already exists.
+        # There is probably no way to recognize it.
+    return None
+
+
+def thehive_edit_user(login: str, user: UserAccount) -> None:
+    """Edit existing user in The Hive (only name can be changed)
+
+    :raise TheHiveUnexpectedReplyError
+    """
+    url = THEHIVE_API_BASE_URL.format(soctools_proxy=config.SOCTOOLSPROXY) + f"/user/{login}"
+    data = {
+        "name": f"{user.firstname} {user.lastname}".strip() if (user.firstname or user.lastname) else user.username,
+        #"organisation": config.THEHIVE_ORG_NAME,
+        #"profile": "analyst",  # TODO allow to set different roles?
+    }
+
+    resp = _send_request("patch", url, data)
+    #print(resp.text)
+    if not resp.ok:
+        if resp.status_code == 404:
+            raise TheHiveUserNotFoundError()
+        print(f"Can't edit user in The Hive: Unexpected reply {resp.status_code}: {resp.text[:500]}")
+        raise TheHiveUnexpectedReplyError(f"Can't edit user in The Hive: Unexpected reply {resp.status_code}")
+    return None
+
+
+def thehive_delete_user(login: str) -> None:
+    """Delete existing user from The Hive
+
+    :raise TheHiveUnexpectedReplyError
+    """
+    url = THEHIVE_API_BASE_URL.format(soctools_proxy=config.SOCTOOLSPROXY) + f"/user/{login}/force?organisation={config.THEHIVE_ORG_NAME}"
+    #print(url)
+    resp = _send_request("delete", url)
+    #print(resp.text)
+    if not resp.ok:
+        if resp.status_code == 404:
+            raise TheHiveUserNotFoundError()
+        print(f"Can't delete user from The Hive: Unexpected reply {resp.status_code}: {resp.text[:500]}")
+        raise TheHiveUnexpectedReplyError(f"Can't delete user from The Hive: Unexpected reply {resp.status_code}")
+    return None
+
+
+# =========================
+# Auxiliary functions
+
+def _send_request(method:str, url:str, data:Optional[dict]=None):
+    return getattr(requests, method)(
+        url,
+        headers={
+            "Authorization": "Bearer " + config.THEHIVE_API_KEY,
+        },
+        verify=config.CA_CERT_FILE,
+        json=data
+    )