Completely reworked to support multiple observable fields

Needs some more testing, but everything seems to work well.

thehive_button/package.json

 {
   "name": "thehive_button",
-  "version": "0.3.0",
+  "version": "1.0.0",
   "description": "Visualisation plugin which creates a simple button to create a new case in The Hive.",
   "main": "index.js",
   "kibana": {
-    "version": "7.2.0"
+    "version": "7.4.2"
   },
   "scripts": {
     "lint": "eslint .",

thehive_button/public/create_case.js

+// Functions to send data to Kibana endpoints
+
+import chrome from 'ui/chrome';
+
+// Create a new Case in The Hive via its API
+// Return a Promise which resolves to object with ID of the new case ('id' attr) or error message ('error' attr)
+export function createTheHiveCase(base_url, api_key, title, descr, severity, startDate, owner, flag, tlp, tags) {
+  // Prepare data
+  var data = JSON.stringify({
+    "base_url": base_url,
+    "api_key": api_key,
+    "body": {
+      "title": title,
+      "description": descr,
+      "severity": severity, // number: 1=low, 2=medium, 3=high
+      "startDate": startDate,
+      "owner": owner, // user name the case will be assigned to
+      "flag": flag, // bool
+      "tlp": tlp, // number: 0=white, 1=green, 2=amber, 3=red
+      "tags": tags, // array of strings
+    }
+  });
+  console.log("TheHiveButton: Sending request to API endpoint 'new_case':", data);
+  var kibana_endpoint_url = chrome.addBasePath('/api/thehive_button/new_case');
+
+  return new Promise(function (resolve, reject) {
+    // Create AJAX request
+    var xhr = new XMLHttpRequest();
+    
+    // Listener to process reply
+    xhr.onreadystatechange = function () {
+      if (this.readyState != 4) {
+        return; // response not ready yet
+      }
+      if (this.status == 200) {
+        const resp = JSON.parse(this.responseText);
+        console.log("TheHiveButton: Response from backend:", resp);
+        if ("error" in resp) {
+          resolve({"error": resp.error});
+        }
+        else if (resp.status_code != 201) {
+          resolve({"error": "Unexpected reply received from The Hive: [" + resp.status_code + "] " + resp.status_msg});
+        }
+        else {
+          resolve({"id": resp.body.id}); // return ID of the new case
+        }
+      }
+      else {
+        console.log("TheHiveButton: Error " + this.status + ": " + this.statusText);
+        resolve({"error": "Error " + this.status + ": " + this.statusText});
+      }
+    }
+
+    // Send the AJAX request
+    xhr.open("POST", kibana_endpoint_url);
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.setRequestHeader("kbn-xsrf", "thehive_plugin"); // this header must be set, although its content is probably irrelevant
+    xhr.send(data);
+  });
+}
+
+// Add observables to an existing Case in The Hive
+// (send the list of observables to our backend endpoint, it pushes them to The Hive)
+export function addCaseObservables(base_url, api_key, caseid, observables) {
+  const kibana_endpoint_url = chrome.addBasePath('/api/thehive_button/add_observables');
+  const data = JSON.stringify({
+    "base_url": base_url,
+    "api_key": api_key,
+    "caseid": caseid,
+    "observables": observables,
+  });
+  console.log("TheHiveButton: Sending request to API endpoint 'add_observables':", data);
+  
+  return new Promise(function (resolve, reject) {
+    // Create AJAX request
+    var xhr = new XMLHttpRequest();
+    
+    // Listener to process reply
+    xhr.onreadystatechange = function () {
+      if (this.readyState != 4) {
+        return; // response not ready yet
+      }
+      if (this.status == 200) {
+        const resp = JSON.parse(this.responseText);
+        console.log("TheHiveButton: Response from backend:", resp);
+        resolve(resp);
+      }
+      else {
+        console.log("TheHiveButton: Error " + this.status + ": " + this.statusText);
+        resolve({"error": "Error " + this.status + ": " + this.statusText});
+      }
+    }
+
+    // Send the AJAX request
+    xhr.open("POST", kibana_endpoint_url);
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.setRequestHeader("kbn-xsrf", "thehive_plugin"); // this header must be set, although its content is probably irrelevant
+    xhr.send(data);
+  });
+}
+

thehive_button/public/main.js

-//import './vis.less';
 import { THEHIVE_API_KEY, THEHIVE_URL, THEHIVE_OWNER } from './env';
-//import { VisController } from './vis_controller';
 import { TheHiveButtonVisComponent } from './vis_controller';
+import { theHiveButtonRequestHandlerProvider } from './request_handler';
+import { optionsEditor } from './options_editor';
 
 import { VisFactoryProvider } from 'ui/vis/vis_factory';
 import { VisTypesRegistryProvider } from 'ui/registry/vis_types';
-import { Schemas } from 'ui/vis/editors/default/schemas';
-//import { Status } from 'ui/vis/update_status';
 import { DefaultEditorSize } from 'ui/vis/editor_size';
 
-import optionsTemplate from './options_template.html';
-
-/*
-TODO: It would be better to compose options tab of EUI React elements,
-but this probably needs at least kibana 7.4.
-See this for example of a custom editor:
-  https://github.com/elastic/kibana/tree/master/src/plugins/vis_type_markdown/public 
-
-import React from 'react';
-import {
-  EuiForm,
-  EuiFormRow,
-  EuiFieldText,
-} from '@elastic/eui';
-
-import { VisOptionsProps } from 'src/plugins/vis_default_editor/public';
-
-function optionsEditor({ stateParams, setValue }) {
-  return (
-  <EuiForm>
-    <EuiFormRow label="Base URL of The Hive">
-      <EuiFieldText value={stateParams.url} />
-    </EuiFormRow>
-    <EuiFormRow label="API key">
-      <EuiFieldText value={stateParams.apikey} />
-    </EuiFormRow>
-    <EuiFormRow label="Username" helpText="Used as the owner of cases created from here">
-      <EuiFieldText value={stateParams.owner} />
-    </EuiFormRow>
-  </EuiForm>
-);
-*/
 
 function TheHiveButtonVisProvider(Private) {
   const VisFactory = Private(VisFactoryProvider);
 
-  console.log("default URL:", THEHIVE_URL);
-  console.log("default API key:", THEHIVE_API_KEY);
+  //console.log("default URL:", THEHIVE_URL);
+  //console.log("default API key:", THEHIVE_API_KEY);
 
   return VisFactory.createReactVisualization({
     name: 'thehive_button',
@@ -61,65 +27,25 @@ function TheHiveButtonVisProvider(Private) {
         url: THEHIVE_URL,
         apikey: THEHIVE_API_KEY,
         owner: THEHIVE_OWNER,
+        obsFields: [], // list of objects, e.g. {name: "clientip", type: "ip", cnt: 100}
       }
     },
-//    editor: optionsEditor,
-     editor: 'default',
-    editorConfig: {
-      optionsTemplate: optionsTemplate,
-      defaultSize: DefaultEditorSize.MEDIUM,
-      //enableAutoApply: true,
-      schemas: new Schemas([
-        // TODO all this (metric and bucket "terms") could be pre-defind
-        // (just a selection of fields to read Observables from would be nice)
-        // But I don't know how and where to pass the values manually.
-        /* Allowed params (from /src/legacy/ui/public/vis/editors/default/schemas.d.ts)
-            aggFilter: string | string[];
-            editor: boolean | string;
-            group: AggGroupNames;
-            max: number;
-            min: number;
-            name: string;
-            params: AggParam[];
-            title: string;
-        */
-        {
-          group: 'metrics',
-          name: 'metric',
-          title: "Metric (not used, ignore this)",
-          min: 1,
-          max: 1,
-          aggFilter: 'count',
-          defaults: [
-            {
-              type: 'count',
-              schema: 'metric',
-            },
-          ],
-          editor: '<div class="hintbox"><i class="fa fa-danger text-info"></i> Some metric must be defined here, but it\'s setting is irrelevant. Just go to "Buckets" below and set up the field to get Observbles from.</div>'
-        },
+    //editor: 'default',
+    editorConfig:  {
+      optionTabs: [
         {
-          group: 'buckets',
-          name: 'group',
-          title: 'Observables',
-          min: 0,
-          //max: 1,
-          aggFilter: ['terms'],
-          defaults: [
-            {
-              type: 'terms',
-              schema: 'group',
-              field: 'ip',
-              size: 1000,
-              orderBy: 'alphabetical', 
-              order: 'ascending',
-            }
-          ]
-        },
-      ]),
+          name: "options",
+          title: "Options",
+          editor: optionsEditor,
+        }
+      ],
+      defaultSize: DefaultEditorSize.LARGE,
     },
-    //requestHandler: 'courier', // default
-    //responseHandler: 'default', // return data as a table, see https://www.elastic.co/guide/en/kibana/7.2/development-visualization-response-handlers.html
+//       optionsTemplate: optionsEditor, //optionsTemplate,
+//       //enableAutoApply: true,
+//     },
+    requestHandler: 'theHiveButtonRequestHandler', // own request handler
+    responseHandler: 'none', // pass data as returned by requestHandler
   });
 }
 

thehive_button/public/options_editor.js

+import React from 'react';
+import {
+  EuiForm,
+  EuiFormRow,
+  EuiTitle,
+  EuiSpacer,
+  EuiFieldText,
+  EuiFieldNumber,
+  EuiSelect,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiButton,
+  EuiButtonIcon,
+} from '@elastic/eui';
+
+// Default data types in The Hive
+const DEFAULT_THE_HIVE_TYPES = [
+  '',
+  'autonomous-system',
+  'domain',
+  'file',
+  'filename',
+  'fqdn',
+  'hash',
+  'ip',
+  'mail',
+  'mail_subject',
+  'regexp',
+  'registry',
+  'uri_path',
+  'url',
+  'user-agent',
+  'other',	
+];
+
+// Options for EuiSelect for selection of field's data type in TheHive
+const typesOptions = DEFAULT_THE_HIVE_TYPES.map( dt => ({value: dt, text: dt}) );
+
+export function optionsEditor(props) {
+  //console.log("editor render(), props:", props);
+  const { stateParams, setValue, setValidity, vis } = props;
+  
+  // onClick/onChange handlers
+  const obsAddNew = () => {
+    const newObsFields = [...stateParams.obsFields, {name: "", type: "", cnt: 100}];
+    // For some reason, first click on the button after editor is loaded does
+    // nothing. Calling setValue twice here fixes it.  
+    setValue("obsFields", newObsFields);
+    setValue("obsFields", newObsFields);
+//     setValidity(false); // since new row is empty, form is always invalid
+  };
+  const obsRemove = (ix) => {
+    let newArray = [...stateParams.obsFields];
+    newArray.splice(ix, 1);
+    setValue("obsFields", newArray);
+//     validate();
+  }
+  const obsSetName = (ix, name) => {
+    let newArray = [...stateParams.obsFields];
+    newArray[ix].name = name;
+    setValue("obsFields", newArray);
+//     validate();
+  } 
+  const obsSetType = (ix, type) => {
+    let newArray = [...stateParams.obsFields];
+    newArray[ix].type = type;
+    setValue("obsFields", newArray);
+//     validate();
+  }
+  const obsSetCnt = (ix, cnt) => {
+    let newArray = [...stateParams.obsFields];
+    newArray[ix].cnt = parseInt(cnt);
+    setValue("obsFields", newArray);
+//     validate();
+  }
+//   const validate = () => {
+//     let valid = true;
+//     for (let field of stateParams.obsFields) {
+//       if (field.name == "" || field.type == "" || field.cnt == "") {
+//         valid = false;
+//         break;
+//       }
+//     }
+//     // TODO check for duplicate fields
+//     setValidity(valid);
+//   }
+  
+  // Get list of all fields in index (except those beginning with "_" or "@")
+  // and create "options" parameter for EuiSelect.
+  // Also, fields with "aggregatable=false" are removed, as they can't be used
+  // with "terms" aggregation we need.
+  // See this for details: https://www.elastic.co/guide/en/elasticsearch/reference/7.x/fielddata.html
+  // Empty field is added at the beginning, meaning "no selection yet".
+  const fieldOptions = [{value: "", text: ""}].concat(
+    vis.indexPattern.fields.raw.filter( f => (f.name[0] != "_" && f.name[0] != "@" && f.aggregatable) ).map( f => ({value: f.name, text: `${f.name} (${f.type})`}) )
+  );
+
+  return <EuiForm>
+    <EuiFormRow fullWidth={true} label="Base URL of The Hive">
+      <EuiFieldText
+        fullWidth={true}
+        value={stateParams.url}
+        onChange={e => setValue('url', e.target.value)}
+        isInvalid={stateParams.url == ""}
+      />
+    </EuiFormRow>
+    <EuiFlexGroup>
+      <EuiFlexItem grow={1}>
+        <EuiFormRow label="API key to access The Hive" helpText="API key of a user with write permission.">
+          <EuiFieldText
+            fullWidth={true}
+            value={stateParams.apikey}
+            onChange={e => setValue('apikey', e.target.value)}
+            isInvalid={stateParams.apikey == ""}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+      <EuiFlexItem grow={1}>
+        <EuiFormRow label="Assignee" helpText="User to assign created cases to. Must be a valid username from The Hive instance.">
+          <EuiFieldText
+            value={stateParams.owner}
+            onChange={e => setValue('owner', e.target.value)}
+            isInvalid={stateParams.owner == ""}
+          />
+        </EuiFormRow>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+    <EuiTitle size="s"><h3>Fields to get potential observables from ...</h3></EuiTitle>
+    <EuiSpacer size="s" />
+    {stateParams.obsFields.map( (field, ix) => (
+      <EuiFlexGroup key={ix} gutterSize="s">
+        <EuiFlexItem grow={3}>
+          <EuiFormRow label="Field name">
+            <EuiSelect
+              options={fieldOptions}
+              value={field.name}
+              onChange={ e => obsSetName(ix, e.target.value) }
+              isInvalid={field.name == ""}
+            />
+          </EuiFormRow>
+        </EuiFlexItem>
+        <EuiFlexItem grow={2}>
+          <EuiFormRow label="Data type in The Hive">
+            <EuiSelect
+              options={typesOptions}
+              value={field.type}
+              onChange={ e => obsSetType(ix, e.target.value) }
+              isInvalid={field.type == ""}
+            />
+          </EuiFormRow>
+        </EuiFlexItem>
+        <EuiFlexItem grow={1}>
+          <EuiFormRow label="Max items shown">
+            <EuiFieldNumber
+              min={1}
+              max={1000}
+              value={parseInt(field.cnt)}
+              onChange={ e => obsSetCnt(ix, e.target.value) }
+              isInvalid={!(field.cnt > 0)}
+            />
+          </EuiFormRow>
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <EuiFormRow hasEmptyLabelSpace>
+            <EuiButtonIcon iconType="trash" iconSize="m" color="danger" aria-label="Remove field" onClick={ e => obsRemove(ix) } />
+          </EuiFormRow>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    ))}
+    <EuiFlexGroup>
+      <EuiFlexItem grow={false}>
+        <EuiButton iconType="plusInCircleFilled" color="primary" onClick={obsAddNew}>Add new field ...</EuiButton>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  </EuiForm>
+}

thehive_button/public/options_template.html

-<div class="form-group">
-  <p><label>Base URL of The Hive</label>
-  <input ng-model="editorState.params.url" class=form-control /></p>
-  <p><label>API key</label>
-  <input ng-model="editorState.params.apikey" class=form-control /></p>
-  <p><label>User name to use as the owner of cases created from here</label>
-  <input ng-model="editorState.params.owner" class=form-control /></p>
-</div>

thehive_button/public/request_handler.js

+import { CourierRequestHandlerProvider as courierRequestHandlerProvider } from 'ui/vis/request_handlers/courier';
+import { SearchSourceProvider } from 'ui/courier/search_source';
+import { RequestAdapter, DataAdapter } from 'ui/inspector/adapters';
+import { VisRequestHandlersRegistryProvider } from 'ui/registry/vis_request_handlers';
+import { AggConfig } from 'ui/vis/agg_config';
+import { AggConfigs } from 'ui/vis/agg_configs';
+import { getTime } from 'ui/timefilter/get_time';
+import { i18n } from '@kbn/i18n';
+import { has } from 'lodash';
+import { calculateObjectHash } from 'ui/vis/lib/calculate_object_hash';
+import { getRequestInspectorStats, getResponseInspectorStats } from 'ui/courier/utils/courier_inspector_utils';
+import chrome from 'ui/chrome';
+
+// Maximum number of unique values of each field (observables) to fetch
+const MAX_NUMBER_OF_TERMS = 5;
+
+const handleCourierRequest = courierRequestHandlerProvider().handler;
+
+// Register new RaquestHandlerProvider 
+const theHiveButtonRequestHandlerProvider = function () {
+  return {
+    name: 'theHiveButtonRequestHandler',
+    handler: theHiveButtonRequestHandler,
+  }
+}
+VisRequestHandlersRegistryProvider.register(theHiveButtonRequestHandlerProvider);
+
+export {theHiveButtonRequestHandlerProvider, theHiveButtonRequestHandler};
+
+
+// The request handler function itself
+async function theHiveButtonRequestHandler(params) {
+  //console.log("theHiveButtonRequestHandler params:", params);
+  
+  let index = params.index;
+  let partialRows = params.partialRows;
+  let metricsAtAllLevels = params.metricsAtAllLevels;
+  let timeRange = params.timeRange;
+  let query = params.query;
+  let filters = params.filters;
+  let inspectorAdapters = params.inspectorAdapters;
+  let queryFilter = params.queryFilter;
+  let forceFetch = params.forceFetch;
+  // our own confiuration:
+  // list of fields to get potential observables from
+  // (each "field" is object {name: str, type: str, cnt: int})
+  let obsFields = params.visParams.obsFields;
+  
+  // filter out invalid field specifications
+  obsFields = obsFields.filter( f => (f.name != "" && f.type != "" && f.cnt > 0) );
+  
+  if (obsFields.length == 0) {
+    //console.log("theHiveButtonRequestHandler: Empty obsFields, nothing to do")
+    return {} // no fields specified, nothing to do
+  }
+
+  // === Prepare request to ask for unique values of all selected fields ===
+
+  // Construct a query for ElasticSearch
+  // Get "terms" (most common unique values) for each field of obsFields
+  const aggs_dsl = {}
+  for (let field of obsFields) {
+    aggs_dsl[field.name] = {
+      terms: {
+          field: field.name,
+          size: field.cnt,
+          order: {_count: "desc"}
+        }
+    };
+  }
+  //console.log("aggs_dsl:", aggs_dsl);
+  
+  // Create empty AggConfigs
+  // (We could pass specifications of a metric and the buckets here,
+  //  but default processing functions assume multiple buckets are sub-buckets,
+  //  which is not what we want. So we must do a "hack" and manually create  
+  //  query directly in format for ElasticSearch)
+  const aggs = new AggConfigs(params.index, []);
+  
+  // === Some magic to get searchSource object ===
+  // (inspired by https://github.com/fbaligand/kibana-enhanced-table/blob/7.4/public/data_load/enhanced-table-request-handler.js)
+  // (I don't understand it, but it works)
+
+  let $injector = await chrome.dangerouslyGetActiveInjector();
+  let Private = $injector.get('Private');
+  let SearchSource = Private(SearchSourceProvider);
+  let searchSource = new SearchSource();
+  searchSource.setField('index', index);
+  searchSource.setField('size', 0);
+
+  inspectorAdapters.requests = new RequestAdapter();
+  inspectorAdapters.data = new DataAdapter();
+  
+
+  // === Execute query ===
+  // We could call standard "courier" here, but it tries to convert the response
+  // to a table, which fails in our case, so we copied the main code of courier
+  // and modified it here.
+  
+  const abortSignal = false;
+  
+  const timeFilterSearchSource = searchSource.createChild({ callParentStartHandlers: true });
+  const requestSearchSource = timeFilterSearchSource.createChild({ callParentStartHandlers: true });
+
+  aggs.setTimeRange(timeRange);
+
+  // For now we need to mirror the history of the passed search source, since
+  // the request inspector wouldn't work otherwise.
+  Object.defineProperty(requestSearchSource, 'history', {
+    get() {
+      return searchSource.history;
+    },
+    set(history) {
+      return searchSource.history = history;
+    }
+  });
+
+  // This has been modified to override DSL format by ours
+//   requestSearchSource.setField('aggs', function () {
+//     return aggs.toDsl(metricsAtAllLevels);
+//   });
+  requestSearchSource.setField('aggs', aggs_dsl); 
+
+  requestSearchSource.onRequestStart((searchSource, searchRequest) => {
+    return aggs.onSearchRequestStart(searchSource, searchRequest);
+  });
+
+  if (timeRange) {
+    timeFilterSearchSource.setField('filter', () => {
+      return getTime(searchSource.getField('index'), timeRange);
+    });
+  }
+
+  requestSearchSource.setField('filter', filters);
+  requestSearchSource.setField('query', query);
+
+  const reqBody = await requestSearchSource.getSearchRequestBody();
+
+  const queryHash = calculateObjectHash(reqBody);
+  // We only need to reexecute the query, if forceFetch was true or the hash of the request body has changed
+  // since the last request
+  const shouldQuery = forceFetch || (searchSource.lastQuery !== queryHash);
+
+  if (shouldQuery) {
+    inspectorAdapters.requests.reset();
+    const request = inspectorAdapters.requests.start(
+      i18n.translate('common.ui.vis.courier.inspector.dataRequest.title', { defaultMessage: 'Data' }),
+      {
+        description: i18n.translate('common.ui.vis.courier.inspector.dataRequest.description',
+          { defaultMessage: 'This request queries Elasticsearch to fetch the data for the visualization.' }),
+      }
+    );
+    request.stats(getRequestInspectorStats(requestSearchSource));
+
+    try {
+      // Abort any in-progress requests before fetching again
+      if (abortSignal) {
+        abortSignal.addEventListener('abort', () => requestSearchSource.cancelQueued());
+      }
+
+      const response = await requestSearchSource.fetch();
+      //console.log("raw response:", response);
+
+      searchSource.lastQuery = queryHash;
+
+      request
+        .stats(getResponseInspectorStats(searchSource, response))
+        .ok({ json: response });
+
+      searchSource.rawResponse = response;
+    } catch(e) {
+      // Log any error during request to the inspector
+      request.error({ json: e });
+      throw e;
+    } finally {
+      // Add the request body no matter if things went fine or not
+      requestSearchSource.getSearchRequestBody().then(req => {
+        request.json(req);
+      });
+    }
+  }
+
+  // === Copy of courier code ends here, now we parse the response ===
+  
+  const resp = searchSource.rawResponse;
+  // Return as object containing a list of unique values (terms) for each 
+  // requested field
+  let unique_values_lists = {}
+  for (let field of obsFields) {
+    unique_values_lists[field.name] = resp.aggregations[field.name].buckets.map( (x) => x.key );
+  }
+
+  //console.log("Final lists:", unique_values_lists);
+  return unique_values_lists;
+}

thehive_button/public/vis.less

-.myvis-container-div {
-  padding: 1em;
-}

thehive_button/server/routes/newcase.js

@@ -34,7 +34,6 @@ function newCaseHandler(req, resp) {
   //if (!base_url.match(/https?:\/\/.*\//)) {
     return {'error': 'Invalid base URL (it must begin with "http[s]" and end with "/")'};
   }
-  // TODO add "/" to the end automatically
   if (!api_key) {
     return {'error': 'API key not set'};
   }
@@ -123,7 +122,7 @@ function addObservablesHandler(req, resp) {
   
 function addObservable(base_url, api_key, caseid, obs) {
   return new Promise( function(resolve, reject) {
-    console.log("Adding observable:", obs);
+    //console.log("Adding observable:", obs);
     request({
         method: 'POST',
         url: base_url + 'api/case/' + caseid + "/artifact",