Skip to content
Snippets Groups Projects
Commit fe04ddda authored by Bozidar Proevski's avatar Bozidar Proevski
Browse files

Added new roles

role  odfees (Open Distro for Elasticsearch) and
role odfekibana (Open Distro for Elasticsearch Kibana)
parent 651242be
No related branches found
No related tags found
No related merge requests found
Showing
with 225 additions and 7 deletions
...@@ -4,7 +4,7 @@ soctools_netname: "dslnifinet" ...@@ -4,7 +4,7 @@ soctools_netname: "dslnifinet"
repo: gn43-dsl repo: gn43-dsl
version: 7 version: 7
suffix: al suffix: a20200516
temp_root: "/tmp/centosbuild" temp_root: "/tmp/centosbuild"
...@@ -35,4 +35,11 @@ ca_cn: "dsldev test ca" ...@@ -35,4 +35,11 @@ ca_cn: "dsldev test ca"
nifiadmin: nifiadmin:
- [ "Bozidar Proevski", "Pass001" ] - [ "Bozidar Proevski", "Pass001" ]
- [ "Arne Oslebo", "Pass002" ] - [ "Arne Oslebo", "Pass002" ]
- [ "NifiELKuser", "Pass003" ]
odfees_img: "{{repo}}/odfees:{{version}}{{suffix}}"
odfekibana_img: "{{repo}}/odfekibana:{{version}}{{suffix}}"
odfees_adminpass: "Pass004"
openid_realm: "GN43WP8T31SOC1"
openid_subjkey: preferred_username
...@@ -6,3 +6,5 @@ ...@@ -6,3 +6,5 @@
- include: openjdk.yml - include: openjdk.yml
- include: zookeeper.yml - include: zookeeper.yml
- include: nifi.yml - include: nifi.yml
- include: odfees.yml
- include: odfekibana.yml
---
- name: Configure elasticsearch Dockerfile
template:
src: odfees/Dockerfile-elastic.j2
dest: "{{role_path}}/files/elasticDockerfile"
- name: Build elasticsearch image
command: docker build -t {{repo}}/elasticsearch:{{version}}{{suffix}} -f {{role_path}}/files/elasticDockerfile {{role_path}}/files
- name: Configure odfe elasticsearch Dockerfile
template:
src: odfees/Dockerfile-odfeelastic.j2
dest: "{{role_path}}/files/odfeesDockerfile"
- name: Build odfe elasticsearch image
command: docker build -t {{repo}}/odfees:{{version}}{{suffix}} -f {{role_path}}/files/odfeesDockerfile {{role_path}}/files
---
- name: Configure kibana Dockerfile
template:
src: odfekibana/Dockerfile-kibana.j2
dest: "{{role_path}}/files/kibanaDockerfile"
- name: Build kibana image
command: docker build -t {{repo}}/kibana:{{version}}{{suffix}} -f {{role_path}}/files/kibanaDockerfile {{role_path}}/files
- name: Configure odfe kibana Dockerfile
template:
src: odfekibana/Dockerfile-odfekibana.j2
dest: "{{role_path}}/files/odfekibanaDockerfile"
- name: Build odfe kibana image
command: docker build -t {{repo}}/odfekibana:{{version}}{{suffix}} -f {{role_path}}/files/odfekibanaDockerfile {{role_path}}/files
...@@ -22,7 +22,7 @@ FROM {{repo}}/openjdk:{{version}}{{suffix}} ...@@ -22,7 +22,7 @@ FROM {{repo}}/openjdk:{{version}}{{suffix}}
ARG UID=1000 ARG UID=1000
ARG GID=1000 ARG GID=1000
ARG NIFI_VERSION=1.9.2 ARG NIFI_VERSION=1.11.4
ARG BASE_URL=https://archive.apache.org/dist ARG BASE_URL=https://archive.apache.org/dist
ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}} ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}}
ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip} ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip}
......
FROM {{repo}}/openjdk:{{version}}{{suffix}}
ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
RUN groupadd -g 1000 elasticsearch && \
adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
WORKDIR /usr/share/elasticsearch
RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
rpm -Uvh https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.6.1-no-jdk-x86_64.rpm && \
cp -a /etc/elasticsearch/ /usr/share/elasticsearch/config/ && \
chown -R elasticsearch /usr/share/elasticsearch/config && \
sed -i -e 's,ES_PATH_CONF=/etc/elasticsearch,ES_PATH_CONF=/usr/share/elasticsearch/config,g' /etc/sysconfig/elasticsearch
EXPOSE 9200 9300
USER elasticsearch
ENTRYPOINT ["/bin/bash"]
FROM {{repo}}/elasticsearch:{{version}}{{suffix}}
ENV PATH="/usr/share/elasticsearch/bin:${PATH}"
USER root
WORKDIR /usr/share/elasticsearch
RUN for PLUGIN in \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-security/opendistro_security-1.6.0.0.zip \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-alerting/opendistro_alerting-1.6.0.0.zip \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-sql/opendistro_sql-1.6.0.0.zip; \
do bin/elasticsearch-plugin install -b ${PLUGIN}; done && \
chown -R elasticsearch plugins/opendistro_security
USER elasticsearch
FROM {{repo}}/centos:{{version}}{{suffix}}
ENV PATH="/usr/share/kibana/bin:${PATH}"
RUN groupadd -g 1000 kibana && \
adduser -u 1000 -g 1000 -d /usr/share/kibana kibana
WORKDIR /usr/share/kibana
RUN rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch && \
rpm -Uvh https://artifacts.elastic.co/downloads/kibana/kibana-oss-7.6.1-x86_64.rpm && \
cp -a /etc/kibana/ /usr/share/kibana/config/ && \
chown -R kibana /usr/share/kibana/config/
EXPOSE 5601
USER kibana
ENTRYPOINT ["/bin/bash"]
FROM {{repo}}/kibana:{{version}}{{suffix}}
ENV PATH="/usr/share/kibana/bin:${PATH}"
USER root
WORKDIR /usr/share/kibana
RUN for PLUGIN in \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-security/opendistro_security_kibana_plugin-1.6.0.0.zip \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-alerting/opendistro-alerting-1.6.0.0.zip \
https://d3g5vo6xdbdb9a.cloudfront.net/downloads/kibana-plugins/opendistro-index-management/opendistro_index_management_kibana-1.6.0.0.zip; \
do bin/kibana-plugin install --allow-root ${PLUGIN}; done
USER kibana
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
- name: Generate truststore - name: Generate truststore
command: > command: >
docker run --rm -v {{role_path}}/files/truststore/:/opt/cafiles/:z docker run --rm -v {{role_path}}/files/truststore/:/opt/cafiles/:z
gn43-dsl/openjdk:7al keytool -import -noprompt -trustcacerts "{{repo}}/openjdk:{{version}}{{suffix}}" keytool -import -noprompt -trustcacerts
-alias "{{item}}" -file "/opt/cafiles/{{item}}.crt" -keystore /opt/cafiles/cacerts.jks -storepass "{{tspass}}" -alias "{{item}}" -file "/opt/cafiles/{{item}}.crt" -keystore /opt/cafiles/cacerts.jks -storepass "{{tspass}}"
with_items: with_items:
- "{{ ca_cn }}" - "{{ ca_cn }}"
...@@ -40,6 +40,8 @@ ...@@ -40,6 +40,8 @@
command: roles/ca/files/easyrsa/easyrsa show-cert {{item}} command: roles/ca/files/easyrsa/easyrsa show-cert {{item}}
with_items: with_items:
- "{{ groups['nificontainers'] }}" - "{{ groups['nificontainers'] }}"
- "{{ groups['odfeescontainers'] }}"
- "{{ groups['odfekibanacontainers'] }}"
environment: environment:
EASYRSA_BATCH: 1 EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA EASYRSA_PKI: roles/ca/files/CA
...@@ -53,6 +55,8 @@ ...@@ -53,6 +55,8 @@
build-serverClient-full {{item}} nopass build-serverClient-full {{item}} nopass
with_items: with_items:
- "{{ groups['nificontainers'] }}" - "{{ groups['nificontainers'] }}"
- "{{ groups['odfeescontainers'] }}"
- "{{ groups['odfekibanacontainers'] }}"
environment: environment:
EASYRSA_BATCH: 1 EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA EASYRSA_PKI: roles/ca/files/CA
...@@ -82,6 +86,8 @@ ...@@ -82,6 +86,8 @@
Enter Export Password: "{{kspass}}" Enter Export Password: "{{kspass}}"
with_items: with_items:
- "{{ groups['nificontainers'] }}" - "{{ groups['nificontainers'] }}"
- "{{ groups['odfeescontainers'] }}"
- "{{ groups['odfekibanacontainers'] }}"
environment: environment:
EASYRSA_BATCH: 1 EASYRSA_BATCH: 1
EASYRSA_PKI: roles/ca/files/CA EASYRSA_PKI: roles/ca/files/CA
...@@ -93,10 +99,42 @@ ...@@ -93,10 +99,42 @@
with_items: with_items:
- "{{ groups['nificontainers'] }}" - "{{ groups['nificontainers'] }}"
- name: Copy nifi truststore to nifi role - name: Copy odfees host certs to odfees role
copy:
src: roles/ca/files/CA/private/{{item}}.p12
dest: roles/odfees/files/{{item}}.p12
with_items:
- "{{ groups['odfeescontainers'] }}"
- name: Copy odfekibana host p12 certs to odfekibana role
copy:
src: roles/ca/files/CA/private/{{item}}.p12
dest: roles/odfekibana/files/{{item}}.p12
with_items:
- "{{ groups['odfekibanacontainers'] }}"
- name: Copy odfekibana host certs to odfekibana role
copy:
src: roles/ca/files/CA/issued/{{item}}.crt
dest: roles/odfekibana/files/{{item}}.crt
with_items:
- "{{ groups['odfekibanacontainers'] }}"
- name: Copy odfekibana host keys to odfekibana role
copy:
src: roles/ca/files/CA/private/{{item}}.key
dest: roles/odfekibana/files/{{item}}.key
with_items:
- "{{ groups['odfekibanacontainers'] }}"
- name: Copy truststore to roles
copy: copy:
src: roles/ca/files/truststore/cacerts.jks src: roles/ca/files/truststore/cacerts.jks
dest: roles/nifi/files/cacerts.jks dest: "roles/{{item}}/files/cacerts.jks"
with_items:
- nifi
- odfees
- odfekibana
- name: Check for existing user certificates - name: Check for existing user certificates
command: roles/ca/files/easyrsa/easyrsa show-cert {{item[0] | regex_escape()}} command: roles/ca/files/easyrsa/easyrsa show-cert {{item[0] | regex_escape()}}
......
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
- include: networkcreate.yml - include: networkcreate.yml
- include: zookeeper.yml - include: zookeeper.yml
- include: nifi.yml - include: nifi.yml
- include: odfees.yml
- include: odfekibana.yml
- include: nginx.yml - include: nginx.yml
- include: networkremove.yml - include: networkremove.yml
...@@ -10,6 +10,9 @@ ...@@ -10,6 +10,9 @@
networks_cli_compatible: yes networks_cli_compatible: yes
published_ports: published_ports:
- "443:443" - "443:443"
- "8443:8443"
- "9443:9443"
- "9200:9200"
- "7750:7750" - "7750:7750"
tags: tags:
- start - start
......
---
- name: Create odfe elasticsearch containers and connect to network
docker_container:
name: "{{ item }}"
hostname: "{{ item }}"
image: "{{ odfees_img }}"
networks:
- name: "{{ soctools_netname }}"
networks_cli_compatible: yes
entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['odfeescontainers'] }}"
tags:
- start
- name: Disconnect odfe elasticsearch containers from network and remove
docker_container:
name: "{{ item }}"
state: absent
with_items: "{{ groups['odfeescontainers'] }}"
tags:
- stop
---
- name: Create odfe kibana containers and connect to network
docker_container:
name: "{{ item }}"
hostname: "{{ item }}"
image: "{{ odfekibana_img }}"
networks:
- name: "{{ soctools_netname }}"
networks_cli_compatible: yes
published_ports:
- "5601:5601"
entrypoint: "/bin/bash"
interactive: "yes"
with_items: "{{ groups['odfekibanacontainers'] }}"
tags:
- start
- name: Disconnect odfe elasticsearch containers from network and remove
docker_container:
name: "{{ item }}"
state: absent
with_items: "{{ groups['odfekibanacontainers'] }}"
tags:
- stop
...@@ -12,15 +12,26 @@ stream { ...@@ -12,15 +12,26 @@ stream {
resolver 127.0.0.11; resolver 127.0.0.11;
upstream nifiserv { upstream nifiserv {
hash $remote_addr consistent;
{% for nifihost in groups['nificontainers'] %} {% for nifihost in groups['nificontainers'] %}
server {{nifihost}}:9443; server {{nifihost}}:9443;
{% endfor %} {% endfor %}
} }
server { server {
listen 443; listen 9443;
proxy_pass nifiserv; proxy_pass nifiserv;
} }
upstream odfeserv {
{% for odfehost in groups['odfeescontainers'] %}
server {{odfehost}}:9200;
{% endfor %}
}
server {
listen 9200;
proxy_pass odfeserv;
}
upstream nifiservtcp7750 { upstream nifiservtcp7750 {
{% for nifihost in groups['nificontainers'] %} {% for nifihost in groups['nificontainers'] %}
server {{nifihost}}:7750; server {{nifihost}}:7750;
......
...@@ -143,7 +143,7 @@ nifi.web.jetty.working.directory=./work/jetty ...@@ -143,7 +143,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200 nifi.web.jetty.threads=200
nifi.web.max.header.size=16 KB nifi.web.max.header.size=16 KB
nifi.web.proxy.context.path=/nifi nifi.web.proxy.context.path=/nifi
nifi.web.proxy.host={{ dslproxy }} nifi.web.proxy.host={{ dslproxy }}:9443
# security properties # # security properties #
nifi.sensitive.props.key= nifi.sensitive.props.key=
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment