Skip to content
Snippets Groups Projects
Commit 288c4563 authored by Bozidar Proevski's avatar Bozidar Proevski
Browse files

New branch added with different approach

parent 18673f66
No related branches found
No related tags found
No related merge requests found
Showing
with 684 additions and 258 deletions
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cat <<EOT > ${nifi_toolkit_props_file}
baseUrl=
keystore=
keystoreType=
keystorePasswd=
keyPasswd=
truststore=
truststoreType=
truststorePasswd=
proxiedEntity=
EOT
cat <<EOT > ${HOME}/.nifi-cli.config
nifi.props=${nifi_toolkit_props_file}
EOT
\ No newline at end of file
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
state_providers_file=${NIFI_HOME}/conf/state-management.xml
property_xpath='/stateManagement/cluster-provider/property'
edit_property() {
property_name=$1
property_value=$2
if [ -n "${property_value}" ]; then
xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}"
fi
}
edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}"
edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}"
\ No newline at end of file
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml
property_xpath='//loginIdentityProviders/provider/property'
# Update a given property in the login-identity-providers file if a value is specified
edit_property() {
property_name=$1
property_value=$2
if [ -n "${property_value}" ]; then
xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}"
fi
}
# Remove comments to enable the ldap-provider
sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}"
edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}"
edit_property 'Manager DN' "${LDAP_MANAGER_DN}"
edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}"
edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}"
edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}"
edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}"
edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}"
edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}"
edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}"
edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}"
edit_property 'Url' "${LDAP_URL}"
edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}"
edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}"
edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}"
\ No newline at end of file
---
# handlers file for nifi
\ No newline at end of file
galaxy_info:
author: your name
description: your description
company: your company (optional)
# If the issue tracker for your role is not on github, uncomment the
# next line and provide a value
# issue_tracker_url: http://example.com/issue/tracker
# Choose a valid license ID from https://spdx.org - some suggested licenses:
# - BSD-3-Clause (default)
# - MIT
# - GPL-2.0-or-later
# - GPL-3.0-only
# - Apache-2.0
# - CC-BY-4.0
license: license (GPL-2.0-or-later, MIT, etc)
min_ansible_version: 2.4
# If this a Container Enabled role, provide the minimum Ansible Container version.
# min_ansible_container_version:
#
# Provide a list of supported platforms, and for each platform a list of versions.
# If you don't wish to enumerate all versions for a particular platform, use 'all'.
# To view available platforms and versions (or releases), visit:
# https://galaxy.ansible.com/api/v1/platforms/
#
# platforms:
# - name: Fedora
# versions:
# - all
# - 25
# - name: SomePlatform
# versions:
# - all
# - 1.0
# - 7
# - 99.99
galaxy_tags: []
# List tags for your role here, one per line. A tag is a keyword that describes
# and categorizes the role. Users find roles by searching for tags. Be sure to
# remove the '[]' above, if you add tags to this list.
#
# NOTE: A tag is limited to a single word comprised of alphanumeric characters.
# Maximum 20 tags per role.
dependencies: []
# List your role dependencies here, one per line. Be sure to remove the '[]' above,
# if you add dependencies to this list.
\ No newline at end of file
--- ---
# tasks file for nifi
- name: Install required packages - name: Copy certificates in NiFi conf dir
apt:
name: ["aptitude","jq","xmlstarlet","procps","sudo","unzip","openjdk-8-jdk"]
state: present
- name: Create system group for Nifi
group:
name: nifi
gid: "{{ nifi_gid }}"
state: present
system: yes
- name: Create system user for Nifi
user:
name: nifi
uid: "{{ nifi_uid }}"
state: present
groups: nifi
system: yes
shell: /bin/false
- name: Create directory {{ nifi_base_dir }}
file:
path: "{{ nifi_base_dir }}"
state: directory
owner: nifi
group: nifi
- name: Download and extract Nifi
unarchive:
src: http://apache.uib.no/nifi/{{ nifi_version }}/nifi-{{ nifi_version }}-bin.tar.gz
remote_src: yes
dest: "{{ nifi_base_dir }}"
tags: download
- name: Download and extract Nifi-toolkit
unarchive:
src: http://apache.uib.no/nifi/{{ nifi_toolkit_version }}/nifi-toolkit-{{ nifi_version }}-bin.zip
remote_src: yes
dest: "{{ nifi_base_dir }}"
tags: download
- name: Link archives to standard locations
file:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
state: link
follow: no
loop:
- { src: "{{ nifi_base_dir }}/nifi-{{ nifi_version }}", dest: "{{ nifi_home }}" }
- { src: "{{ nifi_base_dir }}/nifi-toolkit-{{ nifi_toolkit_version }}", dest: "{{ nifi_toolkit_home }}" }
- name: Create directories
file:
path: "{{ item }}"
state: directory
loop:
- "{{ nifi_home }}/conf"
- "{{ nifi_home }}/database_repository"
- "{{ nifi_home }}/flowfile_repository"
- "{{ nifi_home }}/content_repository"
- "{{ nifi_home }}/provenance_repository"
- "{{ nifi_home }}/state"
- "{{ nifi_log_dir }}"
- "{{ nifi_home }}/scripts"
- name: Modify nifi.env.sh
copy: copy:
src: nifi-env.sh src: "{{ item }}"
dest: "{{ nifi_home }}/bin/nifi-env.sh" dest: "conf/{{ item }}"
mode: "0755" with_items:
when: docker is defined - "{{ inventory_hostname }}.p12"
- cacerts.jks
- name: Copy files tags:
- start
- name: Copy flow in NiFi conf dir
copy: copy:
src: "{{ item }}" src: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
dest: "{{ nifi_home }}/scripts/{{ item }}" dest: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
mode: "0755" when:
loop: - "'{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz' is is_file"
- start.sh tags:
- common.sh - start
- secure.sh
- toolkit.sh - name: Configure NiFi properties for secure servers
- update_cluster_state_management.sh template:
- update_login_providers.sh src: nifi.properties.j2
dest: conf/nifi.properties
tags:
- start
- name: Configure NiFi authorizers for secure servers
template:
src: authorizers.xml.j2
dest: conf/authorizers.xml
tags:
- start
- name: Start NiFi
command: "./bin/nifi.sh start"
tags:
- start
#- name: check reachable hosts
# gather_facts: no
# tasks:
# - command: ping -c1 {{ inventory_hostname }}
# delegate_to: localhost
# register: ping_result
# ignore_errors: yes
# - group_by: key=reachable
# when: ping_result|success
- name: Stop NiFi
command: "./bin/nifi.sh stop"
tags:
- stop
- name: Copy flow from NiFi
fetch:
src: "{{ ansible_facts.env['NIFI_HOME'] }}/conf/flow.xml.gz"
dest: "{{ role_path }}/files/flow-{{ inventory_hostname }}.xml.gz"
flat: yes
tags:
- stop
This diff is collapsed.
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flow.configuration.archive.max.count=
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.queue.backpressure.count=10000
nifi.queue.backpressure.size=1 GB
nifi.authorizer.configuration.file=./conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml
nifi.templates.directory=./conf/templates
nifi.ui.banner.text=
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./lib
nifi.nar.library.autoload.directory=./extensions
nifi.nar.working.directory=./work/nar/
nifi.documentation.working.directory=./work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./conf/zookeeper.properties
# H2 Settings
nifi.database.directory=./database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog
nifi.flowfile.repository.directory=./flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=1 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=../nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
nifi.provenance.repository.debug.frequency=1_000_000
nifi.provenance.repository.encryption.key.provider.implementation=
nifi.provenance.repository.encryption.key.provider.location=
nifi.provenance.repository.encryption.key.id=
nifi.provenance.repository.encryption.key=
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=2
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
nifi.provenance.repository.concurrent.merge.threads=2
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host={{ inventory_hostname }}
nifi.remote.input.secure=true
nifi.remote.input.socket.port=10443
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
nifi.remote.contents.cache.expiration=30 secs
# web properties #
nifi.web.war.directory=./lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.http.network.interface.default=
nifi.web.https.host={{ inventory_hostname }}
nifi.web.https.port=9443
nifi.web.https.network.interface.default=
nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
nifi.web.max.header.size=16 KB
nifi.web.proxy.context.path=/nifi
nifi.web.proxy.host={{ dslproxy }}
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.key.protected=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=
nifi.security.keystore=./conf/{{ inventory_hostname }}.p12
nifi.security.keystoreType=pkcs12
nifi.security.keystorePasswd={{ kspass}}
#nifi.security.keyPasswd=IP7Jgn7amiAYi3LRSRk5LGg3t4zlfh0kEKcAaaoxHDo
nifi.security.truststore=./conf/cacerts.jks
nifi.security.truststoreType=jks
nifi.security.truststorePasswd={{ tspass}}
nifi.security.user.authorizer=managed-authorizer
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# OpenId Connect SSO Properties #
nifi.security.user.oidc.discovery.url=
nifi.security.user.oidc.connect.timeout=5 secs
nifi.security.user.oidc.read.timeout=5 secs
nifi.security.user.oidc.client.id=
nifi.security.user.oidc.client.secret=
nifi.security.user.oidc.preferred.jwsalgorithm=
# Apache Knox SSO Properties #
nifi.security.user.knox.url=
nifi.security.user.knox.publicKey=
nifi.security.user.knox.cookieName=hadoop-jwt
nifi.security.user.knox.audiences=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.transform.dn=NONE
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# nifi.security.identity.mapping.transform.kerb=UPPER
# Group Mapping Properties #
# These properties allow normalizing group names coming from external sources like LDAP. The following example
# lowercases any group name.
#
# nifi.security.group.mapping.pattern.anygroup=^(.*)$
# nifi.security.group.mapping.value.anygroup=$1
# nifi.security.group.mapping.transform.anygroup=LOWER
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=true
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=true
nifi.cluster.node.address={{ inventory_hostname }}
nifi.cluster.node.protocol.port=11443
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.protocol.max.threads=50
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.node.max.concurrent.requests=100
nifi.cluster.firewall.file=
nifi.cluster.flow.election.max.wait.time=1 min
nifi.cluster.flow.election.max.candidates=
# cluster load balancing properties #
nifi.cluster.load.balance.host=
nifi.cluster.load.balance.port=6342
nifi.cluster.load.balance.connections.per.node=4
nifi.cluster.load.balance.max.thread.count=8
nifi.cluster.load.balance.comms.timeout=30 sec
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=dsoclab-zookeeper:2181
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management
# Values supported for nifi.zookeeper.auth.type are "default", which will apply world/anyone rights on znodes
# and "sasl" which will give rights to the sasl/kerberos identity used to authenticate the nifi node
# The identity is determined using the value in nifi.kerberos.service.principal and the removeHostFromPrincipal
# and removeRealmFromPrincipal values (which should align with the kerberos.removeHostFromPrincipal and kerberos.removeRealmFromPrincipal
# values configured on the zookeeper server).
nifi.zookeeper.auth.type=
nifi.zookeeper.kerberos.removeHostFromPrincipal=
nifi.zookeeper.kerberos.removeRealmFromPrincipal=
# kerberos #
nifi.kerberos.krb5.file=
# kerberos service principal #
nifi.kerberos.service.principal=
nifi.kerberos.service.keytab.location=
# kerberos spnego principal #
nifi.kerberos.spnego.principal=
nifi.kerberos.spnego.keytab.location=
nifi.kerberos.spnego.authentication.expiration=12 hours
# external properties files for variable registry
# supports a comma delimited list of file locations
nifi.variable.registry.properties=
localhost
---
- hosts: localhost
remote_user: root
roles:
- nifi
\ No newline at end of file
---
# vars file for nifi
\ No newline at end of file
[dsldev]
localhost ansible_connection=local
[nificontainers]
dsoclab-nifi-1 ansible_connection=docker
dsoclab-nifi-2 ansible_connection=docker
dsoclab-nifi-3 ansible_connection=docker
[nginx]
dsoclab-nginx ansible_connection=docker
---
- name: Start soctools cluster
import_playbook: startsoctools.yml
when: "'start' in ansible_run_tags"
- name: Stop soctools cluster
import_playbook: stopsoctools.yml
when: "'stop' in ansible_run_tags"
---
- name: Start docker containers
hosts: dsldev
roles:
- docker
- name: Reconfigure and start nginx
hosts: nginx
roles:
- nginx
- name: Reconfigure and start NiFi
hosts: nificontainers
roles:
- nifi
---
- name: Get flow from Nifi
hosts: nificontainers
roles:
- nifi
- name: Stop all containers
hosts: dsldev
roles:
- docker
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment