init commit

.dockerignore

+.git
+Dockerfile
+.DS_Store
+.gitignore
+.dockerignore
+
+/credentials
+/cache
+/store
+
+/node_modules
+
+# https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
+
+# General
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
\ No newline at end of file

.gitignore

@@ -11,4 +11,5 @@ dist
 venv
 .vscode
 docs/build
+.env
 

.jenkins/Jenkinsfile

+pipeline {
+  agent any
+
+  environment {
+    ARTIFACTORY_JENKINS_ID = 'geant-project-artifactory'
+    ARTIFACTORY_DOCKER_REPO_NAME = 'geant-swd-docker'
+    ARTIFACTORY_DOCKER_REGISTRY = "artifactory.software.geant.org/${ARTIFACTORY_DOCKER_REPO_NAME}"
+    PROJECT_NAME = 'inventory-provider'
+    VERSION_IDENTIFIER = "${env.TAG_NAME == null ? 'snapshot' : env.TAG_NAME}"
+  }
+
+  stages {
+
+    stage('Build Docker Images') {
+      steps {
+        script {
+                image_tag = "${env.ARTIFACTORY_DOCKER_REGISTRY}/${env.PROJECT_NAME}:${env.VERSION_IDENTIFIER}"
+            }
+         sh "docker build -t ${image_tag} ${env.WORKSPACE}"
+      }
+    }
+
+    stage('Push Docker Images') {
+      steps {
+        rtDockerPush(
+                serverId: "${env.ARTIFACTORY_JENKINS_ID}",
+                image: "${image_tag}",
+                targetRepo: "${env.ARTIFACTORY_DOCKER_REPO_NAME}",
+                properties: "project-name=${env.PROJECT_NAME};status=development"
+            )
+
+      }
+    }
+  }
+}
\ No newline at end of file

.jenkins/scripts/clean

+set -eo pipefail
+source .gitlab/ci/scripts/utils.sh
+
+function main() {
+    docker_prune_images
+}
+
+function docker_prune_images() {
+    docker image prune -a -f --filter="label=name=inventory-provider" --filter "until=72h"
+    docker system prune -f
+}
+
+main $@

.jenkins/scripts/deploy

+set -eo pipefail
+source .gitlab/ci/scripts/utils.sh
+trap 'clean_up' EXIT
+
+function main() {
+    docker login -u $SOKAN_REGISTRY_USER \
+                 -p $SOKAN_REGISTRY_PASSWORD $SOKAN_REGISTRY
+    deploy
+}
+
+# Deploy components to dpeloyment environment
+function deploy() {
+    echo -e "${ORANGE}---${ENV_NAME}---${NOCOLOR}\n"
+    local deployment_image="${HERMES_REGISTRY}/deployment:${IMAGE_TAG}"
+
+    ANSIBLE_INVENTORY="/deployment/ansible/inventories/${ENV_NAME}"
+
+    environment=$(get_environment)
+
+    options="-e ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY} \
+             --network host"
+    
+    options="${options} -v /opt/vault:/tmp/vault \
+                        -e ANSIBLE_VAULT_PASSWORD_FILE=/tmp/vault/${environment}-vault-pass"
+
+    API_IMAGE="${HERMES_REGISTRY}/api:${IMAGE_TAG}"
+    UI_IMAGE="${HERMES_REGISTRY}/ui:${IMAGE_TAG}"
+    options="${options} -e API_VERSION=${API_VERSION} -e UI_VERSION=${UI_VERSION}"
+
+    if is_nightly || is_clean_deploy; then
+        docker run --rm $options $deployment_image /bin/bash -c \
+                    "ansible-playbook /deployment/ansible/playbooks/install/main.yml \
+                     --extra-vars 'vagrant_setup_vms_cleanup=true'"
+
+    else
+        docker run --rm $options $deployment_image update.sh
+
+    fi
+}
+
+function clean_up() {
+    echo -e "${PURPLE}Cleaning Up...${NOCOLOR}\n"
+    local deployment_image="${HERMES_REGISTRY}/deployment:${IMAGE_TAG}"
+
+    docker image rm -f $deployment_image
+}
+
+main $@

.jenkins/scripts/push

+set -eo pipefail
+source .gitlab/ci/scripts/utils.sh
+
+function main() {
+    if is_local || is_nightly || is_test_deploy || is_clean_deploy; then
+        prepare
+        build
+        push "${IMAGE}:${IMAGE_TAG}"
+        if push_international; then
+            push "${IMAGE_INTERNATIONAL}:${IMAGE_TAG}"
+        fi
+        if [ $BUILD_CONTEXT == "api/" ]; then
+            build_hermes_base
+        fi
+    fi
+}
+
+function prepare() {
+    echo -e "${PURPLE}Preparing...${NOCOLOR}\n"
+    docker_login
+    eval "${EXTRA_PREPARE_COMMANDS}"
+}
+
+function build() {
+    echo -e "${PURPLE}Building ${IMAGE}...${NOCOLOR}\n"
+    docker build --tag "${IMAGE}:${IMAGE_TAG}" \
+                 --file ${DOCKERFILE} ${EXTRA_BUILD_OPTIONS} \
+                 ${BUILD_CONTEXT}
+}
+
+function build_hermes_base() {
+    echo -e "${PURPLE}Building hermes-base ...${NOCOLOR}\n"
+    docker build --tag "hermes-base:${IMAGE_TAG}" \
+                 --file ${DOCKERFILE} ${EXTRA_BUILD_OPTIONS} \
+                 --target base-image \
+                 ${BUILD_CONTEXT}
+}
+
+
+main $@

.jenkins/scripts/release

+set -eo pipefail
+source .gitlab/ci/scripts/utils.sh
+
+REGISTRIES="$HERMES_REGISTRY $HERMES_REGISTRY_INTERNATIONAL"
+
+function main() {
+    local images=$@
+
+    docker_login
+
+    for image in $images;
+    do
+        for registry in $REGISTRIES;
+        do
+            release "$image" "$registry"
+
+        done
+
+    done
+}
+
+function release() {
+    local image=${1}
+    local registry=${2}
+
+    local source_image="${registry}/${image}:dev-${SUCCESSFUL_PIPELINE_NUMBER}"
+    local versioned_image="${registry}/${image}:${IMAGE_TAG}"
+    local latest_image="${registry}/${image}:latest"
+
+    change_tag $source_image $versioned_image
+    push  $versioned_image
+
+    change_tag $source_image $latest_image
+    push  $latest_image
+}
+
+main $@

.jenkins/scripts/utils.sh

+NOCOLOR='\033[0m'
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+ORANGE='\033[0;33m'
+PURPLE='\033[0;35m'
+
+function run_timed_command() {
+  local cmd="${1}"
+  local start=$(date +%s)
+  echo_success "\$ ${cmd}"
+  eval "${cmd}"
+  local ret=$?
+  local end=$(date +%s)
+  local runtime=$((end-start))
+
+  if [[ $ret -eq 0 ]]; then
+    echo_success "==> '${cmd}' succeeded in ${runtime} seconds."
+    return 0
+  else
+    echo_err "==> '${cmd}' failed (${ret}) in ${runtime} seconds."
+    exit 1
+  fi
+}
+
+function echo_success() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n${GREEN}** %s **\n${NOCOLOR}" "${1}" >&2;
+  else
+    printf "${GREEN}%s\n${NOCOLOR}" "${1}" >&2;
+  fi
+}
+
+function echo_err() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n${RED}** %s **\n${NOCOLOR}" "${1}" >&2;
+  else
+    printf "${RED}%s\n${NOCOLOR}" "${1}" >&2;
+  fi
+}
+
+function is_clean_deploy() {
+  [ "$CLEAN_DEPLOY" = "true" ]
+
+  local result=$?
+  return $result
+}
+
+function is_test_deploy() {
+  [ "$TEST_DEPLOY" = "true" ]
+
+  local result=$?
+  return $result
+}
+
+function is_local() {
+    [ ! -z $CI_MERGE_REQUEST_IID ]
+    
+    local result=$?
+    return $result
+}
+
+function is_lab() {
+    [ "$CI_COMMIT_REF_NAME" = "master" ] && \
+    [ "$ENV" == "lab" ] && \
+    [ $SUCCESSFUL_PIPELINE_NUMBER ] 
+
+    local result=$?
+    return $result
+}
+
+function is_production() {
+    [ "$ENV" == "production" ] && \
+    [ $SUCCESSFUL_PIPELINE_NUMBER ] && \
+    [ $TAG ]
+
+    local result=$?
+    return $result
+}
+
+function is_nightly() {
+    [ "$CI_PIPELINE_SOURCE" = "schedule" ] && \
+    [ "$FREQUENCY" = "nightly" ]
+
+    local result=$?
+    return $result
+}
+
+function push_international() {
+    [ "$PUSH_INTERNATIONAL" = "true" ] && \
+    [ $IMAGE_INTERNATIONAL ]
+
+    local result=$?
+    return $result
+}
+
+function get_environment() {
+  if is_local || is_clean_deploy || is_test_deploy || is_nightly; then
+    echo "local"
+
+  elif is_lab; then
+    echo "lab"
+
+  elif is_production; then
+    echo "production"
+
+  fi
+}
+
+function docker_login() {
+    echo -e "${PURPLE}Docker login...${NOCOLOR}\n"
+    docker login -u $SOKAN_REGISTRY_USER \
+                 -p $SOKAN_REGISTRY_PASSWORD $SOKAN_REGISTRY
+    if push_international; then
+      docker login -u $MARKIGHT_REGISTRY_USER \
+                   -p $MARKIGHT_REGISTRY_PASSWORD $MARKIGHT_REGISTRY
+    fi
+}
+
+function change_tag() {
+    local source_image="${1}"
+    local target_image="${2}"
+
+    echo -e "${PURPLE}Changing Tag ${source_image} to ${target_image}...${NOCOLOR}\n"
+    docker pull $source_image
+    docker image tag $source_image \
+                     $target_image
+    docker image rm -f $source_image
+}
+
+function push() {
+    local image="${1}"
+    
+    echo -e "${PURPLE}Pushing ${image}...${NOCOLOR}\n"
+    docker push $image
+}

Dockerfile

+### Build and install packages ###
+FROM python:3.6.15 as base-image
+
+LABEL name="inventory-provider"
+
+# Install Python dependencies
+COPY ./requirements.txt /requirements.txt
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /requirements.txt
+
+### Final image ###
+FROM python:3.6.15-slim-buster
+
+LABEL name="inventory-provider"
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+# Create entrypoints
+COPY ./start-web-app.sh ./start-worker.sh ./start-monitoring.sh /
+RUN chmod +x /start-web-app.sh /start-worker.sh /start-monitoring.sh
+
+# Create a dedicated user and group for the app
+RUN groupadd -r inventory_provider && \
+    useradd -r -g inventory_provider inventory_provider
+
+# Copy the Python dependencies from the base image
+COPY --from=base-image /usr/local/lib/python3.6/site-packages/ /usr/local/lib/python3.6/site-packages/
+COPY --from=base-image /usr/local/bin/ /usr/local/bin/
+
+# Copy the app's source code
+COPY ./ /app
+
+# Set ownership and permissions for the app
+RUN chown -R inventory_provider:inventory_provider /app/
+
+# Switch to the inventory_provider user
+USER inventory_provider
+
+# Set the working directory
+WORKDIR /app
+
+# Expose the appropriate port
+EXPOSE 8000
+
+# Define the entrypoint command
+ENTRYPOINT ["/start-web-app.sh"]

inventory_provider/config.py

 import json
 import jsonschema
+from dotenv import load_dotenv
 
 CONFIG_SCHEMA = {
     '$schema': 'https://json-schema.org/draft-07/schema#',
@@ -262,6 +263,10 @@ def load(f):
     :param f: file-like object that produces the config file
     :return: a dict containing the parsed configuration parameters
     """
+
+    # Load environment variables from .env file
+    load_dotenv()
+
     config = json.loads(f.read())
     jsonschema.validate(config, CONFIG_SCHEMA)
     return config

inventory_provider/gunicorn.py

+import os
+
+syslog = True
+syslog_addr = 'unix:///dev/log'
+syslog_facility = os.environ.get('SYSLOG_FACILITY', '')
+capture_output = False
+statsd_host = 'localhost:8125'
+statsd_prefix = 'gunicorn'
+user = os.environ.get('USERNAME', '')
+group = os.environ.get('GROUPNAME', '')
+timeout = 60
+workers = 4
+bind = os.environ.get('BIND_ADDRESS', '')
+pidfile = os.environ.get('GUNICORN_PID_FILENAME', '')
+raw_env = [
+    f'FLASK_SETTINGS_FILENAME={os.environ.get("app_conf_filename", "")}',
+    f'INVENTORY_PROVIDER_CONFIG_FILENAME={os.environ.get("inventory_conf_filename", "")}',
+    f'LOGGING_CONFIG={os.environ.get("logging_conf_filename", "")}'
+]
\ No newline at end of file

requirements.txt

@@ -15,6 +15,8 @@ lxml
 requests
 netifaces
 tree-format
+gunicorn
+python-dotenv
 
 pytest
 pytest-mock

start-monitoring.sh

+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+cd /app
+

start-web-app.sh

+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+gunicorn -c inventory_provider/gunicorn.py inventory_provider.app:app

start-worker.sh

+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+cd /app
+
+celery -A inventory_provider.tasks.worker worker --pidfile "$worker_pid_filename" --concurrency 10
\ No newline at end of file