diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..610ee52942e16eec75bfbfba11c4597458a290e4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,142 @@ +.git +Dockerfile +.DS_Store +.gitignore +.dockerignore + +/credentials +/cache +/store + +/node_modules + +# https://github.com/github/gitignore/blob/master/Global/macOS.gitignore + +# General +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ \ No newline at end of file diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.gitignore b/.gitignore index 4aff35c896206640b789c6493cbb1b1257042322..ebf9343bbba6473cccdb3f0e16c2323d114ac67e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ dist venv .vscode docs/build +.env diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..658e2c330a97291d537e6ff4cd9560facf633b12 --- /dev/null +++ b/.jenkins/Jenkinsfile @@ -0,0 +1,35 @@ +pipeline { + agent any + + environment { + ARTIFACTORY_JENKINS_ID = 'geant-project-artifactory' + ARTIFACTORY_DOCKER_REPO_NAME = 'geant-swd-docker' + ARTIFACTORY_DOCKER_REGISTRY = "artifactory.software.geant.org/${ARTIFACTORY_DOCKER_REPO_NAME}" + PROJECT_NAME = 'inventory-provider' + VERSION_IDENTIFIER = "${env.TAG_NAME == null ? 'snapshot' : env.TAG_NAME}" + } + + stages { + + stage('Build Docker Images') { + steps { + script { + image_tag = "${env.ARTIFACTORY_DOCKER_REGISTRY}/${env.PROJECT_NAME}:${env.VERSION_IDENTIFIER}" + } + sh "docker build -t ${image_tag} ${env.WORKSPACE}" + } + } + + stage('Push Docker Images') { + steps { + rtDockerPush( + serverId: "${env.ARTIFACTORY_JENKINS_ID}", + image: "${image_tag}", + targetRepo: "${env.ARTIFACTORY_DOCKER_REPO_NAME}", + properties: "project-name=${env.PROJECT_NAME};status=development" + ) + + } + } + } +} \ No newline at end of file diff --git a/.jenkins/scripts/clean b/.jenkins/scripts/clean new file mode 100644 index 0000000000000000000000000000000000000000..fab9b6e66791fbbe5234048f591cac4535937efe --- /dev/null +++ b/.jenkins/scripts/clean @@ -0,0 +1,13 @@ +set -eo pipefail +source .gitlab/ci/scripts/utils.sh + +function main() { + docker_prune_images +} + +function docker_prune_images() { + docker image prune -a -f --filter="label=name=inventory-provider" --filter "until=72h" + docker system prune -f +} + +main $@ diff --git a/.jenkins/scripts/deploy b/.jenkins/scripts/deploy new file mode 100644 index 0000000000000000000000000000000000000000..01951ba2bc8589384dcb82bcbfe6866a0e32fca6 --- /dev/null +++ b/.jenkins/scripts/deploy @@ -0,0 +1,48 @@ +set -eo pipefail +source .gitlab/ci/scripts/utils.sh +trap 'clean_up' EXIT + +function main() { + docker login -u $SOKAN_REGISTRY_USER \ + -p $SOKAN_REGISTRY_PASSWORD $SOKAN_REGISTRY + deploy +} + +# Deploy components to dpeloyment environment +function deploy() { + echo -e "${ORANGE}---${ENV_NAME}---${NOCOLOR}\n" + local deployment_image="${HERMES_REGISTRY}/deployment:${IMAGE_TAG}" + + ANSIBLE_INVENTORY="/deployment/ansible/inventories/${ENV_NAME}" + + environment=$(get_environment) + + options="-e ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY} \ + --network host" + + options="${options} -v /opt/vault:/tmp/vault \ + -e ANSIBLE_VAULT_PASSWORD_FILE=/tmp/vault/${environment}-vault-pass" + + API_IMAGE="${HERMES_REGISTRY}/api:${IMAGE_TAG}" + UI_IMAGE="${HERMES_REGISTRY}/ui:${IMAGE_TAG}" + options="${options} -e API_VERSION=${API_VERSION} -e UI_VERSION=${UI_VERSION}" + + if is_nightly || is_clean_deploy; then + docker run --rm $options $deployment_image /bin/bash -c \ + "ansible-playbook /deployment/ansible/playbooks/install/main.yml \ + --extra-vars 'vagrant_setup_vms_cleanup=true'" + + else + docker run --rm $options $deployment_image update.sh + + fi +} + +function clean_up() { + echo -e "${PURPLE}Cleaning Up...${NOCOLOR}\n" + local deployment_image="${HERMES_REGISTRY}/deployment:${IMAGE_TAG}" + + docker image rm -f $deployment_image +} + +main $@ diff --git a/.jenkins/scripts/push b/.jenkins/scripts/push new file mode 100644 index 0000000000000000000000000000000000000000..9a26952f1b8ce3137e263f58316309d2966cb130 --- /dev/null +++ b/.jenkins/scripts/push @@ -0,0 +1,40 @@ +set -eo pipefail +source .gitlab/ci/scripts/utils.sh + +function main() { + if is_local || is_nightly || is_test_deploy || is_clean_deploy; then + prepare + build + push "${IMAGE}:${IMAGE_TAG}" + if push_international; then + push "${IMAGE_INTERNATIONAL}:${IMAGE_TAG}" + fi + if [ $BUILD_CONTEXT == "api/" ]; then + build_hermes_base + fi + fi +} + +function prepare() { + echo -e "${PURPLE}Preparing...${NOCOLOR}\n" + docker_login + eval "${EXTRA_PREPARE_COMMANDS}" +} + +function build() { + echo -e "${PURPLE}Building ${IMAGE}...${NOCOLOR}\n" + docker build --tag "${IMAGE}:${IMAGE_TAG}" \ + --file ${DOCKERFILE} ${EXTRA_BUILD_OPTIONS} \ + ${BUILD_CONTEXT} +} + +function build_hermes_base() { + echo -e "${PURPLE}Building hermes-base ...${NOCOLOR}\n" + docker build --tag "hermes-base:${IMAGE_TAG}" \ + --file ${DOCKERFILE} ${EXTRA_BUILD_OPTIONS} \ + --target base-image \ + ${BUILD_CONTEXT} +} + + +main $@ diff --git a/.jenkins/scripts/release b/.jenkins/scripts/release new file mode 100644 index 0000000000000000000000000000000000000000..2a0f88886658388561f83c02e2d713df1b6c79a5 --- /dev/null +++ b/.jenkins/scripts/release @@ -0,0 +1,37 @@ +set -eo pipefail +source .gitlab/ci/scripts/utils.sh + +REGISTRIES="$HERMES_REGISTRY $HERMES_REGISTRY_INTERNATIONAL" + +function main() { + local images=$@ + + docker_login + + for image in $images; + do + for registry in $REGISTRIES; + do + release "$image" "$registry" + + done + + done +} + +function release() { + local image=${1} + local registry=${2} + + local source_image="${registry}/${image}:dev-${SUCCESSFUL_PIPELINE_NUMBER}" + local versioned_image="${registry}/${image}:${IMAGE_TAG}" + local latest_image="${registry}/${image}:latest" + + change_tag $source_image $versioned_image + push $versioned_image + + change_tag $source_image $latest_image + push $latest_image +} + +main $@ diff --git a/.jenkins/scripts/utils.sh b/.jenkins/scripts/utils.sh new file mode 100644 index 0000000000000000000000000000000000000000..ce6718374c56b1cb084637065787aebfa689c7be --- /dev/null +++ b/.jenkins/scripts/utils.sh @@ -0,0 +1,139 @@ +NOCOLOR='\033[0m' +RED='\033[0;31m' +GREEN='\033[0;32m' +ORANGE='\033[0;33m' +PURPLE='\033[0;35m' + +function run_timed_command() { + local cmd="${1}" + local start=$(date +%s) + echo_success "\$ ${cmd}" + eval "${cmd}" + local ret=$? + local end=$(date +%s) + local runtime=$((end-start)) + + if [[ $ret -eq 0 ]]; then + echo_success "==> '${cmd}' succeeded in ${runtime} seconds." + return 0 + else + echo_err "==> '${cmd}' failed (${ret}) in ${runtime} seconds." + exit 1 + fi +} + +function echo_success() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n${GREEN}** %s **\n${NOCOLOR}" "${1}" >&2; + else + printf "${GREEN}%s\n${NOCOLOR}" "${1}" >&2; + fi +} + +function echo_err() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n${RED}** %s **\n${NOCOLOR}" "${1}" >&2; + else + printf "${RED}%s\n${NOCOLOR}" "${1}" >&2; + fi +} + +function is_clean_deploy() { + [ "$CLEAN_DEPLOY" = "true" ] + + local result=$? + return $result +} + +function is_test_deploy() { + [ "$TEST_DEPLOY" = "true" ] + + local result=$? + return $result +} + +function is_local() { + [ ! -z $CI_MERGE_REQUEST_IID ] + + local result=$? + return $result +} + +function is_lab() { + [ "$CI_COMMIT_REF_NAME" = "master" ] && \ + [ "$ENV" == "lab" ] && \ + [ $SUCCESSFUL_PIPELINE_NUMBER ] + + local result=$? + return $result +} + +function is_production() { + [ "$ENV" == "production" ] && \ + [ $SUCCESSFUL_PIPELINE_NUMBER ] && \ + [ $TAG ] + + local result=$? + return $result +} + +function is_nightly() { + [ "$CI_PIPELINE_SOURCE" = "schedule" ] && \ + [ "$FREQUENCY" = "nightly" ] + + local result=$? + return $result +} + +function push_international() { + [ "$PUSH_INTERNATIONAL" = "true" ] && \ + [ $IMAGE_INTERNATIONAL ] + + local result=$? + return $result +} + +function get_environment() { + if is_local || is_clean_deploy || is_test_deploy || is_nightly; then + echo "local" + + elif is_lab; then + echo "lab" + + elif is_production; then + echo "production" + + fi +} + +function docker_login() { + echo -e "${PURPLE}Docker login...${NOCOLOR}\n" + docker login -u $SOKAN_REGISTRY_USER \ + -p $SOKAN_REGISTRY_PASSWORD $SOKAN_REGISTRY + if push_international; then + docker login -u $MARKIGHT_REGISTRY_USER \ + -p $MARKIGHT_REGISTRY_PASSWORD $MARKIGHT_REGISTRY + fi +} + +function change_tag() { + local source_image="${1}" + local target_image="${2}" + + echo -e "${PURPLE}Changing Tag ${source_image} to ${target_image}...${NOCOLOR}\n" + docker pull $source_image + docker image tag $source_image \ + $target_image + docker image rm -f $source_image +} + +function push() { + local image="${1}" + + echo -e "${PURPLE}Pushing ${image}...${NOCOLOR}\n" + docker push $image +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..9a405d525aae2db09bfd50a87e5e0b569f7f87ac --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +### Build and install packages ### +FROM python:3.6.15 as base-image + +LABEL name="inventory-provider" + +# Install Python dependencies +COPY ./requirements.txt /requirements.txt +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /requirements.txt + +### Final image ### +FROM python:3.6.15-slim-buster + +LABEL name="inventory-provider" + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# Create entrypoints +COPY ./start-web-app.sh ./start-worker.sh ./start-monitoring.sh / +RUN chmod +x /start-web-app.sh /start-worker.sh /start-monitoring.sh + +# Create a dedicated user and group for the app +RUN groupadd -r inventory_provider && \ + useradd -r -g inventory_provider inventory_provider + +# Copy the Python dependencies from the base image +COPY --from=base-image /usr/local/lib/python3.6/site-packages/ /usr/local/lib/python3.6/site-packages/ +COPY --from=base-image /usr/local/bin/ /usr/local/bin/ + +# Copy the app's source code +COPY ./ /app + +# Set ownership and permissions for the app +RUN chown -R inventory_provider:inventory_provider /app/ + +# Switch to the inventory_provider user +USER inventory_provider + +# Set the working directory +WORKDIR /app + +# Expose the appropriate port +EXPOSE 8000 + +# Define the entrypoint command +ENTRYPOINT ["/start-web-app.sh"] diff --git a/env.example b/env.example new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/inventory_provider/config.py b/inventory_provider/config.py index 1a753c7ae03c14d857ad7e91e9e6177b3c1b2afb..049d9bbe6ee8cd7292493f062524a3d083b5302d 100644 --- a/inventory_provider/config.py +++ b/inventory_provider/config.py @@ -1,5 +1,6 @@ import json import jsonschema +from dotenv import load_dotenv CONFIG_SCHEMA = { '$schema': 'https://json-schema.org/draft-07/schema#', @@ -262,6 +263,10 @@ def load(f): :param f: file-like object that produces the config file :return: a dict containing the parsed configuration parameters """ + + # Load environment variables from .env file + load_dotenv() + config = json.loads(f.read()) jsonschema.validate(config, CONFIG_SCHEMA) return config diff --git a/inventory_provider/gunicorn.py b/inventory_provider/gunicorn.py new file mode 100644 index 0000000000000000000000000000000000000000..1aabbfdee044e085ff66a65dca8ed4eefebd3f7f --- /dev/null +++ b/inventory_provider/gunicorn.py @@ -0,0 +1,19 @@ +import os + +syslog = True +syslog_addr = 'unix:///dev/log' +syslog_facility = os.environ.get('SYSLOG_FACILITY', '') +capture_output = False +statsd_host = 'localhost:8125' +statsd_prefix = 'gunicorn' +user = os.environ.get('USERNAME', '') +group = os.environ.get('GROUPNAME', '') +timeout = 60 +workers = 4 +bind = os.environ.get('BIND_ADDRESS', '') +pidfile = os.environ.get('GUNICORN_PID_FILENAME', '') +raw_env = [ + f'FLASK_SETTINGS_FILENAME={os.environ.get("app_conf_filename", "")}', + f'INVENTORY_PROVIDER_CONFIG_FILENAME={os.environ.get("inventory_conf_filename", "")}', + f'LOGGING_CONFIG={os.environ.get("logging_conf_filename", "")}' +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6e19d444bcb5a01be2cd7c45e1a59f44833b2e91..c3e89881bdb92b5450f2afd4aed01253a9de25dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,8 @@ lxml requests netifaces tree-format +gunicorn +python-dotenv pytest pytest-mock diff --git a/start-monitoring.sh b/start-monitoring.sh new file mode 100644 index 0000000000000000000000000000000000000000..637e834d28fe50d37c8adc99446015c481882739 --- /dev/null +++ b/start-monitoring.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +cd /app + diff --git a/start-web-app.sh b/start-web-app.sh new file mode 100644 index 0000000000000000000000000000000000000000..b9ed7ac7c9d7213b078a638b7c932d01f45e0bdc --- /dev/null +++ b/start-web-app.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +gunicorn -c inventory_provider/gunicorn.py inventory_provider.app:app diff --git a/start-worker.sh b/start-worker.sh new file mode 100644 index 0000000000000000000000000000000000000000..9f948cc465ee5dab1dcfb7227ac9db363fd97c03 --- /dev/null +++ b/start-worker.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +cd /app + +celery -A inventory_provider.tasks.worker worker --pidfile "$worker_pid_filename" --concurrency 10 \ No newline at end of file