Add hotfix for filtering via slugs

.dockerignore

+# Git
+.git
+.gitignore
+.gitattributes
+
+# CI
+.codeclimate.yml
+.travis.yml
+.taskcluster.yml
+
+# Docker
+docker-compose.yml
+Dockerfile
+.docker
+.dockerignore
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+**/*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Virtual environment
+.env
+.venv/
+venv/
+
+# PyCharm
+.idea
+
+# Python mode for VIM
+.ropeproject
+**/.ropeproject
+
+# Vim swap files
+**/*.swp
+
+# VS Code
+.vscode/
\ No newline at end of file

.env.example

+NETBOX_URL=
+NETBOX_TOKEN=
+REPO_URL=https://github.com/netbox-community/devicetype-library.git
+REPO_BRANCH=master
+IGNORE_SSL_ERRORS=False
+#SLUGS=c9300-48u isr4431 isr4331

.github/dependabot.yml

+---
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: monthly
+      time: '02:00'
+      timezone: America/New_York
+    labels:
+      - dependencies
+    target-branch: master
+    assignees:
+        - "danner26"
\ No newline at end of file

.github/workflows/docker.yml

+---
+name: ci
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'main'
+  pull_request:
+    branches:
+      - 'master'
+      - 'main'
+  workflow_dispatch:
+  release:
+    types: [published, edited]
+
+jobs:
+  build-and-push-images:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/minitriga/Netbox-Device-Type-Library-Import
+          tags: |
+            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+      - 
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1 
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
\ No newline at end of file

.github/workflows/stale.yml

+---
+#close-stale-issues (https://github.com/marketplace/actions/close-stale-issues)
+name: Close stale PRs
+on:  # yamllint disable-line rule:truthy
+  schedule:
+    - cron: 0 4 * * *
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v5
+        with:
+          close-pr-message: >
+            This PR has been automatically closed due to lack of activity.
+          days-before-stale: 30
+          days-before-close: 7
+          operations-per-run: 100
+          remove-stale-when-updated: false
+          stale-pr-label: stale
+          stale-pr-message: >
+            This PR has been automatically marked as stale because it has not
+            had recent activity. It will be closed automatically if no further
+            progress is made.
\ No newline at end of file

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env*
+!.env.example
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Editor
+.vscode
+
+repo

Dockerfile

+FROM python:3.9-alpine
+
+ENV REPO_URL=https://github.com/netbox-community/devicetype-library.git
+WORKDIR /app
+COPY requirements.txt .
+
+# Install dependencies
+RUN apk add --no-cache git ca-certificates && \
+    python3 -m pip install --upgrade pip && \
+    pip3 install -r requirements.txt
+
+# Copy over src code
+COPY *.py ./
+
+# -u to avoid stdout buffering
+CMD ["python3","-u","nb-dt-import.py"]

LICENSE

+MIT License
+
+Copyright (c) 2021 Alexander Gittings
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

+# Netbox Device Type Import
+
+This library is intended to be your friend and help you import all the device-types defined within the the [NetBox Device Type Library Repository](https://github.com/netbox-community/devicetype-library).
+
+> Tested working with 2.9.4, 2.10.4
+
+## 🪄 Description
+
+This script will clone a copy of the `netbox-community/devicetype-library` repository to your machine to allow it to import the device types you would like without copy and pasting them into the Netbox UI.
+
+## 🚀 Getting Started
+
+1. This script is written in Python, so lets setup a virtual environment.
+
+```
+git clone https://github.com/netbox-community/Device-Type-Library-Import.git
+cd Netbox-Device-Type-Library-Import
+python3 -m venv venv
+source venv/bin/activate
+```
+
+2. Now that we have the basics setup, we'll need to install the requirements.
+
+```
+pip install -r requirements.txt
+```
+
+3. There are two variables that are required when using this script to import device types into your Netbox installation. (1) Your Netbox instance URL and (2) a token with **write rights**.
+
+Copy the existing `.env.example` to your own `.env` file, and fill in the variables.
+
+```
+cp .env.example .env
+vim .env
+```
+
+Finally, we are able to execute the script and import some device templates!
+
+## 🔌 Usage
+
+To use the script, simply execute the script as follows. Make sure you're still in the activated virtual environment we created before.
+
+```
+./nb-dt-import.py
+```
+
+This will clone the latest master branch from the `netbox-community/devicetype-library` from Github and install it into the `repo` subdirectory. If this directory already exists, it will perform a `git pull` to update the reposity instead.
+
+Next, it will loop over every manufacturer and every device of every manufacturer and begin checking if your Netbox install already has them, and if not, creates them. It will skip preexisting manufacturers, devices, interfaces, etc. so as to not end up with duplicate entries in your Netbox instance.
+
+### 🧰 Arguments
+
+This script currently accepts a list of vendors as an arugment, so that you can selectively import devices.
+
+To import only device by APC, for example:
+
+```
+./nb-dt-import.py --vendors apc
+```
+
+`--vendors` can also accept a comma separated list of vendors if you want to import multiple.
+
+```
+./nb-dt-import.py --vendors apc,juniper
+```
+
+## Docker build
+
+It's possible to use this project as a docker container.
+
+To build :
+
+```
+docker build -t netbox-devicetype-import-library .
+```
+
+Alternatively you can pull a pre-built image from Github Container Registry (ghcr.io):
+
+```
+docker pull ghcr.io/minitriga/netbox-device-type-library-import
+```
+
+The container supports the following env var as configuration :
+
+- `REPO_URL`, the repo to look for device types (defaults to _https://github.com/netbox-community/devicetype-library.git_)
+- `REPO_BRANCH`, the branch to check out if appropriate, defaults to master.
+- `NETBOX_URL`, used to access netbox
+- `NETBOX_TOKEN`, token for accessing netbox
+- `VENDORS`, a comma-separated list of vendors to import (defaults to None)
+
+To run :
+
+```
+docker run -e "NETBOX_URL=http://netbox:8080/" -e "NETBOX_TOKEN=98765434567890" ghcr.io/minitriga/netbox-device-type-library-import
+```
+
+## 🧑‍💻 Contributing
+
+We're happy about any pull requests!
+
+## 📜 License
+
+MIT

log_handler.py

+from sys import exit as system_exit
+
+
+class LogHandler:
+    def __new__(cls, *args, **kwargs):
+        return super().__new__(cls)
+
+    def __init__(self, args):
+        self.args = args
+
+    def exception(self, exception_type, exception, stack_trace=None):
+        exception_dict = {
+            "EnvironmentError": f'Environment variable "{exception}" is not set.',
+            "SSLError": f'SSL verification failed. IGNORE_SSL_ERRORS is {exception}. Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING.',
+            "GitCommandError": f'The repo "{exception}" is not a valid git repo.',
+            "GitInvalidRepositoryError": f'The repo "{exception}" is not a valid git repo.',
+            "Exception": f'An unknown error occurred: "{exception}"'
+        }
+
+        if self.args.verbose and stack_trace:
+            print(stack_trace)
+        print(exception_dict[exception_type])
+        system_exit(1)
+
+    def verbose_log(self, message):
+        if self.args.verbose:
+            print(message)
+
+    def log(self, message):
+        print(message)
+
+    def log_device_ports_created(self, created_ports: list = [], port_type: str = "port"):
+        for port in created_ports:
+            self.verbose_log(f'{port_type} Template Created: {port.name} - '
+                             + f'{port.type if hasattr(port, "type") else ""} - {port.device_type.id} - '
+                             + f'{port.id}')
+        return len(created_ports)
+
+    def log_module_ports_created(self, created_ports: list = [], port_type: str = "port"):
+        for port in created_ports:
+            self.verbose_log(f'{port_type} Template Created: {port.name} - '
+                             + f'{port.type if hasattr(port, "type") else ""} - {port.module_type.id} - '
+                             + f'{port.id}')
+        return len(created_ports)

nb-dt-import.py

+#!/usr/bin/env python3
+from collections import Counter
+from datetime import datetime
+import yaml
+import pynetbox
+from glob import glob
+import os
+
+import settings
+from netbox_api import NetBox
+
+
+def main():
+    startTime = datetime.now()
+    args = settings.args
+
+    netbox = NetBox(settings)
+    files, vendors = settings.dtl_repo.get_devices(
+        f'{settings.dtl_repo.repo_path}/device-types/', args.vendors)
+
+    settings.handle.log(f'{len(vendors)} Vendors Found')
+    device_types = settings.dtl_repo.parse_files(files, slugs=args.slugs)
+    settings.handle.log(f'{len(device_types)} Device-Types Found')
+    netbox.create_manufacturers(vendors)
+    netbox.create_device_types(device_types)
+
+    if netbox.modules:
+        settings.handle.log("Modules Enabled. Creating Modules...")
+        files, vendors = settings.dtl_repo.get_devices(
+            f'{settings.dtl_repo.repo_path}/module-types/', args.vendors)
+        settings.handle.log(f'{len(vendors)} Module Vendors Found')
+        module_types = settings.dtl_repo.parse_files(files, slugs=args.slugs)
+        settings.handle.log(f'{len(module_types)} Module-Types Found')
+        netbox.create_manufacturers(vendors)
+        netbox.create_module_types(module_types)
+
+    settings.handle.log('---')
+    settings.handle.verbose_log(
+        f'Script took {(datetime.now() - startTime)} to run')
+    settings.handle.log(f'{netbox.counter["added"]} devices created')
+    settings.handle.log(f'{netbox.counter["images"]} images uploaded')
+    settings.handle.log(
+        f'{netbox.counter["updated"]} interfaces/ports updated')
+    settings.handle.log(
+        f'{netbox.counter["manufacturer"]} manufacturers created')
+    if settings.NETBOX_FEATURES['modules']:
+        settings.handle.log(
+            f'{netbox.counter["module_added"]} modules created')
+        settings.handle.log(
+            f'{netbox.counter["module_port_added"]} module interface / ports created')
+
+
+if __name__ == "__main__":
+    main()

repo.py

+import os
+from glob import glob
+from re import sub as re_sub
+from git import Repo, exc
+import yaml
+
+
+class DTLRepo:
+    def __new__(cls, *args, **kwargs):
+        return super().__new__(cls)
+
+    def __init__(self, args, repo_path, exception_handler):
+        self.handle = exception_handler
+        self.yaml_extensions = ['yaml', 'yml']
+        self.url = args.url
+        self.repo_path = repo_path
+        self.branch = args.branch
+        self.repo = None
+        self.cwd = os.getcwd()
+
+        if os.path.isdir(self.repo_path):
+            self.pull_repo()
+        else:
+            self.clone_repo()
+
+    def get_relative_path(self):
+        return self.repo_path
+
+    def get_absolute_path(self):
+        return os.path.join(self.cwd, self.repo_path)
+
+    def get_devices_path(self):
+        return os.path.join(self.get_absolute_path(), 'device-types')
+
+    def get_modules_path(self):
+        return os.path.join(self.get_absolute_path(), 'module-types')
+
+    def slug_format(self, name):
+        return re_sub('\W+', '-', name.lower())
+
+    def pull_repo(self):
+        try:
+            self.handle.log("Package devicetype-library is already installed, "
+                            + f"updating {self.get_absolute_path()}")
+            self.repo = Repo(self.repo_path)
+            if not self.repo.remotes.origin.url.endswith('.git'):
+                self.handle.exception("GitInvalidRepositoryError", self.repo.remotes.origin.url,
+                                      f"Origin URL {self.repo.remotes.origin.url} does not end with .git")
+            self.repo.remotes.origin.pull()
+            self.repo.git.checkout(self.branch)
+            self.handle.verbose_log(
+                f"Pulled Repo {self.repo.remotes.origin.url}")
+        except exc.GitCommandError as git_error:
+            self.handle.exception(
+                "GitCommandError", self.repo.remotes.origin.url, git_error)
+        except Exception as git_error:
+            self.handle.exception(
+                "Exception", 'Git Repository Error', git_error)
+
+    def clone_repo(self):
+        try:
+            self.repo = Repo.clone_from(
+                self.url, self.get_absolute_path(), branch=self.branch)
+            self.handle.log(
+                f"Package Installed {self.repo.remotes.origin.url}")
+        except exc.GitCommandError as git_error:
+            self.handle.exception("GitCommandError", self.url, git_error)
+        except Exception as git_error:
+            self.handle.exception(
+                "Exception", 'Git Repository Error', git_error)
+
+    def get_devices(self, base_path, vendors: list = None):
+        files = []
+        discovered_vendors = []
+        vendor_dirs = os.listdir(base_path)
+
+        for folder in [vendor for vendor in vendor_dirs if not vendors or vendor.casefold() in vendors]:
+            if folder.casefold() != "testing":
+                discovered_vendors.append({'name': folder,
+                                           'slug': self.slug_format(folder)})
+                for extension in self.yaml_extensions:
+                    files.extend(glob(base_path + folder + f'/*.{extension}'))
+        return files, discovered_vendors
+
+    def parse_files(self, files: list, slugs: list = None):
+        deviceTypes = []
+        for file in files:
+            with open(file, 'r') as stream:
+                try:
+                    data = yaml.safe_load(stream)
+                except yaml.YAMLError as excep:
+                    self.handle.verbose_log(excep)
+                    continue
+                manufacturer = data['manufacturer']
+                data['manufacturer'] = {
+                    'name': manufacturer, 'slug': self.slug_format(manufacturer)}
+
+                # Save file location to resolve any relative paths for images
+                data['src'] = file
+            
+            if data.get('slug') is None:
+                continue
+
+            if slugs and True not in [True if s.casefold() in data['slug'].casefold() else False for s in slugs]:
+                self.handle.verbose_log(f"Skipping {data['model']}")
+                continue
+
+            deviceTypes.append(data)
+        return deviceTypes

requirements.txt

+GitPython==3.1.32
+pynetbox==7.0.1
+python-dotenv==1.0.0
+PyYAML==6.0.1
\ No newline at end of file

settings.py

+from argparse import ArgumentParser
+import os
+from log_handler import LogHandler
+from repo import DTLRepo
+from dotenv import load_dotenv
+load_dotenv()
+
+REPO_URL = os.getenv("REPO_URL",
+                     default="https://github.com/netbox-community/devicetype-library.git")
+REPO_BRANCH = os.getenv("REPO_BRANCH", default="master")
+NETBOX_URL = os.getenv("NETBOX_URL")
+NETBOX_TOKEN = os.getenv("NETBOX_TOKEN")
+IGNORE_SSL_ERRORS = (os.getenv("IGNORE_SSL_ERRORS", default="False") == "True")
+REPO_PATH = f"{os.path.dirname(os.path.realpath(__file__))}/repo"
+
+# optionally load vendors through a comma separated list as env var
+VENDORS = list(filter(None, os.getenv("VENDORS", "").split(",")))
+
+# optionally load device types through a space separated list as env var
+SLUGS = os.getenv("SLUGS", "").split()
+
+NETBOX_FEATURES = {
+    'modules': False,
+}
+
+parser = ArgumentParser(description='Import Netbox Device Types')
+parser.add_argument('--vendors', nargs='+', default=VENDORS,
+                    help="List of vendors to import eg. apc cisco")
+parser.add_argument('--url', '--git', default=REPO_URL,
+                    help="Git URL with valid Device Type YAML files")
+parser.add_argument('--slugs', nargs='+', default=SLUGS,
+                    help="List of device-type slugs to import eg. ap4431 ws-c3850-24t-l")
+parser.add_argument('--branch', default=REPO_BRANCH,
+                    help="Git branch to use from repo")
+parser.add_argument('--verbose', action='store_true', default=False,
+                    help="Print verbose output")
+
+args = parser.parse_args()
+
+args.vendors = [v.casefold()
+                for vendor in args.vendors for v in vendor.split(",") if v.strip()]
+args.slugs = [s for slug in args.slugs for s in slug.split(",") if s.strip()]
+
+handle = LogHandler(args)
+# Evaluate environment variables and exit if one of the mandatory ones are not set
+MANDATORY_ENV_VARS = ["REPO_URL", "NETBOX_URL", "NETBOX_TOKEN"]
+for var in MANDATORY_ENV_VARS:
+    if var not in os.environ:
+        handle.exception("EnvironmentError", var,
+                         f'Environment variable "{var}" is not set.\n\nMANDATORY_ENV_VARS: {str(MANDATORY_ENV_VARS)}.\n\nCURRENT_ENV_VARS: {str(os.environ)}')
+
+dtl_repo = DTLRepo(args, REPO_PATH, handle)