feat: diode data plugin 1.0 rc (#88)

Author: mfiedorowicz

.github/CODEOWNERS

@@ -1 +1 @@
-*                     @leoparente @mfiedorowicz
+* @leoparente @ltucker @mfiedorowicz

Makefile

@@ -14,10 +14,10 @@ docker-compose-netbox-plugin-down:
 
 .PHONY: docker-compose-netbox-plugin-test
 docker-compose-netbox-plugin-test:
-	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run -u root --rm netbox ./manage.py test --keepdb netbox_diode_plugin
+	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run -u root --rm netbox ./manage.py test $(TEST_FLAGS) --keepdb netbox_diode_plugin
 	@$(MAKE) docker-compose-netbox-plugin-down
 
 .PHONY: docker-compose-netbox-plugin-test-cover
 docker-compose-netbox-plugin-test-cover:
-	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run --rm -u root -e COVERAGE_FILE=/opt/netbox/netbox/coverage/.coverage netbox sh -c "coverage run --source=netbox_diode_plugin --omit=*_pb2*,*/migrations/* ./manage.py test --keepdb netbox_diode_plugin && coverage xml -o /opt/netbox/netbox/coverage/report.xml && coverage report -m | tee /opt/netbox/netbox/coverage/report.txt"
+	-@$(DOCKER_COMPOSE) -f docker/docker-compose.yaml -f docker/docker-compose.test.yaml run --rm -u root -e COVERAGE_FILE=/opt/netbox/netbox/coverage/.coverage netbox sh -c "coverage run --source=netbox_diode_plugin --omit=*/migrations/* ./manage.py test --keepdb netbox_diode_plugin && coverage xml -o /opt/netbox/netbox/coverage/report.xml && coverage report -m | tee /opt/netbox/netbox/coverage/report.txt"
 	@$(MAKE) docker-compose-netbox-plugin-down

README.md

@@ -16,6 +16,7 @@ at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in
 |:--------------:|:--------------:|
 |    >= 3.7.2    |     0.1.0      |
 |    >= 4.1.0    |     0.4.0      |
+|    >= 4.2.3    |     1.0.0      |
 
 ## Installation
 
@@ -46,32 +47,18 @@ Also in your `configuration.py` file, in order to customise the plugin settings,
 ```python
 PLUGINS_CONFIG = {
     "netbox_diode_plugin": {
-        # Auto-provision users for Diode plugin
-        "auto_provision_users": False,
-
         # Diode gRPC target for communication with Diode server
         "diode_target_override": "grpc://localhost:8080/diode",
 
-        # User allowed for Diode to NetBox communication
-        "diode_to_netbox_username": "diode-to-netbox",
-
-        # User allowed for NetBox to Diode communication
-        "netbox_to_diode_username": "netbox-to-diode",
-
-        # User allowed for data ingestion
-        "diode_username": "diode-ingestion",
+        # Username associated with changes applied via plugin
+        "diode_username": "diode",
     },
 }
 ```
 
 Note: Once you customise usernames with PLUGINS_CONFIG during first installation, you should not change or remove them
 later on. Doing so will cause the plugin to stop working properly.
 
-`auto_provision_users` is a boolean flag (default: `False`) that determines whether the plugin should automatically
-create the users during
-migration. If set to `False`, you will need to provision Diode users with their API keys manually via the plugin's setup
-page in the NetBox UI.
-
 Restart NetBox services to load the plugin:
 
 ```
@@ -89,28 +76,6 @@ cd /opt/netbox
 source venv/bin/activate
 ```
 
-Three API keys will be needed (these are random 40 character long alphanumeric strings). They can be generated and set
-to the appropriate environment variables with the following commands:
-
-```shell
-# API key for the Diode service to interact with NetBox
-export DIODE_TO_NETBOX_API_KEY=$(head -c20 </dev/urandom|xxd -p); env | grep DIODE_TO_NETBOX_API_KEY
-# API key for the NetBox service to interact with Diode
-export NETBOX_TO_DIODE_API_KEY=$(head -c20 </dev/urandom|xxd -p); env | grep NETBOX_TO_DIODE_API_KEY
-# API key for Diode SDKs to ingest data into Diode
-export DIODE_API_KEY=$(head -c20 </dev/urandom|xxd -p); env | grep DIODE_API_KEY
-```
-
-**Note:** store these API key strings in a safe place as they will be needed later to configure the Diode server.
-
-If you don't set these environment variables, the plugin will generate random API keys for you either during the
-migration process (with `auto_provision_users` set to `True`) or when you manually create the users in the plugin's
-setup page in the NetBox UI.
-
-It's important to note that environment variables with API keys should be populated in the Diode server's
-environment variables (see [docs](https://github.com/netboxlabs/diode/tree/develop/diode-server#running-the-diode-server)) 
-as well to ensure proper communication between the Diode SDK, Diode server and the NetBox plugin.
-
 Run migrations to create all necessary resources:
 
 ```shell

docker/Dockerfile-diode-netbox-plugin

@@ -1,8 +1,12 @@
-FROM netboxcommunity/netbox:v4.1-3.0.1
+FROM netboxcommunity/netbox:v4.2.3-3.1.1
 
 COPY ./netbox/configuration/ /etc/netbox/config/
 RUN chmod 755 /etc/netbox/config/* && \
     chown unit:root /etc/netbox/config/*
 
+COPY ./netbox/local_settings.py /opt/netbox/netbox/netbox/local_settings.py
+RUN chmod 755 /opt/netbox/netbox/netbox/local_settings.py && \
+    chown unit:root /opt/netbox/netbox/netbox/local_settings.py
+
 COPY ./requirements-diode-netbox-plugin.txt /opt/netbox/
 RUN /opt/netbox/venv/bin/pip install  --no-warn-script-location -r /opt/netbox/requirements-diode-netbox-plugin.txt

docker/docker-compose.yaml

@@ -1,7 +1,7 @@
 name: diode-netbox-plugin
 services:
-  netbox:
-    image: netboxcommunity/netbox:v4.1-3.0.1-diode-netbox-plugin
+  netbox: &netbox
+    image: netboxcommunity/netbox:v4.2.3-3.1.1-diode-netbox-plugin
     build:
       context: .
       dockerfile: Dockerfile-diode-netbox-plugin
@@ -22,13 +22,30 @@ services:
       - ./netbox/nginx-unit.json:/opt/netbox/nginx-unit.json:z,ro
       - ../netbox_diode_plugin:/opt/netbox/netbox/netbox_diode_plugin:ro
       - ./netbox/launch-netbox.sh:/opt/netbox/launch-netbox.sh:z,ro
+      - ./netbox/plugins_dev.py:/etc/netbox/config/plugins.py:z,ro
       - ./coverage:/opt/netbox/netbox/coverage:z,rw
       - netbox-media-files:/opt/netbox/netbox/media:rw
       - netbox-reports-files:/opt/netbox/netbox/reports:rw
       - netbox-scripts-files:/opt/netbox/netbox/scripts:rw
     ports:
       - "8000:8080"
-
+  
+  netbox-worker:
+    <<: *netbox
+    depends_on:
+      netbox:
+        condition: service_healthy
+    command:
+      - /opt/netbox/venv/bin/python
+      - /opt/netbox/netbox/manage.py
+      - rqworker
+    healthcheck:
+      test: ps -aux | grep -v grep | grep -q rqworker || exit 1
+      start_period: 20s
+      timeout: 3s
+      interval: 15s
+    ports: []
+  
   # postgres
   netbox-postgres:
     image: docker.io/postgres:16-alpine

docker/netbox/configuration/configuration.py

@@ -44,9 +44,12 @@ def _environ_get_and_map(variable_name: str, default: str | None = None,
     return map_fn(env_value)
 
 
-_AS_BOOL = lambda value: value.lower() == 'true'
-_AS_INT = lambda value: int(value)
-_AS_LIST = lambda value: list(filter(None, value.split(' ')))
+def _AS_BOOL(value):
+    return value.lower() == 'true'
+def _AS_INT(value):
+    return int(value)
+def _AS_LIST(value):
+    return list(filter(None, value.split(' ')))
 
 _BASE_DIR = dirname(dirname(abspath(__file__)))
 

docker/netbox/configuration/logging.py

@@ -1,3 +1,20 @@
+from os import environ
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        '': {  # root logger
+            'handlers': ['console'],
+            'level': 'DEBUG' if environ.get('DEBUG', 'false').lower() == 'true' else 'INFO',
+        },
+    },
+}
 # # Remove first comment(#) on each line to implement this working logging example.
 # # Add LOGLEVEL environment variable to netbox if you use this example & want a different log level.
 # from os import environ

docker/netbox/configuration/plugins.py

@@ -4,7 +4,10 @@
 # To learn how to build images with your required plugins
 # See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins
 
-PLUGINS = ["netbox_diode_plugin"]
+PLUGINS = [
+    "netbox_diode_plugin",
+    "netbox_branching",
+]
 
 # PLUGINS_CONFIG = {
 #     "netbox_diode_plugin": {

docker/netbox/env/netbox.env

@@ -37,7 +37,5 @@ SUPERUSER_NAME=admin
 SUPERUSER_PASSWORD=admin
 WEBHOOKS_ENABLED=true
 RELOAD_NETBOX_ON_DIODE_PLUGIN_CHANGE=false
-DIODE_TO_NETBOX_API_KEY=1368dbad13e418d5a443d93cf255edde03a2a754
-NETBOX_TO_DIODE_API_KEY=1e99338b8cab5fc637bc55f390bda1446f619c42
-DIODE_API_KEY=5a52c45ee8231156cb620d193b0291912dd15433
 BASE_PATH=netbox/
+DEBUG=False

docker/netbox/local_settings.py

@@ -0,0 +1,13 @@
+from netbox_branching.utilities import DynamicSchemaDict
+
+from .configuration import DATABASE
+
+# Wrap DATABASES with DynamicSchemaDict for dynamic schema support
+DATABASES = DynamicSchemaDict({
+    'default': DATABASE,
+})
+
+# Employ our custom database router
+DATABASE_ROUTERS = [
+    'netbox_branching.database.BranchAwareRouter',
+]