Feature: Secrets integration, including Secrets Providers and Secrets…

… Groups (#868)

* Initial model, UI, and REST API for Secrets

* Secrets providers API, initial TextFile and EnvironmentVariable provider implementations (#887)

* Add Secret.value property, add EnvironmentVariable provider, add dummy-plugin Constant provider, add tests

* Add TextFileSecretProvider

* Add docs

* Improve display of secret providers in the UI

* Refactor SecretsProvider registration to use the Nautobot registry instead of python entry_points

* Refactor slightly

* Add ability for secrets providers to define an HTML form for parameter inputs

* Fix default value for JSONField and add error handling in JS

* Add username_secret and token_secret support to GitRepository

* Docs updates

* Review feedback - add description field, etc.

* Revise secrets docs; add SecretError exceptions instead of returning None on various failures

* One of these days I'll remember to run flake8 before pushing

* Review comments

* SecretsGroup feature (#1042)

* WIP

* More WIP

* WIP remove SecretType model

* Such WIP. Wow

* WIP: working secretsgroup-edit UI

* More WIP

* Change Category/Meaning to Access Type/Secret Type

* Add SecretsGroup key to Device model; get tests passing

* Add test coverage for REST API and filters

* Add SecretsGroup view tests

* Linting fixes

* Docs updates

* Cleanup leftover SecretType cruft

* Update nautobot/docs/user-guides/git-data-source.md

Co-authored-by: Jathan McCollum <jathan@gmail.com>

* Fix egregious issues

Co-authored-by: Jathan McCollum <jathan@gmail.com>

* Support Jinja2 templating of secret parameters (#1058)

* Support Jinja2 templating of secret parameters

* Add secrets providers to plugin detail view

* Doc updates

* Include SecretsGroupAssociation in GraphQL

* Move 'Secrets' to a top-level menu

* Don't try to sort `SecretsProvider` class objects in plugin config features registry (#1065)

* Fix TypeError when trying to sort `SecretsProvider` class objects
* Don't sort `secrets_providers` when added to features.

* Add release-note content for Secrets

* Update nautobot/extras/views.py

Co-authored-by: John Anderson <lampwins@gmail.com>

* Change FK to SecretsGroup behavior to SET_NULL

* Use render_jinja2() in rendered_parameters()

Co-authored-by: Jathan McCollum <jathan@gmail.com>
Co-authored-by: John Anderson <lampwins@gmail.com>

examples/dummy_plugin/dummy_plugin/secrets.py

@@ -0,0 +1,41 @@
+from django import forms
+
+from nautobot.utilities.forms import BootstrapMixin
+from nautobot.extras.secrets import SecretsProvider
+
+
+class ConstantValueSecretsProvider(SecretsProvider):
+    """
+    Example of a plugin-provided SecretsProvider - this one just uses a user-specified constant value.
+
+    Obviously this is insecure and not something you'd want to actually use!
+    """
+
+    slug = "constant-value"
+    name = "Constant Value"
+
+    class ParametersForm(BootstrapMixin, forms.Form):
+        """
+        User-friendly form for specifying the required parameters of this provider.
+        """
+
+        constant = forms.CharField(
+            required=True,
+            help_text="Constant secret value. <strong>Example Only - DO NOT USE FOR REAL SENSITIVE DATA</strong>",
+        )
+
+    @classmethod
+    def get_value_for_secret(cls, secret, obj=None, **kwargs):
+        """
+        Return the value defined in the Secret.parameters "constant" key.
+
+        A more realistic SecretsProvider would make calls to external APIs, etc. to retrieve a secret from storage.
+
+        Args:
+            secret (nautobot.extras.models.Secret): The secret whose value should be retrieved.
+            obj (object): The object (Django model or similar) providing context for the secret's parameters.
+        """
+        return secret.rendered_parameters(obj=obj).get("constant")
+
+
+secrets_providers = [ConstantValueSecretsProvider]

mkdocs.yml

@@ -56,16 +56,17 @@ nav:
         - IP Address Management: 'user-guides/getting-started/ipam.md'
         - The Search Bar: 'user-guides/getting-started/search-bar.md'
     - Core Functionality:
-        - IP Address Management: 'core-functionality/ipam.md'
-        - VLAN Management: 'core-functionality/vlans.md'
-        - Sites and Racks: 'core-functionality/sites-and-racks.md'
+        - Circuits: 'core-functionality/circuits.md'
         - Devices and Cabling: 'core-functionality/devices.md'
         - Device Types: 'core-functionality/device-types.md'
-        - Virtualization: 'core-functionality/virtualization.md'
-        - Service Mapping: 'core-functionality/services.md'
-        - Circuits: 'core-functionality/circuits.md'
+        - IP Address Management: 'core-functionality/ipam.md'
         - Power Tracking: 'core-functionality/power.md'
+        - Secrets: 'core-functionality/secrets.md'
+        - Service Mapping: 'core-functionality/services.md'
+        - Sites and Racks: 'core-functionality/sites-and-racks.md'
         - Tenancy: 'core-functionality/tenancy.md'
+        - Virtualization: 'core-functionality/virtualization.md'
+        - VLAN Management: 'core-functionality/vlans.md'
     - Additional Features:
         - Caching: 'additional-features/caching.md'
         - Change Logging: 'additional-features/change-logging.md'

nautobot/dcim/api/serializers.py

@@ -73,7 +73,7 @@
     StatusModelSerializerMixin,
     TaggedObjectSerializer,
 )
-from nautobot.extras.api.nested_serializers import NestedConfigContextSchemaSerializer
+from nautobot.extras.api.nested_serializers import NestedConfigContextSchemaSerializer, NestedSecretsGroupSerializer
 from nautobot.ipam.api.nested_serializers import (
     NestedIPAddressSerializer,
     NestedVLANSerializer,
@@ -750,6 +750,7 @@ class DeviceSerializer(TaggedObjectSerializer, StatusModelSerializerMixin, Custo
     primary_ip4 = NestedIPAddressSerializer(required=False, allow_null=True)
     primary_ip6 = NestedIPAddressSerializer(required=False, allow_null=True)
     parent_device = serializers.SerializerMethodField()
+    secrets_group = NestedSecretsGroupSerializer(required=False, allow_null=True)
     cluster = NestedClusterSerializer(required=False, allow_null=True)
     virtual_chassis = NestedVirtualChassisSerializer(required=False, allow_null=True)
     local_context_schema = NestedConfigContextSchemaSerializer(required=False, allow_null=True)
@@ -775,6 +776,7 @@ class Meta:
             "primary_ip",
             "primary_ip4",
             "primary_ip6",
+            "secrets_group",
             "cluster",
             "virtual_chassis",
             "vc_position",
@@ -838,6 +840,7 @@ class Meta(DeviceSerializer.Meta):
             "primary_ip",
             "primary_ip4",
             "primary_ip6",
+            "secrets_group",
             "cluster",
             "virtual_chassis",
             "vc_position",

nautobot/dcim/api/views.py

@@ -2,6 +2,7 @@
 from collections import OrderedDict
 
 from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import F
 from django.http import HttpResponseForbidden, HttpResponse
 from django.shortcuts import get_object_or_404
@@ -60,6 +61,8 @@
     CustomFieldModelViewSet,
     StatusViewSetMixin,
 )
+from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices
+from nautobot.extras.secrets.exceptions import SecretError
 from nautobot.ipam.models import Prefix, VLAN
 from nautobot.utilities.api import get_serializer_for_model
 from nautobot.utilities.utils import count_related
@@ -467,12 +470,49 @@ def napalm(self, request, pk):
 
         napalm_methods = request.GET.getlist("method")
         response = OrderedDict([(m, None) for m in napalm_methods])
-        username = settings.NAPALM_USERNAME
-        password = settings.NAPALM_PASSWORD
+
+        # Get NAPALM credentials for the device, or fall back to the legacy global NAPALM credentials
+        if device.secrets_group:
+            try:
+                try:
+                    username = device.secrets_group.get_secret_value(
+                        SecretsGroupAccessTypeChoices.TYPE_GENERIC,
+                        SecretsGroupSecretTypeChoices.TYPE_USERNAME,
+                        obj=device,
+                    )
+                except ObjectDoesNotExist:
+                    # No defined secret, fall through to legacy behavior
+                    username = settings.NAPALM_USERNAME
+                try:
+                    password = device.secrets_group.get_secret_value(
+                        SecretsGroupAccessTypeChoices.TYPE_GENERIC,
+                        SecretsGroupSecretTypeChoices.TYPE_PASSWORD,
+                        obj=device,
+                    )
+                except ObjectDoesNotExist:
+                    # No defined secret, fall through to legacy behavior
+                    password = settings.NAPALM_PASSWORD
+            except SecretError as exc:
+                raise ServiceUnavailable(f"Unable to retrieve device credentials: {exc.message}") from exc
+
         optional_args = settings.NAPALM_ARGS.copy()
         if device.platform.napalm_args is not None:
             optional_args.update(device.platform.napalm_args)
 
+        # Get NAPALM enable-secret from the device if present
+        if device.secrets_group:
+            try:
+                optional_args["secret"] = device.secrets_group.get_secret_value(
+                    SecretsGroupAccessTypeChoices.TYPE_GENERIC,
+                    SecretsGroupSecretTypeChoices.TYPE_SECRET,
+                    obj=device,
+                )
+            except ObjectDoesNotExist:
+                # No defined secret, this is OK
+                pass
+            except SecretError as exc:
+                raise ServiceUnavailable(f"Unable to retrieve device credentials: {exc.message}") from exc
+
         # Update NAPALM parameters according to the request headers
         for header in request.headers:
             if header[:9].lower() != "x-napalm-":

nautobot/dcim/filters.py

@@ -8,6 +8,7 @@
     CreatedUpdatedFilterSet,
     StatusModelFilterSetMixin,
 )
+from nautobot.extras.models import SecretsGroup
 from nautobot.tenancy.filters import TenancyFilterSet
 from nautobot.tenancy.models import Tenant
 from nautobot.utilities.choices import ColorChoices
@@ -690,6 +691,17 @@ class DeviceFilterSet(
         method="_has_primary_ip",
         label="Has a primary IP",
     )
+    secrets_group_id = django_filters.ModelMultipleChoiceFilter(
+        field_name="secrets_group",
+        queryset=SecretsGroup.objects.all(),
+        label="Secrets group (ID)",
+    )
+    secrets_group = django_filters.ModelMultipleChoiceFilter(
+        field_name="secrets_group__slug",
+        queryset=SecretsGroup.objects.all(),
+        to_field_name="slug",
+        label="Secrets group (slug)",
+    )
     virtual_chassis_id = django_filters.ModelMultipleChoiceFilter(
         field_name="virtual_chassis",
         queryset=VirtualChassis.objects.all(),

nautobot/dcim/forms.py

@@ -27,7 +27,7 @@
     StatusModelCSVFormMixin,
     StatusFilterFormMixin,
 )
-from nautobot.extras.models import Tag
+from nautobot.extras.models import SecretsGroup, Tag
 from nautobot.ipam.constants import BGP_ASN_MAX, BGP_ASN_MIN
 from nautobot.ipam.models import IPAddress, VLAN
 from nautobot.tenancy.forms import TenancyFilterForm, TenancyForm
@@ -1687,6 +1687,7 @@ class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldModelForm, Relationship
         required=False,
         query_params={"manufacturer_id": ["$manufacturer", "null"]},
     )
+    secrets_group = DynamicModelChoiceField(queryset=SecretsGroup.objects.all(), required=False)
     cluster_group = DynamicModelChoiceField(
         queryset=ClusterGroup.objects.all(),
         required=False,
@@ -1717,6 +1718,7 @@ class Meta:
             "platform",
             "primary_ip4",
             "primary_ip6",
+            "secrets_group",
             "cluster_group",
             "cluster",
             "tenant_group",
@@ -1841,6 +1843,12 @@ class BaseDeviceCSVForm(StatusModelCSVFormMixin, CustomFieldModelCSVForm):
         required=False,
         help_text="Virtualization cluster",
     )
+    secrets_group = CSVModelChoiceField(
+        queryset=SecretsGroup.objects.all(),
+        required=False,
+        to_field_name="name",
+        help_text="Secrets group",
+    )
 
     class Meta:
         fields = []
@@ -1975,6 +1983,7 @@ class DeviceBulkEditForm(
     tenant = DynamicModelChoiceField(queryset=Tenant.objects.all(), required=False)
     platform = DynamicModelChoiceField(queryset=Platform.objects.all(), required=False)
     serial = forms.CharField(max_length=50, required=False, label="Serial Number")
+    secrets_group = DynamicModelChoiceField(queryset=SecretsGroup.objects.all(), required=False)
 
     class Meta:
         nullable_fields = [
@@ -1983,6 +1992,7 @@ class Meta:
             "serial",
             "rack",
             "rack_group",
+            "secrets_group",
         ]
 
 

nautobot/dcim/migrations/0007_device_secrets_group.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.1.13 on 2021-11-15 13:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("extras", "0016_secret"),
+        ("dcim", "0006_auto_slug"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="device",
+            name="secrets_group",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="extras.secretsgroup",
+            ),
+        ),
+    ]

nautobot/dcim/models/devices.py

@@ -535,6 +535,14 @@ class Device(PrimaryModel, ConfigContextModel, StatusModel):
     comments = models.TextField(blank=True)
     images = GenericRelation(to="extras.ImageAttachment")
 
+    secrets_group = models.ForeignKey(
+        to="extras.SecretsGroup",
+        on_delete=models.SET_NULL,
+        default=None,
+        blank=True,
+        null=True,
+    )
+
     objects = ConfigContextModelQuerySet.as_manager()
 
     csv_headers = [
@@ -552,6 +560,7 @@ class Device(PrimaryModel, ConfigContextModel, StatusModel):
         "rack_name",
         "position",
         "face",
+        "secrets_group",
         "comments",
     ]
     clone_fields = [
@@ -563,6 +572,7 @@ class Device(PrimaryModel, ConfigContextModel, StatusModel):
         "rack",
         "status",
         "cluster",
+        "secrets_group",
     ]
 
     class Meta:

nautobot/dcim/tables/devices.py

@@ -187,6 +187,7 @@ class DeviceTable(StatusTableMixin, BaseTable):
     virtual_chassis = tables.LinkColumn(viewname="dcim:virtualchassis", args=[Accessor("virtual_chassis__pk")])
     vc_position = tables.Column(verbose_name="VC Position")
     vc_priority = tables.Column(verbose_name="VC Priority")
+    secrets_group = tables.Column(linkify=True)
     tags = TagColumn(url_name="dcim:device_list")
 
     class Meta(BaseTable.Meta):
@@ -212,6 +213,7 @@ class Meta(BaseTable.Meta):
             "virtual_chassis",
             "vc_position",
             "vc_priority",
+            "secrets_group",
             "tags",
         )
         default_columns = (

nautobot/dcim/templates/dcim/device.html

@@ -194,6 +194,16 @@
                                             {% endif %}
                                         </td>
                                     </tr>
+                                    <tr>
+                                        <td>Secrets Group</td>
+                                        <td>
+                                            {% if object.secrets_group %}
+                                                <a href="{{ object.secrets_group.get_absolute_url }}">{{ object.secrets_group }}</a>
+                                            {% else %}
+                                                <span class="text-muted">&mdash;</span>
+                                            {% endif %}
+                                        </td>
+                                    </tr>
                                     {% if object.cluster %}
                                         <tr>
                                             <td>Cluster</td>

nautobot/dcim/templates/dcim/device_edit.html

@@ -62,6 +62,7 @@
                 {% render_field form.primary_ip4 %}
                 {% render_field form.primary_ip6 %}
             {% endif %}
+            {% render_field form.secrets_group %}
         </div>
     </div>
     <div class="panel panel-default">