GraphQL as a Git Data source provider. (#6752)

--------- Co-authored-by: “DJ <“dj.howard@networktocode.com> Co-authored-by: Glenn Matthews <glenn.matthews@networktocode.com> Co-authored-by: Hanlin Miao <46973263+HanlinMiao@users.noreply.github.com>
nautobot · Jan 23, 2025 · ce92919 · ce92919
1 parent 36f882a
commit ce92919
Show file tree

Hide file tree

Showing 13 changed files with 260 additions and 0 deletions.
diff --git a/changes/4702.added b/changes/4702.added
@@ -0,0 +1 @@
+Added support for loading GraphQL queries from a Git repository.
diff --git a/nautobot/docs/user-guide/feature-guides/git-data-source.md b/nautobot/docs/user-guide/feature-guides/git-data-source.md
@@ -19,6 +19,7 @@ The feature uses the concept of a `provides` field to identify the provided data
 |[Jobs](../platform-functionality/jobs/index.md)|A way for users to execute custom logic on demand from within the Nautobot UI, to accomplish various Nautobot data creation, modification, and validation tasks.|
 |[Config Contexts](../core-data-model/extras/configcontext.md)|Config contexts can be used to provide additional data that you can't natively store in Nautobot.|
 |[Config Context Schemas](../core-data-model/extras/configcontextschema.md)|Schemas enforce data validation on config contexts.|
+|[GraphQL queries](../feature-guides/graphql.md)|GraphQL reduces the complexity of performing multiple API calls|
 
 ### Examples of Apps Defining Additional Providers
 
@@ -282,6 +283,19 @@ config_context_schemas
 ├── schema_2.json
 ```
 
+### GraphQL Queries
+
+See also the [GraphQL feature guide](../feature-guides/graphql.md).
+
+GraphQL queries can be used to reduce the complexity of performing multiple API calls while also correlating results by empowering the user to create their own query that provides the user exactly what they want and nothing that they don't, in a single API call.
+
+```no-highlight
+▶ tree graphql_queries
+graphql_queries
+├── query1.gql
+├── query2.gql
+```
+
 ## Additional Git Data Sources
 
 As seen in [Fill out Repository Details](#fill-out-repository-details), the standard installation of Nautobot will come natively with export templates, jobs, and config contexts.  Additional data sources can be incorporated using the Nautobot App system.  For example, the [nautobot-golden-config](https://github.com/nautobot/nautobot-app-golden-config) App implements four additional data sources.
@@ -304,6 +318,7 @@ For more information for the Golden Configuration specific data sources, navigat
 |Jobs|`jobs`|
 |Config Contexts|`config_contexts`|
 |Config Context Schemas|`config_context_schemas`|
+|GraphQL Queries|`graphql_queries`|
 
 - Synchronization Status Failures.
     - Validate branch is correct and exists in the remote repository.

diff --git a/nautobot/extras/api/serializers.py b/nautobot/extras/api/serializers.py
@@ -480,11 +480,32 @@ class Meta:
 
 class GraphQLQuerySerializer(ValidatedModelSerializer, NotesSerializerMixin):
     variables = serializers.DictField(read_only=True)
+    owner_content_type = ContentTypeField(
+        queryset=ContentType.objects.filter(FeatureQuery("graphql_query_owners").get_query()),
+        required=False,
+        allow_null=True,
+        default=None,
+    )
+    owner = serializers.SerializerMethodField(read_only=True)
 
     class Meta:
         model = GraphQLQuery
         fields = "__all__"
 
+    @extend_schema_field(
+        PolymorphicProxySerializer(
+            component_name="GraphQLQueryOwner",
+            resource_type_field_name="object_type",
+            serializers=lambda: nested_serializers_for_models(FeatureQuery("graphql_query_owners").list_subclasses()),
+            allow_null=True,
+        )
+    )
+    def get_owner(self, obj):
+        if obj.owner is None:
+            return None
+        depth = get_nested_serializer_depth(self)
+        return return_nested_serializer_data_based_on_depth(self, depth, obj, obj.owner, "owner")
+
 
 class GraphQLQueryInputSerializer(serializers.Serializer):
     variables = serializers.DictField(allow_null=True, default={})

diff --git a/nautobot/extras/constants.py b/nautobot/extras/constants.py
@@ -11,6 +11,7 @@
     "dynamic_groups",
     "export_template_owners",
     "export_templates",
+    "graphql_query_owners",
     "graphql",
     "job_results",  # No longer used
     "locations",

diff --git a/nautobot/extras/datasources/git.py b/nautobot/extras/datasources/git.py
@@ -30,6 +30,7 @@
     DynamicGroup,
     ExportTemplate,
     GitRepository,
+    GraphQLQuery,
     Job,
     JobQueue,
     JobResult,
@@ -935,6 +936,120 @@ def delete_git_export_templates(repository_record, job_result, preserve=None):
             job_result.log(msg, level_choice=LogLevelChoices.LOG_WARNING, grouping="export templates")
 
 
+#
+# GraphQL handling
+#
+
+
+def refresh_git_graphql_queries(repository_record, job_result, delete=False):
+    """Callback function for GitRepository updates - refresh all GraphQLQuery managed by this repository."""
+    if "extras.graphqlquery" in repository_record.provided_contents and not delete:
+        update_git_graphql_queries(repository_record, job_result)
+    else:
+        delete_git_graphql_queries(repository_record, job_result)
+
+
+logger = logging.getLogger(__name__)
+
+
+def update_git_graphql_queries(repository_record, job_result):
+    """Refresh any GraphQL queries provided by this Git repository."""
+    graphql_query_path = os.path.join(repository_record.filesystem_path, "graphql_queries")
+    git_repository_content_type = ContentType.objects.get_for_model(GitRepository)
+    graphql_queries = []
+
+    if os.path.isdir(graphql_query_path):
+        for file in os.listdir(graphql_query_path):
+            file_path = os.path.join(graphql_query_path, file)
+            if not os.path.isfile(file_path):
+                continue
+
+            # Remove `.gql` extension from the name if it exists
+            query_name = file.rsplit(".gql", 1)[0] if file.endswith(".gql") else file
+
+            try:
+                with open(file_path, "r") as fd:
+                    query_content = fd.read().strip()
+
+                graphql_query, created = GraphQLQuery.objects.get_or_create(
+                    name=query_name,
+                    owner_content_type=git_repository_content_type,
+                    owner_object_id=repository_record.pk,
+                    defaults={"query": query_content},
+                )
+                modified = graphql_query.query != query_content
+                graphql_queries.append(query_name)
+                # Only attempt to update if the content has changed
+                if modified:
+                    try:
+                        graphql_query.query = query_content
+                        graphql_query.validated_save()
+                        msg = (
+                            f"Successfully created GraphQL query: {query_name}"
+                            if created
+                            else f"Successfully updated GraphQL query: {query_name}"
+                        )
+                        logger.info(msg)
+                        job_result.log(
+                            msg, obj=graphql_query, level_choice=LogLevelChoices.LOG_INFO, grouping="graphql queries"
+                        )
+                    except Exception as exc:
+                        # Log validation error and retain the existing query
+                        error_msg = (
+                            f"Invalid GraphQL syntax for query '{query_name}'. "
+                            f"Retaining the existing query. Error: {exc}"
+                        )
+                        logger.error(error_msg)
+                        job_result.log(error_msg, level_choice=LogLevelChoices.LOG_ERROR, grouping="graphql queries")
+                        continue
+                else:
+                    msg = f"No changes to GraphQL query: {query_name}"
+                    logger.info(msg)
+                    job_result.log(
+                        msg, obj=graphql_query, level_choice=LogLevelChoices.LOG_INFO, grouping="graphql queries"
+                    )
+
+            except Exception as exc:
+                # Check if a query with the same name already exists
+                existing_query = GraphQLQuery.objects.filter(name=query_name).first()
+                if existing_query and existing_query.owner_object_id != repository_record.pk:
+                    error_msg = (
+                        f"GraphQL query '{query_name}' already exists "
+                        f"Please rename the query in the repository and try again."
+                    )
+                else:
+                    error_msg = f"Error processing GraphQL query file '{file}': {exc}"
+
+                # Log the error
+                logger.error(error_msg)
+                job_result.log(error_msg, level_choice=LogLevelChoices.LOG_ERROR, grouping="graphql queries")
+
+    # Delete any queries not in the preserved list
+    delete_git_graphql_queries(repository_record, job_result, preserve=graphql_queries)
+
+
+def delete_git_graphql_queries(repository_record, job_result, preserve=None):
+    """Delete GraphQL queries owned by the given Git repository that are not in the preserve list."""
+    git_repository_content_type = ContentType.objects.get_for_model(GitRepository)
+    if preserve is None:
+        preserve = []
+
+    for graphql_query in GraphQLQuery.objects.filter(
+        owner_content_type=git_repository_content_type,
+        owner_object_id=repository_record.pk,
+    ):
+        if graphql_query.name not in preserve:
+            try:
+                graphql_query.delete()
+                msg = f"Deleted GraphQL query: {graphql_query.name}"
+                logger.warning(msg)
+                job_result.log(msg, level_choice=LogLevelChoices.LOG_WARNING, grouping="graphql queries")
+            except Exception as exc:
+                error_msg = f"Unable to delete '{graphql_query.name}': {exc}"
+                logger.error(error_msg)
+                job_result.log(error_msg, level_choice=LogLevelChoices.LOG_ERROR, grouping="graphql queries")
+
+
 # Register built-in callbacks for data types potentially provided by a GitRepository
 register_datasource_contents(
     [
@@ -978,5 +1093,15 @@ def delete_git_export_templates(repository_record, job_result, preserve=None):
                 callback=refresh_git_export_templates,
             ),
         ),
+        (
+            "extras.gitrepository",
+            DatasourceContent(
+                name="graphql queries",
+                content_identifier="extras.graphqlquery",
+                icon="mdi-graphql",
+                weight=400,
+                callback=refresh_git_graphql_queries,
+            ),
+        ),
     ]
 )
diff --git a/nautobot/extras/migrations/0122_add_graphqlquery_owner_content_type.py b/nautobot/extras/migrations/0122_add_graphqlquery_owner_content_type.py
@@ -0,0 +1,34 @@
+# Generated by Django 4.2.17 on 2025-01-08 16:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+import nautobot.extras.utils
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("extras", "0121_alter_team_contacts"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="graphqlquery",
+            name="owner_content_type",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                limit_choices_to=nautobot.extras.utils.FeatureQuery("graphql_query_owners"),
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="graphql_queries",
+                to="contenttypes.contenttype",
+            ),
+        ),
+        migrations.AddField(
+            model_name="graphqlquery",
+            name="owner_object_id",
+            field=models.UUIDField(blank=True, default=None, null=True),
+        ),
+    ]
diff --git a/nautobot/extras/models/datasources.py b/nautobot/extras/models/datasources.py
@@ -18,6 +18,7 @@
 @extras_features(
     "config_context_owners",
     "export_template_owners",
+    "graphql_query_owners",
     "graphql",
     "job_results",
     "webhooks",

diff --git a/nautobot/extras/models/models.py b/nautobot/extras/models/models.py
@@ -680,6 +680,21 @@ class GraphQLQuery(
     SavedViewMixin,
     BaseModel,
 ):
+    # A Graphql Query *may* be owned by another model, such as a GitRepository, or it may be un-owned
+    owner_content_type = models.ForeignKey(
+        to=ContentType,
+        on_delete=models.CASCADE,
+        limit_choices_to=FeatureQuery("graphql_query_owners"),
+        default=None,
+        null=True,
+        blank=True,
+        related_name="graphql_queries",
+    )
+    owner_object_id = models.UUIDField(default=None, null=True, blank=True)
+    owner = GenericForeignKey(
+        ct_field="owner_content_type",
+        fk_field="owner_object_id",
+    )
     name = models.CharField(max_length=CHARFIELD_MAX_LENGTH, unique=True)
     query = models.TextField()
     variables = models.JSONField(encoder=DjangoJSONEncoder, default=dict, blank=True)

diff --git a/nautobot/extras/tests/git_data/01-valid-files/graphql_queries/device_interfaces.gql b/nautobot/extras/tests/git_data/01-valid-files/graphql_queries/device_interfaces.gql
@@ -0,0 +1,8 @@
+query {
+  devices {
+    name
+    interfaces {
+      name
+    }
+  }
+}
diff --git a/nautobot/extras/tests/git_data/01-valid-files/graphql_queries/device_names.gql b/nautobot/extras/tests/git_data/01-valid-files/graphql_queries/device_names.gql
@@ -0,0 +1,5 @@
+{
+    device {
+        name
+    }
+}
diff --git a/nautobot/extras/tests/git_data/02-invalid-files/graphql_queries/bad_device_names.gql b/nautobot/extras/tests/git_data/02-invalid-files/graphql_queries/bad_device_names.gql
@@ -0,0 +1,5 @@
+{
+    device {
+
+
+}
diff --git a/nautobot/extras/tests/test_api.py b/nautobot/extras/tests/test_api.py
@@ -1241,6 +1241,8 @@ class GraphQLQueryTest(APIViewTestCases.APIViewTestCase):
         },
     ]
 
+    choices_fields = ["owner_content_type"]
+
     @classmethod
     def setUpTestData(cls):
         cls.graphqlqueries = (
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Added support for loading GraphQL queries from a Git repository.