Support resource definition using manifest URL (ansible-collections#478)

Support resource definition using manifest URL

SUMMARY

Closes ansible-collections#451

ISSUE TYPE


Feature Pull Request

COMPONENT NAME

k8s
k8s_scale
k8s_service

Reviewed-by: Mike Graves <mgraves@redhat.com>
Reviewed-by: Abhijeet Kasurde <None>
Reviewed-by: Bikouo Aubin <None>

changelogs/fragments/478-add-support-for-manifest-url.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+  - k8s, k8s_scale, k8s_service - add support for resource definition as manifest via. (https://github.com/ansible-collections/kubernetes.core/issues/451).

plugins/action/k8s_info.py

@@ -359,7 +359,7 @@ def run(self, tmp=None, task_vars=None):
         # find the file in the expected search path
         src = self._task.args.get("src", None)
 
-        if src:
+        if src and not src.startswith(("http://", "https://", "ftp://")):
             if remote_transport:
                 # src is on remote node
                 result.update(

plugins/doc_fragments/k8s_resource_options.py

@@ -29,6 +29,7 @@ class ModuleDocFragment(object):
     - Reads from the local file system. To read from the Ansible controller's file system, including vaulted files, use the file lookup
       plugin or template lookup plugin, combined with the from_yaml filter, and pass the result to
       I(resource_definition). See Examples below.
+    - The URL to manifest files that can be used to create the resource. Added in version 2.4.0.
     - Mutually exclusive with I(template) in case of M(kubernetes.core.k8s) module.
     type: path
 """

plugins/module_utils/common.py

@@ -26,6 +26,7 @@
 import sys
 import hashlib
 from datetime import datetime
+from tempfile import NamedTemporaryFile
 
 from ansible_collections.kubernetes.core.plugins.module_utils.version import (
     LooseVersion,
@@ -47,6 +48,7 @@
 from ansible.module_utils._text import to_native, to_bytes, to_text
 from ansible.module_utils.common.dict_transformations import dict_merge
 from ansible.module_utils.parsing.convert_bool import boolean
+from ansible.module_utils.urls import Request
 
 K8S_IMP_ERR = None
 try:
@@ -342,6 +344,28 @@ def _load_config():
 get_api_client._pool = {}
 
 
+def fetch_file_from_url(module, url):
+    # Download file
+    bufsize = 65536
+    file_name, file_ext = os.path.splitext(str(url.rsplit("/", 1)[1]))
+    temp_file = NamedTemporaryFile(
+        dir=module.tmpdir, prefix=file_name, suffix=file_ext, delete=False
+    )
+    module.add_cleanup_file(temp_file.name)
+    try:
+        rsp = Request().open("GET", url)
+        if not rsp:
+            module.fail_json(msg="Failure downloading %s" % url)
+        data = rsp.read(bufsize)
+        while data:
+            temp_file.write(data)
+            data = rsp.read(bufsize)
+        temp_file.close()
+    except Exception as e:
+        module.fail_json(msg="Failure downloading %s, %s" % (url, to_native(e)))
+    return temp_file.name
+
+
 class K8sAnsibleMixin(object):
     def __init__(self, module, pyyaml_required=True, *args, **kwargs):
         module.deprecate(
@@ -529,8 +553,15 @@ def remove_aliases(self):
                     if alias in self.params:
                         self.params.pop(alias)
 
-    def load_resource_definitions(self, src):
+    def load_resource_definitions(self, src, module=None):
         """Load the requested src path"""
+        if module and (
+            src.startswith("https://")
+            or src.startswith("http://")
+            or src.startswith("ftp://")
+        ):
+            src = fetch_file_from_url(module, src)
+
         result = None
         path = os.path.normpath(src)
         if not os.path.exists(path):
@@ -745,7 +776,7 @@ def set_resource_definitions(self, module):
 
         src = module.params.get("src")
         if src:
-            self.resource_definitions = self.load_resource_definitions(src)
+            self.resource_definitions = self.load_resource_definitions(src, module)
         try:
             self.resource_definitions = [
                 item for item in self.resource_definitions if item
@@ -853,9 +884,9 @@ def set_defaults(self, resource, definition):
         definition["apiVersion"] = resource.group_version
         metadata = definition.get("metadata", {})
         if not metadata.get("name") and not metadata.get("generateName"):
-            if self.name:
+            if hasattr(self, "name") and self.name:
                 metadata["name"] = self.name
-            elif self.generate_name:
+            elif hasattr(self, "generate_name") and self.generate_name:
                 metadata["generateName"] = self.generate_name
         if resource.namespaced and self.namespace and not metadata.get("namespace"):
             metadata["namespace"] = self.namespace

plugins/module_utils/k8s/resource.py

@@ -5,6 +5,7 @@
 from typing import cast, Dict, Iterable, List, Optional, Union
 
 from ansible.module_utils.six import string_types
+from ansible.module_utils.urls import Request
 
 try:
     import yaml
@@ -53,7 +54,11 @@ def create_definitions(params: Dict) -> List[ResourceDefinition]:
         definitions = from_yaml(d)
     elif params.get("src"):
         d = cast(str, params.get("src"))
-        definitions = from_file(d)
+        if hasattr(d, "startswith") and d.startswith(("https://", "http://", "ftp://")):
+            data = Request().open("GET", d).read().decode("utf8")
+            definitions = from_yaml(data)
+        else:
+            definitions = from_file(d)
     else:
         # We'll create an empty definition and let merge_params set values
         # from the module parameters.

plugins/module_utils/k8s/service.py

@@ -305,7 +305,6 @@ def find(
     def create(self, resource: Resource, definition: Dict) -> Dict:
         namespace = definition["metadata"].get("namespace")
         name = definition["metadata"].get("name")
-        results = {"changed": False, "result": {}}
 
         if self.module.check_mode and not self.client.dry_run:
             k8s_obj = _encode_stringdata(definition)
@@ -327,7 +326,7 @@ def create(self, resource: Resource, definition: Dict) -> Dict:
                         name
                     )
                 )
-                return results
+                return dict()
             except Exception as e:
                 reason = e.body if hasattr(e, "body") else e
                 msg = "Failed to create object: {0}".format(reason)

plugins/modules/k8s_scale.py

@@ -250,7 +250,7 @@ def _continue_or_exit(warn):
             module.exit_json(warning=warn, **return_attributes)
 
     for existing in existing_items:
-        if module.params["kind"].lower() == "job":
+        if kind.lower() == "job":
             existing_count = existing.spec.parallelism
         elif hasattr(existing.spec, "replicas"):
             existing_count = existing.spec.replicas
@@ -285,7 +285,7 @@ def _continue_or_exit(warn):
             continue
 
         if existing_count != replicas:
-            if module.params["kind"].lower() == "job":
+            if kind.lower() == "job":
                 existing.spec.parallelism = replicas
                 result = {"changed": True}
                 if module.check_mode:

plugins/modules/k8s_service.py

@@ -169,6 +169,9 @@
 from ansible_collections.kubernetes.core.plugins.module_utils.k8s.resource import (
     create_definitions,
 )
+from ansible_collections.kubernetes.core.plugins.module_utils.k8s.runner import (
+    perform_action,
+)
 
 
 SERVICE_ARG_SPEC = {
@@ -195,7 +198,7 @@ def merge_dicts(x, y):
             if isinstance(x[k], dict) and isinstance(y[k], dict):
                 yield (k, dict(merge_dicts(x[k], y[k])))
             else:
-                yield (k, y[k])
+                yield (k, y[k] if y[k] else x[k])
         elif k in x:
             yield (k, x[k])
         else:
@@ -211,32 +214,6 @@ def argspec():
     return argument_spec
 
 
-def perform_action(svc, resource, definition, params):
-    state = params.get("state", None)
-    result = {}
-
-    existing = svc.retrieve(resource, definition)
-
-    if state == "absent":
-        result = svc.delete(resource, definition, existing)
-        result["method"] = "delete"
-    else:
-        if params.get("apply"):
-            result = svc.apply(resource, definition, existing)
-            result["method"] = "apply"
-        elif not existing:
-            result = svc.create(resource, definition)
-            result["method"] = "create"
-        elif params.get("force", False):
-            result = svc.replace(resource, definition, existing)
-            result["method"] = "replace"
-        else:
-            result = svc.update(resource, definition, existing)
-            result["method"] = "update"
-
-    return result
-
-
 def execute_module(svc):
     """Module execution"""
     module = svc.module
@@ -263,9 +240,8 @@ def execute_module(svc):
 
     # 'resource_definition:' has lower priority than module parameters
     definition = dict(merge_dicts(definitions[0], definition))
-    resource = svc.find_resource("Service", api_version, fail=True)
 
-    result = perform_action(svc, resource, definition, module.params)
+    result = perform_action(svc, definition, module.params)
 
     module.exit_json(**result)
 

tests/integration/targets/k8s_manifest_url/aliases

@@ -0,0 +1,4 @@
+k8s_service
+k8s
+k8s_scale
+time=40

tests/integration/targets/k8s_manifest_url/defaults/main.yml

@@ -0,0 +1,64 @@
+---
+test_namespace: "k8s-manifest-url"
+file_server_container_name: "nginx-server"
+file_server_published_port: 30001
+file_server_container_image: "docker.io/nginx"
+
+pod_manifest:
+  file_name: pod.yaml
+  definition: |
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: yaml-pod
+    spec:
+      containers:
+      - name: busy
+        image: busybox
+        command:
+          - /bin/sh
+          - -c
+          - while true;do date;sleep 5; done
+
+deployment_manifest:
+  file_name: deployment.yaml
+  definition: |
+    ---
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: nginx-deployment
+      labels:
+        app: nginx
+    spec:
+      replicas: 3
+      selector:
+        matchLabels:
+          app: nginx
+      template:
+        metadata:
+          labels:
+            app: nginx
+        spec:
+          containers:
+          - name: nginx
+            image: nginx
+
+service_manifest:
+  file_name: service.yaml
+  definition: |
+    ---
+    apiVersion: v1
+    kind: Service
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      ports:
+        - name: http
+          port: 80
+      selector:
+        app: nginx
+    status:
+      loadBalancer: {}

tests/integration/targets/k8s_manifest_url/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - setup_namespace