minor

JaeAeich · JaeAeich · commit 6fa35acf1421 · 2024-08-28T14:24:24.000+05:30
diff --git a/deployment/config.yaml b/deployment/config.yaml
@@ -126,7 +126,7 @@ custom:
     # Taskmaster image name
     imageName: docker.io/elixircloud/tesk-core-taskmaster
     # Taskmaster image version
-    imageVersion: fix-null-volumes3
+    imageVersion: fix-null-volumes5
     # Filer image name
     filerImageName: docker.io/elixircloud/tesk-core-filer
     # Filer image version
diff --git a/json_dump_tm.json b/json_dump_tm.json
@@ -0,0 +1 @@
+{"api_version": "batch/v1", "kind": "Job", "metadata": {"annotations": {"json-input": "{\n  \"id\": null,\n  \"state\": \"UNKNOWN\",\n  \"name\": \"string\",\n  \"description\": \"string\",\n  \"inputs\": [\n    {\n      \"name\": null,\n      \"description\": null,\n      \"url\": \"s3://my-object-store/file1\",\n      \"path\": \"/data/file1\",\n      \"type\": \"FILE\",\n      \"content\": null,\n      \"streamable\": null\n    }\n  ],\n  \"outputs\": [\n    {\n      \"name\": null,\n      \"description\": null,\n      \"url\": \"s3://my-object-store/outfile-1\",\n      \"path\": \"/data/outfile\",\n      \"path_prefix\": null,\n      \"type\": \"FILE\"\n    }\n  ],\n  \"resources\": {\n    \"cpu_cores\": 4,\n    \"preemptible\": false,\n    \"ram_gb\": 8.0,\n    \"disk_gb\": 40.0,\n    \"zones\": [\n      \"us-west-1\"\n    ],\n    \"backend_parameters\": {\n      \"VmSize\": \"Standard_D64_v3\"\n    },\n    \"backend_parameters_strict\": false\n  },\n  \"executors\": [\n    {\n      \"image\": \"ubuntu:20.04\",\n      \"command\": [\n        \"/bin/md5\",\n        \"/data/file1\"\n      ],\n      \"workdir\": \"/data/\",\n      \"stdin\": \"/data/file1\",\n      \"stdout\": \"/tmp/stdout.log\",\n      \"stderr\": \"/tmp/stderr.log\",\n      \"env\": {\n        \"BLASTDB\": \"/data/GRC38\",\n        \"HMMERDB\": \"/data/hmmer\"\n      },\n      \"ignore_error\": true\n    }\n  ],\n  \"volumes\": [\n    \"/vol/A/\"\n  ],\n  \"tags\": {\n    \"PROJECT_GROUP\": \"alice-lab\",\n    \"WORKFLOW_ID\": \"cwl-01234\"\n  },\n  \"logs\": null,\n  \"creation_time\": null\n}", "tes-task-name": "string"}, "creation_timestamp": null, "deletion_grace_period_seconds": null, "deletion_timestamp": null, "finalizers": null, "generate_name": null, "generation": null, "labels": {"app": "taskmaster", "job-type": "taskmaster"}, "managed_fields": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "namespace": null, "owner_references": null, "resource_version": null, "self_link": null, "uid": null}, "spec": {"active_deadline_seconds": null, "backoff_limit": null, "backoff_limit_per_index": null, "completion_mode": null, "completions": null, "manual_selector": null, "max_failed_indexes": null, "parallelism": null, "pod_failure_policy": null, "pod_replacement_policy": null, "selector": null, "suspend": null, "template": {"metadata": {"annotations": null, "creation_timestamp": null, "deletion_grace_period_seconds": null, "deletion_timestamp": null, "finalizers": null, "generate_name": null, "generation": null, "labels": null, "managed_fields": null, "name": "taskmaster", "namespace": null, "owner_references": null, "resource_version": null, "self_link": null, "uid": null}, "spec": {"active_deadline_seconds": null, "affinity": null, "automount_service_account_token": null, "containers": [{"args": ["-f", "/jsoninput/JSON_INPUT.gz", "-n", "tesk", "-fn", "docker.io/elixircloud/tesk-core-filer", "-fv", "v0.10.2"], "command": null, "env": [{"name": "TESK_FTP_USERNAME", "value": null, "value_from": {"config_map_key_ref": null, "field_ref": null, "resource_field_ref": null, "secret_key_ref": {"key": "username", "name": "account-secret", "optional": true}}}, {"name": "TESK_FTP_PASSWORD", "value": null, "value_from": {"config_map_key_ref": null, "field_ref": null, "resource_field_ref": null, "secret_key_ref": {"key": "password", "name": "account-secret", "optional": true}}}, {"name": "KEY", "value": "value", "value_from": null}, {"name": "FILER_BACKOFF_LIMIT", "value": "2", "value_from": null}, {"name": "EXECUTOR_BACKOFF_LIMIT", "value": "2", "value_from": null}], "env_from": null, "image": "docker.io/elixircloud/tesk-core-taskmaster:fix-null-volumes5", "image_pull_policy": null, "lifecycle": null, "liveness_probe": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "ports": null, "readiness_probe": null, "resize_policy": null, "resources": null, "restart_policy": null, "security_context": null, "startup_probe": null, "stdin": null, "stdin_once": null, "termination_message_path": null, "termination_message_policy": null, "tty": null, "volume_devices": null, "volume_mounts": [{"mount_path": "/podinfo", "mount_propagation": null, "name": "podinfo", "read_only": true, "sub_path": null, "sub_path_expr": null}, {"mount_path": "/jsoninput", "mount_propagation": null, "name": "jsoninput", "read_only": true, "sub_path": null, "sub_path_expr": null}], "working_dir": null}], "dns_config": null, "dns_policy": null, "enable_service_links": null, "ephemeral_containers": null, "host_aliases": null, "host_ipc": null, "host_network": null, "host_pid": null, "host_users": null, "hostname": null, "image_pull_secrets": null, "init_containers": null, "node_name": null, "node_selector": null, "os": null, "overhead": null, "preemption_policy": null, "priority": null, "priority_class_name": null, "readiness_gates": null, "resource_claims": null, "restart_policy": "Never", "runtime_class_name": null, "scheduler_name": null, "scheduling_gates": null, "security_context": null, "service_account": null, "service_account_name": "taskmaster", "set_hostname_as_fqdn": null, "share_process_namespace": null, "subdomain": null, "termination_grace_period_seconds": null, "tolerations": null, "topology_spread_constraints": null, "volumes": [{"aws_elastic_block_store": null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null, "config_map": null, "csi": null, "downward_api": {"default_mode": null, "items": [{"field_ref": {"api_version": null, "field_path": "metadata.labels"}, "mode": null, "path": "labels", "resource_field_ref": null}]}, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume": null, "flocker": null, "gce_persistent_disk": null, "git_repo": null, "glusterfs": null, "host_path": null, "iscsi": null, "name": "podinfo", "nfs": null, "persistent_volume_claim": null, "photon_persistent_disk": null, "portworx_volume": null, "projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret": null, "storageos": null, "vsphere_volume": null}, {"aws_elastic_block_store": null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null, "config_map": {"default_mode": null, "items": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "optional": null}, "csi": null, "downward_api": null, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume": null, "flocker": null, "gce_persistent_disk": null, "git_repo": null, "glusterfs": null, "host_path": null, "iscsi": null, "name": "jsoninput", "nfs": null, "persistent_volume_claim": null, "photon_persistent_disk": null, "portworx_volume": null, "projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret": null, "storageos": null, "vsphere_volume": null}]}}, "ttl_seconds_after_finished": null}, "status": null}
diff --git a/tesk/api/ga4gh/tes/task/create_task.py b/tesk/api/ga4gh/tes/task/create_task.py
@@ -36,7 +36,8 @@ def create_task(self) -> TesCreateTaskResponse:
             try:
                 attempts_no += 1
                 resources = self.task.resources
-
+                limits = self.kubernetes_client_wrapper.list_limits()
+                print("limits", limits)
                 minimum_ram_gb = self.kubernetes_client_wrapper.minimum_ram_gb()
 
                 if not self.task.resources:
@@ -49,22 +50,65 @@ def create_task(self) -> TesCreateTaskResponse:
                     # self.user
                 )
 
-                print(taskmaster_job)
                 taskmaster_config_map = (
                     self.tes_kubernetes_converter.from_tes_task_to_k8s_config_map(
                         self.task,
                         taskmaster_job,
                         # self.user
                     )
                 )
-                print(taskmaster_config_map)
-
-                # Create ConfigMap and Job
-                _ = self.kubernetes_client_wrapper.create_config_map(
-                    taskmaster_config_map
-                )
-                created_job = self.kubernetes_client_wrapper.create_job(taskmaster_job)
-
+                # print("*********************************")
+                # print(
+                #     f"Manifest consumed to create taskmaster job as {type(taskmaster_job)}"
+                # )
+                # print(taskmaster_config_map)
+                # print("*********************************")
+
+                # print("*********************************")
+                # print(
+                #     f"Manifest consumed to create config map as {type(taskmaster_config_map)}"
+                # )
+                # print(taskmaster_config_map)
+                # print("*********************************")
+
+                # # Create ConfigMap and Job
+                # configmap = self.kubernetes_client_wrapper.create_config_map(
+                #     taskmaster_config_map
+                # )
+                # print("********************")
+                # print(f"Created ConfigMap as {type(configmap)}")
+                # print(configmap)
+                # print("********************")
+                # created_job = self.kubernetes_client_wrapper.create_job(taskmaster_job)
+
+                # print("********************")
+                # print(f"Created taskmaster job as {type(created_job)}")
+                # print(created_job)
+                # print("********************")
+                # Open a file to write the output
+                with open("output.log", "w") as f:
+                    f.write("*********************************\n")
+                    f.write(f"Manifest consumed to create taskmaster job as {type(taskmaster_job)}\n")
+                    f.write(str(taskmaster_job) + "\n")
+                    f.write("*********************************\n")
+
+                    f.write("*********************************\n")
+                    f.write(f"Manifest consumed to create config map as {type(taskmaster_config_map)}\n")
+                    f.write(str(taskmaster_config_map) + "\n")
+                    f.write("*********************************\n")
+
+                    # Create ConfigMap and Job
+                    configmap = self.kubernetes_client_wrapper.create_config_map(taskmaster_config_map)
+                    f.write("********************\n")
+                    f.write(f"Created ConfigMap as {type(configmap)}\n")
+                    f.write(str(configmap) + "\n")
+                    f.write("********************\n")
+                    
+                    created_job = self.kubernetes_client_wrapper.create_job(taskmaster_job)
+                    f.write("********************\n")
+                    f.write(f"Created taskmaster job as {type(created_job)}\n")
+                    f.write(str(created_job) + "\n")
+                    f.write("********************\n")
                 assert created_job.metadata is not None
                 assert created_job.metadata.name is not None
 
diff --git a/tesk/api/kubernetes/convert/converter.py b/tesk/api/kubernetes/convert/converter.py
@@ -146,14 +146,8 @@ def from_tes_task_to_k8s_config_map(
             taskmaster_config_map.metadata is not None
         ), "Taskmaster metadata is should have already been set!"
 
-        if taskmaster_config_map.metadata.labels is None:
-            taskmaster_config_map.metadata.labels = {}
 
-        if taskmaster_config_map.metadata.annotations is None:
-            taskmaster_config_map.metadata.annotations = {}
-
-        # FIXME: Figure out what to do if task.name is none.
-        task_name = task.name or "String"
+        task_name = task.name
 
         taskmaster_config_map.metadata.annotations[
             self.constants.ann_testask_name_key
@@ -185,11 +179,6 @@ def from_tes_task_to_k8s_config_map(
             for idx, executor in enumerate(task.executors)
         ]
 
-        print("nice")
-        print(task)
-        print(task.volumes)
-        print("nice")
-
         taskmaster_input: dict[str, Any] = {
             "inputs": pydantic_model_list_dict(task.inputs) if task.inputs else [],
             "outputs": pydantic_model_list_dict(task.outputs) if task.outputs else [],
@@ -200,16 +189,13 @@ def from_tes_task_to_k8s_config_map(
                 else 10.0
             },
         }
-        print(taskmaster_input)
         taskmaster_input[self.constants.taskmaster_input_exec_key] = [
             exec_job.to_dict() for exec_job in executors_as_jobs
         ]
 
-        print(taskmaster_input)
         taskmaster_input_as_json = json.loads(
             json.dumps(taskmaster_input, default=decimal_to_float)
         )
-        print(taskmaster_input_as_json)
 
         try:
             with BytesIO() as obj:
@@ -246,9 +232,13 @@ def from_tes_executor_to_k8s_job(  # noqa: PLR0913
         executor_job: V1Job = (
             self.template_supplier.get_executor_template_with_value_from_config()
         )
+        print("Before processing")
+        print(executor_job)
 
         # Set executors name based on taskmaster's job name
-        Job(executor_job).change_job_name(Task(taskmaster_name=generated_task_id).get_executor_name(executor_index))
+        Job(executor_job).change_job_name(
+            Task(taskmaster_name=generated_task_id).get_executor_name(executor_index)
+        )
 
         if executor_job.metadata is None:
             executor_job.metadata = V1ObjectMeta()
@@ -304,6 +294,9 @@ def from_tes_executor_to_k8s_job(  # noqa: PLR0913
 
         assert container.resources.requests is not None
 
+        # container.restart_policy = "Never"
+        print(container)
+
         if resources.cpu_cores:
             container.resources.requests["cpu"] = parse_quantity(
                 str(resources.cpu_cores)
@@ -314,14 +307,20 @@ def from_tes_executor_to_k8s_job(  # noqa: PLR0913
                 f"{resources.ram_gb:.6f}Gi"
             )
 
-            # # Workaround
-            # # Check if volumes is None and set it to an empty list if it is
-            # if (
-            #     executor_job.spec
-            #     and executor_job.spec.spec
-            #     and executor_job.spec.spec.volumes is None
-            # ):
-            #     executor_job.spec.spec.volumes = []
+        executor_job.spec.template.spec.containers[
+            0
+        ].resources.limits = executor_job.spec.template.spec.containers[
+            0
+        ].resources.requests
+
+        # # Workaround
+        # # Check if volumes is None and set it to an empty list if it is
+        # if (
+        #     executor_job.spec
+        #     and executor_job.spec.spec
+        #     and executor_job.spec.spec.volumes is None
+        # ):
+        #     executor_job.spec.spec.volumes = []
 
         return executor_job
 
diff --git a/tesk/api/kubernetes/convert/executor_command_wrapper.py b/tesk/api/kubernetes/convert/executor_command_wrapper.py
@@ -51,4 +51,4 @@ def get_commands_with_stream_redirects(self) -> List[str]:
 
         result.append(" ".join(command_parts))
 
-        return result
+        return result
diff --git a/tesk/api/kubernetes/convert/template.py b/tesk/api/kubernetes/convert/template.py
@@ -197,4 +197,6 @@ def get_executor_template_with_value_from_config(self) -> V1Job:
                 )
             ]
 
+        job.spec.template.spec.containers[0].restart_policy = "Never"
+
         return job

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+{"api_version": "batch/v1", "kind": "Job", "metadata": {"annotations": {"json-input": "{\n \"id\": null,\n \"state\": \"UNKNOWN\",\n \"name\": \"string\",\n \"description\": \"string\",\n \"inputs\": [\n {\n \"name\": null,\n \"description\": null,\n \"url\": \"s3://my-object-store/file1\",\n \"path\": \"/data/file1\",\n \"type\": \"FILE\",\n \"content\": null,\n \"streamable\": null\n }\n ],\n \"outputs\": [\n {\n \"name\": null,\n \"description\": null,\n \"url\": \"s3://my-object-store/outfile-1\",\n \"path\": \"/data/outfile\",\n \"path_prefix\": null,\n \"type\": \"FILE\"\n }\n ],\n \"resources\": {\n \"cpu_cores\": 4,\n \"preemptible\": false,\n \"ram_gb\": 8.0,\n \"disk_gb\": 40.0,\n \"zones\": [\n \"us-west-1\"\n ],\n \"backend_parameters\": {\n \"VmSize\": \"Standard_D64_v3\"\n },\n \"backend_parameters_strict\": false\n },\n \"executors\": [\n {\n \"image\": \"ubuntu:20.04\",\n \"command\": [\n \"/bin/md5\",\n \"/data/file1\"\n ],\n \"workdir\": \"/data/\",\n \"stdin\": \"/data/file1\",\n \"stdout\": \"/tmp/stdout.log\",\n \"stderr\": \"/tmp/stderr.log\",\n \"env\": {\n \"BLASTDB\": \"/data/GRC38\",\n \"HMMERDB\": \"/data/hmmer\"\n },\n \"ignore_error\": true\n }\n ],\n \"volumes\": [\n \"/vol/A/\"\n ],\n \"tags\": {\n \"PROJECT_GROUP\": \"alice-lab\",\n \"WORKFLOW_ID\": \"cwl-01234\"\n },\n \"logs\": null,\n \"creation_time\": null\n}", "tes-task-name": "string"}, "creation_timestamp": null, "deletion_grace_period_seconds": null, "deletion_timestamp": null, "finalizers": null, "generate_name": null, "generation": null, "labels": {"app": "taskmaster", "job-type": "taskmaster"}, "managed_fields": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "namespace": null, "owner_references": null, "resource_version": null, "self_link": null, "uid": null}, "spec": {"active_deadline_seconds": null, "backoff_limit": null, "backoff_limit_per_index": null, "completion_mode": null, "completions": null, "manual_selector": null, "max_failed_indexes": null, "parallelism": null, "pod_failure_policy": null, "pod_replacement_policy": null, "selector": null, "suspend": null, "template": {"metadata": {"annotations": null, "creation_timestamp": null, "deletion_grace_period_seconds": null, "deletion_timestamp": null, "finalizers": null, "generate_name": null, "generation": null, "labels": null, "managed_fields": null, "name": "taskmaster", "namespace": null, "owner_references": null, "resource_version": null, "self_link": null, "uid": null}, "spec": {"active_deadline_seconds": null, "affinity": null, "automount_service_account_token": null, "containers": [{"args": ["-f", "/jsoninput/JSON_INPUT.gz", "-n", "tesk", "-fn", "docker.io/elixircloud/tesk-core-filer", "-fv", "v0.10.2"], "command": null, "env": [{"name": "TESK_FTP_USERNAME", "value": null, "value_from": {"config_map_key_ref": null, "field_ref": null, "resource_field_ref": null, "secret_key_ref": {"key": "username", "name": "account-secret", "optional": true}}}, {"name": "TESK_FTP_PASSWORD", "value": null, "value_from": {"config_map_key_ref": null, "field_ref": null, "resource_field_ref": null, "secret_key_ref": {"key": "password", "name": "account-secret", "optional": true}}}, {"name": "KEY", "value": "value", "value_from": null}, {"name": "FILER_BACKOFF_LIMIT", "value": "2", "value_from": null}, {"name": "EXECUTOR_BACKOFF_LIMIT", "value": "2", "value_from": null}], "env_from": null, "image": "docker.io/elixircloud/tesk-core-taskmaster:fix-null-volumes5", "image_pull_policy": null, "lifecycle": null, "liveness_probe": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "ports": null, "readiness_probe": null, "resize_policy": null, "resources": null, "restart_policy": null, "security_context": null, "startup_probe": null, "stdin": null, "stdin_once": null, "termination_message_path": null, "termination_message_policy": null, "tty": null, "volume_devices": null, "volume_mounts": [{"mount_path": "/podinfo", "mount_propagation": null, "name": "podinfo", "read_only": true, "sub_path": null, "sub_path_expr": null}, {"mount_path": "/jsoninput", "mount_propagation": null, "name": "jsoninput", "read_only": true, "sub_path": null, "sub_path_expr": null}], "working_dir": null}], "dns_config": null, "dns_policy": null, "enable_service_links": null, "ephemeral_containers": null, "host_aliases": null, "host_ipc": null, "host_network": null, "host_pid": null, "host_users": null, "hostname": null, "image_pull_secrets": null, "init_containers": null, "node_name": null, "node_selector": null, "os": null, "overhead": null, "preemption_policy": null, "priority": null, "priority_class_name": null, "readiness_gates": null, "resource_claims": null, "restart_policy": "Never", "runtime_class_name": null, "scheduler_name": null, "scheduling_gates": null, "security_context": null, "service_account": null, "service_account_name": "taskmaster", "set_hostname_as_fqdn": null, "share_process_namespace": null, "subdomain": null, "termination_grace_period_seconds": null, "tolerations": null, "topology_spread_constraints": null, "volumes": [{"aws_elastic_block_store": null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null, "config_map": null, "csi": null, "downward_api": {"default_mode": null, "items": [{"field_ref": {"api_version": null, "field_path": "metadata.labels"}, "mode": null, "path": "labels", "resource_field_ref": null}]}, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume": null, "flocker": null, "gce_persistent_disk": null, "git_repo": null, "glusterfs": null, "host_path": null, "iscsi": null, "name": "podinfo", "nfs": null, "persistent_volume_claim": null, "photon_persistent_disk": null, "portworx_volume": null, "projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret": null, "storageos": null, "vsphere_volume": null}, {"aws_elastic_block_store": null, "azure_disk": null, "azure_file": null, "cephfs": null, "cinder": null, "config_map": {"default_mode": null, "items": null, "name": "task-aa423895-42c2-417e-b080-2287f667f856", "optional": null}, "csi": null, "downward_api": null, "empty_dir": null, "ephemeral": null, "fc": null, "flex_volume": null, "flocker": null, "gce_persistent_disk": null, "git_repo": null, "glusterfs": null, "host_path": null, "iscsi": null, "name": "jsoninput", "nfs": null, "persistent_volume_claim": null, "photon_persistent_disk": null, "portworx_volume": null, "projected": null, "quobyte": null, "rbd": null, "scale_io": null, "secret": null, "storageos": null, "vsphere_volume": null}]}}, "ttl_seconds_after_finished": null}, "status": null}
Original file line number	Diff line number	Diff line change
`@@ -51,4 +51,4 @@ def get_commands_with_stream_redirects(self) -> List[str]:`
`51`	`51`
`52`	`52`	`result.append(" ".join(command_parts))`
`53`	`53`
`54`		`- return result`
	`54`	`+ return result`
Original file line number	Diff line number	Diff line change
`@@ -197,4 +197,6 @@ def get_executor_template_with_value_from_config(self) -> V1Job:`
`197`	`197`	`)`
`198`	`198`	`]`
`199`	`199`
	`200`	`+ job.spec.template.spec.containers[0].restart_policy = "Never"`
	`201`	`+`
`200`	`202`	`return job`