Include project and spider name in job logs on object storage (#47, PR #51)

This is the behaviour of scrapyd, and makes it much easier to find logs by accessing object storage.

Author: vlerkin

kubernetes.yaml

@@ -97,7 +97,7 @@ data:
     env_config   = spider-example-env
     repository   = ghcr.io/q-m/scrapyd-k8s-spider-example
 
-    # It is strongly recomended to set resource requests and limits on production.
+    # It is strongly recommended to set resource requests and limits on production.
     # They can be overridden on the project and spider level.
     [default.resources]
     requests_cpu = 0.2

scrapyd_k8s/joblogs/log_handler_k8s.py

@@ -298,6 +298,8 @@ def handle_events(self, event):
             if pod.metadata.labels.get("org.scrapy.job_id"):
                 job_id = pod.metadata.labels.get("org.scrapy.job_id")
                 pod_name = pod.metadata.name
+                spider = pod.metadata.labels.get("org.scrapy.spider")
+                project = pod.metadata.labels.get("org.scrapy.project")
                 thread_name = f"{self.namespace}_{pod_name}"
                 if pod.status.phase == 'Running':
                     if (thread_name in self.watcher_threads
@@ -320,7 +322,7 @@ def handle_events(self, event):
                                 logger.info(
                                     f"Removed local log file '{log_filename}' since it already exists in storage.")
                         else:
-                            self.object_storage_provider.upload_file(log_filename)
+                            self.object_storage_provider.upload_file(project, spider, log_filename)
                             os.remove(log_filename)
                             logger.info(f"Removed local log file '{log_filename}' after successful upload.")
                     else:

scrapyd_k8s/object_storage/libcloud_driver.py

@@ -10,6 +10,7 @@
 from libcloud.storage.providers import get_driver
 
 logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
 
 class LibcloudObjectStorage:
     """
@@ -113,14 +114,18 @@ def replace_var(match):
         result = result.replace(r'\${', '${')
         return result
 
-    def upload_file(self, local_path):
+    def upload_file(self, project, spider, local_path):
         """
         Uploads a file to the object storage container.
 
         Parameters
         ----------
         local_path : str
-            The local file path of the file to be uploaded.
+            The job_id that is passed as a local path.
+        project : str
+            The name of the project.
+        spider : str
+            The name of the spider.
 
         Returns
         -------
@@ -130,7 +135,8 @@ def upload_file(self, local_path):
         ----
         Logs information about the upload status or errors encountered.
         """
-        object_name = os.path.basename(local_path)
+        job_id = os.path.basename(local_path).replace('.txt', '')
+        object_name = f"logs/{project}/{spider}/{job_id}.log"
         try:
             container = self.driver.get_container(container_name=self._container_name)
             self.driver.upload_object(