kvrocks loading + other template config fixes:

- load: don't attempt to overwrite kvrocks db if already exists
- load: download into temp dir first, then rename if successful
- load: skip loading if remote file doesn't exist, nothing to do but reset index
- save: bump max memory for save pod
- index import: set high backoff to ensure index jobs don't fail
- kvrocks: remove --appendonly flag, unsupported in kvrocks

Author: ikreymer

chart/app-templates/index-import-job.yaml

@@ -13,7 +13,9 @@ metadata:
 
 spec:
   ttlSecondsAfterFinished: 300
-  backoffLimit: 6
+
+  # keep retrying indefinitely
+  backoffLimit: 1000000
 
 # if purging, set completions to disallow changing parallelism
 # for import, omit to allow increasing later

chart/app-templates/redis.yaml

@@ -104,7 +104,7 @@ spec:
       imagePullPolicy: {{ redis_image_pull_policy }}
 
       {% if use_kvrocks %}
-      args: ["-c", "/redis-conf/kvrocks.conf", "--dir", "/data", "--appendonly", "{{ 'no' if load_dump else 'yes' }}"]
+      args: ["-c", "/redis-conf/kvrocks.conf", "--dir", "/data"]
       {% else %}
       args: ["/redis-conf/redis.conf", "--appendonly", "{{ 'no' if load_dump else 'yes' }}"]
       {% endif %}
@@ -237,7 +237,7 @@ spec:
 
       resources: &rclone_resources
         limits:
-          memory: "200Mi"
+          memory: "1000Mi"
 
         requests:
           memory: "200Mi"
@@ -253,11 +253,33 @@ spec:
         - |
           set -e
 
+          # if remote file doesn't exist, just return success (nothing to load)
+          if ! rclone lsf remote:{{ remote_file_path }} > /dev/null 2>&1; then
+            echo "Remote {{ remote_file_path }} does not exist, nothing to load, done"
+            exit 0
+          fi
+
         {% if use_kvrocks %}
-          mkdir -p /data/{{ local_file_dest }}
-          cd /data/{{ local_file_dest }}
+          # only run if target dir doesn't exist yet
+          if [ ! -d "/data/{{ local_file_dest }}" ]; then
+
+            # remove existing + create new empty tmp dir
+            rm -rf /data/{{ local_file_dest }}.tmp
+            mkdir -p /data/{{ local_file_dest }}.tmp
+
+            cd /data/{{ local_file_dest }}.tmp
 
-          rclone --config "" cat remote:{{ remote_file_path }} | tar xzf -
+            echo "Downloading from {{ remote_file_path }}"
+
+            rclone --config "" cat remote:{{ remote_file_path }} | tar xzf -
+
+            # move to final dest only if tar extract succeeds
+            mv /data/{{ local_file_dest }}.tmp /data/{{ local_file_dest }}
+
+            echo "Downloaded to /data/{{ local_file_dest }} successfully!"
+          else
+            echo "/data/{{ local_file_dest }} already present (likely pod restarted), not overwriting"
+          fi
 
         {% else %}
           rclone --config "" copyto remote:{{ remote_file_path }} /data/{{ local_file_dest }}