set redis queues

Author: YuWei-CH

INSTALL/conf/dev_reform.ini

@@ -0,0 +1,13 @@
+[program:dev-reform]
+directory=/home/dev/reformWeb
+command=/home/dev/venv/bin/gunicorn app:app -b localhost:9000
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/reform/dev-reform.err.log
+stderr_logfile_maxbytes=10MB
+stderr_logfile_backups=5
+stdout_logfile=/var/log/reform/dev-reform.out.log
+stdout_logfile_maxbytes=10MB
+stdout_logfile_backups=5
+user=dev
+environment=REFORM_ENV="development"

INSTALL/conf/dev_worker.ini

@@ -0,0 +1,14 @@
+[program:dev-worker]
+directory=/home/dev/reformWeb
+environment=PATH="/home/dev/venv/bin:%(ENV_PATH)s"
+command=/home/dev/venv/bin/python /home/dev/venv/bin/rq worker dev_queue --logging_level=INFO
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/reform/dev-worker.err.log
+stderr_logfile_maxbytes=10MB
+stderr_logfile_backups=5
+stdout_logfile=/var/log/reform/dev-worker.out.log
+stdout_logfile_maxbytes=10MB
+stdout_logfile_backups=5
+user=dev
+chown=dev:dev

INSTALL/conf/supervisor_reform.ini INSTALL/conf/reform.iniINSTALL/conf/supervisor_reform.ini renamed to INSTALL/conf/reform.ini

@@ -10,3 +10,4 @@ stdout_logfile=/var/log/reform/reform.out.log
 stdout_logfile_maxbytes=10MB
 stdout_logfile_backups=5
 user=reform
+environment=REFORM_ENV="production"

INSTALL/conf/supervisor_worker.ini INSTALL/conf/worker.iniINSTALL/conf/supervisor_worker.ini renamed to INSTALL/conf/worker.ini

@@ -1,7 +1,7 @@
 [program:worker]
 directory=/home/reform/reformWeb
 environment=PATH="/home/reform/venv/bin:%(ENV_PATH)s"
-command=/home/reform/venv/bin/python /home/reform/venv/bin/rq worker --logging_level=ERROR
+command=/home/reform/venv/bin/python /home/reform/venv/bin/rq worker prod_queue --logging_level=ERROR
 autostart=true
 autorestart=true
 stderr_logfile=/var/log/reform/worker.err.log

app.py

@@ -85,8 +85,25 @@ def submit():
 
             # (4) Send the job to the backend
             # Connect to the Redis server and intial a queue
-            redis_conn = Redis()
-            q = Queue(connection=redis_conn, default_timeout=3000)
+            redis_conn = Redis() # This will connect to localhost:6379, db 0 by default
+
+            # Determine the environment based on the REFORM_ENV environment variable
+            reform_env = os.environ.get("REFORM_ENV")
+
+            if reform_env == "development":
+                queue_name = "dev_queue"
+            elif reform_env == "production":
+                queue_name = "prod_queue"
+            else:
+                # Fallback for safety, though it should ideally be set by Supervisor
+                print(f"WARNING: REFORM_ENV not set or unknown value: {reform_env}. Defaulting to 'default' queue.")
+                queue_name = "default"
+
+            # Initialize the Queue with the determined queue_name
+            q = Queue(queue_name, connection=redis_conn, default_timeout=3000)
+            # Connect to the Redis server and intial a queue
+            #redis_conn = Redis()
+            #q = Queue(connection=redis_conn, default_timeout=3000)
 
             # Push job function and parameters into RQ
             job = q.enqueue(redisjob, args=(target_dir,
@@ -199,9 +216,25 @@ def submit_test():
 
             # (4) Send job to the backend
             # Use the redis queue as same as production site
-            redis_conn = Redis()
+            redis_conn = Redis() # This will connect to localhost:6379, db 0 by default
+
+            # Determine the environment based on the REFORM_ENV environment variable
+            reform_env = os.environ.get("REFORM_ENV")
+
+            if reform_env == "development":
+                queue_name = "dev_queue"
+            elif reform_env == "production":
+                queue_name = "prod_queue"
+            else:
+                # Fallback for safety, though it should ideally be set by Supervisor
+                print(f"WARNING: REFORM_ENV not set or unknown value: {reform_env}. Defaulting to 'default' queue.")
+                queue_name = "default"
+
+            # Initialize the Queue with the determined queue_name
+            q = Queue(queue_name, connection=redis_conn, default_timeout=3000)
+            #redis_conn = Redis()
 
-            q = Queue(connection=redis_conn, default_timeout=3000)
+            #q = Queue(connection=redis_conn, default_timeout=3000)
 
             job = q.enqueue(redisjob, args=(target_dir,
                                             timestamp,