Skip to content

Commit 6281933

Browse files
genericdatagenericdata
committed
Correct error logic and logs in emails
1 parent 75bcda5 commit 6281933

File tree

2 files changed

+104
-60
lines changed

2 files changed

+104
-60
lines changed

‎job.py‎

Lines changed: 100 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -25,23 +25,38 @@ def redisjob(target_dir, timestamp, email, chrom, upstream_fasta, downstream_fas
2525
python_exec_path = os.path.expanduser("~/venv/bin/python")
2626

2727
if position:
28-
command = "bash ./run.sh {} {} {} {} {} {} {} {} {} {}".format(python_exec_path, target_dir, timestamp, email, chrom,
29-
ref_fastaURL, ref_gffURL, in_fasta,
30-
in_gff, position)
28+
command_list = ["bash", "./run.sh", python_exec_path, target_dir, timestamp, email, chrom,
29+
ref_fastaURL, ref_gffURL, in_fasta, in_gff, position]
3130
else:
32-
command = "bash ./run.sh {} {} {} {} {} {} {} {} {} {} {}".format(python_exec_path, target_dir, timestamp, email, chrom,
33-
ref_fastaURL, ref_gffURL, in_fasta,
34-
in_gff, upstream_fasta,
35-
downstream_fasta)
31+
command_list = ["bash", "./run.sh", python_exec_path, target_dir, timestamp, email, chrom,
32+
ref_fastaURL, ref_gffURL, in_fasta, in_gff, upstream_fasta, downstream_fasta]
33+
34+
status=1
3635
try:
37-
os.system(command)
38-
os.system("echo Emailing")
39-
send_email(email, timestamp)
40-
os.system("echo Emailed")
36+
# Using subprocess.run with check=True to raise an exception on non-zero exit codes
37+
# You might want to capture stdout/stderr for debugging:
38+
result = subprocess.run(command_list, check=True, capture_output=True, text=True)
39+
print("Command output:", result.stdout)
40+
print("Command error:", result.stderr)
41+
#subprocess.run(command_list, check=True) # This will raise CalledProcessError if run.sh fails
42+
43+
# os.system("echo Emailing") # Consider replacing os.system with subprocess.run here too
44+
# send_email(email, timestamp)
45+
# os.system("echo Emailed")
46+
status=0
4147
db_update(timestamp, "status", "complete")
42-
except:
43-
os.system("echo Command Failed")
44-
send_email_error(email)
48+
except subprocess.CalledProcessError as e:
49+
print(f"Command failed with exit code {e.returncode}")
50+
print(f"Error executing command: {e}") # Print the error for debugging
51+
db_update(timestamp, "status", "failed")
52+
except Exception as e: # Catch other potential exceptions during execution
53+
os.system("echo An unexpected error occurred")
54+
print(f"Unexpected error: {e}")
55+
db_update(timestamp, "status", "failed")
56+
57+
os.system("echo Emailing") # Consider replacing os.system with subprocess.run here too
58+
send_email(email, timestamp, status)
59+
os.system("echo Emailed")
4560

4661
# Determine if this file is accepted by checking the file extension
4762
def allowed_file(filename):
@@ -111,55 +126,82 @@ def download(target_dir, URL):
111126
if URL:
112127
return wget.download(URL, target_dir)
113128

114-
def send_email(email, timestamp):
115-
# Set DDL to 1 week later (168hrs)
129+
def send_email(email, timestamp, status):
130+
# Set DDL to 1 week later (168hrs) - only relevant for success email
116131
deadline = datetime.now() + timedelta(hours=168)
117132
deadline_str = deadline.strftime('%B %d, %Y')
118133

119-
# paths to the log files
120134
err_log_path = f"./downloads/{timestamp}/{timestamp}-worker-err.log"
121-
out_log_path = f"./downloads/{timestamp}/{timestamp}-worker-out.log"
122-
123-
# read the content of the log files
124-
with open(err_log_path, 'r') as file:
125-
err_log_content = file.read()
126-
127-
with open(out_log_path, 'r') as file:
128-
out_log_content = file.read()
129-
130-
with j.app_context():
131-
subject = f"Reform Results - Download Deadline: {deadline_str}"
132-
msg = Message(subject, sender='[email protected]', recipients=[email])
133-
msg.html = f"""<i>ref</i>orm job complete.
134-
<a href='https://reform.bio.nyu.edu/download/{timestamp}'>Click here to download results</a>.
135-
The file will be available for the next 7 days until {deadline_str}.<br><br>
136-
137-
If you use <i>ref</i>orm in your research, please cite the GitHub repository:<br>
138-
reform: https://github.com/gencorefacility/reform<br><br>
139-
140-
You may also cite our article:<br>
141-
Mohammed Khalfan, Eric Borenstein, Pieter Spealman, Farah Abdul-Rahman, and David Gresham (2021).<br>
142-
<i>Modifying Reference Sequence and Annotation Files Quickly and Reproducibly with reform.</i><br><br>
143-
144-
<b>Reform.py Output Log:</b><br><pre>{err_log_content}</pre><br>
145-
<b>Worker Output Log:</b><br><pre>{out_log_content}</pre>
146-
"""
147-
mail.send(msg)
148-
149-
# Remove the log files from the download folder
150-
os.remove(err_log_path)
151-
os.remove(out_log_path)
152-
153-
154-
def send_email_error(email):
155-
with j.app_context():
156-
msg = Message('reform results - error', sender='[email protected]', recipients=[email])
157-
msg.html = f""""<i>ref</i>orm job had an error. Please review and resubmit. <br><br>
158-
<b>Reform.py Output Log:</b><br><pre>{err_log_content}</pre><br>
159-
<b>Worker Output Log:</b><br><pre>{out_log_content}</pre>
160-
"""
161-
mail.send(msg)
135+
out_log_path = f"./results/{timestamp}/{timestamp}-worker-out.log" # Assuming a worker-out.log is also generated
136+
137+
err_log_content = "No error log content available."
138+
out_log_content = "No output log content available."
139+
140+
# Read the content of the log files (safely)
141+
if os.path.exists(err_log_path):
142+
try:
143+
with open(err_log_path, 'r') as file:
144+
err_log_content = file.read()
145+
except Exception as e:
146+
err_log_content = f"Could not read error log file: {e}"
147+
else:
148+
err_log_content = f"Error log file not found at: {err_log_path}"
149+
150+
151+
if os.path.exists(out_log_path):
152+
try:
153+
with open(out_log_path, 'r') as file:
154+
out_log_content = file.read()
155+
except Exception as e:
156+
out_log_content = f"Could not read output log file: {e}"
157+
else:
158+
out_log_content = f"Output log file not found at: {out_log_path}"
159+
160+
161+
with j.app_context(): # Ensure this context is available where send_email is called
162+
if status == 0:
163+
# Success Email
164+
subject = f"Reform Results - Download Deadline: {deadline_str}"
165+
msg = Message(subject, sender='[email protected]', recipients=[email])
166+
msg.html = f"""<i>ref</i>orm job complete.
167+
<a href='https://reform.bio.nyu.edu/download/{timestamp}'>Click here to download results</a>.
168+
The file will be available for the next 7 days until {deadline_str}.<br><br>
169+
170+
If you use <i>ref</i>orm in your research, please cite the GitHub repository:<br>
171+
https://github.com/gencorefacility/reform<br><br>
172+
173+
<b>Reform Output:</b><br><pre>{err_log_content}</pre><br>
174+
"""
175+
else:
176+
# Error Email
177+
subject = 'Reform Results - ERROR'
178+
msg = Message(subject, sender='[email protected]', recipients=[email])
179+
msg.html = f"""<i>ref</i>orm job had an error. Please review and resubmit. <br><br>
180+
<b>Output:</b><br><pre>{out_log_content}</pre>
181+
"""
182+
try:
183+
j.extensions['mail'].send(msg) # Correct way to access Flask-Mail instance
184+
print(f"Email sent successfully for job {timestamp} (Status: {status}) to {email}")
185+
except Exception as mail_e:
186+
print(f"Failed to send email for job {timestamp} (Status: {status}) to {email}: {mail_e}")
187+
188+
189+
# Remove the log files - this should happen regardless of email status
190+
# Be careful with os.remove if the files are critical for debugging or auditing.
191+
# It might be better to move them to an archive folder or clean up later.
192+
if os.path.exists(err_log_path):
193+
try:
194+
os.remove(err_log_path)
195+
print(f"Removed {err_log_path}")
196+
except Exception as e:
197+
print(f"Error removing {err_log_path}: {e}")
162198

199+
if os.path.exists(out_log_path):
200+
try:
201+
os.remove(out_log_path)
202+
print(f"Removed {out_log_path}")
203+
except Exception as e:
204+
print(f"Error removing {out_log_path}: {e}")
163205

164206
def db_create():
165207
db = sqlite3.connect('database.db')

‎run.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
22

3+
set -eo pipefail
4+
35
PYTHON_EXEC_PATH="$1"
46
shift
57

@@ -96,7 +98,7 @@ if [ ! -z "$position" ]; then
9698

9799
$PYTHON_EXEC_PATH reform.py --chrom $chrom --position $position --in_fasta ./uploads/$timestamp/$in_fasta \
98100
--in_gff ./uploads/$timestamp/$in_gff --ref_fasta "$ref_fasta_path" --ref_gff "$ref_gff_path" \
99-
--output_dir "./results/$timestamp/" 2>&1 | tee ./results/$timestamp/$timestamp-worker-err.log
101+
--output_dir "./results/$timestamp/" 2>&1 | tee ./results/$timestamp/$timestamp-worker-err.log || exit 1
100102
else
101103
echo $PYTHON_EXEC_PATH reform.py --chrom $chrom --upstream_fasta ./uploads/$timestamp/$upstream_fasta \
102104
--downstream_fasta ./uploads/$timestamp/$downstream_fasta --in_fasta ./uploads/$timestamp/$in_fasta \
@@ -106,7 +108,7 @@ else
106108
$PYTHON_EXEC_PATH reform.py --chrom $chrom --upstream_fasta ./uploads/$timestamp/$upstream_fasta \
107109
--downstream_fasta ./uploads/$timestamp/$downstream_fasta --in_fasta ./uploads/$timestamp/$in_fasta \
108110
--in_gff ./uploads/$timestamp/$in_gff --ref_fasta "$ref_fasta_path" --ref_gff "$ref_gff_path" \
109-
--output_dir "./results/$timestamp/" 2>&1 | tee ./results/$timestamp/$timestamp-worker-err.log
111+
--output_dir "./results/$timestamp/" 2>&1 | tee ./results/$timestamp/$timestamp-worker-err.log || exit 1
110112
fi
111113

112114
# remove upload folder

0 commit comments

Comments
 (0)