From b3b3c8c5fb697b3e83f5053009245bfa1655750a Mon Sep 17 00:00:00 2001 From: Ludovic Bellier Date: Thu, 25 Apr 2024 16:44:11 -0700 Subject: [PATCH 1/4] initial implementation of isxd reading --- suite2p/io/__init__.py | 1 + suite2p/io/isxd.py | 106 +++++++++++++++++++++++++++++++++++++++++ suite2p/io/utils.py | 30 +++++++++++- suite2p/run_s2p.py | 14 +++++- 4 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 suite2p/io/isxd.py diff --git a/suite2p/io/__init__.py b/suite2p/io/__init__.py index cf1f71208..b24f757ea 100644 --- a/suite2p/io/__init__.py +++ b/suite2p/io/__init__.py @@ -12,3 +12,4 @@ from .dcam import dcimg_to_binary from .binary import BinaryFile, BinaryFileCombined from .server import send_jobs +from .isxd import isxd_to_binary diff --git a/suite2p/io/isxd.py b/suite2p/io/isxd.py new file mode 100644 index 000000000..dac2791e6 --- /dev/null +++ b/suite2p/io/isxd.py @@ -0,0 +1,106 @@ +""" +Copyright © 2024 Inscopix, Inc., a Bruker company. Authored by Ludovic Bellier. +""" +# import os +import numpy as np +from . import utils + +try: + import isx + HAS_ISX = True +except (ModuleNotFoundError, ImportError): + HAS_ISX = False + + +def isxd_to_binary(ops): + """ finds Inscopix isxd files and writes them to binaries + + Parameters + ---------- + ops : dictionary + "nplanes", "data_path", "save_path", "save_folder", "fast_disk", + "nchannels", "keep_movie_raw", "look_one_level_down" + + Returns + ------- + ops : dictionary of first plane + "Ly", "Lx", ops["reg_file"] or ops["raw_file"] is created binary + + """ + if not HAS_ISX: + raise ImportError("Inscopix isx is required for this file type, please 'pip install isx'") + + ops1 = utils.init_ops(ops) + # the following should be taken from the metadata and not needed but the files are initialized before... + nplanes = ops1[0]["nplanes"] + nchannels = ops1[0]["nchannels"] + # open all binary files for writing + ops1, file_list, reg_file, reg_file_chan2 = utils.find_files_open_binaries(ops1) + iall = 0 + for j in range(ops1[0]["nplanes"]): + ops1[j]["nframes_per_folder"] = np.zeros(len(file_list), np.int32) + ik = 0 + + for ifile, fname in enumerate(file_list): + f = isx.Movie.read(fname) + nplanes = 1 #f.shape[1] + nchannels = 1 #f.shape[2] + nframes = f.timing.num_samples + iblocks = np.arange(0, nframes, ops1[0]["batch_size"]) + if iblocks[-1] < nframes: + iblocks = np.append(iblocks, nframes) + + # data = nframes x nplanes x nchannels x pixels x pixels + if nchannels > 1: + nfunc = ops1[0]["functional_chan"] - 1 + else: + nfunc = 0 + # loop over all frames + for ichunk, onset in enumerate(iblocks[:-1]): + offset = iblocks[ichunk + 1] + im = np.array([f.get_frame_data(x) for x in np.arange(onset, offset)]) + im2mean = im.mean(axis=0).astype(np.float32) / len(iblocks) + for ichan in range(nchannels): + nframes = im.shape[0] + im2write = im[:] + for j in range(0, nplanes): + if iall == 0: + ops1[j]["meanImg"] = np.zeros((im.shape[1], im.shape[2]), + np.float32) + if nchannels > 1: + ops1[j]["meanImg_chan2"] = np.zeros( + (im.shape[1], im.shape[2]), np.float32) + ops1[j]["nframes"] = 0 + if ichan == nfunc: + ops1[j]["meanImg"] += np.squeeze(im2mean) + reg_file[j].write( + bytearray(im2write[:].astype("int16"))) + else: + ops1[j]["meanImg_chan2"] += np.squeeze(im2mean) + reg_file_chan2[j].write( + bytearray(im2write[:].astype("int16"))) + + ops1[j]["nframes"] += im2write.shape[0] + ops1[j]["nframes_per_folder"][ifile] += im2write.shape[0] + ik += nframes + iall += nframes + + # write ops files + do_registration = ops1[0]["do_registration"] + do_nonrigid = ops1[0]["nonrigid"] + for ops in ops1: + ops["Ly"] = im.shape[1] + ops["Lx"] = im.shape[2] + if not do_registration: + ops["yrange"] = np.array([0, ops["Ly"]]) + ops["xrange"] = np.array([0, ops["Lx"]]) + #ops["meanImg"] /= ops["nframes"] + #if nchannels>1: + # ops["meanImg_chan2"] /= ops["nframes"] + np.save(ops["ops_path"], ops) + # close all binary files and write ops files + for j in range(0, nplanes): + reg_file[j].close() + if nchannels > 1: + reg_file_chan2[j].close() + return ops1[0] diff --git a/suite2p/io/utils.py b/suite2p/io/utils.py index c317fd25a..743f6b45e 100644 --- a/suite2p/io/utils.py +++ b/suite2p/io/utils.py @@ -226,6 +226,7 @@ def get_nd2_list(ops): print("** Found %d nd2 files - converting to binary **" % (len(fsall))) return fsall, ops + def get_dcimg_list(ops): """ make list of dcimg files to process if ops["look_one_level_down"], then all dcimg"s in all folders + one level down @@ -247,6 +248,29 @@ def get_dcimg_list(ops): print("** Found %d dcimg files - converting to binary **" % (len(fsall))) return fsall, ops + +def get_isxd_list(ops): + """ make list of isxd files to process + if ops["look_one_level_down"], then all isxd"s in all folders + one level down + """ + froot = ops["data_path"] + fold_list = ops["data_path"] + fsall = [] + nfs = 0 + first_tiffs = [] + for k, fld in enumerate(fold_list): + fs, ftiffs = list_files(fld, ops["look_one_level_down"], ["*.isxd"]) + fsall.extend(fs) + first_tiffs.extend(list(ftiffs)) + if len(fs) == 0: + print("Could not find any isxd files") + raise Exception("no isxds") + else: + ops["first_tiffs"] = np.array(first_tiffs).astype("bool") + print("** Found %d isxd files - converting to binary **" % (len(fsall))) + return fsall, ops + + def find_files_open_binaries(ops1, ish5=False): """ finds tiffs or h5 files and opens binaries for writing @@ -295,7 +319,7 @@ def find_files_open_binaries(ops1, ish5=False): # find h5"s else: raise Exception("No h5 files found") - + elif input_format == "sbx": # find sbx fs, ops2 = get_sbx_list(ops1[0]) @@ -315,6 +339,10 @@ def find_files_open_binaries(ops1, ish5=False): fs, ops2 = get_dcimg_list(ops1[0]) print("DCAM image files:") print("\n".join(fs)) + elif input_format == "isxd": + fs, ops2 = get_isxd_list(ops1[0]) + print("Inscopix files:") + print("\n".join(fs)) else: # find tiffs fs, ops2 = get_tif_list(ops1[0]) diff --git a/suite2p/run_s2p.py b/suite2p/run_s2p.py index af2d68921..de230223f 100644 --- a/suite2p/run_s2p.py +++ b/suite2p/run_s2p.py @@ -50,6 +50,12 @@ except ImportError: HAS_DCIMG = False +try: + import isx + HAS_ISX = True +except ImportError: + HAS_ISX = False + from functools import partial from pathlib import Path @@ -409,7 +415,7 @@ def run_s2p(ops={}, db={}, server={}): plane_folders = natsorted([ f.path for f in os.scandir(save_folder) if f.is_dir() and f.name[:5] == "plane" ]) - + if len(plane_folders) > 0 and (ops.get("input_format") and ops["input_format"]=="binary"): # binary file is already made, will use current ops ops_paths = [os.path.join(f, "ops.npy") for f in plane_folders] @@ -470,6 +476,10 @@ def run_s2p(ops={}, db={}, server={}): ops["input_format"] = "dcimg" if not HAS_DCIMG: raise ImportError("dcimg not found; pip install dcimg") + elif ops.get("isxd"): + ops["input_format"] = "isxd" + if not HAS_ISX: + raise ImportError("isx not found; pip install isx") elif not "input_format" in ops: ops["input_format"] = "tif" elif ops["input_format"] == "movie": @@ -496,6 +506,8 @@ def run_s2p(ops={}, db={}, server={}): io.movie_to_binary, "dcimg": io.dcimg_to_binary, + "isxd": + io.isxd_to_binary, } if ops["input_format"] in convert_funs: ops0 = convert_funs[ops["input_format"]](ops.copy()) From 8ca94f46da40f69f46062a2bca2054f32aec8724 Mon Sep 17 00:00:00 2001 From: Ludovic Bellier Date: Wed, 19 Jun 2024 23:28:33 -0700 Subject: [PATCH 2/4] NWB saving safeguard for series of tif frames --- suite2p/io/nwb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/suite2p/io/nwb.py b/suite2p/io/nwb.py index 1f009764b..9f211f8e4 100644 --- a/suite2p/io/nwb.py +++ b/suite2p/io/nwb.py @@ -349,6 +349,8 @@ def save_nwb(save_folder): ) # link to external data external_data = ops["filelist"] if "filelist" in ops else [""] + if ops["filelist"] != [""] and ".ome.tif" in ops["filelist"][0]: + ops["filelist"] = [ops["filelist"][0]] image_series = TwoPhotonSeries( name="TwoPhotonSeries", dimension=[ops["Ly"], ops["Lx"]], From 053d63414ee140e332ffaf180e89eff3a19de642 Mon Sep 17 00:00:00 2001 From: Ludovic Bellier Date: Thu, 20 Jun 2024 00:33:56 -0700 Subject: [PATCH 3/4] fix nwb saving --- suite2p/io/nwb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suite2p/io/nwb.py b/suite2p/io/nwb.py index 9f211f8e4..effda9a2e 100644 --- a/suite2p/io/nwb.py +++ b/suite2p/io/nwb.py @@ -350,7 +350,7 @@ def save_nwb(save_folder): # link to external data external_data = ops["filelist"] if "filelist" in ops else [""] if ops["filelist"] != [""] and ".ome.tif" in ops["filelist"][0]: - ops["filelist"] = [ops["filelist"][0]] + external_data = [ops["filelist"][0]] image_series = TwoPhotonSeries( name="TwoPhotonSeries", dimension=[ops["Ly"], ops["Lx"]], From 69d5573feb7106bdc114e5cc55ff4f17f1bed2e0 Mon Sep 17 00:00:00 2001 From: Ludovic Bellier Date: Fri, 2 Aug 2024 12:52:32 -0700 Subject: [PATCH 4/4] fix frames_include --- suite2p/registration/register.py | 1 + 1 file changed, 1 insertion(+) diff --git a/suite2p/registration/register.py b/suite2p/registration/register.py index d0ce9f5f5..4d78ea4a4 100644 --- a/suite2p/registration/register.py +++ b/suite2p/registration/register.py @@ -643,6 +643,7 @@ def registration_wrapper(f_reg, f_raw=None, f_reg_chan2=None, f_raw_chan2=None, refImg=refImg, ops=ops) refImg, rmin, rmax, mean_img, rigid_offsets, nonrigid_offsets, zest = outputs yoff, xoff, corrXY = rigid_offsets + n_frames = len(yoff) if ops["nonrigid"]: yoff1, xoff1, corrXY1 = nonrigid_offsets