diff --git a/slicops/config.py b/slicops/config.py index 27abe8b..9b2f70e 100644 --- a/slicops/config.py +++ b/slicops/config.py @@ -40,7 +40,7 @@ def cfg(): vue_port=(8008, pykern.pkasyncio.cfg_port, "port of Vue dev server"), ), package_path=( - tuple(["slicops"]), + ("slicops",), tuple, "Names of root packages that should be checked for codes and resources. Order is important, the first package with a matching code/resource will be used.", ), diff --git a/slicops/ctx.py b/slicops/ctx.py index 5a6e4a6..b66eac7 100644 --- a/slicops/ctx.py +++ b/slicops/ctx.py @@ -10,6 +10,7 @@ import pykern.fconf import pykern.pkresource import pykern.util +import slicops.config import slicops.field import slicops.ui_layout @@ -32,10 +33,19 @@ def _check_raw(got): step = "yaml" try: - r = pykern.fconf.parse_all( - path or pykern.pkresource.file_path("sliclet"), - glob=f"{name}*", - ) + n = f"sliclet/{name}.yaml" + r = pykern.fconf.Parser( + [ + ( + path.join(n) + if path + else pykern.pkresource.file_path( + n, + packages=slicops.config.cfg().package_path, + ) + ) + ] + ).result _check_raw(r) step = "fields" self.fields = self.__parse(r[step], PKDict(), slicops.field.prototypes()) @@ -165,6 +175,9 @@ def _update(old, new): def group_get(self, field, group, attr=None): return self.__ctx.fields[field].group_get(group, attr) + def multi_get(self, fields): + return PKDict((k, self.__field(k).value_get()) for k in fields) + def multi_set(self, *args): def _args(): if len(args) > 1: diff --git a/slicops/package_data/.gitignore b/slicops/package_data/.gitignore index 5970902..3ed971e 100644 --- a/slicops/package_data/.gitignore +++ b/slicops/package_data/.gitignore @@ -1,2 +1,2 @@ -vue/ +vue ng-build/ diff --git a/slicops/package_data/sliclet/screen.yaml b/slicops/package_data/sliclet/screen.yaml index 0c7e516..b1f096d 100644 --- a/slicops/package_data/sliclet/screen.yaml +++ b/slicops/package_data/sliclet/screen.yaml @@ -24,6 +24,18 @@ fields: Gaussian: gaussian "Super Gaussian": super_gaussian value: gaussian + images_to_average: + prototype: Enum + constraints: + choices: + - 1 + - 2 + - 3 + - 4 + - 5 + ui: + label: Images to average + value: 1 plot: prototype: Dict ui: @@ -56,6 +68,11 @@ fields: ui: css_kind: danger label: Stop + save_to_file: + prototype: Button + ui: + css_kind: outline-info + label: Save to File target_in_button: prototype: Button ui: @@ -69,11 +86,12 @@ fields: ui_layout: - cols: - - css: col-sm-3 + - css: col-lg-3 rows: - beam_path - camera - pv + - images_to_average - cell_group: - start_button - stop_button @@ -82,13 +100,16 @@ ui_layout: - target_in_button - target_out_button - target_status - - css: col-sm-9 col-xxl-7 + - css: col-lg-9 col-xxl-7 rows: - plot - cols: - - css: col-sm-3 + - css: col-lg-3 rows: - curve_fit_method - - css: col-sm-3 + - css: col-lg-3 rows: - color_map + - css: col-md-1 + rows: + - save_to_file diff --git a/slicops/pkcli/service.py b/slicops/pkcli/service.py index 9e599c6..ab603e2 100644 --- a/slicops/pkcli/service.py +++ b/slicops/pkcli/service.py @@ -24,56 +24,67 @@ def ui_api(self, tcp_port=None, prod=False): """ from pykern import pkconfig, pkresource from pykern.api import server - from slicops import config, ui_api, quest - from tornado import web + from slicops import config, quest, sliclet, ui_api - def _tcp_port(): - return ( - PKDict(tcp_port=pkconfig.parse_positive_int(tcp_port)) - if tcp_port - else PKDict() - ) + from tornado import web - def _uri_map(config): - if prod: - return [ - ( - # very specific so we control the name space - r"^/(assets/[^/.]+\.(?:css|js)|favicon.png|index.html|)$", - web.StaticFileHandler, - PKDict( - path=str(pkresource.file_path("vue")), - default_filename="index.html", - ), - ), - ] + def _dev_uri_map(config): return [ ( # send any non-api call to the proxy rf"^(?!{config.api_uri}).*", - ProxyHandler, + _ProxyHandler, PKDict( proxy_url=f"http://localhost:{config.vue_port}", ), ), ] + def _prod_uri_map(config): + d = PKDict( + # TODO(robnagler) package_path + path=str(pkresource.file_path("vue")), + default_filename="index.html", + ) + return [ + # NOTE: StaticFileHandler requires match returns a group + ( + # very specific so we control the name space + r"^/(assets/[^/.]+\.(?:css|js)|favicon.png|index.html|)$", + web.StaticFileHandler, + d, + ), + ( + # vue index.html is returned for sliclet URLs + rf"^/($|(?:{'|'.join(sliclet.names())})(?:$|/.*))", + _VueIndexHandler, + d, + ), + ] + + def _tcp_port(): + return ( + PKDict(tcp_port=pkconfig.parse_positive_int(tcp_port)) + if tcp_port + else PKDict() + ) + c = config.cfg().ui_api.copy() server.start( attr_classes=quest.attr_classes(), api_classes=ui_api.api_classes(), http_config=c.pkupdate( PKDict( - uri_map=_uri_map(c), + uri_map=_prod_uri_map(c) if prod else _dev_uri_map(c), **_tcp_port(), ) ), ) -class ProxyHandler(tornado.web.RequestHandler): +class _ProxyHandler(tornado.web.RequestHandler): def initialize(self, proxy_url, **kwargs): - super(ProxyHandler, self).initialize(**kwargs) + super().initialize(**kwargs) self.http_client = tornado.httpclient.AsyncHTTPClient() self.proxy_url = proxy_url @@ -82,3 +93,8 @@ async def get(self): self.set_status(r.code) self.set_header("Content-Type", r.headers["Content-Type"]) self.write(r.body) + + +class _VueIndexHandler(tornado.web.StaticFileHandler): + def get_absolute_path(self, root, path, *args, **kwargs): + return super().get_absolute_path(root, self.default_filename) diff --git a/slicops/plot.py b/slicops/plot.py index 1317577..9126225 100644 --- a/slicops/plot.py +++ b/slicops/plot.py @@ -6,10 +6,115 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp +import h5py import numpy +import pykern.pkio import scipy.optimize +class ImageSet: + """Fits images, possibly averaging. + + Can take arbitrary meta data, e.g. pv and it will be written by + `save_file`. + + Args: + meta (PKDict): images_to_average, camera, curve_fit_method, pv + + """ + + def __init__(self, meta): + self.meta = meta + self._frames = [] + self._timestamps = [] + self._prev = None + + def add_frame(self, frame, timestamp): + """Add and update fit + + Args: + frame (ndarray): new image + timestamp (datetime): time of frame + Returns: + PKDict: frame and fit or None if not enough frames + """ + + def _mean(): + if self.meta.images_to_average == 1: + return self._frames[-1] + return numpy.mean(self._frames, axis=0) + + self._frames.append(frame) + self._timestamps.append(timestamp) + if len(self._frames) != self.meta.images_to_average: + return None + self._prev = PKDict( + fit=fit_image(_mean(), self.meta.curve_fit_method), + frames=self._frames, + timestamps=self._timestamps, + ) + self._frames = [] + self._timestamps = [] + return self._prev.fit + + def save_file(self, dir_path): + # TODO(robnagler) the naming is a bit goofy, possibly frames/{images,timestamps} and analysis. + """Creates a hdf5 file with the structure:: + /image Group + /frames Dataset {images_to_average, ysize, xsize} + /mean Dataset {ysize, xsize} + /timestamps Dataset {images_to_average} + /x Group + /fit Dataset {xsize} + /profile Dataset {xsize} + /y Group + /fit Dataset {ysize} + /profile Dataset {ysize} + /meta Group (camera, curve_fit_method, images_to_average, etc.) + + Args: + dir_path (py.path): directory + """ + + def _image_dim(meta_group, dim): + g = meta_group.create_group(dim) + f = self._prev.fit[dim] + g.create_dataset("profile", data=f.lineout) + if not f.fit.results: + return + g.attrs.update(f.fit.results) + g.create_dataset("fit", data=f.fit.fit_line) + + def _meta(h5_file): + g = h5_file.create_group("meta") + g.attrs.update(self.meta) + g.create_dataset("frames", data=self._prev.frames) + g.create_dataset( + "timestamps", + data=[d.timestamp() for d in self._prev.timestamps], + ) + + def _path(): + # TODO(robnagler) centralize timestamp format + t = self._prev.timestamps[-1] + rv = dir_path.join( + t.strftime("%Y-%m"), + f"{t.strftime('%Y%m%dT%H%M%SZ')}-{self.meta.camera}.h5", + ) + rv.dirpath().ensure(dir=True) + return rv + + def _writer(path): + with h5py.File(path, "w") as f: + _meta(f) + g = f.create_group("image") + g.create_dataset("mean", data=self._prev.fit.raw_pixels) + _image_dim(g, "x") + _image_dim(g, "y") + + pykern.pkio.atomic_write(_path(), writer=_writer) + + def fit_image(image, method): """Attemp an analytical fit for the sum along the x and y dimensions diff --git a/slicops/sliclet/__init__.py b/slicops/sliclet/__init__.py index 06d20f0..0e0b60e 100644 --- a/slicops/sliclet/__init__.py +++ b/slicops/sliclet/__init__.py @@ -11,9 +11,15 @@ import enum import importlib import inspect +import itertools +import pykern.pkconfig +import pykern.pkconst +import pykern.pkinspect +import pykern.pkio import pykern.util import queue import re +import slicops.config import slicops.ctx import slicops.field import threading @@ -30,11 +36,50 @@ class _Work(enum.IntEnum): _CTX_WRITE_ARGS = frozenset(["field_values"]) +_NAMES = None + def instance(name, queue): + def _import(name): + # TODO(robnagler) move to pykern, copied from sirepo.util + # NOTE: fixed a bug (s = None) + s = None + for p in slicops.config.cfg().package_path: + n = None + try: + n = f"{p}.sliclet.{name}" + return importlib.import_module(n) + except ModuleNotFoundError as e: + if n is not None and n != e.name: + # import is failing due to ModuleNotFoundError in a sub-import + # not the module we are looking for + raise + s = pkdexc() + pass + # gives more debugging info (perhaps more confusion) + if s: + pkdc(s) + raise AssertionError( + f"cannot find sliclet={name} in package_path={slicops.config.cfg().package_path}" + ) + if not name: name = _cfg.default - return importlib.import_module(f"slicops.sliclet.{name}").CLASS(name, queue) + return _import(name).CLASS(name, queue) + + +def names(): + + def _find(): + # TODO(robnagler) move to pykern, copied from sirepo + for p in slicops.config.cfg().package_path: + yield pykern.pkinspect.package_module_names(f"{p}.sliclet") + + global _NAMES + + if _NAMES is None: + _NAMES = tuple(sorted(itertools.chain.from_iterable(_find()))) + return _NAMES class Base: @@ -115,6 +160,9 @@ def lock_for_update(self, log_op=None): def put_exception(self, exc): self.__put_work(_Work.error, exc) + def save_file_path(self): + return _cfg.save_file_root.join(self.__class__.__name__).ensure(dir=True) + def session_end(self): self.__put_work(_Work.session_end, None) @@ -210,12 +258,28 @@ def _work_start(self, unused): return True +def _cfg_py_path(value): + if isinstance(value, str): + if not os.path.isabs(value): + pykern.pkconfig.raise_error(f"not an absolute path={value}") + return pykern.pkio.py_path(value) + if not isinstance(value, pykern.pkconst.PY_PATH_LOCAL_TYPE): + pykern.pkconfig.raise_error(f"not a py.path type={type(value)} value={value}") + return value + + def _init(): global _cfg - from pykern import pkconfig - _cfg = pkconfig.init( + def _path(): + rv = (_cfg_py_path, "root path for screen save files") + if pykern.pkconfig.in_dev_mode(): + return (pykern.util.dev_run_dir(_path).join("save").ensure(dir=True),) + rv + return pykern.pkconfig.Required(rv) + + _cfg = pykern.pkconfig.init( default=("screen", str, "default sliclet"), + save_file_root=_path(), ) diff --git a/slicops/sliclet/screen.py b/slicops/sliclet/screen.py index 105e265..e1628b2 100644 --- a/slicops/sliclet/screen.py +++ b/slicops/sliclet/screen.py @@ -6,7 +6,7 @@ from pykern.pkcollections import PKDict from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp -from slicops.device.screen import TargetStatus +import pykern.pkcompat import pykern.pkconfig import pykern.util import queue @@ -22,9 +22,9 @@ _cfg = None _BUTTONS_DISABLE = ( + ("single_button.ui.enabled", False), ("start_button.ui.enabled", False), ("stop_button.ui.enabled", False), - ("single_button.ui.enabled", False), ) _TARGET_DISABLE = ( @@ -51,6 +51,7 @@ ) _BUTTONS_VISIBLE = ( + ("images_to_average.ui.visible", True), ("single_button.ui.visible", True), ("start_button.ui.visible", True), ("stop_button.ui.visible", True), @@ -67,6 +68,8 @@ ("plot.value", None), ("pv.ui.visible", False), ("pv.value", None), + ("save_to_file.ui.enabled", False), + ("save_to_file.ui.visible", False), ) + _BUTTONS_DISABLE + _BUTTONS_INVISIBLE @@ -82,6 +85,8 @@ ("curve_fit_method.ui.enabled", True), ("curve_fit_method.ui.visible", True), ("plot.ui.visible", True), + ("save_to_file.ui.enabled", True), + ("save_to_file.ui.visible", True), ) @@ -99,8 +104,16 @@ def on_change_camera(self, txn, value, **kwargs): def on_change_beam_path(self, txn, value, **kwargs): self.__beam_path_change(txn, value) - def on_change_curve_fit_method(self, txn, **kwargs): - self.__update_plot(txn) + def on_change_curve_fit_method(self, txn, value, **kwargs): + # TODO(robnagler) optimize with ImageSet.update_curve_fit_method() + self.__new_image_set(txn) + + def on_change_images_to_average(self, txn, value, **kwargs): + # TODO(robnagler) optimize with ImageSet.update_images_to_average() + self.__new_image_set(txn) + + def on_click_save_to_file(self, txn, **kwargs): + self.__image_set.save_file(self.save_file_path()) def on_click_single_button(self, txn, **kwargs): self.__single_button = True @@ -174,6 +187,7 @@ def __device_change(self, txn, beam_path, camera): def __device_destroy(self, txn=None): if not self.__device: return + self.__image_set = None self.__single_button = False self.__handler.destroy() self.__handler = None @@ -246,13 +260,26 @@ def __handle_image(self, image): ("start_button.ui.enabled", True), ) + def __new_image_set(self, txn): + self.__image_set = slicops.plot.ImageSet( + txn.multi_get( + ("beam_path", "camera", "curve_fit_method", "images_to_average", "pv") + ), + ) + def __handle_target_status(self, status): with self.lock_for_update() as txn: self.__current_value["target"] = status txn.multi_set( ("target_status", status.name), - ("target_in_button.ui.enabled", status == TargetStatus.OUT), - ("target_out_button.ui.enabled", status == TargetStatus.IN), + ( + "target_in_button.ui.enabled", + status == slicops.device.screen.TargetStatus.OUT, + ), + ( + "target_out_button.ui.enabled", + status == slicops.device.screen.TargetStatus.IN, + ), ) def __set(self, txn, accessor, value, txn_set, method=None): diff --git a/slicops/unit_util.py b/slicops/unit_util.py index 1671bf7..9f27529 100644 --- a/slicops/unit_util.py +++ b/slicops/unit_util.py @@ -18,6 +18,9 @@ def __init__(self, sliclet, *args, **kwargs): self.__sliclet = sliclet super().__init__(*args, **kwargs) self.__update_q = asyncio.Queue() + self.__http_uri = ( + f"http://{self.http_config.tcp_ip}:{self.http_config.tcp_port}" + ) async def ctx_update(self): from pykern import pkunit @@ -36,6 +39,24 @@ async def ctx_field_set(self, **kwargs): self.__caller() await self.client.call_api("ui_ctx_write", PKDict(field_values=PKDict(kwargs))) + async def http_get(self, rel_uri): + from tornado import httpclient + + def _client(): + return httpclient.AsyncHTTPClient(force_instance=True) + + self.__caller() + return ( + await _client().fetch( + httpclient.HTTPRequest( + connect_timeout=_TIMEOUT, + method="GET", + request_timeout=_TIMEOUT, + url=self.__http_uri + rel_uri, + ), + ) + ).body + async def __aenter__(self): await super().__aenter__() asyncio.create_task(self.__subscribe()) @@ -59,8 +80,12 @@ def _server_config(self, *args, **kwargs): def _server_start(self, *args, **kwargs): from slicops.pkcli import service + from pykern.pkcollections import PKDict - service.Commands().ui_api() + k = PKDict() + if self.server_config.get("prod"): + k.prod = True + service.Commands().ui_api(**k) def __caller(self): from pykern import pkdebug, pkinspect diff --git a/tests/ctx_data/simple.in/input.yaml b/tests/ctx_data/simple.in/sliclet/input.yaml similarity index 100% rename from tests/ctx_data/simple.in/input.yaml rename to tests/ctx_data/simple.in/sliclet/input.yaml diff --git a/tests/plot_test.py b/tests/plot_test.py new file mode 100644 index 0000000..b98fe6c --- /dev/null +++ b/tests/plot_test.py @@ -0,0 +1,144 @@ +"""Test plot + +:copyright: Copyright (c) 2025 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +from pykern import pkunit + + +def test_imageset_save_to_file(): + from glob import glob + from pykern import pkio + import h5py + + i = _imageset() + with pkio.save_chdir(pkunit.work_dir()) as w: + i.imageset.save_file(w) + with h5py.File(glob("2024-09/*.h5")[0]) as h: + pkunit.pkeq( + h["/image/mean"][:].tolist(), + i.expected_mean, + ) + + +def test_imageset_stats(): + i = _imageset() + f = i.frame + pkunit.pkeq( + f.raw_pixels.tolist(), + i.expected_mean, + ) + pkunit.pkeq( + f.x.lineout.tolist(), + [0.5, 2.5, 12.5, 2.5, 0.5], + ) + pkunit.pkeq( + f.y.lineout.tolist(), + [3, 5, 7.5, 3], + ) + pkunit.pkeq(round(f.x.fit.results.sig, 2), 0.53) + pkunit.pkeq([round(v, 2) for v in f.x.fit.fit_line], [0.5, 2.5, 12.5, 2.5, 0.5]) + + +def _imageset(): + from datetime import datetime + from pykern.pkcollections import PKDict + from slicops.plot import ImageSet + import numpy + + def _image(values): + return numpy.array(values).reshape(4, 5) + + i = ImageSet( + PKDict( + images_to_average=2, + camera="Test", + curve_fit_method="gaussian", + pv="test", + ) + ) + pkunit.pkeq( + i.add_frame( + _image( + [ + 0, + 0, + 5, + 0, + 0, + 0, + 5, + 10, + 0, + 0, + 0, + 0, + 5, + 5, + 0, + 0, + 0, + 5, + 0, + 0, + ] + ), + datetime.strptime("2024-09-19 15:45:30", "%Y-%m-%d %H:%M:%S"), + ), + None, + ) + return PKDict( + imageset=i, + frame=i.add_frame( + _image( + [ + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + ] + ), + datetime.strptime("2024-09-19 15:45:31", "%Y-%m-%d %H:%M:%S"), + ), + expected_mean=_image( + [ + 0.5, + 0, + 2.5, + 0, + 0, + 0, + 2.5, + 5, + 0, + 0, + 0, + 0, + 2.5, + 2.5, + 0, + 0, + 0, + 2.5, + 0, + 0.5, + ] + ).tolist(), + ) diff --git a/tests/sliclet_data/sliclet/unit.yaml b/tests/sliclet_data/sliclet/unit.yaml new file mode 100644 index 0000000..5f6bf21 --- /dev/null +++ b/tests/sliclet_data/sliclet/unit.yaml @@ -0,0 +1,10 @@ +fields: + one: + prototype: Integer + value: 1 + +ui_layout: + - cols: + - css: col-lg-6 + rows: + - one diff --git a/tests/sliclet_data/somepkg/__init__.py b/tests/sliclet_data/somepkg/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/tests/sliclet_data/somepkg/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/sliclet_data/somepkg/sliclet/__init__.py b/tests/sliclet_data/somepkg/sliclet/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/tests/sliclet_data/somepkg/sliclet/__init__.py @@ -0,0 +1 @@ +# diff --git a/tests/sliclet_data/somepkg/sliclet/unit.py b/tests/sliclet_data/somepkg/sliclet/unit.py new file mode 100644 index 0000000..85021c9 --- /dev/null +++ b/tests/sliclet_data/somepkg/sliclet/unit.py @@ -0,0 +1,16 @@ +""" + +:copyright: Copyright (c) 2025 The Board of Trustees of the Leland Stanford Junior University, through SLAC National Accelerator Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All Rights Reserved. +:license: http://github.com/slaclab/slicops/LICENSE +""" + +from pykern.pkcollections import PKDict +from pykern.pkdebug import pkdc, pkdlog, pkdp +import slicops.sliclet + + +class Unit(slicops.sliclet.Base): + pass + + +CLASS = Unit diff --git a/tests/sliclet_data/vue/index.html b/tests/sliclet_data/vue/index.html new file mode 100644 index 0000000..cf7711b --- /dev/null +++ b/tests/sliclet_data/vue/index.html @@ -0,0 +1 @@ +xyzzy diff --git a/tests/sliclet_test.py b/tests/sliclet_test.py new file mode 100644 index 0000000..d659860 --- /dev/null +++ b/tests/sliclet_test.py @@ -0,0 +1,45 @@ +"""Test sliclet + +:copyright: Copyright (c) 2025 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +import pytest + + +@pytest.mark.asyncio(loop_scope="module") +async def test_all(monkeypatch): + prev_file_path = None + + def _file_path(path, *args, **kwargs): + nonlocal prev_file_path + from pykern import pkinspect, pkdebug + + if path in ("vue", "sliclet/unit.yaml"): + return pkunit.data_dir().join(path) + return prev_file_path(path, caller_context=pkinspect.caller_module()) + + def _init(): + nonlocal prev_file_path + + import os, sys + + os.environ["SLICOPS_CONFIG_PACKAGE_PATH"] = "somepkg:slicops" + + from pykern import pkunit + + sys.path.insert(0, str(pkunit.data_dir())) + + from pykern import pkresource + + prev_file_path = pkresource.file_path + monkeypatch.setattr(pkresource, "file_path", _file_path) + + _init() + + from slicops import unit_util + from pykern import pkunit, pkdebug + + async with unit_util.SlicletSetup("unit", prod=True) as s: + pkunit.pkeq(b"xyzzy\n", await s.http_get("")) + pkunit.pkeq(b"xyzzy\n", await s.http_get("/screen")) diff --git a/ui/src/components/VApp.vue b/ui/src/components/VApp.vue index c5175c0..b34551d 100644 --- a/ui/src/components/VApp.vue +++ b/ui/src/components/VApp.vue @@ -92,8 +92,8 @@ document.title = result.sliclet_title; if (! props.sliclet) { const u = '/' + result.sliclet_name; - if (route.path !== u) { - // avoid a possible redirect loop + // if default route or new app, just switch to it + if (route.path == '/' || ! (new RegExp(`^${u}(?:/|$)`)).test(route.path)) { router.replace(u); } }