From 290b4af647996aeefb98053a6952b3f5115d23fd Mon Sep 17 00:00:00 2001 From: Florian Rupprecht Date: Mon, 20 Feb 2023 10:05:27 -0500 Subject: [PATCH 1/2] Avoid cyclic dependency --- src/cpac/backends/__init__.py | 29 +---------------------------- src/cpac/backends/core.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 28 deletions(-) create mode 100644 src/cpac/backends/core.py diff --git a/src/cpac/backends/__init__.py b/src/cpac/backends/__init__.py index 60821047..faee3672 100644 --- a/src/cpac/backends/__init__.py +++ b/src/cpac/backends/__init__.py @@ -1,28 +1 @@ -class BackendMapper(object): - - parameters = {} - - def __init__(self, *args, **kwargs): - self.parameters = kwargs - - def __call__(self, platform, parent=None): - return self._clients[platform.__class__]( - platform=platform, - **self.parameters, - parent=parent - ) - - -def Backends(platform, **kwargs): - """ - Given a string, return a Backend - """ - from .docker import Docker - from .singularity import Singularity - - return( - { - 'docker': Docker, - 'singularity': Singularity - }[platform](**kwargs) - ) +from .core import Backends, BackendMapper diff --git a/src/cpac/backends/core.py b/src/cpac/backends/core.py new file mode 100644 index 00000000..c05cd27b --- /dev/null +++ b/src/cpac/backends/core.py @@ -0,0 +1,34 @@ +from cpac.utils.osutils import IS_PLATFORM_WINDOWS + + +class BackendMapper(object): + + parameters = {} + + def __init__(self, *args, **kwargs): + self.parameters = kwargs + + def __call__(self, platform, parent=None): + return self._clients[platform.__class__]( + platform=platform, + **self.parameters, + parent=parent + ) + + +def Backends(platform, **kwargs): + """ + Given a string, return a Backend + """ + from .docker import Docker + if not IS_PLATFORM_WINDOWS: + from .singularity import Singularity + else: + Singularity = None + + return( + { + 'docker': Docker, + 'singularity': Singularity + }[platform](**kwargs) + ) \ No newline at end of file From 3d3d71fc93298da3b74bad794d36f2c5ef5fa42c Mon Sep 17 00:00:00 2001 From: Florian Rupprecht Date: Mon, 20 Feb 2023 10:05:46 -0500 Subject: [PATCH 2/2] Basic windows support --- requirements.txt | 2 +- setup.cfg | 6 +++--- src/cpac/__main__.py | 23 ++++++++++++-------- src/cpac/backends/docker.py | 40 +++++++++++++++++++++++------------ src/cpac/backends/platform.py | 18 +++++++++++----- src/cpac/checks/__init__.py | 0 src/cpac/checks/checks.py | 32 ++++++++++++++++++++++++++++ src/cpac/utils/osutils.py | 3 +++ src/cpac/utils/utils.py | 19 +++++++++++++++++ 9 files changed, 112 insertions(+), 31 deletions(-) create mode 100644 src/cpac/checks/__init__.py create mode 100644 src/cpac/checks/checks.py create mode 100644 src/cpac/utils/osutils.py diff --git a/requirements.txt b/requirements.txt index 969a7576..fffcce50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ docker-pycreds pandas >= 0.23.4 pyyaml setuptools -spython >= 0.0.81 +spython >= 0.0.81 ; platform_system!="Windows" tabulate >= 0.8.6 tornado websocket-client diff --git a/setup.cfg b/setup.cfg index ed86cbc8..a1d8b872 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,8 +9,8 @@ author = C-PAC developers author-email = cpac@cnl.childmind.org license = mit url = https://github.com/FCP-INDI/cpac -long-description = file: README.rst -long-description-content-type: text/x-rst +#long-description = file: README.rst +#long-description-content-type: text/x-rst; charset=UTF-8 # Change if running only on Windows, Mac or Linux (comma-separated) platforms = any # Add here all kinds of additional classifiers as defined under @@ -40,7 +40,7 @@ install_requires = dockerpty docker-pycreds pandas >= 0.23.4 - spython >= 0.0.81 + spython >= 0.0.81 ; platform_system!="Windows" pyyaml rich tabulate >= 0.8.6 diff --git a/src/cpac/__main__.py b/src/cpac/__main__.py index 62e166aa..2b80521f 100644 --- a/src/cpac/__main__.py +++ b/src/cpac/__main__.py @@ -11,6 +11,7 @@ from cpac import __version__ from cpac.backends import Backends +from cpac.utils.osutils import IS_PLATFORM_WINDOWS from cpac.helpers import cpac_parse_resources as parse_resources, TODOs _logger = logging.getLogger(__name__) @@ -220,11 +221,12 @@ def _parser(): parse_resources.set_args(subparsers.add_parser( 'parse-resources', add_help=True, aliases=['parse_resources'], - help='\n'.join([parse_resources.__doc__.split( - parse_resources.__file__.split('/', maxsplit=-1)[-1], - maxsplit=1)[-1].strip().replace( - r'`cpac_parse_resources`', '"parse-resources"'), - 'See "cpac parse-resources --help" for more information.']))) + help=parse_resources.__doc__ + .replace('cpac_parse_resources.py', '') + .strip() + .replace(r'`cpac_parse_resources`', '"parse-resources"') + + '\nSee "cpac parse-resources --help" for more information.' + )) crash_parser = subparsers.add_parser( 'crash', add_help=True, @@ -438,16 +440,19 @@ def run(): # parse args parsed = parse_args(args) if not parsed.platform and "--platform" not in args: - if parsed.image and os.path.exists(parsed.image): + if not IS_PLATFORM_WINDOWS and parsed.image and os.path.exists(parsed.image): parsed.platform = 'singularity' else: parsed.platform = 'docker' try: main(parsed) # fall back on Singularity if Docker not found - except (DockerException, NotFound): # pragma: no cover - parsed.platform = 'singularity' - main(parsed) + except (DockerException, NotFound) as exc: # pragma: no cover + if IS_PLATFORM_WINDOWS: + raise exc + else: + parsed.platform = 'singularity' + main(parsed) else: main(parsed) diff --git a/src/cpac/backends/docker.py b/src/cpac/backends/docker.py index 98d2f247..551a110a 100644 --- a/src/cpac/backends/docker.py +++ b/src/cpac/backends/docker.py @@ -1,11 +1,15 @@ import os +from cpac.utils.osutils import IS_PLATFORM_WINDOWS +from cpac.utils.utils import windows_path_to_docker + import docker -import dockerpty from docker.errors import ImageNotFound from cpac.backends.platform import Backend, PlatformMeta +if not IS_PLATFORM_WINDOWS: + import dockerpty class Docker(Backend): def __init__(self, **kwargs): @@ -57,8 +61,8 @@ def _collect_config(self, **kwargs): if isinstance(self.pipeline_config, str): container_kwargs = {'image': self.image} if os.path.exists(self.pipeline_config): - container_kwargs['volumes'] = {self.pipeline_config: { - 'bind': self.pipeline_config, + container_kwargs['volumes'] = {windows_path_to_docker(self.pipeline_config): { + 'bind': windows_path_to_docker(self.pipeline_config), 'mode': 'ro', }} try: @@ -68,7 +72,7 @@ def _collect_config(self, **kwargs): self.pull(**kwargs) container = self.client.containers.create( **container_kwargs) - stream = container.get_archive(path=self.pipeline_config)[0] + stream = container.get_archive(path=windows_path_to_docker(self.pipeline_config))[0] self.config = b''.join([ l for l in stream # noqa E741 ]).split(b'\x000000000')[-1].replace(b'\x00', b'').decode() @@ -143,6 +147,13 @@ def _execute(self, command, run_type='run', **kwargs): **self.docker_kwargs } + if IS_PLATFORM_WINDOWS: + shared_kwargs['working_dir'] = windows_path_to_docker(shared_kwargs['working_dir']) + + for i in range(len(command)): + if len(command[i]) > 1 and command[i][1] == ':': + command[i] = windows_path_to_docker(command[i]) + if run_type == 'run': self.container = self.client.containers.run( **shared_kwargs, @@ -170,15 +181,18 @@ def _execute(self, command, run_type='run', **kwargs): stream=True )[1] elif run_type == 'enter': - self.container = self.client.containers.create( - **shared_kwargs, - auto_remove=True, - entrypoint='/bin/bash', - stdin_open=True, - tty=True, - detach=False - ) - dockerpty.start(self.client.api, self.container.id) + if IS_PLATFORM_WINDOWS: + raise NotImplementedError() + else: + self.container = self.client.containers.create( + **shared_kwargs, + auto_remove=True, + entrypoint='/bin/bash', + stdin_open=True, + tty=True, + detach=False + ) + dockerpty.start(self.client.api, self.container.id) return container_return def get_response(self, command, **kwargs): diff --git a/src/cpac/backends/platform.py b/src/cpac/backends/platform.py index a612effc..24c585e7 100644 --- a/src/cpac/backends/platform.py +++ b/src/cpac/backends/platform.py @@ -1,7 +1,10 @@ """Base classes for platform-specific implementations""" +from cpac.utils.osutils import IS_PLATFORM_WINDOWS + import atexit import os -import pwd +if not IS_PLATFORM_WINDOWS: + import pwd import tempfile import textwrap @@ -411,10 +414,15 @@ def _set_bindings(self, **kwargs): if kwargs.get('config_bindings'): for binding in kwargs['config_bindings']: self._bind_volume(binding) - self.uid = os.getuid() - pwuid = pwd.getpwuid(self.uid) - self.username = getattr(pwuid, 'pw_name', - getattr(pwuid, 'pw_gecos', str(self.uid))) + + if IS_PLATFORM_WINDOWS: + self.username = os.getlogin() + self.uid = hash(self.username) % 100 # TODO: Is there something we have to keep in mind here? + else: + self.uid = os.getuid() + pwuid = pwd.getpwuid(self.uid) + self.username = getattr(pwuid, 'pw_name', + getattr(pwuid, 'pw_gecos', str(self.uid))) self.bindings.update({ 'tag': tag, 'uid': self.uid, diff --git a/src/cpac/checks/__init__.py b/src/cpac/checks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cpac/checks/checks.py b/src/cpac/checks/checks.py new file mode 100644 index 00000000..2d85a018 --- /dev/null +++ b/src/cpac/checks/checks.py @@ -0,0 +1,32 @@ +"""Functions to check things like the in-container C-PAC version.""" +from semver import VersionInfo + +from cpac.backends import Backends + + +def check_version_at_least(min_version, platform, image=None, tag=None): + """Function to check the in-container C-PAC version + + Parameters + ---------- + min_version : str + Semantic version + + platform : str or None + + image : str or None + + tag : str or None + + Returns + ------- + bool + Is the version at least the minimum version? + """ + if platform is None: + platform = 'docker' + arg_vars = {'platform': platform, 'image': image, 'tag': tag, + 'command': 'version'} + return VersionInfo.parse(min_version) <= VersionInfo.parse( + Backends(**arg_vars).run( + run_type='version').versions.CPAC.lstrip('v')) diff --git a/src/cpac/utils/osutils.py b/src/cpac/utils/osutils.py new file mode 100644 index 00000000..67401dc8 --- /dev/null +++ b/src/cpac/utils/osutils.py @@ -0,0 +1,3 @@ +import platform + +IS_PLATFORM_WINDOWS = platform.system() == 'Windows' diff --git a/src/cpac/utils/utils.py b/src/cpac/utils/utils.py index 46303398..44079257 100644 --- a/src/cpac/utils/utils.py +++ b/src/cpac/utils/utils.py @@ -2,6 +2,10 @@ import os +import pathlib as pl +from os import PathLike +from typing import Union + from itertools import permutations from typing import Iterator, overload from warnings import warn @@ -9,6 +13,7 @@ import yaml from cpac import dist_name +from cpac.utils.osutils import IS_PLATFORM_WINDOWS class LocalsToBind: @@ -175,6 +180,18 @@ def _warn_if_undefined(self): return False +def windows_path_to_docker(path: Union[str, PathLike]): + if isinstance(path, str) and ':' not in path: + return path + + win_path = pl.Path(path) + if win_path.is_absolute(): + win_drive = win_path.drive + win_dir = win_path.relative_to(win_drive).as_posix() + return f'/{win_drive[0].lower()}{win_dir}' + return win_path.as_posix() + + class Volume: '''Class to store bind volume information''' @overload @@ -201,6 +218,8 @@ def __repr__(self): return str(self) def __str__(self): + if IS_PLATFORM_WINDOWS: + return f'{windows_path_to_docker(self.local)}:{windows_path_to_docker(self.bind)}:{self.mode}' return f'{self.local}:{self.bind}:{self.mode}'