From b4c608a4b53cab7ddbce6098d6539af42a09b7c6 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Fri, 1 Aug 2025 12:11:54 +0200 Subject: [PATCH 1/6] use requirements files for package installs and make lint envs consistent --- .github/workflows/python-sanity-check.yml | 6 ++++-- local-requirements.txt | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-sanity-check.yml b/.github/workflows/python-sanity-check.yml index ada340686..6429fed73 100644 --- a/.github/workflows/python-sanity-check.yml +++ b/.github/workflows/python-sanity-check.yml @@ -76,7 +76,9 @@ jobs: run: | sudo apt install -y libenchant-2-dev libcrack2-dev libssl-dev python -m pip install --upgrade pip - pip install flake8 pylint #pytest + if [ -f local-requirements.txt ]; then pip install -r local-requirements.txt; fi + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f recommended.txt ]; then pip install -r recommended.txt; fi # We may need git installed to get a full repo clone rather than unpacked archive - name: Check out source repository uses: actions/checkout@v4 @@ -125,7 +127,7 @@ jobs: run: | dnf install -y enchant cracklib openssl-devel python -m pip install --upgrade pip - pip install flake8 pylint #pytest + if [ -f local-requirements.txt ]; then pip install -r local-requirements.txt; fi if [ -f requirements.txt ]; then pip install -r requirements.txt; fi if [ -f recommended.txt ]; then pip install -r recommended.txt; fi # We need git installed to get a full repo clone rather than unpacked archive diff --git a/local-requirements.txt b/local-requirements.txt index 7faf3b026..14fd4527b 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -3,8 +3,10 @@ # This list is mainly used to specify addons needed for the unit tests. # We only need autopep8 on py 3 as it's used in 'make fmt' (with py3) autopep8;python_version >= "3" +flake8 # We need paramiko for the ssh unit tests # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support paramiko;python_version >= "3.7" paramiko<3;python_version < "3.7" +pylint werkzeug From 98d70c945a610da5f215f9fb1aee9753e4145bc0 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Wed, 28 Aug 2024 11:43:03 +0200 Subject: [PATCH 2/6] Implement an intitial server based on flask. Do the core work necessary to have a first response served to a GET request and some basic wiring to allow submission of JSON via POST. CHECKPOINT: running coreapi server overrides? not yet sure why this was needed Implement a user creation endpoint in the server. Back a POST handler onto the refactored createuser functionality file which exposes a function that can be invoked programatically with a configuration. Split out the response data decoding chunk. axe depedency pn reworkings on userapi fixups - move closer to the server working without createuser changes Implement finding users by email address. relocate fixup carve out payloads and rework them to make use of the validation helper shut up flake complaints about things that have changed further fixup another --- mig/lib/__init__.py | 0 mig/lib/coresvc/__init__.py | 2 + mig/lib/coresvc/__main__.py | 30 ++++ mig/lib/coresvc/server.py | 248 ++++++++++++++++++++++++++++++++++ mig/shared/useradm.py | 4 +- requirements.txt | 1 + tests/support/httpsupp.py | 98 ++++++++++++++ tests/support/serversupp.py | 3 + tests/test_mig_lib_coresvc.py | 245 +++++++++++++++++++++++++++++++++ 9 files changed, 630 insertions(+), 1 deletion(-) create mode 100644 mig/lib/__init__.py create mode 100644 mig/lib/coresvc/__init__.py create mode 100644 mig/lib/coresvc/__main__.py create mode 100755 mig/lib/coresvc/server.py create mode 100644 tests/support/httpsupp.py create mode 100644 tests/test_mig_lib_coresvc.py diff --git a/mig/lib/__init__.py b/mig/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/coresvc/__init__.py b/mig/lib/coresvc/__init__.py new file mode 100644 index 000000000..412a33c62 --- /dev/null +++ b/mig/lib/coresvc/__init__.py @@ -0,0 +1,2 @@ +from mig.lib.coresvc.server import ThreadedApiHttpServer, \ + _create_and_expose_server diff --git a/mig/lib/coresvc/__main__.py b/mig/lib/coresvc/__main__.py new file mode 100644 index 000000000..1a8155104 --- /dev/null +++ b/mig/lib/coresvc/__main__.py @@ -0,0 +1,30 @@ +from argparse import ArgumentError +from getopt import getopt +import sys + +from mig.shared.conf import get_configuration_object +from mig.services.coreapi.server import main as server_main + + +def _getopt_opts_to_options(opts): + options = {} + for k, v in opts: + options[k[1:]] = v + return options + + +def _required_argument_error(option, argument_name): + raise ArgumentError(None, 'Missing required argument: %s %s' % + (option, argument_name.upper())) + + +if __name__ == '__main__': + (opts, args) = getopt(sys.argv[1:], 'c:') + opts_dict = _getopt_opts_to_options(opts) + + if 'c' not in opts_dict: + raise _required_argument_error('-c', 'config_file') + + configuration = get_configuration_object(opts_dict['c'], + skip_log=True, disable_auth_log=True) + server_main(configuration) diff --git a/mig/lib/coresvc/server.py b/mig/lib/coresvc/server.py new file mode 100755 index 000000000..bdc1f5f29 --- /dev/null +++ b/mig/lib/coresvc/server.py @@ -0,0 +1,248 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# mig/services/coreapi/server - coreapi service server internals +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + + +"""HTTP server parts of the coreapi service.""" + +from __future__ import print_function +from __future__ import absolute_import + +from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import ThreadingMixIn + +from flask import Flask, request, Response +import json +import os +import sys +import threading +import time +import werkzeug.exceptions as httpexceptions +from wsgiref.simple_server import WSGIRequestHandler + +from mig.lib.coresvc.payloads import PayloadException, \ + PAYLOAD_POST_USER +from mig.shared.base import canonical_user, keyword_auto, force_native_str_rec +from mig.shared.useradm import fill_user, \ + create_user as useradm_create_user, search_users as useradm_search_users + + +httpexceptions_by_code = { + exc.code: exc for exc in httpexceptions.__dict__.values() if hasattr(exc, 'code')} + + +def http_error_from_status_code(http_status_code, http_url, description=None): + return httpexceptions_by_code[http_status_code](description) + + +def json_reponse_from_status_code(http_status_code, content): + json_content = json.dumps(content) + return Response(json_content, http_status_code, { 'Content-Type': 'application/json' }) + + +def _create_user(configuration, payload): + user_dict = canonical_user( + configuration, payload, PAYLOAD_POST_USER._fields) + fill_user(user_dict) + force_native_str_rec(user_dict) + + try: + useradm_create_user(user_dict, configuration, keyword_auto, default_renew=True) + except: + raise http_error_from_status_code(500, None) + user_email = user_dict['email'] + objects = search_users(configuration, { + 'email': user_email + }) + if len(objects) != 1: + raise http_error_from_status_code(400, None) + return objects[0] + + +def search_users(configuration, search_filter): + _, hits = useradm_search_users(search_filter, configuration, keyword_auto) + return list((obj for _, obj in hits)) + + +def _create_and_expose_server(server, configuration): + app = Flask('coreapi') + + @app.get('/user') + def GET_user(): + raise http_error_from_status_code(400, None) + + @app.get('/user/') + def GET_user_username(username): + return 'FOOBAR' + + @app.get('/user/find') + def GET_user_find(): + query_params = request.args + + objects = search_users(configuration, { + 'email': query_params['email'] + }) + + if len(objects) != 1: + raise http_error_from_status_code(404, None) + + return dict(objects=objects) + + @app.post('/user') + def POST_user(): + payload = request.get_json() + + try: + payload = PAYLOAD_POST_USER.ensure(payload) + except PayloadException as vr: + return http_error_from_status_code(400, None, vr.serialize()) + + user = _create_user(configuration, payload) + return json_reponse_from_status_code(201, user) + + return app + + +class ApiHttpServer(HTTPServer): + """ + http(s) server that contains a reference to an OpenID Server and + knows its base URL. + Extended to fork on requests to avoid one slow or broken login stalling + the rest. + """ + + def __init__(self, configuration, logger=None, host=None, port=None, **kwargs): + self.configuration = configuration + self.logger = logger if logger else configuration.logger + self.server_app = None + self._on_start = kwargs.pop('on_start', lambda _: None) + + addr = (host, port) + HTTPServer.__init__(self, addr, ApiHttpRequestHandler, **kwargs) + + @property + def base_environ(self): + return {} + + def get_app(self): + return self.server_app + + def server_activate(self): + HTTPServer.server_activate(self) + self._on_start(self) + + +class ThreadedApiHttpServer(ThreadingMixIn, ApiHttpServer): + """Multi-threaded version of the ApiHttpServer""" + + @property + def base_url(self): + proto = 'http' + return '%s://%s:%d/' % (proto, self.server_name, self.server_port) + + +class ApiHttpRequestHandler(WSGIRequestHandler): + """TODO: docstring""" + + def __init__(self, socket, addr, server, **kwargs): + self.server = server + + # NOTE: drop idle clients after N seconds to clean stale connections. + # Does NOT include clients that connect and do nothing at all :-( + self.timeout = 120 + + self._http_url = None + self.parsed_uri = None + self.path_parts = None + self.retry_url = '' + + WSGIRequestHandler.__init__(self, socket, addr, server, **kwargs) + + @property + def configuration(self): + return self.server.configuration + + @property + def daemon_conf(self): + return self.server.configuration.daemon_conf + + @property + def logger(self): + return self.server.logger + + +def start_service(configuration, host=None, port=None): + assert host is not None, "required kwarg: host" + assert port is not None, "required kwarg: port" + + logger = configuration.logger + + def _on_start(server, *args, **kwargs): + server.server_app = _create_and_expose_server( + None, server.configuration) + + httpserver = ThreadedApiHttpServer( + configuration, host=host, port=port, on_start=_on_start) + + serve_msg = 'Server running at: %s' % httpserver.base_url + logger.info(serve_msg) + print(serve_msg) + while True: + logger.debug('handle next request') + httpserver.handle_request() + logger.debug('done handling request') + httpserver.expire_volatile() + + +def main(configuration=None): + if not configuration: + from mig.shared.conf import get_configuration_object + # Force no log init since we use separate logger + configuration = get_configuration_object(skip_log=True) + + logger = configuration.logger + + # Allow e.g. logrotate to force log re-open after rotates + #register_hangup_handler(configuration) + + # FIXME: + host = 'localhost' # configuration.user_openid_address + port = 5555 # configuration.user_openid_port + server_address = (host, port) + + info_msg = "Starting coreapi..." + logger.info(info_msg) + print(info_msg) + + try: + start_service(configuration, host=host, port=port) + except KeyboardInterrupt: + info_msg = "Received user interrupt" + logger.info(info_msg) + print(info_msg) + info_msg = "Leaving with no more workers active" + logger.info(info_msg) + print(info_msg) diff --git a/mig/shared/useradm.py b/mig/shared/useradm.py index eac6cfe7c..73d60e4eb 100644 --- a/mig/shared/useradm.py +++ b/mig/shared/useradm.py @@ -2322,7 +2322,9 @@ def search_users(search_filter, conf_path, db_path, fnmatch for. """ - if conf_path: + if isinstance(conf_path, Configuration): + configuration = conf_path + elif conf_path: if isinstance(conf_path, basestring): configuration = Configuration(conf_path) else: diff --git a/requirements.txt b/requirements.txt index 5c2b1bc8f..8c398a2ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ # migrid core dependencies on a format suitable for pip install as described on # https://pip.pypa.io/en/stable/reference/requirement-specifiers/ +flask future # cgi was removed from the standard library in Python 3.13 diff --git a/tests/support/httpsupp.py b/tests/support/httpsupp.py new file mode 100644 index 000000000..4a115c489 --- /dev/null +++ b/tests/support/httpsupp.py @@ -0,0 +1,98 @@ +import codecs +import json + +from tests.support._env import PY2 + +if PY2: + from urllib2 import HTTPError, Request, urlopen + from urllib import urlencode +else: + from urllib.error import HTTPError + from urllib.parse import urlencode + from urllib.request import urlopen, Request + + +def attempt_to_decode_response_data(data, response_encoding=None): + if data is None: + return None + elif response_encoding == 'textual': + data = codecs.decode(data, 'utf8') + + try: + return json.loads(data) + except Exception as e: + return data + elif response_encoding == 'binary': + return data + else: + raise AssertionError( + 'issue_POST: unknown response_encoding "%s"' % (response_encoding,)) + + +class HttpAssertMixin: + + def _issue_GET(self, server_address, request_path, query_dict=None, response_encoding='textual'): + assert isinstance(server_address, tuple) and len( + server_address) == 2, "require server address tuple" + assert isinstance(request_path, str) and request_path.startswith( + '/'), "require http path starting with /" + request_url = ''.join( + ('http://', server_address[0], ':', str(server_address[1]), request_path)) + + if query_dict is not None: + query_string = urlencode(query_dict) + request_url = ''.join((request_url, '?', query_string)) + + status = 0 + data = None + + try: + response = urlopen(request_url, None, timeout=2000) + + status = response.getcode() + data = response.read() + except HTTPError as httpexc: + status = httpexc.code + data = None + + content = attempt_to_decode_response_data(data, response_encoding) + return (status, content) + + def _issue_POST(self, server_address, request_path, request_data=None, request_json=None, response_encoding='textual'): + assert isinstance(server_address, tuple) and len( + server_address) == 2, "require server address tuple" + assert isinstance(request_path, str) and request_path.startswith( + '/'), "require http path starting with /" + request_url = ''.join( + ('http://', server_address[0], ':', str(server_address[1]), request_path)) + + if request_data and request_json: + raise ValueError( + "only one of data or json request data may be specified") + + status = 0 + data = None + + try: + if request_json is not None: + request_data = codecs.encode(json.dumps(request_json), 'utf8') + request_headers = { + 'Content-Type': 'application/json' + } + request = Request(request_url, request_data, + headers=request_headers) + elif request_data is not None: + request = Request(request_url, request_data) + else: + request = Request(request_url) + + response = urlopen(request, timeout=2000) + + status = response.getcode() + data = response.read() + except HTTPError as httpexc: + status = httpexc.code + data = httpexc.file.read() + + content = attempt_to_decode_response_data(data, response_encoding) + return (status, content) diff --git a/tests/support/serversupp.py b/tests/support/serversupp.py index 0e0fd4b94..74baa2a62 100644 --- a/tests/support/serversupp.py +++ b/tests/support/serversupp.py @@ -41,6 +41,7 @@ class ServerWithinThreadExecutor: def __init__(self, ServerClass, *args, **kwargs): self._serverclass = ServerClass + self._serverclass_on_instance = kwargs.pop('on_instance') self._arguments = (args, kwargs) self._started = ThreadEvent() self._thread = None @@ -53,6 +54,8 @@ def run(self): server_kwargs['on_start'] = lambda _: self._started.set() self._wrapped = self._serverclass(*server_args, **server_kwargs) + if self._serverclass_on_instance: + self._serverclass_on_instance(self._wrapped) try: self._wrapped.serve_forever() diff --git a/tests/test_mig_lib_coresvc.py b/tests/test_mig_lib_coresvc.py new file mode 100644 index 000000000..75ca71fa2 --- /dev/null +++ b/tests/test_mig_lib_coresvc.py @@ -0,0 +1,245 @@ +from __future__ import print_function +import codecs +import errno +import json +import os +import shutil +import sys +import unittest +from threading import Thread +from unittest import skip + +from tests.support import PY2, MigTestCase, testmain, temppath, \ + make_wrapped_server +from tests.support.httpsupp import HttpAssertMixin + +from mig.shared.base import keyword_auto +from mig.shared.useradm import create_user +from mig.lib.coresvc import ThreadedApiHttpServer, \ + _create_and_expose_server + +_TAG_P_OPEN = '

' +_TAG_P_CLOSE = '

' +_USERADM_PATH_KEYS = ('user_cache', 'user_db_home', 'user_home', + 'user_settings', 'mrsl_files_dir', 'resource_pending') + + +def _extend_configuration(*args): + pass + + +def ensure_dirs_needed_by_create_user(configuration): + for config_key in _USERADM_PATH_KEYS: + dir_path = getattr(configuration, config_key)[0:-1] + try: + os.mkdir(dir_path) + except OSError as exc: + pass + + +def extract_error_description_from_html(content): + open_tag_index = content.find(_TAG_P_OPEN) + start_index = open_tag_index + len(_TAG_P_OPEN) + end_index = content.find(_TAG_P_CLOSE) + error_desription = content[start_index:end_index] + return error_desription + + +class MigServerGrid_openid(MigTestCase, HttpAssertMixin): + def before_each(self): + self.server_addr = None + self.server_thread = None + + ensure_dirs_needed_by_create_user(self.configuration) + + self.server_addr = ('localhost', 4567) + self.server_thread = self._make_server( + self.configuration, self.logger, self.server_addr) + + def _provide_configuration(self): + return 'testconfig' + + def after_each(self): + if self.server_thread: + self.server_thread.stop() + + def issue_GET(self, request_path): + return self._issue_GET(self.server_addr, request_path) + + def issue_POST(self, request_path, **kwargs): + return self._issue_POST(self.server_addr, request_path, **kwargs) + + @unittest.skipIf(PY2, "Python 3 only") + def test__GET_returns_not_found_for_missing_path(self): + self.server_thread.start_wait_until_ready() + + status, _ = self.issue_GET('/nonexistent') + + self.assertEqual(status, 404) + + @unittest.skipIf(PY2, "Python 3 only") + def test_GET_user__top_level_request(self): + self.server_thread.start_wait_until_ready() + + status, _ = self.issue_GET('/user') + + self.assertEqual(status, 400) + + @unittest.skipIf(PY2, "Python 3 only") + def test_GET__user_userid_request_succeeds_with_status_ok(self): + example_username = 'dummy-user' + example_username_home_dir = temppath( + 'state/user_home/%s' % example_username, self, ensure_dir=True) + test_user_home = os.path.dirname( + example_username_home_dir) # strip user from path + test_state_dir = os.path.dirname(test_user_home) + test_user_db_home = os.path.join(test_state_dir, "user_db_home") + self.server_thread.start_wait_until_ready() + + status, content = self.issue_GET('/user/dummy-user') + + self.assertEqual(status, 200) + self.assertEqual(content, 'FOOBAR') + + @unittest.skipIf(PY2, "Python 3 only") + def test_GET_openid_user_username(self): + self.server_thread.start_wait_until_ready() + + status, content = self.issue_GET('/user/dummy-user') + + self.assertEqual(status, 200) + self.assertEqual(content, 'FOOBAR') + + @unittest.skipIf(PY2, "Python 3 only") + def test_POST_user__bad_input_data(self): + self.server_thread.start_wait_until_ready() + + status, content = self.issue_POST('/user', request_json={ + 'greeting': 'provocation' + }) + + self.assertEqual(status, 400) + error_description = extract_error_description_from_html(content) + error_description_lines = error_description.split('
') + self.assertEqual( + error_description_lines[0], 'payload failed to validate:') + + @unittest.skipIf(PY2, "Python 3 only") + def test_POST_user(self): + self.server_thread.start_wait_until_ready() + + status, content = self.issue_POST('/user', response_encoding='textual', request_json=dict( + full_name="Test User", + organization="Test Org", + state="NA", + country="DK", + email="user@example.com", + comment="This is the create comment", + password="password", + )) + + self.assertEqual(status, 201) + self.assertIsInstance(content, dict) + self.assertIn('unique_id', content) + + def _make_configuration(self, test_logger, server_addr): + configuration = self.configuration + _extend_configuration( + configuration, + server_addr[0], + server_addr[1], + logger=test_logger, + expandusername=False, + host_rsa_key='', + nossl=True, + show_address=False, + show_port=False, + ) + return configuration + + @staticmethod + def _make_server(configuration, logger=None, server_address=None): + def _on_instance(server): + server.server_app = _create_and_expose_server( + server, server.configuration) + + (host, port) = server_address + server_thread = make_wrapped_server(ThreadedApiHttpServer, + configuration, logger, host, port, on_instance=_on_instance) + return server_thread + + +class MigServerGrid_openid__existing_user(MigTestCase, HttpAssertMixin): + def before_each(self): + ensure_dirs_needed_by_create_user(self.configuration) + + user_dict = { + 'full_name': "Test User", + 'organization': "Test Org", + 'state': "NA", + 'country': "DK", + 'email': "user@example.com", + 'comment': "This is the create comment", + 'password': "password", + } + create_user(user_dict, self.configuration, + keyword_auto, default_renew=True) + + self.server_addr = ('localhost', 4567) + self.server_thread = self._make_server( + self.configuration, self.logger, self.server_addr) + + def _provide_configuration(self): + return 'testconfig' + + def after_each(self): + if self.server_thread: + self.server_thread.stop() + + @unittest.skipIf(PY2, "Python 3 only") + def test_GET_openid_user_find(self): + self.server_thread.start_wait_until_ready() + + status, content = self._issue_GET(self.server_addr, '/user/find', { + 'email': 'user@example.com' + }) + + self.assertEqual(status, 200) + + self.assertIsInstance(content, dict) + self.assertIn('objects', content) + self.assertIsInstance(content['objects'], list) + + user = content['objects'][0] + # check we received the correct user + self.assertEqual(user['full_name'], 'Test User') + + def _make_configuration(self, test_logger, server_addr): + configuration = self.configuration + _extend_configuration( + configuration, + server_addr[0], + server_addr[1], + logger=test_logger, + expandusername=False, + host_rsa_key='', + nossl=True, + show_address=False, + show_port=False, + ) + return configuration + + @staticmethod + def _make_server(configuration, logger=None, server_address=None): + def _on_instance(server): + server.server_app = _create_and_expose_server( + server, server.configuration) + + (host, port) = server_address + server_thread = make_wrapped_server(ThreadedApiHttpServer, + configuration, logger, host, port, on_instance=_on_instance) + return server_thread + + +if __name__ == '__main__': + testmain() From fe15a09d2d3e09bc2fb42959b40ce8261a7a8b82 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Mon, 28 Jul 2025 16:03:52 +0200 Subject: [PATCH 3/6] move the user routes into a separate file making us of flask Blueprint provide access within those routes to the server-wide configuration using the documented "app_context" provisions --- mig/lib/coresvc/respond.py | 18 +++++++ mig/lib/coresvc/routes/__init__.py | 0 mig/lib/coresvc/routes/user.py | 71 ++++++++++++++++++++++++++ mig/lib/coresvc/server.py | 81 +++--------------------------- 4 files changed, 96 insertions(+), 74 deletions(-) create mode 100644 mig/lib/coresvc/respond.py create mode 100644 mig/lib/coresvc/routes/__init__.py create mode 100644 mig/lib/coresvc/routes/user.py diff --git a/mig/lib/coresvc/respond.py b/mig/lib/coresvc/respond.py new file mode 100644 index 000000000..79cc4b994 --- /dev/null +++ b/mig/lib/coresvc/respond.py @@ -0,0 +1,18 @@ +from flask import Response +import json +import werkzeug.exceptions as httpexceptions + +httpexceptions_by_code = { + exc.code: exc for exc in httpexceptions.__dict__.values() if hasattr(exc, "code") +} + + +def http_error_from_status_code(http_status_code, http_url, description=None): + return httpexceptions_by_code[http_status_code](description) + + +def json_reponse_from_status_code(http_status_code, content): + json_content = json.dumps(content) + return Response( + json_content, http_status_code, {"Content-Type": "application/json"} + ) diff --git a/mig/lib/coresvc/routes/__init__.py b/mig/lib/coresvc/routes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/coresvc/routes/user.py b/mig/lib/coresvc/routes/user.py new file mode 100644 index 000000000..3f8a12a55 --- /dev/null +++ b/mig/lib/coresvc/routes/user.py @@ -0,0 +1,71 @@ +from flask import Blueprint, request, current_app + +from mig.lib.coresvc.payloads import PayloadException, PAYLOAD_POST_USER +from mig.lib.coresvc.respond import \ + http_error_from_status_code, \ + json_reponse_from_status_code +from mig.shared.base import canonical_user, keyword_auto, force_native_str_rec +from mig.shared.useradm import fill_user, \ + create_user as useradm_create_user, search_users as useradm_search_users + +def _create_user(configuration, payload): + user_dict = canonical_user( + configuration, payload, PAYLOAD_POST_USER._fields) + fill_user(user_dict) + force_native_str_rec(user_dict) + + try: + useradm_create_user(user_dict, configuration, keyword_auto, default_renew=True) + except: + raise http_error_from_status_code(500, None) + user_email = user_dict['email'] + objects = search_users(configuration, { + 'email': user_email + }) + if len(objects) != 1: + raise http_error_from_status_code(400, None) + return objects[0] + + +def search_users(configuration, search_filter): + _, hits = useradm_search_users(search_filter, configuration, keyword_auto) + return list((obj for _, obj in hits)) + + +bp = Blueprint('user', __name__) + + +@bp.get('/user') +def GET_user(): + raise http_error_from_status_code(400, None) + +@bp.get('/user/') +def GET_user_username(username): + return 'FOOBAR' + +@bp.get('/user/find') +def GET_user_find(): + configuration, = current_app.migctx + query_params = request.args + + objects = search_users(configuration, { + 'email': query_params['email'] + }) + + if len(objects) != 1: + raise http_error_from_status_code(404, None) + + return dict(objects=objects) + +@bp.post('/user') +def POST_user(): + configuration, = current_app.migctx + payload = request.get_json() + + try: + payload = PAYLOAD_POST_USER.ensure(payload) + except PayloadException as vr: + return http_error_from_status_code(400, None, vr.serialize()) + + user = _create_user(configuration, payload) + return json_reponse_from_status_code(201, user) diff --git a/mig/lib/coresvc/server.py b/mig/lib/coresvc/server.py index bdc1f5f29..89922936f 100755 --- a/mig/lib/coresvc/server.py +++ b/mig/lib/coresvc/server.py @@ -34,94 +34,27 @@ from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn -from flask import Flask, request, Response +from collections import namedtuple +from flask import Flask, current_app import json import os import sys import threading import time -import werkzeug.exceptions as httpexceptions from wsgiref.simple_server import WSGIRequestHandler -from mig.lib.coresvc.payloads import PayloadException, \ - PAYLOAD_POST_USER -from mig.shared.base import canonical_user, keyword_auto, force_native_str_rec -from mig.shared.useradm import fill_user, \ - create_user as useradm_create_user, search_users as useradm_search_users - -httpexceptions_by_code = { - exc.code: exc for exc in httpexceptions.__dict__.values() if hasattr(exc, 'code')} - - -def http_error_from_status_code(http_status_code, http_url, description=None): - return httpexceptions_by_code[http_status_code](description) - - -def json_reponse_from_status_code(http_status_code, content): - json_content = json.dumps(content) - return Response(json_content, http_status_code, { 'Content-Type': 'application/json' }) - - -def _create_user(configuration, payload): - user_dict = canonical_user( - configuration, payload, PAYLOAD_POST_USER._fields) - fill_user(user_dict) - force_native_str_rec(user_dict) - - try: - useradm_create_user(user_dict, configuration, keyword_auto, default_renew=True) - except: - raise http_error_from_status_code(500, None) - user_email = user_dict['email'] - objects = search_users(configuration, { - 'email': user_email - }) - if len(objects) != 1: - raise http_error_from_status_code(400, None) - return objects[0] - - -def search_users(configuration, search_filter): - _, hits = useradm_search_users(search_filter, configuration, keyword_auto) - return list((obj for _, obj in hits)) +MigCtx = namedtuple('MigCtx', ['configuration']) def _create_and_expose_server(server, configuration): app = Flask('coreapi') - @app.get('/user') - def GET_user(): - raise http_error_from_status_code(400, None) - - @app.get('/user/') - def GET_user_username(username): - return 'FOOBAR' - - @app.get('/user/find') - def GET_user_find(): - query_params = request.args - - objects = search_users(configuration, { - 'email': query_params['email'] - }) - - if len(objects) != 1: - raise http_error_from_status_code(404, None) - - return dict(objects=objects) - - @app.post('/user') - def POST_user(): - payload = request.get_json() - - try: - payload = PAYLOAD_POST_USER.ensure(payload) - except PayloadException as vr: - return http_error_from_status_code(400, None, vr.serialize()) + with app.app_context(): + current_app.migctx = MigCtx(configuration=configuration) - user = _create_user(configuration, payload) - return json_reponse_from_status_code(201, user) + from .routes import user + app.register_blueprint(user.bp) return app From 59aa945f61e51e25fda11f4c87bd547cd43d7f8a Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Wed, 5 Mar 2025 20:50:37 +0100 Subject: [PATCH 4/6] Put together first pass of a self contained core service API client. --- mig/lib/coreapi/__init__.py | 110 ++++++++++++++++++++++++++++++++++ tests/support/serversupp.py | 18 ++++-- tests/test_mig_lib_coreapi.py | 104 ++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 mig/lib/coreapi/__init__.py create mode 100644 tests/test_mig_lib_coreapi.py diff --git a/mig/lib/coreapi/__init__.py b/mig/lib/coreapi/__init__.py new file mode 100644 index 000000000..995053549 --- /dev/null +++ b/mig/lib/coreapi/__init__.py @@ -0,0 +1,110 @@ +import codecs +import json +import werkzeug.exceptions as httpexceptions + +from tests.support._env import PY2 + +if PY2: + from urllib2 import HTTPError, Request, urlopen + from urllib import urlencode +else: + from urllib.error import HTTPError + from urllib.parse import urlencode + from urllib.request import urlopen, Request + +from mig.lib.coresvc.payloads import PAYLOAD_POST_USER + + +httpexceptions_by_code = { + exc.code: exc for exc in httpexceptions.__dict__.values() if hasattr(exc, 'code')} + + +def attempt_to_decode_response_data(data, response_encoding=None): + if data is None: + return None + elif response_encoding == 'textual': + data = codecs.decode(data, 'utf8') + + try: + return json.loads(data) + except Exception as e: + return data + elif response_encoding == 'binary': + return data + else: + raise AssertionError( + 'issue_POST: unknown response_encoding "%s"' % (response_encoding,)) + + +def http_error_from_status_code(http_status_code, description=None): + return httpexceptions_by_code[http_status_code](description) + + +class CoreApiClient: + def __init__(self, base_url): + self._base_url = base_url + + def _issue_GET(self, request_path, query_dict=None, response_encoding='textual'): + request_url = ''.join((self._base_url, request_path)) + + if query_dict is not None: + query_string = urlencode(query_dict) + request_url = ''.join((request_url, '?', query_string)) + + status = 0 + data = None + + try: + response = urlopen(request_url, None, timeout=2000) + + status = response.getcode() + data = response.read() + except HTTPError as httpexc: + status = httpexc.code + data = None + + content = attempt_to_decode_response_data(data, response_encoding) + return (status, content) + + def _issue_POST(self, request_path, request_data=None, request_json=None, response_encoding='textual'): + request_url = ''.join((self._base_url, request_path)) + + if request_data and request_json: + raise ValueError( + "only one of data or json request data may be specified") + + status = 0 + data = None + + try: + if request_json is not None: + request_data = codecs.encode(json.dumps(request_json), 'utf8') + request_headers = { + 'Content-Type': 'application/json' + } + request = Request(request_url, request_data, + headers=request_headers) + elif request_data is not None: + request = Request(request_url, request_data) + else: + request = Request(request_url) + + response = urlopen(request, timeout=2000) + + status = response.getcode() + data = response.read() + except HTTPError as httpexc: + status = httpexc.code + data = httpexc.fp.read() + + content = attempt_to_decode_response_data(data, response_encoding) + return (status, content) + + def createUser(self, user_dict): + payload = PAYLOAD_POST_USER.ensure(user_dict) + + status, output = self._issue_POST('/user', request_json=dict(payload)) + if status != 201: + description = output if isinstance(output, str) else None + raise http_error_from_status_code(status, description) + return output diff --git a/tests/support/serversupp.py b/tests/support/serversupp.py index 74baa2a62..f7d78bffd 100644 --- a/tests/support/serversupp.py +++ b/tests/support/serversupp.py @@ -41,12 +41,16 @@ class ServerWithinThreadExecutor: def __init__(self, ServerClass, *args, **kwargs): self._serverclass = ServerClass - self._serverclass_on_instance = kwargs.pop('on_instance') + self._serverclass_on_instance = kwargs.pop('on_instance', None) self._arguments = (args, kwargs) self._started = ThreadEvent() self._thread = None self._wrapped = None + def __getattr__(self, attr): + assert self._wrapped, "wrapped instance was not created" + return getattr(self._wrapped, attr) + def run(self): """Mimic the same method from the standard thread API""" server_args, server_kwargs = self._arguments @@ -76,14 +80,16 @@ def start_wait_until_ready(self): def stop(self): """Mimic the same method from the standard thread API""" self.stop_server() - self._wrapped = None - self._thread.join() - self._thread = None + if self._thread: + self._thread.join() + self._thread = None def stop_server(self): """Stop server thread""" - self._wrapped.shutdown() - self._wrapped.server_close() + if self._wrapped: + self._wrapped.shutdown() + self._wrapped.server_close() + self._wrapped = None def make_wrapped_server(ServerClass, *args, **kwargs): diff --git a/tests/test_mig_lib_coreapi.py b/tests/test_mig_lib_coreapi.py new file mode 100644 index 000000000..06a2e0a8c --- /dev/null +++ b/tests/test_mig_lib_coreapi.py @@ -0,0 +1,104 @@ +import codecs +import json +from http.server import HTTPServer, BaseHTTPRequestHandler + +from tests.support import MigTestCase, testmain +from tests.support.serversupp import make_wrapped_server + +from mig.lib.coreapi import CoreApiClient + + +class TestRequestHandler(BaseHTTPRequestHandler): + def do_POST(self): + test_server = self.server + + if test_server._programmed_response: + status, content = test_server._programmed_response + elif test_server._programmed_error: + status, content = test_server._programmed_error + + self.send_response(status) + self.end_headers() + self.wfile.write(content) + + +class TestHTTPServer(HTTPServer): + def __init__(self, addr, **kwargs): + self._programmed_error = None + self._programmed_response = None + self._on_start = kwargs.pop('on_start', lambda _: None) + + HTTPServer.__init__(self, addr, TestRequestHandler, **kwargs) + + def clear_programmed(self): + self._programmed_error = None + + def set_programmed_error(self, status, content): + assert self._programmed_response is None + assert isinstance(content, bytes) + self._programmed_error = (status, content) + + def set_programmed_response(self, status, content): + assert self._programmed_error is None + assert isinstance(content, bytes) + self._programmed_response = (status, content) + + def set_programmed_json_response(self, status, content): + self.set_programmed_response(status, codecs.encode(json.dumps(content), 'utf8')) + + def server_activate(self): + HTTPServer.server_activate(self) + self._on_start(self) + + +class TestMigLibCoreapi(MigTestCase): + def before_each(self): + self.server_addr = ('localhost', 4567) + self.server = make_wrapped_server(TestHTTPServer, self.server_addr) + + def after_each(self): + server = getattr(self, 'server', None) + setattr(self, 'server', None) + if server: + server.stop() + + def test_raises_in_the_absence_of_success(self): + self.server.start_wait_until_ready() + self.server.set_programmed_error(418, b'tea; earl grey; hot') + instance = CoreApiClient("http://%s:%s/" % self.server_addr) + + with self.assertRaises(Exception): + instance.createUser({ + 'full_name': "Test User", + 'organization': "Test Org", + 'state': "NA", + 'country': "DK", + 'email': "user@example.com", + 'comment': "This is the create comment", + 'password': "password", + }) + + def test_returs_a_user_object(self): + test_content = { + 'foo': 1, + 'bar': True + } + self.server.start_wait_until_ready() + self.server.set_programmed_json_response(201, test_content) + instance = CoreApiClient("http://%s:%s/" % self.server_addr) + + content = instance.createUser({ + 'full_name': "Test User", + 'organization': "Test Org", + 'state': "NA", + 'country': "DK", + 'email': "user@example.com", + 'comment': "This is the create comment", + 'password': "password", + }) + + self.assertIsInstance(content, dict) + self.assertEqual(content, test_content) + +if __name__ == '__main__': + testmain() From c4c998185f8d8f51d43de539d7116c99a36d48fa Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Fri, 1 Aug 2025 12:10:24 +0200 Subject: [PATCH 5/6] PY2 cleanup --- mig/lib/coreapi/__init__.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/mig/lib/coreapi/__init__.py b/mig/lib/coreapi/__init__.py index 995053549..be669f584 100644 --- a/mig/lib/coreapi/__init__.py +++ b/mig/lib/coreapi/__init__.py @@ -1,17 +1,10 @@ import codecs import json +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import urlopen, Request import werkzeug.exceptions as httpexceptions -from tests.support._env import PY2 - -if PY2: - from urllib2 import HTTPError, Request, urlopen - from urllib import urlencode -else: - from urllib.error import HTTPError - from urllib.parse import urlencode - from urllib.request import urlopen, Request - from mig.lib.coresvc.payloads import PAYLOAD_POST_USER From f9e990b314ff035f8869d6737e4ed74e89f1acb7 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Fri, 1 Aug 2025 12:28:44 +0200 Subject: [PATCH 6/6] fixup --- recommended.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/recommended.txt b/recommended.txt index 930b4390d..d4d998453 100644 --- a/recommended.txt +++ b/recommended.txt @@ -1,5 +1,6 @@ # migrid full dependencies on a format suitable for pip install as described on # https://pip.pypa.io/en/stable/reference/requirement-specifiers/ +flask future # NOTE: python-3.6 and earlier versions require older pyotp, whereas 3.7+ # should work with any modern version. We tested 2.9.0 to work.