diff --git a/.gitignore b/.gitignore index 2868a55d5..a422a36ac 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,14 @@ docs/.cache # Don't store poetry.lock file libs/cln-version-manager/poetry.lock + +# Ignore files generated by gltestserver +.env +.gltestserver +uv.lock +metadata.json + +# JS Examples +examples/javascript/node_modules +examples/javascript/response.bin +examples/javascript/package-lock.json diff --git a/Makefile b/Makefile index 76ae1d19d..246711042 100644 --- a/Makefile +++ b/Makefile @@ -192,8 +192,9 @@ docs-publish: docs --branch gh-pages \ --remote origin -gltestserver-image: docker/gl-testserver/Dockerfile - docker build \ +gltestserver-image: ${REPO_ROOT}/docker/gl-testserver/Dockerfile + docker buildx build \ + --load \ --build-arg DOCKER_USER=$(shell whoami) \ --build-arg UID=$(shell id -u) \ --build-arg GID=$(shell id -g) \ @@ -203,8 +204,8 @@ gltestserver-image: docker/gl-testserver/Dockerfile . gltestserver: gltestserver-image - mkdir -p /tmp/gltestserver docker run \ + --rm \ --user $(shell id -u):$(shell id -g) \ -e DOCKER_USER=$(shell whoami) \ --net=host \ diff --git a/docker/gl-testserver/Dockerfile b/docker/gl-testserver/Dockerfile index 3f736a7fb..58c17e744 100644 --- a/docker/gl-testserver/Dockerfile +++ b/docker/gl-testserver/Dockerfile @@ -80,6 +80,6 @@ RUN cargo build --bin gl-signerproxy RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN uv sync --locked -v +RUN uv lock && uv sync --locked -v RUN uv run clnvm get-all CMD uv run gltestserver run --metadata ${REPO}/ --directory ${REPO}/.gltestserver diff --git a/examples/javascript/README.md b/examples/javascript/README.md new file mode 100644 index 000000000..e20e6a0f9 --- /dev/null +++ b/examples/javascript/README.md @@ -0,0 +1,76 @@ +# How to run javascript examples with gltestserver + +## Step 1 (Terminal 1): Start the Server +```bash +make gltestserver +``` + +## Step 2 (Terminal 2): Register the Node +```bash +GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \ +GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \ +GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \ +GL_SCHEDULER_GRPC_URI=https://localhost:38067 \ +cargo run --bin glcli scheduler register --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver +``` + +## Step 3 (Terminal 2): Schedule the Node +```bash +GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \ +GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \ +GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \ +GL_SCHEDULER_GRPC_URI=https://localhost:38067 \ +cargo run --bin glcli scheduler schedule --verbose --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver +``` + +## Step 4 (Terminal 2): Start the Signer +```bash +GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \ +GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \ +GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \ +GL_SCHEDULER_GRPC_URI=https://localhost:38067 \ +cargo run --bin glcli signer run --verbose --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver +``` + +## Step 5 (Terminal 3): Run the Example +### 5.1: Navigate and Install Dependencies for the Example +```bash +cd ./examples/javascript +npm install +``` + +### 5.2: Get Node ID +```bash +lightning-hsmtool getnodeid $HOME/greenlight/.gltestserver/gl-testserver/hsm_secret +``` +Sample Output: 034c46b632a9ff3975fb7cd4e764a36ec476b522be2555e83a3183ab1ee3e36e93 + +### 5.3: Encode Node ID to Base64 +```python +import binascii +import base64 +print(base64.b64encode(binascii.unhexlify("")).decode('utf-8')) +``` +Sample Output: A0xGtjKp/zl1+3zU52SjbsR2tSK+JVXoOjGDqx7j426T + +### 5.4: Modify Default Values +- Open the file `./examples/javascript/grpc-web-proxy-client.js`. + +- Locate the line defining `AUTH_PUBKEY` and replace its value with the Base64-encoded public key output from Step 5.3: + + ```javascript + const AUTH_PUBKEY = 'replace+this+with+your+base64+encoded+pubkey'; + ``` + +- Replace the default PORT value `1111` with the port number from `grpc_web_proxy_uri` obtained in Step 1: + ```javascript + const PORT = process.argv[2] || '1111'; + ``` + Alternatively, the port number can be passed as a command-line argument when running the nodejs script in the next step. + +- Save the changes to the file. + +### 5.5: Run the Example +```bash +node grpc-web-proxy-client.js +``` diff --git a/examples/javascript/grpc-web-proxy-client.js b/examples/javascript/grpc-web-proxy-client.js new file mode 100644 index 000000000..10acde6ff --- /dev/null +++ b/examples/javascript/grpc-web-proxy-client.js @@ -0,0 +1,135 @@ +const path = require('path'); +const axios = require('axios'); +const protobuf = require('protobufjs'); + +const PORT = process.argv[2] || '1111'; +const AUTH_PUBKEY = 'replace+this+with+your+base64+encoded+pubkey'; +const AUTH_SIGNATURE = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; +const PROTO_PATHS = [ + path.join(process.cwd(), '../../libs/gl-client/.resources/proto/node.proto'), + path.join(process.cwd(), '../../libs/gl-client/.resources/proto/primitives.proto') +]; + +function getGrpcErrorMessage(grpcStatusCode) { + const grpcStatusMessages = { + 0: 'OK: The operation completed successfully.', + 1: 'CANCELLED: The operation was cancelled (typically by the caller).', + 2: 'UNKNOWN: Unknown error. Usually means an internal error occurred.', + 3: 'INVALID_ARGUMENT: The client specified an invalid argument.', + 4: 'DEADLINE_EXCEEDED: The operation took too long and exceeded the time limit.', + 5: 'NOT_FOUND: A specified resource was not found.', + 6: 'ALREADY_EXISTS: The resource already exists.', + 7: 'PERMISSION_DENIED: The caller does not have permission to execute the operation.', + 8: 'RESOURCE_EXHAUSTED: A resource (such as quota) was exhausted.', + 9: 'FAILED_PRECONDITION: The operation was rejected due to a failed precondition.', + 10: 'ABORTED: The operation was aborted, typically due to a concurrency issue.', + 11: 'OUT_OF_RANGE: The operation attempted to access an out-of-range value.', + 12: 'UNIMPLEMENTED: The operation is not implemented or supported by the server.', + 13: 'INTERNAL: Internal server error.', + 14: 'UNAVAILABLE: The service is unavailable (e.g., network issues, server down).', + 15: 'DATA_LOSS: Unrecoverable data loss or corruption.', + 16: 'UNAUTHENTICATED: The request is missing or has invalid authentication credentials.' + } + return grpcStatusMessages[grpcStatusCode] || "UNKNOWN_STATUS_CODE: The status code returned by gRPC server is not in the list."; +} + +async function encodePayload(clnNode, method, payload) { + const methodRequest = clnNode.lookupType(`cln.${method}Request`); + const errMsg = methodRequest.verify(payload); + if (errMsg) throw new Error(errMsg); + const header = Buffer.alloc(4); + header.writeUInt8(0, 0); + const requestPayload = methodRequest.create(payload); + const encodedPayload = methodRequest.encodeDelimited(requestPayload).finish(); + return Buffer.concat([header, encodedPayload]); +} + +async function sendRequest(methodUrl, encodedPayload) { + const buffer = Buffer.alloc(8); + buffer.writeUInt32BE(Math.floor(Date.now() / 1000), 4); + const axiosConfig = { + responseType: 'arraybuffer', + headers: { + 'content-type': 'application/grpc', + 'accept': 'application/grpc', + 'glauthpubkey': AUTH_PUBKEY, + 'glauthsig': AUTH_SIGNATURE, + 'glts': buffer.toString('base64'), + }, + }; + return await axios.post(`http://localhost:${PORT}/cln.Node/${methodUrl}`, encodedPayload, axiosConfig); +} + +function transformValue(key, value) { + if ((value.type && value.type === "Buffer") || value instanceof Buffer || value instanceof Uint8Array) { + return Buffer.from(value).toString('hex'); + } + if (value.msat && !Number.isNaN(parseInt(value.msat))) { + // FIXME: Amount.varify check will work with 0 NOT '0'. Amount default is '0'. + return parseInt(value.msat); + } + return value; +} + +function decodeResponse(clnNode, method, response) { + const methodResponse = clnNode.lookupType(`cln.${method}Response`) + const offset = 5; + const responseData = new Uint8Array(response.data).slice(offset); + const grpcStatus = +response.headers['grpc-status']; + if (grpcStatus !== 0) { + let errorDecoded = new TextDecoder("utf-8").decode(responseData); + if (errorDecoded !== 'None') { + errorDecoded = JSON.parse(errorDecoded.replace(/([a-zA-Z0-9_]+):/g, '"$1":')); + } else { + errorDecoded = {code: grpcStatus, message: getGrpcErrorMessage(grpcStatus)}; + } + return { grpc_code: grpcStatus, grpc_error: getGrpcErrorMessage(grpcStatus), error: errorDecoded}; + } else { + // FIXME: Use decodeDelimited + const decodedRes = methodResponse.decode(responseData); + const decodedResObject = methodResponse.toObject(decodedRes, { + longs: String, + enums: String, + bytes: Buffer, + defaults: true, + arrays: true, + objects: true, + }); + return JSON.parse(JSON.stringify(decodedResObject, transformValue)); + } +} + +async function fetchNodeData() { + try { + const clnNode = new protobuf.Root().loadSync(PROTO_PATHS, { keepCase: true }); + const FeeratesStyle = clnNode.lookupEnum('cln.FeeratesStyle'); + const NewaddrAddresstype = clnNode.lookupEnum('cln.NewaddrAddresstype'); + const methods = ['Getinfo', 'Feerates', 'NewAddr', 'Invoice', 'ListInvoices']; + const method_payloads = [{}, {style: FeeratesStyle.values.PERKW}, {addresstype: NewaddrAddresstype.values.ALL}, {amount_msat: {amount: {msat: 500000}}, description: 'My coffee', label: 'coffeeinvat' + Date.now()}, {}]; + for (let i = 0; i < methods.length; i++) { + console.log('--------------------------------------------\n', (i + 1), '-', methods[i], '\n--------------------------------------------'); + console.log('Payload Raw:\n', method_payloads[i]); + const CapitalizedMethodName = methods[i].charAt(0).toUpperCase() + methods[i].slice(1).toLowerCase(); + const encodedPayload = await encodePayload(clnNode, CapitalizedMethodName, method_payloads[i]); + console.log('\nPayload Encoded:\n', encodedPayload); + try { + const response = await sendRequest(methods[i], encodedPayload); + console.log('\nResponse Headers:\n', response.headers); + console.log('\nResponse Data:\n', response.data); + const responseJSON = decodeResponse(clnNode, CapitalizedMethodName, response); + console.log('\nResponse Decoded:'); + console.dir(responseJSON, { depth: null, color: true }); + } catch (error) { + console.error('\nResponse Error:\n', error.response.status, ' - ', error.response.statusText); + } + } + } catch (error) { + console.error('Error:', error.message); + if (error.response) { + console.error('Error status:', error.response.status); + console.error('Error data:', error.response.data); + } + } +} + +fetchNodeData(); diff --git a/examples/javascript/package.json b/examples/javascript/package.json new file mode 100644 index 000000000..c43406abc --- /dev/null +++ b/examples/javascript/package.json @@ -0,0 +1,19 @@ +{ + "name": "grpc-web-proxy-client", + "version": "1.0.0", + "description": "Example for grpc web proxy client", + "main": "grpc-web-proxy-client.js", + "directories": { + "doc": "doc", + "test": "tests" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^1.7.9", + "protobufjs": "^7.4.0" + } +} diff --git a/libs/gl-client-py/pyproject.toml b/libs/gl-client-py/pyproject.toml index bc7aa4269..637e66bbd 100644 --- a/libs/gl-client-py/pyproject.toml +++ b/libs/gl-client-py/pyproject.toml @@ -1,5 +1,6 @@ [project] name = "gl-client" +version = "0.3.0" dependencies = [ "protobuf>=3", diff --git a/libs/gl-testing/gltesting/certs.py b/libs/gl-testing/gltesting/certs.py index dbbc80138..7a7dd966c 100644 --- a/libs/gl-testing/gltesting/certs.py +++ b/libs/gl-testing/gltesting/certs.py @@ -4,7 +4,7 @@ import tempfile import json import os -from sh import cfssl, openssl, cfssljson +import subprocess from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat._oid import NameOID @@ -108,9 +108,13 @@ def path_to_identity(path): ) def postprocess_private_key(keyfile): - converted = openssl("pkcs8", "-topk8", "-nocrypt", "-in", keyfile).stdout - with open(keyfile, "wb") as f: - f.write(converted) + result = subprocess.run(["openssl", "pkcs8", "-topk8", "-nocrypt", "-in", keyfile], capture_output=True, text=True) + if result.returncode == 0: + converted = result.stdout + with open(keyfile, "wb") as f: + f.write(converted.encode()) + else: + raise RuntimeError(f"OpenSSL command failed with error: {result.stderr}") def parent_ca(path): @@ -167,24 +171,15 @@ def genca(idpath): if not os.path.exists(directory): os.makedirs(directory) - cfssljson(cfssl("gencert", "-initca", tmpcsr.name), "-bare", path[3]) - + certs_json = subprocess.check_output(["cfssl", "gencert", "-initca", tmpcsr.name]) + subprocess.run(["cfssljson", "-bare", path[3]], input=certs_json) + # Write config tmpconfig = tempfile.NamedTemporaryFile(mode="w") tmpconfig.write(config) tmpconfig.flush() - cfssljson( - cfssl( - "sign", - f"-ca={parent[0]}", - f"-ca-key={parent[1]}", - f"-config={tmpconfig.name}", - f"-profile={profile}", - path[3] + ".csr", - ), - "-bare", - path[3], - ) + sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", f"-config={tmpconfig.name}", f"-profile={profile}", path[3] + ".csr"]) + subprocess.run(["cfssljson", "-bare", path[3]], input=sign_certs_json) # Cleanup the temporary certificate signature request os.remove(path[3] + ".csr") @@ -225,18 +220,8 @@ def gencert(idpath): if not os.path.exists(directory): os.makedirs(directory) - cfssljson( - cfssl( - "gencert", - f"-ca={parent[0]}", - f"-ca-key={parent[1]}", - f"-config={tmpconfig.name}", - f"-profile={profile}", - tmpcsr.name, - ), - "-bare", - path[3], - ) + certs_json = subprocess.check_output(["cfssl", "gencert", f"-ca={parent[0]}", f"-ca-key={parent[1]}", f"-config={tmpconfig.name}", f"-profile={profile}", tmpcsr.name]) + subprocess.run(["cfssljson", "-bare", path[3]], input=certs_json) # Cleanup the temporary certificate signature request os.remove(path[3] + ".csr") @@ -300,28 +285,11 @@ def gencert_from_csr(csr: bytes, recover=False, pairing=False): os.makedirs(directory) if pairing: - cfssljson( - cfssl( - "sign", - f"-ca={parent[0]}", - f"-ca-key={parent[1]}", - tmpcsr.name, - tmpsubject.name, - ), - "-bare", - path[3], - ) + sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", tmpcsr.name, tmpsubject.name]) else: - cfssljson( - cfssl( - "sign", - f"-ca={parent[0]}", - f"-ca-key={parent[1]}", - tmpcsr.name, - ), - "-bare", - path[3], - ) + sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", tmpcsr.name]) + + subprocess.run(["cfssljson", "-bare", path[3]], input=sign_certs_json) # Cleanup the temporary certificate signature request os.remove(path[3] + ".csr") @@ -333,4 +301,3 @@ def gencert_from_csr(csr: bytes, recover=False, pairing=False): cert = certf.read() certf.close() return cert - diff --git a/libs/gl-testing/gltesting/grpcweb.py b/libs/gl-testing/gltesting/grpcweb.py index 5aaa231a3..f0ca65d08 100644 --- a/libs/gl-testing/gltesting/grpcweb.py +++ b/libs/gl-testing/gltesting/grpcweb.py @@ -8,15 +8,13 @@ from gltesting.scheduler import Scheduler from ephemeral_port_reserve import reserve -from threading import Thread, Event +from threading import Thread from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler import logging import struct import httpx from dataclasses import dataclass from typing import Dict -import ssl - class GrpcWebProxy(object): def __init__(self, scheduler: Scheduler, grpc_port: int): @@ -67,6 +65,7 @@ class Request: @dataclass class Response: body: bytes + headers: Dict[str, str] class Handler(BaseHTTPRequestHandler): @@ -91,9 +90,10 @@ def proxy(self, request) -> Response: headers=headers, content=content, ) - client = httpx.Client(http1=False, http2=True) + timeout = httpx.Timeout(10.0, connect=5.0) + client = httpx.Client(http1=False, http2=True, timeout=timeout) res = client.send(req) - return Response(body=res.content) + return Response(body=res.content, headers=res.headers) def auth(self, request: Request) -> bool: """Authenticate the request. True means allow.""" @@ -118,11 +118,18 @@ def do_POST(self): req = Request(body=body, headers=self.headers, flags=flags, length=length) if not self.auth(req): - self.wfile.write(b"HTTP/1.1 401 Unauthorized\r\n\r\n") + self.send_response(401) + self.send_header("Content-Type", "application/grpc") + self.send_header("grpc-status", "16") + self.end_headers() + self.wfile.write(b"Unauthorized") return - + response = self.proxy(req) - self.wfile.write(b"HTTP/1.0 200 OK\n\n") + self.send_response(200) + self.send_header("Content-Type", "application/grpc") + self.send_header("grpc-status", response.headers.get("grpc-status", "0")) + self.end_headers() self.wfile.write(response.body) self.wfile.flush() @@ -203,7 +210,23 @@ def proxy(self, request: Request): headers=headers, content=content, ) - res = client.send(req) - - # Return response - return Response(body=res.content) + + try: + res = client.send(req) + # Capture the error from header and send it in the body as well + if res.headers.get("grpc-status", "0") != "0": + grpc_status = res.headers.get("grpc-status") + error_message = res.headers.get("grpc-message", "None") + self.logger.warning(f"gRPC status code received: {grpc_status}") + self.logger.warning(f"gRPC message received: {error_message}") + error = error_message.encode("utf-8") + error_res = struct.pack("!cI", request.flags, len(error)) + error + return Response(body=error_res, headers=res.headers) + # Return successful response + return Response(body=res.content, headers=res.headers) + except Exception as e: + self.logger.warning(f"gRPC request error received: {str(e)}") + error_message = f"Internal Server Error: {str(e)}" + error = error_message.encode("utf-8") + error_res = struct.pack("!cI", request.flags, len(error)) + error + return Response(body=error_res, headers={"Content-Type": "application/grpc", "grpc-status": "13"})