Grpc web proxy client example

.gitignore

@@ -20,3 +20,14 @@ docs/.cache
 
 # Don't store poetry.lock file
 libs/cln-version-manager/poetry.lock
+
+# Ignore files generated by gltestserver
+.env
+.gltestserver
+uv.lock
+metadata.json
+
+# JS Examples 
+examples/javascript/node_modules
+examples/javascript/response.bin
+examples/javascript/package-lock.json

Makefile

@@ -192,8 +192,9 @@ docs-publish: docs
 	  --branch gh-pages \
 	  --remote origin
 
-gltestserver-image: docker/gl-testserver/Dockerfile
-	docker build \
+gltestserver-image: ${REPO_ROOT}/docker/gl-testserver/Dockerfile
+	docker buildx build \
+	  --load \
 	  --build-arg DOCKER_USER=$(shell whoami) \
 	  --build-arg UID=$(shell id -u) \
 	  --build-arg GID=$(shell id -g) \
@@ -203,8 +204,8 @@ gltestserver-image: docker/gl-testserver/Dockerfile
 	  .
 
 gltestserver: gltestserver-image
-	mkdir -p /tmp/gltestserver
 	docker run \
+	  --rm \
 	  --user $(shell id -u):$(shell id -g) \
 	  -e DOCKER_USER=$(shell whoami) \
 	  --net=host \

docker/gl-testserver/Dockerfile

@@ -80,6 +80,6 @@ RUN cargo build --bin gl-signerproxy
 
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 
-RUN uv sync --locked -v
+RUN uv lock && uv sync --locked -v
 RUN uv run clnvm get-all
 CMD uv run gltestserver run --metadata ${REPO}/ --directory ${REPO}/.gltestserver

examples/javascript/README.md

@@ -0,0 +1,76 @@
+# How to run javascript examples with gltestserver
+
+## Step 1 (Terminal 1): Start the Server
+```bash
+make gltestserver
+```
+
+## Step 2 (Terminal 2): Register the Node
+```bash
+GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \
+GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \
+GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \
+GL_SCHEDULER_GRPC_URI=https://localhost:38067 \
+cargo run --bin glcli scheduler register --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver
+```
+
+## Step 3 (Terminal 2): Schedule the Node
+```bash
+GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \
+GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \
+GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \
+GL_SCHEDULER_GRPC_URI=https://localhost:38067 \
+cargo run --bin glcli scheduler schedule --verbose --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver
+```
+
+## Step 4 (Terminal 2): Start the Signer
+```bash
+GL_CA_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/ca.crt \
+GL_NOBODY_CRT=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody.crt \
+GL_NOBODY_KEY=$HOME/greenlight/.gltestserver/gl-testserver/certs/users/nobody-key.pem \
+GL_SCHEDULER_GRPC_URI=https://localhost:38067 \
+cargo run --bin glcli signer run --verbose --network=regtest --data-dir=$HOME/greenlight/.gltestserver/gl-testserver
+```
+
+## Step 5 (Terminal 3): Run the Example
+### 5.1: Navigate and Install Dependencies for the Example
+```bash
+cd ./examples/javascript
+npm install
+```
+
+### 5.2: Get Node ID
+```bash
+lightning-hsmtool getnodeid $HOME/greenlight/.gltestserver/gl-testserver/hsm_secret
+```
+Sample Output:	034c46b632a9ff3975fb7cd4e764a36ec476b522be2555e83a3183ab1ee3e36e93
+
+### 5.3: Encode Node ID to Base64
+```python
+import binascii
+import base64
+print(base64.b64encode(binascii.unhexlify("<node id from step 5.2>")).decode('utf-8'))
+```
+Sample Output: A0xGtjKp/zl1+3zU52SjbsR2tSK+JVXoOjGDqx7j426T
+
+### 5.4: Modify Default Values
+- Open the file `./examples/javascript/grpc-web-proxy-client.js`.
+
+- Locate the line defining `AUTH_PUBKEY` and replace its value with the Base64-encoded public key output from Step 5.3:
+
+    ```javascript
+    const AUTH_PUBKEY = 'replace+this+with+your+base64+encoded+pubkey';
+    ```
+
+- Replace the default PORT value `1111` with the port number from `grpc_web_proxy_uri` obtained in Step 1:
+    ```javascript
+    const PORT = process.argv[2] || '1111';
+    ```
+    Alternatively, the port number can be passed as a command-line argument when running the nodejs script in the next step.
+
+- Save the changes to the file.
+
+### 5.5: Run the Example
+```bash
+node grpc-web-proxy-client.js
+```

examples/javascript/grpc-web-proxy-client.js

@@ -0,0 +1,135 @@
+const path = require('path');
+const axios = require('axios');
+const protobuf = require('protobufjs');
+
+const PORT = process.argv[2] || '1111';
+const AUTH_PUBKEY = 'replace+this+with+your+base64+encoded+pubkey';
+const AUTH_SIGNATURE = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+const PROTO_PATHS = [
+  path.join(process.cwd(), '../../libs/gl-client/.resources/proto/node.proto'),
+  path.join(process.cwd(), '../../libs/gl-client/.resources/proto/primitives.proto')
+];
+
+function getGrpcErrorMessage(grpcStatusCode) {
+  const grpcStatusMessages = {
+    0: 'OK: The operation completed successfully.',
+    1: 'CANCELLED: The operation was cancelled (typically by the caller).',
+    2: 'UNKNOWN: Unknown error. Usually means an internal error occurred.',
+    3: 'INVALID_ARGUMENT: The client specified an invalid argument.',
+    4: 'DEADLINE_EXCEEDED: The operation took too long and exceeded the time limit.',
+    5: 'NOT_FOUND: A specified resource was not found.',
+    6: 'ALREADY_EXISTS: The resource already exists.',
+    7: 'PERMISSION_DENIED: The caller does not have permission to execute the operation.',
+    8: 'RESOURCE_EXHAUSTED: A resource (such as quota) was exhausted.',
+    9: 'FAILED_PRECONDITION: The operation was rejected due to a failed precondition.',
+    10: 'ABORTED: The operation was aborted, typically due to a concurrency issue.',
+    11: 'OUT_OF_RANGE: The operation attempted to access an out-of-range value.',
+    12: 'UNIMPLEMENTED: The operation is not implemented or supported by the server.',
+    13: 'INTERNAL: Internal server error.',
+    14: 'UNAVAILABLE: The service is unavailable (e.g., network issues, server down).',
+    15: 'DATA_LOSS: Unrecoverable data loss or corruption.',
+    16: 'UNAUTHENTICATED: The request is missing or has invalid authentication credentials.'
+  }
+  return grpcStatusMessages[grpcStatusCode] || "UNKNOWN_STATUS_CODE: The status code returned by gRPC server is not in the list.";
+}
+
+async function encodePayload(clnNode, method, payload) {
+  const methodRequest = clnNode.lookupType(`cln.${method}Request`);
+  const errMsg = methodRequest.verify(payload);
+  if (errMsg) throw new Error(errMsg);
+  const header = Buffer.alloc(4);
+  header.writeUInt8(0, 0);
+  const requestPayload = methodRequest.create(payload);
+  const encodedPayload = methodRequest.encodeDelimited(requestPayload).finish();
+  return Buffer.concat([header, encodedPayload]);
+}
+
+async function sendRequest(methodUrl, encodedPayload) {
+  const buffer = Buffer.alloc(8);
+  buffer.writeUInt32BE(Math.floor(Date.now() / 1000), 4);
+  const axiosConfig = {
+    responseType: 'arraybuffer',
+    headers: {
+      'content-type': 'application/grpc',
+      'accept': 'application/grpc',
+      'glauthpubkey': AUTH_PUBKEY,
+      'glauthsig': AUTH_SIGNATURE,
+      'glts': buffer.toString('base64'),
+    },
+  };
+  return await axios.post(`http://localhost:${PORT}/cln.Node/${methodUrl}`, encodedPayload, axiosConfig);
+}
+
+function transformValue(key, value) {
+  if ((value.type && value.type === "Buffer") || value instanceof Buffer || value instanceof Uint8Array) {
+    return Buffer.from(value).toString('hex');
+  }
+  if (value.msat && !Number.isNaN(parseInt(value.msat))) {
+    // FIXME: Amount.varify check will work with 0 NOT '0'. Amount default is '0'.
+    return parseInt(value.msat);
+  }
+  return value;
+}
+
+function decodeResponse(clnNode, method, response) {
+  const methodResponse = clnNode.lookupType(`cln.${method}Response`)  
+  const offset = 5;
+  const responseData = new Uint8Array(response.data).slice(offset);
+  const grpcStatus = +response.headers['grpc-status'];
+  if (grpcStatus !== 0) {
+    let errorDecoded = new TextDecoder("utf-8").decode(responseData);
+    if (errorDecoded !== 'None') {
+      errorDecoded = JSON.parse(errorDecoded.replace(/([a-zA-Z0-9_]+):/g, '"$1":'));
+    } else {
+      errorDecoded = {code: grpcStatus, message: getGrpcErrorMessage(grpcStatus)};
+    }
+    return { grpc_code: grpcStatus, grpc_error: getGrpcErrorMessage(grpcStatus), error: errorDecoded};
+  } else {
+    // FIXME: Use decodeDelimited
+    const decodedRes = methodResponse.decode(responseData);
+    const decodedResObject = methodResponse.toObject(decodedRes, {
+      longs: String,
+      enums: String,
+      bytes: Buffer,
+      defaults: true,
+      arrays: true,
+      objects: true,
+    });
+    return JSON.parse(JSON.stringify(decodedResObject, transformValue));
+  }
+}
+
+async function fetchNodeData() {
+  try {
+    const clnNode = new protobuf.Root().loadSync(PROTO_PATHS, { keepCase: true });
+    const FeeratesStyle = clnNode.lookupEnum('cln.FeeratesStyle');
+    const NewaddrAddresstype = clnNode.lookupEnum('cln.NewaddrAddresstype');
+    const methods = ['Getinfo', 'Feerates', 'NewAddr', 'Invoice', 'ListInvoices'];
+    const method_payloads = [{}, {style: FeeratesStyle.values.PERKW}, {addresstype: NewaddrAddresstype.values.ALL}, {amount_msat: {amount: {msat: 500000}}, description: 'My coffee', label: 'coffeeinvat' + Date.now()}, {}];
+    for (let i = 0; i < methods.length; i++) {
+      console.log('--------------------------------------------\n', (i + 1), '-', methods[i], '\n--------------------------------------------');
+      console.log('Payload Raw:\n', method_payloads[i]);
+      const CapitalizedMethodName = methods[i].charAt(0).toUpperCase() + methods[i].slice(1).toLowerCase();
+      const encodedPayload = await encodePayload(clnNode, CapitalizedMethodName, method_payloads[i]);
+      console.log('\nPayload Encoded:\n', encodedPayload);
+      try {
+        const response = await sendRequest(methods[i], encodedPayload);
+        console.log('\nResponse Headers:\n', response.headers);
+        console.log('\nResponse Data:\n', response.data);
+        const responseJSON = decodeResponse(clnNode, CapitalizedMethodName, response);
+        console.log('\nResponse Decoded:');
+        console.dir(responseJSON, { depth: null, color: true });
+      } catch (error) {
+        console.error('\nResponse Error:\n', error.response.status, ' - ', error.response.statusText);
+      }
+    }
+  } catch (error) {
+    console.error('Error:', error.message);
+    if (error.response) {
+      console.error('Error status:', error.response.status);
+      console.error('Error data:', error.response.data);
+    }
+  }
+}
+
+fetchNodeData();

examples/javascript/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "grpc-web-proxy-client",
+  "version": "1.0.0",
+  "description": "Example for grpc web proxy client",
+  "main": "grpc-web-proxy-client.js",
+  "directories": {
+    "doc": "doc",
+    "test": "tests"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "axios": "^1.7.9",
+    "protobufjs": "^7.4.0"
+  }
+}

libs/gl-client-py/pyproject.toml

@@ -1,5 +1,6 @@
 [project]
 name = "gl-client"
+version = "0.3.0"
 
 dependencies = [
     "protobuf>=3",

libs/gl-testing/gltesting/certs.py

@@ -4,7 +4,7 @@
 import tempfile
 import json
 import os
-from sh import cfssl, openssl, cfssljson
+import subprocess
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat._oid import NameOID
@@ -108,9 +108,13 @@ def path_to_identity(path):
     )
 
 def postprocess_private_key(keyfile):
-    converted = openssl("pkcs8", "-topk8", "-nocrypt", "-in", keyfile).stdout
-    with open(keyfile, "wb") as f:
-        f.write(converted)
+    result = subprocess.run(["openssl", "pkcs8", "-topk8", "-nocrypt", "-in", keyfile], capture_output=True, text=True)
+    if result.returncode == 0:
+        converted = result.stdout
+        with open(keyfile, "wb") as f:
+            f.write(converted.encode())
+    else:
+        raise RuntimeError(f"OpenSSL command failed with error: {result.stderr}")
 
 
 def parent_ca(path):
@@ -167,24 +171,15 @@ def genca(idpath):
     if not os.path.exists(directory):
         os.makedirs(directory)
 
-    cfssljson(cfssl("gencert", "-initca", tmpcsr.name), "-bare", path[3])
-
+    certs_json = subprocess.check_output(["cfssl", "gencert", "-initca", tmpcsr.name])
+    subprocess.run(["cfssljson", "-bare", path[3]], input=certs_json)
+
     # Write config
     tmpconfig = tempfile.NamedTemporaryFile(mode="w")
     tmpconfig.write(config)
     tmpconfig.flush()
-    cfssljson(
-        cfssl(
-            "sign",
-            f"-ca={parent[0]}",
-            f"-ca-key={parent[1]}",
-            f"-config={tmpconfig.name}",
-            f"-profile={profile}",
-            path[3] + ".csr",
-        ),
-        "-bare",
-        path[3],
-    )
+    sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", f"-config={tmpconfig.name}", f"-profile={profile}", path[3] + ".csr"])
+    subprocess.run(["cfssljson", "-bare", path[3]], input=sign_certs_json)
     # Cleanup the temporary certificate signature request
     os.remove(path[3] + ".csr")
 
@@ -225,18 +220,8 @@ def gencert(idpath):
     if not os.path.exists(directory):
         os.makedirs(directory)
 
-    cfssljson(
-        cfssl(
-            "gencert",
-            f"-ca={parent[0]}",
-            f"-ca-key={parent[1]}",
-            f"-config={tmpconfig.name}",
-            f"-profile={profile}",
-            tmpcsr.name,
-        ),
-        "-bare",
-        path[3],
-    )
+    certs_json = subprocess.check_output(["cfssl", "gencert", f"-ca={parent[0]}", f"-ca-key={parent[1]}", f"-config={tmpconfig.name}", f"-profile={profile}", tmpcsr.name])
+    subprocess.run(["cfssljson", "-bare", path[3]], input=certs_json)
     # Cleanup the temporary certificate signature request
     os.remove(path[3] + ".csr")
 
@@ -300,28 +285,11 @@ def gencert_from_csr(csr: bytes, recover=False, pairing=False):
         os.makedirs(directory)
 
     if pairing:
-        cfssljson(
-            cfssl(
-                "sign",
-                f"-ca={parent[0]}",
-                f"-ca-key={parent[1]}",
-                tmpcsr.name,
-                tmpsubject.name,
-            ),
-            "-bare",
-            path[3],
-        )
+        sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", tmpcsr.name, tmpsubject.name])
     else:
-        cfssljson(
-            cfssl(
-                "sign",
-                f"-ca={parent[0]}",
-                f"-ca-key={parent[1]}",
-                tmpcsr.name,
-            ),
-            "-bare",
-            path[3],
-        )
+        sign_certs_json = subprocess.check_output(["cfssl", "sign", f"-ca={parent[0]}", f"-ca-key={parent[1]}", tmpcsr.name])
+
+    subprocess.run(["cfssljson", "-bare", path[3]], input=sign_certs_json)
 
     # Cleanup the temporary certificate signature request
     os.remove(path[3] + ".csr")
@@ -333,4 +301,3 @@ def gencert_from_csr(csr: bytes, recover=False, pairing=False):
     cert = certf.read()
     certf.close()
     return cert
-