diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 19f4b5adf..21f5f2738 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] jdk: [24] include: - os: windows-latest @@ -49,12 +49,13 @@ jobs: - name: Run tests run: bb test + jvm-integration-test: runs-on: ${{matrix.os}} strategy: fail-fast: false matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] jdk: [24] include: - os: windows-latest @@ -84,6 +85,8 @@ jobs: # AccessDeniedException exception when renaming temp files # while downloading dependencies. Workaround: offload some # of the deps download to the clojure CLI tool. + # + # See https://clojure.atlassian.net/browse/TBUILD-47 clojure -P bb prod-cli @@ -91,7 +94,7 @@ jobs: run: bb integration-test graalvm-build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: @@ -130,7 +133,7 @@ jobs: graalvm-integration-test: needs: graalvm-build - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false steps: @@ -148,6 +151,11 @@ jobs: cli: '1.12.0.1530' bb: '1.12.200' + - name: Install tinyproxy + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq tinyproxy + - uses: actions/download-artifact@v4 name: eca-native @@ -156,8 +164,8 @@ jobs: cp -rf eca-native/eca eca chmod +x eca - - name: Run integration tests - run: bb integration-test + - name: Run integration tests (via proxy) + run: bb integration-test --proxy nix-build: needs: unit-test diff --git a/.gitignore b/.gitignore index 3179ccfa4..ea2203f1f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,5 @@ docs/README.md docs/CHANGELOG.md docs/images/ -integration-test/stderr.txt +/integration-test/out/ reachability-metadata.json diff --git a/CHANGELOG.md b/CHANGELOG.md index db6c2116a..8040dd533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Add dynamic model discovery via `fetchModels` provider config for OpenAI-compatible `/models` endpoints - Improve error handling for incompatible models messages in chat. #209 +- Add basic username/password proxy authentication support and recognize lowercase http[s]_proxy env var alongside HTTP[S]_PROXY. #248 ## 0.87.2 diff --git a/bb.edn b/bb.edn index 9db9485bb..5bc0e203d 100644 --- a/bb.edn +++ b/bb.edn @@ -16,6 +16,7 @@ tag make/tag get-last-changelog-entry make/get-last-changelog-entry + ;; run with -h to see the available options integration-test make/integration-test local-webpage make/local-webpage}} diff --git a/deps-lock.json b/deps-lock.json index 1a4e97e69..7db657ebd 100644 --- a/deps-lock.json +++ b/deps-lock.json @@ -292,6 +292,26 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-K3q8dApBM5bzLK6nQPIYevjHqnKDN+E7THIyf0bhD3o=" }, + { + "mvn-path": "com/barchart/base/barchart-archon/2.5.10/barchart-archon-2.5.10.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-ttNvNLKQMA1wTtCZlFYJd7VR6JFI2MdOGHlMpPGccvg=" + }, + { + "mvn-path": "com/barchart/udt/barchart-udt-archon/2.3.0/barchart-udt-archon-2.3.0.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-pYvFwljO3N6hn60O9/6o7QWcNuGOmYzk3U2PIB7v8TE=" + }, + { + "mvn-path": "com/barchart/udt/barchart-udt-bundle/2.3.0/barchart-udt-bundle-2.3.0.jar", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-5/xl5m09UQOkrMgQ1HFcUHU1GzLGwvXe/UPFSYlhE60=" + }, + { + "mvn-path": "com/barchart/udt/barchart-udt-bundle/2.3.0/barchart-udt-bundle-2.3.0.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-/l5UPQXp1ogQxax0tEMr7MwiKyYSt73bMCaY/O82vUw=" + }, { "mvn-path": "com/cognitect/aws/api/0.8.723/api-0.8.723.jar", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -767,6 +787,21 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-4J00XnPKP7yn8+BfMN63Tp053Wt5qT/ujFEfI0F7aCg=" }, + { + "mvn-path": "com/google/guava/guava-parent/20.0/guava-parent-20.0.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-8SJv0H/HKvjWIyvfpwvzHYg6GgHLxUfyOnTpBmxpLfE=" + }, + { + "mvn-path": "com/google/guava/guava/20.0/guava-20.0.jar", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-NqZm47ca5/Dw3KI2VLZ+CG5sk9GS9gul39VRnbbCiMg=" + }, + { + "mvn-path": "com/google/guava/guava/20.0/guava-20.0.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-NjzIN2e3YNelZNUwHglGfm1I/BwcFmSx4YxQgVzhkHY=" + }, { "mvn-path": "com/google/inject/guice-parent/4.2.2/guice-parent-4.2.2.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -1157,6 +1192,16 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-A0+cuKcLwC7X/0DCTkyNXaNXCedXic5/Q5/m4ipb/h0=" }, + { + "mvn-path": "io/netty/netty-all/4.0.44.Final/netty-all-4.0.44.Final.jar", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-jCvFNQVsv2b5ZWucTpLbjSoXMlKCNGMc7oC8J7yR8hg=" + }, + { + "mvn-path": "io/netty/netty-all/4.0.44.Final/netty-all-4.0.44.Final.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-/ZZHTGaIw7m2gboN8KEzeT7DZn/7k8TGrmnrmZAZV8w=" + }, { "mvn-path": "io/netty/netty-bom/4.2.0.Final/netty-bom-4.2.0.Final.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -1182,6 +1227,11 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-YBEweqaNRjZ99sUyr7NiemOh+krBYgPhnrFQWoDfOvo=" }, + { + "mvn-path": "io/netty/netty-parent/4.0.44.Final/netty-parent-4.0.44.Final.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-Zc9FQCl5Hav9RaeRz+Zf2ge0bkbV2sVx/DcsmNImTSU=" + }, { "mvn-path": "io/netty/netty-parent/4.1.30.Final/netty-parent-4.1.30.Final.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -1712,6 +1762,16 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-gtMfHcxFg+/9dE6XkWWxbaZL+GvKYj/F0bA+2U9FyFo=" }, + { + "mvn-path": "org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.jar", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-islvxoZRLXd/yoXhRPGWzXz+DArsIxJyKUl9Gjj/ZRw=" + }, + { + "mvn-path": "org/apache/commons/commons-lang3/3.5/commons-lang3-3.5.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-Ref7ssIx25A6XVqtr8Y2oXOk1UVg94oR/0mAKO+eNF4=" + }, { "mvn-path": "org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -1727,6 +1787,11 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-Oi5p0G1kHR87KTEm3J4uTqZWO/jDbIfgq2+kKS0Et5w=" }, + { + "mvn-path": "org/apache/commons/commons-parent/41/commons-parent-41.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-sod8gBb4sokkyOkN1a5AzRHzKNAqHemNgN4iV0qzbsc=" + }, { "mvn-path": "org/apache/commons/commons-parent/42/commons-parent-42.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -3207,6 +3272,16 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-C5sUo9YhBvr+jGinF7h7h60YaFiZRRt1PAT6QbaFd4Q=" }, + { + "mvn-path": "org/littleshoot/littleproxy/1.1.2/littleproxy-1.1.2.jar", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-eWS9BW4UF5212m9Qv6j2AkGTVxrECCA0wTq/osNDWg8=" + }, + { + "mvn-path": "org/littleshoot/littleproxy/1.1.2/littleproxy-1.1.2.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-PADML6MSw6aEKjeZR2JdEc0APvP6HimHnq/3lV5bk/I=" + }, { "mvn-path": "org/msgpack/msgpack/0.6.12/msgpack-0.6.12.jar", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -3297,6 +3372,11 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-7WHEYf3Yy2lBLALnF28b953DPxKNXuuY9198rdT/fEM=" }, + { + "mvn-path": "org/slf4j/slf4j-api/1.7.24/slf4j-api-1.7.24.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-16FSZtIvrcHIG7w7YmN4AdEW5ITrbIMSF9w2xTe+SNI=" + }, { "mvn-path": "org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom", "mvn-repo": "https://repo1.maven.org/maven2/", @@ -3357,6 +3437,11 @@ "mvn-repo": "https://repo1.maven.org/maven2/", "hash": "sha256-3qe7VIVu9pOIkiqtCiU4qfNd8nk/iUF5S8smUYp0CK8=" }, + { + "mvn-path": "org/slf4j/slf4j-parent/1.7.24/slf4j-parent-1.7.24.pom", + "mvn-repo": "https://repo1.maven.org/maven2/", + "hash": "sha256-NjZ9EnpYbVArYL9U2y1zl+yUVzYg7/Ujhz7BKhLlSDk=" + }, { "mvn-path": "org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom", "mvn-repo": "https://repo1.maven.org/maven2/", diff --git a/deps.edn b/deps.edn index a9af9114b..add20558b 100644 --- a/deps.edn +++ b/deps.edn @@ -24,12 +24,22 @@ :aliases {:dev {:extra-paths ["dev"] :extra-deps {cider/cider-nrepl {:mvn/version "0.53.2"} - nrepl/nrepl {:mvn/version "1.3.0"}}} + nrepl/nrepl {:mvn/version "1.3.0"}} + ;; Properties useful for low-level debugging + ;; :jvm-opts ["-Djdk.httpclient.HttpClient.log=headers,errors" + ;; "-Dclojure.storm.instrumentOnlyPrefixes=eca,hato" + ;; "-Dflowstorm.startRecording=true"] + } + :test {:extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"} org.clojure/test.check {:mvn/version "1.1.1"} nubank/matcher-combinators {:mvn/version "3.9.1"} - http-kit/http-kit {:mvn/version "2.8.0"}} - :jvm-opts ["-XX:-OmitStackTraceInFastThrow"] + http-kit/http-kit {:mvn/version "2.8.0"} + org.littleshoot/littleproxy {:mvn/version "1.1.2"}} + :jvm-opts ["-XX:-OmitStackTraceInFastThrow" + ;; required for channeling https requirests via http + ;; proxy with basic authentication scheme. + "-Djdk.http.auth.tunneling.disabledSchemes="] :extra-paths ["test"] :main-opts ["-m" "kaocha.runner"]} :build {:extra-paths ["resources"] @@ -43,4 +53,4 @@ "-Dborkdude.dynaload.aot=true" "-Dclojure.compiler.direct-linking=true" "-Dclojure.spec.skip-macros=true"] - :extra-deps {com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"}}}}} + :extra-deps {com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"}}}}} diff --git a/docs/configuration.md b/docs/configuration.md index 13c9b2cf4..f431f4192 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -57,6 +57,19 @@ It's possible to retrieve content of any configs with a string value using the ` - `classpath`: `${classpath:path/to/eca/file}` to get a file content from [ECA's classpath](https://github.com/editor-code-assistant/eca/tree/master/resources) - `netrc`: Support Unix RC [credential files](./models.md#credential-file-authentication) +## Proxy Configuration + +ECA supports proxies with basic cleartext authentication via the de-facto env vars: + +```bash +HTTP_PROXY="http://user:pass@host:port" +HTTPS_PROXY="http://user:pass@host:port" +http_proxy="http://user:pass@host:port" +https_proxy="http://user:pass@host:port" +``` + +Lowercase var wins if both are set. Credentials (if used) must match for HTTP and HTTPS. + ## Providers / Models For providers and models configuration check the [dedicated models section](./models.md#custom-providers). diff --git a/flake.nix b/flake.nix index cf13cf28d..ee3ce02f5 100644 --- a/flake.nix +++ b/flake.nix @@ -40,6 +40,7 @@ clojure clojure-lsp git + tinyproxy ]; env = { diff --git a/integration-test/entrypoint.clj b/integration-test/entrypoint.clj index ea1052be4..d07c08f72 100755 --- a/integration-test/entrypoint.clj +++ b/integration-test/entrypoint.clj @@ -1,6 +1,8 @@ (ns entrypoint (:require - [babashka.process :refer [shell]] + [babashka.fs :as fs] + [babashka.process :refer [shell] :as p] + [clojure.java.io :as io] [clojure.test :as t] [integration.eca :as eca] [integration.chat.hooks-test] @@ -39,28 +41,105 @@ t/report log-tail-report] ~@body)) + +(def tinyproxy-dir-env-var + "Env var name pointing to the directory of the Tinyproxy executable." + "ECA_TINYPROXY_DIR") +(def proxy-conf + "Default Tinyproxy host, port, and credentials." + {:host "127.0.0.1" :port 8864 :user "tiny" :pass "pass"}) +(def tinyproxy-conf + "Tinyproxy configuration content." + (format "Listen %s +Port %d +Timeout 600 +Allow 127.0.0.1 +BasicAuth %s %s +LogLevel Info" (:host proxy-conf) (:port proxy-conf) (:user proxy-conf) (:pass proxy-conf))) +(def proxy-http + "Proxy URL with credentials." + (format "http://%s:%s@%s:%s" + (:user proxy-conf) (:pass proxy-conf) (:host proxy-conf) (:port proxy-conf))) + +(defn tinyproxy-start! + "Start a transient Tinyproxy process using the `tinyrpoxy-conf`; looks + for the executable `tinyproxy` in `tinyproxy-dir-env-var` or system PATH. + + Writes `tinyproxy.conf` and `tinyproxy.log` in `eca/*eca-out-dir*`." + [] + + (let [tp-str "tinyproxy" + tp (or (System/getenv tinyproxy-dir-env-var) (fs/which tp-str))] + (if-not tp + (throw (ex-info "No tinyproxy executables found." {:searched-for tp-str + (keyword tinyproxy-dir-env-var) (System/getenv tinyproxy-dir-env-var)})) + (let [out-conf (str (fs/path eca/*eca-out-dir* "tinyproxy.conf")) + out-log (str (fs/path eca/*eca-out-dir* "tinyproxy.log")) + cmd-full [(str tp) "-d" "-c" out-conf]] + (spit out-conf tinyproxy-conf) + (println :--entrypoint.tinyproxy-server/starting :cmd cmd-full :log-path out-log) + (println :---tinyproxy :conf out-conf) + (println (slurp out-conf)) + (println :---tinyrpoxy :end) + (p/process cmd-full + {:out (io/file out-log) + :exit-fn + (fn [{:keys [cmd exit]}] + (when (not= exit 0) + (println :--entrypoint.tinyproxy-server/exited :cmd cmd :exit-status exit) + (try + (println :log (slurp out-log)) + (catch Exception _e)) + (System/exit exit))) + :shutdown p/destroy-tree + :err :out}))))) + #_{:clojure-lsp/ignore [:clojure-lsp/unused-public-var]} -(defn run-all [binary] - (alter-var-root #'eca/*eca-binary-path* (constantly binary)) - (apply require namespaces) - - (println "Preparing mcp-server-sample") - (shell {:out nil :dir "integration-test/mcp-server-sample"} "clojure -Stree") - (llm-mock.server/start!) - - (let [timeout-minutes (if (re-find #"(?i)win|mac" (System/getProperty "os.name")) - 10 ;; win and mac ci runs take longer - 5) - test-results (timeout (* timeout-minutes 60 1000) - #(with-log-tail-report - (apply t/run-tests namespaces)))] - - (llm-mock.server/stop!) - - (when (= test-results :timed-out) - (println) - (println (format "Timeout after %d minutes running integration tests!" timeout-minutes)) - (System/exit 1)) - - (let [{:keys [fail error]} test-results] - (System/exit (+ fail error))))) +(defn run-all + "Run all tests in given namespaces NSES or all `namespaces` if not provided, + using the specified BINARY eca server. + + Sets `eca/*eca-binary-path*` to BINARY and `eca/*eca-out-dir*` to + `./integration-test/out` + + If PROXY is provided, sets `eca/*http-proxy*` to it and starts a + transient Tinyproxy server. + + Runs tests with a timeout, exiting with the sum of failures and + errors, if any." + + [binary {:keys [nses proxy]}] + (let [nses (or nses namespaces)] + + (doseq [namespace nses] + (println :entrypoint.run-all/ns-loading namespace) + (require namespace)) + (alter-var-root #'eca/*eca-binary-path* (constantly binary)) + (let [out-dir (str (fs/absolutize (fs/path "integration-test" "out")))] + (fs/create-dirs out-dir) + (alter-var-root #'eca/*eca-out-dir* (constantly out-dir))) + (when proxy + (alter-var-root #'eca/*http-proxy* (constantly proxy-http)) + (println "Routing requests through proxy:" eca/*http-proxy*) + (tinyproxy-start!)) + + (println "Preparing mcp-server-sample") + (shell {:out nil :dir "integration-test/mcp-server-sample"} "clojure -Stree") + (llm-mock.server/start!) + + (let [timeout-minutes (if (re-find #"(?i)win|mac" (System/getProperty "os.name")) + 10 ;; win and mac ci runs take longer + 5) + test-results (timeout (* timeout-minutes 60 1000) + #(with-log-tail-report + (apply t/run-tests nses)))] + + (llm-mock.server/stop!) + + (when (= test-results :timed-out) + (println) + (println (format "Timeout after %d minutes running integration tests!" timeout-minutes)) + (System/exit 1)) + + (let [{:keys [fail error]} test-results] + (System/exit (+ fail error)))))) diff --git a/integration-test/integration/eca.clj b/integration-test/integration/eca.clj index 8c0d089db..db1cb7b9b 100644 --- a/integration-test/integration/eca.clj +++ b/integration-test/integration/eca.clj @@ -1,23 +1,50 @@ (ns integration.eca (:require + [babashka.fs :as fs] [babashka.process :as p] [clojure.core.async :as async] - [clojure.java.io :as io] [clojure.test :refer [use-fixtures]] [integration.client :as client] [llm-mock.mocks :as llm.mocks])) (def ^:dynamic *eca-binary-path* nil) (def ^:dynamic *eca-process* nil) +(def ^:dynamic *eca-out-dir* + "Directory for transient files, including logs and runtime configurations." + nil) (def ^:dynamic *mock-client* nil) +(def ^:dynamic *http-proxy* + "Current HTTP proxy URL for routing requests, if set." + nil) (defn start-server - ([binary] - (start-server binary [])) - ([binary args] - (p/process (into [(.getCanonicalPath (io/file binary)) "server" "--log-level" "debug"] args) - {:err (io/file (.getParent (io/file *eca-binary-path*)) "integration-test" "stderr.txt") - :dir "integration-test/sample-test/"}))) + "Start the ECA server from the given BINARY. + +If BINARY is `clojure`, invokes the server from source via the Clojure CLI. + +Logs output to `*eca-out-dir*/server.stderr.txt`. + +If `*http-proxy*` is set, passes it as the `HTTP_PROXY` environment variable." + [binary] + (let [binary-path (str (fs/absolutize (or (fs/which binary) (fs/which (str "./" binary)) + (throw (Exception. (str "Cannot locate eca binary: " binary)))))) + args-extra (when (= binary "clojure") + ["-M" "-m" "eca.main"]) + cmd-full (concat [binary-path] args-extra ["server" "--log-level" "debug"]) + log-path (str (fs/path *eca-out-dir* "server.stderr.txt"))] + + (println :--eca.integration.start-server/starting :cmd cmd-full :log-path log-path :http-proxy *http-proxy*) + (p/process cmd-full + (cond-> {:err log-path + :exit-fn (fn [{:keys [cmd exit]}] + (when (not= exit 0) + (println :--eca.integration.start-server/exited :cmd cmd :exit-status exit) + (try + (println :log (slurp log-path)) + (catch Exception _e)) + (System/exit exit)))} + *http-proxy* + (assoc :extra-env {"HTTP_PROXY" *http-proxy*}))))) (defn start-process! [] (let [server (start-server *eca-binary-path*) diff --git a/integration-test/llm_mock/server.clj b/integration-test/llm_mock/server.clj index 0d461dc80..62a0f9520 100644 --- a/integration-test/llm_mock/server.clj +++ b/integration-test/llm_mock/server.clj @@ -1,5 +1,6 @@ (ns llm-mock.server (:require + [integration.eca :as eca] [llm-mock.anthropic :as llm-mock.anthropic] [llm-mock.ollama :as llm-mock.ollama] [llm-mock.openai :as llm-mock.openai] @@ -10,42 +11,51 @@ (defonce ^:private server* (atom nil)) -(defn ^:private app [req] - (let [{:keys [request-method uri]} req] - (cond - (and (= :post request-method) - (= uri "/openai/v1/responses")) - (llm-mock.openai/handle-openai-responses req) +(defn ^:private app + "Handle mock HTTP request REQ - (and (= :post request-method) - (= uri "/openai-chat/chat/completions")) - (llm-mock.openai-chat/handle-openai-chat req) + If `eca/*http-proxy*` is set, validates that the request came through the proxy; + otherwise responds with 403." + [req] + (let [{:keys [request-method uri headers]} req] + (if (and eca/*http-proxy* (not (get headers "via"))) + {:status 403 + :headers {"Content-Type" "text/plain"} + :body (str "requests must go through a proxy: " req)} + (cond + (and (= :post request-method) + (= uri "/openai/v1/responses")) + (llm-mock.openai/handle-openai-responses req) - (and (= :post request-method) - (= uri "/github-copilot/chat/completions")) - (llm-mock.openai-chat/handle-openai-chat req) + (and (= :post request-method) + (= uri "/openai-chat/chat/completions")) + (llm-mock.openai-chat/handle-openai-chat req) - (and (= :post request-method) - (= uri "/google/chat/completions")) - (llm-mock.openai-chat/handle-openai-chat req) + (and (= :post request-method) + (= uri "/github-copilot/chat/completions")) + (llm-mock.openai-chat/handle-openai-chat req) - (and (= :post request-method) - (= uri "/anthropic/v1/messages")) - (llm-mock.anthropic/handle-anthropic-messages req) + (and (= :post request-method) + (= uri "/google/chat/completions")) + (llm-mock.openai-chat/handle-openai-chat req) - (and (= :get request-method) - (= uri "/ollama/api/tags")) - (llm-mock.ollama/handle-ollama-tags req) + (and (= :post request-method) + (= uri "/anthropic/v1/messages")) + (llm-mock.anthropic/handle-anthropic-messages req) - (and (= :post request-method) - (= uri "/ollama/api/show")) - (llm-mock.ollama/handle-ollama-show req) + (and (= :get request-method) + (= uri "/ollama/api/tags")) + (llm-mock.ollama/handle-ollama-tags req) - (and (= :post request-method) - (= uri "/ollama/api/chat")) - (llm-mock.ollama/handle-ollama-chat req) + (and (= :post request-method) + (= uri "/ollama/api/show")) + (llm-mock.ollama/handle-ollama-show req) - :else {:status 404 :headers {} :body "not found"}))) + (and (= :post request-method) + (= uri "/ollama/api/chat")) + (llm-mock.ollama/handle-ollama-chat req) + + :else {:status 404 :headers {} :body "not found"})))) (defn start! [] (when-not @server* diff --git a/scripts/make.clj b/scripts/make.clj index e6708cc5f..6ae9ca488 100644 --- a/scripts/make.clj +++ b/scripts/make.clj @@ -1,5 +1,6 @@ (ns make (:require + [babashka.cli :as cli] [babashka.deps :as deps] [babashka.fs :as fs] [babashka.process :as p] @@ -92,18 +93,59 @@ (clj! ["-M:test"]) (println)) +(def integration-test-cli-spec + "Command-line options supported by the `integration-test` task." + {:spec + {:dev {:desc "Invoke the ECA server from the source files using the `clojure` CLI rather than the ECA binary."} + :help {:desc "Display help." + :alias :h} + :list-ns {:desc "List available test namespaces." + :alias :l} + + :ns {:desc "Comma-separated list of test namespaces to run." + :type :string + :validate (fn [s] + (every? #(some #{%} entrypoint/namespaces) + (map symbol (string/split s #","))))} + :proxy + {:desc (str "Send requests through a transient Tinyproxy instance and verify routing. The Tinyproxy executable path can be set via the " entrypoint/tinyproxy-dir-env-var " environment variable if it is not on the PATH.") + :type :string}}}) + (defn integration-test "Run the integration tests in 'test/integration-test/' using `./eca[.bat|.exe]`. + Parses command-line options, run with `-h|--help` to see all available flags. + There should only be one eca executable found, throws error otherwise." - [] - (let [eca-bins (->> [:native :script] (map eca-bin-filename) distinct) - eca-bins-found (->> eca-bins (filter fs/exists?) (into #{}))] + [& args] + (let [{:keys [dev help list-ns ns proxy]} + (cli/parse-opts args integration-test-cli-spec) + eca-bins (->> [:native :script] (map eca-bin-filename) distinct) + eca-bins-found (->> eca-bins (filter fs/exists?) (into #{})) + eca-bins-found (if dev + #{"clojure"} + eca-bins-found) + nses (when ns (map symbol (string/split ns #",")))] + + (when help + (println "Run the integration tests using the ./eca[.bat].exe] server.") + (println (let [spec integration-test-cli-spec] + (cli/format-opts (merge spec {:order (vec (keys (:spec spec)))})))) + (System/exit 0)) + + (when list-ns + (println "Available test namespaces:") + (doseq [n entrypoint/namespaces] (println " - " n)) + (System/exit 0)) (case (count eca-bins-found) 0 (throw (ex-info "No eca executables found." {:searched-for eca-bins})) - 1 (entrypoint/run-all (str (first eca-bins-found))) + 1 (entrypoint/run-all (str (first eca-bins-found)) (cond-> {} + proxy + (assoc :proxy true) + nses + (assoc :nses nses))) (throw (ex-info "More than one eca executables found. Can only work with one." {:bin-found eca-bins-found}))))) diff --git a/src/eca/client_http.clj b/src/eca/client_http.clj new file mode 100644 index 000000000..c6d891404 --- /dev/null +++ b/src/eca/client_http.clj @@ -0,0 +1,96 @@ +(ns eca.client-http + "Support for the HTTP client to make outbound requests." + (:require [eca.logger :as logger] + [eca.proxy :as proxy]) + (:import [java.net URI Proxy Proxy$Type InetSocketAddress ProxySelector] + [java.io IOException])) + +(defn hato-client-make + "Builds an options map for creating a Hato HTTP client. + + Accepts all OPTS supported by `hato.client-http/build-http-client`. + + It also supports optional specialized keys + `:eca.client-http/proxy-http` and `:eca.client-http/proxy-https`, + which define proxy servers for outgoing HTTP and HTTPS requests; + proxy credentials are used from either proxy when provided, and if + both are present, they must be identical, otherwise an exception is + thrown. + + Each proxy map includes: + :host - the proxy host + :port - the proxy port + :username - optional username for proxy authentication + :password - optional password for proxy authentication + + Returns a map suitable for passing to `hato.client-http/build-http-client`." + [{:eca.client-http/keys [proxy-http proxy-https] :as opts}] + (logger/debug "hato-client-config: " opts) + (let [{http-host :host http-port :port http-user :username http-pass :password} proxy-http + {https-host :host https-port :port https-user :username https-pass :password} proxy-https + opts (apply dissoc opts [:eca.client-http/proxy-http :eca.client-http/proxy-https]) + proxy-http-addr (and http-host http-port (InetSocketAddress. ^String http-host ^int http-port)) + proxy-https-addr (and https-host https-port (InetSocketAddress. ^String https-host ^int https-port)) + proxy-selector (when (or proxy-http-addr proxy-https-addr) + (proxy [ProxySelector] [] + (select [^URI uri] + (let [scheme (.getScheme uri)] + (cond + (and proxy-http-addr (= scheme "http")) + [(Proxy. Proxy$Type/HTTP proxy-http-addr)] + (and proxy-https-addr (= scheme "https")) + [(Proxy. Proxy$Type/HTTP proxy-https-addr)] + :else + []))) + (connectFailed [_ _ ^IOException e] + (.printStackTrace e)))) + http-creds (when (and http-user http-pass) + {:user http-user :pass http-pass}) + https-creds (when (and https-user https-pass) + {:user https-user :pass https-pass}) + proxy-creds (cond + (and http-creds https-creds) + (if (= http-creds https-creds) + http-creds + (throw (ex-info "HTTP and HTTPS proxy credentials must be identical" + {:http http-creds + :https https-creds}))) + http-creds http-creds + https-creds https-creds + :else nil)] + (cond-> opts + proxy-selector + (assoc :proxy proxy-selector) + proxy-creds + (assoc :authenticator proxy-creds)))) + +(def ^:dynamic *hato-http-client* + "Global Hato HTTP client used throughout the application for making + HTTP requests" + nil) + +(defn merge-with-global-http-client + "Merge the given Hato HTTP client options with the global + `*hato-http-client*` and return the result." + [http-client] + (merge *hato-http-client* http-client)) + +(defn hato-client-global-setup! + "Builds the Hato HTTP client used throughout the application for making + HTTP requests from HATO-OPTS and stores it in + `eca.client-http/*hato-http-client*`. + + HATO-OPTS are the same options accepted by Hato's + `hato.client-http/build-http-client`. In addition, if HTTP or HTTPS proxy + settings are present in the environment + variables (`http_proxy`/`HTTP_PROXY` and `https_proxy`/`HTTPS_PROXY`), + the corresponding proxy configuration is added to the build." + [hato-opts] + (let [{:keys [http https] :as _env-proxies} (proxy/env-proxy-urls-parse) + opts (cond-> hato-opts + http + (assoc :eca.client-http/proxy-http http) + https + (assoc :eca.client-http/proxy-https https)) + hato-http-client (hato-client-make opts)] + (alter-var-root #'*hato-http-client* (constantly hato-http-client)))) diff --git a/src/eca/llm_providers/anthropic.clj b/src/eca/llm_providers/anthropic.clj index fb2d61b12..1dabdff2c 100644 --- a/src/eca/llm_providers/anthropic.clj +++ b/src/eca/llm_providers/anthropic.clj @@ -3,6 +3,7 @@ [cheshire.core :as json] [clojure.java.io :as io] [clojure.string :as string] + [eca.client-http :as client] [eca.config :as config] [eca.features.login :as f.login] [eca.llm-util :as llm-util] @@ -89,7 +90,7 @@ :body (json/generate-string body) :throw-exceptions? false :async? true - :http-client http-client + :http-client (client/merge-with-global-http-client http-client) :as (if on-stream :stream :json)} (fn [{:keys [status body]}] (try @@ -289,9 +290,12 @@ :code_challenge_method "S256" :state verifier}))})) +(def ^:private oauth-token-url + "https://console.anthropic.com/v1/oauth/token") + (defn ^:private oauth-authorize [code verifier] (let [[code state] (string/split code #"#") - url "https://console.anthropic.com/v1/oauth/token" + url oauth-token-url body {:grant_type "authorization_code" :code code :state state @@ -302,6 +306,7 @@ url {:headers {"Content-Type" "application/json"} :body (json/generate-string body) + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) {:refresh-token (:refresh_token body) @@ -312,7 +317,7 @@ :body body}))))) (defn ^:private oauth-refresh [refresh-token] - (let [url "https://console.anthropic.com/v1/oauth/token" + (let [url oauth-token-url body {:grant_type "refresh_token" :refresh_token refresh-token :client_id client-id} @@ -321,6 +326,7 @@ {:headers {"Content-Type" "application/json"} :body (json/generate-string body) :throw-exceptions? false + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) {:refresh-token (:refresh_token body) @@ -330,13 +336,18 @@ {:status status :body body}))))) + +(def ^:private create-api-key-url + "https://api.anthropic.com/api/oauth/claude_cli/create_api_key") + (defn ^:private create-api-key [access-token] - (let [url "https://api.anthropic.com/api/oauth/claude_cli/create_api_key" + (let [url create-api-key-url {:keys [status body]} (http/post url {:headers {"Authorization" (str "Bearer " access-token) "Content-Type" "application/x-www-form-urlencoded" "Accept" "application/json, text/plain, */*"} + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) (let [raw-key (:raw_key body)] diff --git a/src/eca/llm_providers/copilot.clj b/src/eca/llm_providers/copilot.clj index a01ff38fc..7cc382412 100644 --- a/src/eca/llm_providers/copilot.clj +++ b/src/eca/llm_providers/copilot.clj @@ -1,6 +1,7 @@ (ns eca.llm-providers.copilot (:require [cheshire.core :as json] + [eca.client-http :as client] [eca.config :as config] [eca.features.login :as f.login] [hato.client :as http])) @@ -13,25 +14,33 @@ "editor-plugin-version" "eca/*" "editor-version" (str "eca/" (config/eca-version))}) +(def ^:private oauth-login-device-url + "https://github.com/login/device/code") + (defn ^:private oauth-url [] (let [{:keys [body]} (http/post - "https://github.com/login/device/code" + oauth-login-device-url {:headers (auth-headers) :body (json/generate-string {:client_id client-id :scope "read:user"}) + :http-client (client/merge-with-global-http-client {}) :as :json})] {:user-code (:user_code body) :device-code (:device_code body) :url (:verification_uri body)})) +(def ^:private oauth-login-access-token-url + "https://github.com/login/oauth/access_token") + (defn ^:private oauth-access-token [device-code] (let [{:keys [status body]} (http/post - "https://github.com/login/oauth/access_token" + oauth-login-access-token-url {:headers (auth-headers) :body (json/generate-string {:client_id client-id :device_code device-code :grant_type "urn:ietf:params:oauth:grant-type:device_code"}) :throw-exceptions? false + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) (:access_token body) @@ -39,12 +48,16 @@ {:status status :body body}))))) +(def ^:private oauth-copilot-token-url + "https://api.github.com/copilot_internal/v2/token") + (defn ^:private oauth-renew-token [access-token] (let [{:keys [status body]} (http/get - "https://api.github.com/copilot_internal/v2/token" + oauth-copilot-token-url {:headers (merge (auth-headers) {"authorization" (str "token " access-token)}) :throw-exceptions? false + :http-client (client/merge-with-global-http-client {}) :as :json})] (if-let [token (:token body)] {:api-key token diff --git a/src/eca/llm_providers/ollama.clj b/src/eca/llm_providers/ollama.clj index 668e24c2e..b5660349c 100644 --- a/src/eca/llm_providers/ollama.clj +++ b/src/eca/llm_providers/ollama.clj @@ -2,6 +2,7 @@ (:require [cheshire.core :as json] [clojure.java.io :as io] + [eca.client-http :as client] [eca.llm-util :as llm-util] [eca.logger :as logger] [eca.shared :refer [deep-merge]] @@ -21,6 +22,7 @@ {:keys [status body]} (http/get (format list-models-url api-url) {:throw-exceptions? false + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) (do @@ -39,6 +41,7 @@ (format show-model-url api-url) {:throw-exceptions? false :body (json/generate-string {:model model}) + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) (do @@ -66,6 +69,7 @@ {:body (json/generate-string body) :throw-exceptions? false :async? true + :http-client (client/merge-with-global-http-client {}) :as (if on-stream :stream :json)} (fn [{:keys [status body]}] (try diff --git a/src/eca/llm_providers/openai.clj b/src/eca/llm_providers/openai.clj index 2f5aa7351..9add35126 100644 --- a/src/eca/llm_providers/openai.clj +++ b/src/eca/llm_providers/openai.clj @@ -3,6 +3,7 @@ [cheshire.core :as json] [clojure.java.io :as io] [clojure.string :as string] + [eca.client-http :as client] [eca.config :as config] [eca.features.login :as f.login] [eca.llm-util :as llm-util] @@ -56,7 +57,7 @@ :body (json/generate-string body) :throw-exceptions? false :async? true - :http-client http-client + :http-client (client/merge-with-global-http-client http-client) :as (if on-stream :stream :json)} (fn [{:keys [status body]}] (try @@ -283,9 +284,12 @@ :code_challenge_method "S256" :state verifier}))})) +(def ^:private oauth-token-url + "https://auth.openai.com/oauth/token") + (defn ^:private oauth-authorize [server-url code verifier] (let [{:keys [status body]} (http/post - "https://auth.openai.com/oauth/token" + oauth-token-url {:headers {"Content-Type" "application/json"} :body (json/generate-string {:grant_type "authorization_code" @@ -293,6 +297,7 @@ :code code :code_verifier verifier :redirect_uri server-url}) + :http-client (client/merge-with-global-http-client {}) :as :json})] (if (= 200 status) {:refresh-token (:refresh_token body) diff --git a/src/eca/llm_providers/openai_chat.clj b/src/eca/llm_providers/openai_chat.clj index ccab3da28..a9c475c7e 100644 --- a/src/eca/llm_providers/openai_chat.clj +++ b/src/eca/llm_providers/openai_chat.clj @@ -3,6 +3,7 @@ [cheshire.core :as json] [clojure.java.io :as io] [clojure.string :as string] + [eca.client-http :as client] [eca.llm-util :as llm-util] [eca.logger :as logger] [eca.shared :refer [assoc-some deep-merge]] @@ -111,7 +112,7 @@ :body (json/generate-string body) :throw-exceptions? false :async? true - :http-client http-client + :http-client (client/merge-with-global-http-client http-client) :as (if on-stream :stream :json)} (fn [{:keys [status body]}] (try diff --git a/src/eca/main.clj b/src/eca/main.clj index def2e9da1..6fa38dbcb 100644 --- a/src/eca/main.clj +++ b/src/eca/main.clj @@ -7,7 +7,7 @@ [clojure.string :as string] [eca.config :as config] [eca.logger :as logger] - [eca.proxy :as proxy] + [eca.client-http :as client] [eca.server :as server])) (set! *warn-on-reflection* true) @@ -94,7 +94,7 @@ (defn ^:private handle-action! [action options] - (proxy/load!) + (client/hato-client-global-setup! {}) (when (= "server" action) (when-some [cfg-file (:config-file options)] (reset! config/custom-config-file-path* cfg-file)) diff --git a/src/eca/proxy.clj b/src/eca/proxy.clj index 0bb91f67d..7937275db 100644 --- a/src/eca/proxy.clj +++ b/src/eca/proxy.clj @@ -1,23 +1,91 @@ (ns eca.proxy (:require + [clojure.string :as string] [eca.config :as config]) (:import - [java.net URL])) + [java.net URI URLDecoder])) (set! *warn-on-reflection* true) -(defn load! [] - (when-let [^URL url (try (some-> (not-empty (config/get-env "HTTP_PROXY")) - (URL.)) - (catch Exception _ nil))] - (System/setProperty "http.proxyHost" (.getHost url)) - (let [port (.getPort url)] - (when (not= -1 port) - (System/setProperty "http.proxyPort" (str (.getPort url)))))) - (when-let [^URL url (try (some-> (not-empty (config/get-env "HTTPS_PROXY")) - (URL.)) - (catch Exception _ nil))] - (System/setProperty "https.proxyHost" (.getHost url)) - (let [port (.getPort url)] - (when (not= -1 port) - (System/setProperty "https.proxyPort" (str (.getPort url))))))) +(defn parse-proxy-url + "Parses a proxy URL-STR of the form `scheme://[user:pass@]host:port` into a map. + + Returns a map with the following keys: + + :host - the host part of the URL + + :port - the port part. Defaults to 80 for HTTP, 443 for HTTPS, + and throws `IllegalArgumentException` if the scheme is neither + + :username - the user part of the URL, if present; URL-decoded + + :password - the password part of the URL, if present; URL-decoded + + Both username and password are automatically URL-decoded if they contain percent encoded characters." + [url-str] + (when (and url-str (not (clojure.string/blank? url-str))) + (let [^URI uri (URI. url-str) + host (.getHost uri) + port (if (neg? (.getPort uri)) + (case (.getScheme uri) + "http" 80 + "https" 443 + (throw (IllegalArgumentException. (str "Unsupported scheme: " (.getScheme uri))))) + (.getPort uri)) + userinfo (.getUserInfo uri) + [user pass] (when userinfo + (clojure.string/split userinfo #":" 2)) + decode (fn [^String s] (when s (URLDecoder/decode s "UTF-8")))] + {:host host + :port port + :username (decode user) + :password (decode pass)}))) + +(defn proxy-urls-system-env-get + "Returns a map of the HTTP and HTTPS proxy environment variables, + preferring lowercase variable names over uppercase. + + The map includes: + + :http - string value of http_proxy or HTTP_PROXY. + :https - string value of https_proxy or HTTPS_PROXY." + [] + {:http (or (config/get-env "http_proxy") + (config/get-env "HTTP_PROXY")) + :https (or (config/get-env "https_proxy") + (config/get-env "HTTPS_PROXY"))}) + +(defn proxy-urls-parse + "Parses the HTTP and HTTPS proxy URL strings from URLS-MAP and returns a map of parsed values. + + The input map may contain the keys: + + :http - the HTTP proxy URL string, or nil + :https - the HTTPS proxy URL string, or nil + + Returns a map with the same keys: + + :http - a map with keys :host, :port, :username, :password for the HTTP proxy, or nil if not set + :https - a map with keys :host, :port, :username, :password for the HTTPS proxy, or nil if not set + + Ports default to 80 for HTTP and 443 for HTTPS if not specified. Usernames and passwords are URL-decoded if present." + [urls-map] + (update-vals (select-keys urls-map [:http :https]) parse-proxy-url)) + +(defn env-proxy-urls-parse + "Fetches the HTTP and HTTPS proxy URLs from environment variables and returns a map of parsed values. + + The function looks for these environment variables, preferring lowercase over uppercase: + + :http - from `http_proxy` or `HTTP_PROXY` + :https - from `https_proxy` or `HTTPS_PROXY` + + The returned map contains: + + :http - a map with keys :host, :port, :username, :password for the HTTP proxy, or nil if not set + :https - a map with keys :host, :port, :username, :password for the HTTPS proxy, or nil if not set + + Port defaults to 80 for HTTP and 443 for HTTPS if not specified. Usernames and passwords are URL-decoded if present." + [] + (proxy-urls-parse (proxy-urls-system-env-get))) + diff --git a/test/eca/client_http_test.clj b/test/eca/client_http_test.clj new file mode 100644 index 000000000..e0c97cbe4 --- /dev/null +++ b/test/eca/client_http_test.clj @@ -0,0 +1,199 @@ +(ns eca.client-http-test + (:require [clojure.test :refer [deftest is testing]] + [eca.client-http :as client] + [eca.client-test-helpers :refer [with-proxy *proxy-host* *proxy-port*]] + [eca.config :as config] + [hato.client :as hato]) + (:import [java.io IOException])) + +(deftest hato-client-make-test + (testing "proxy http setup" + (with-proxy {} + (fn [_req] + {:status 200 + :body "hello"}) + + (let [client (client/hato-client-make {:eca.client-http/proxy-http {:host *proxy-host* + :port *proxy-port*}}) + response (hato/post + "http://localhost:99/now" + {:http-client client})] + (is (= {:uri "http://localhost:99/now", + :status 200, + :body "hello", + :version :http-1.1 + :request {:user-info nil, + :http-client {}, + :headers {"accept-encoding" "gzip, deflate"}, + :server-port 99, + :url "http://localhost:99/now", + :uri "/now", + :server-name "localhost", + :query-string nil, + :scheme :http, + :request-method :post}} + (-> response + (select-keys [:uri :status :body :version :request]) + (update-in [:request :http-client] dissoc :proxy) + (update-in [:request] dissoc :http-request))))))) + + (testing "Handles both HTTP and HTTPS proxies and selects based on URI scheme" + (let [reqs* (atom [])] + (with-proxy {} + (fn [req] + (swap! reqs* conj req) + {:status 200 :body (:uri req)}) + + (let [client (client/hato-client-make {:eca.client-http/proxy-http {:host *proxy-host* :port *proxy-port*} + :eca.client-http/proxy-https {:host *proxy-host* :port *proxy-port*}})] + ;; HTTP request + (let [http-resp (hato/post "http://localhost:99/http" {:http-client client})] + (is (= 200 (:status http-resp))) + (is (= "/http" (:body http-resp)))) + + ;; HTTPS request + (is (thrown-with-msg? ;; expected as we only testing rerouting through proxy + Exception + #"Unrecognized SSL message, plaintext connection?" (hato/post "https://localhost/https" {:http-client client}))) + (let [req2 (second @reqs*)] + (is (= {:method "CONNECT" :uri "localhost:443"} (select-keys req2 [:method :uri])))))))) + + (testing "Includes an authenticator when proxy username and password are provided" + (with-proxy {:user "u" :pass "p"} + (fn [req] {:status 200 :body (:uri req)}) + (let [client (client/hato-client-make {:eca.client-http/proxy-http {:host *proxy-host* :port *proxy-port* :username "u" :password "p"}}) + response (hato/post "http://localhost:99/auth" {:http-client client})] + (is (= 200 (:status response))) + (is (= "/auth" (:body response)))))) + + (testing "Includes an authenticator when proxy username and password are provided (HTTPS)" + (let [req* (atom nil)] + (with-proxy {:user "us" :pass "ps"} + (fn [req] + (reset! req* req) + {:status 200 :body (:uri req)}) + (let [client (client/hato-client-make {:eca.client-http/proxy-https {:host *proxy-host* :port *proxy-port* :username "us" :password "ps"}})] + (is (thrown-with-msg? + Exception + #"Unrecognized SSL message, plaintext connection?" (hato/post "https://localhost/auth" {:http-client client}))) + (is (= {:method "CONNECT" :uri "localhost:443"} (select-keys @req* [:method :uri])) @req*))))) + + (testing "Rejects requests when proxy username and password are incorrect" + (with-proxy {:user "correct-user" :pass "correct-pass"} + (fn [req] + {:status 200 :body (:uri req)}) + (let [client (client/hato-client-make + {:eca.client-http/proxy-http + {:host *proxy-host* + :port *proxy-port* + :username "wrong-user" + :password "wrong-pass"}})] + (is (thrown-with-msg? ;; expected as we only testing rerouting through proxy + IOException + #"too many authentication attempts" + (hato/post "http://localhost:99/fail-auth" {:http-client client})))))) + + (testing "Rejects requests when proxy username and password are incorrect (HTTPS)" + (let [req* (atom nil)] + (with-proxy {:user "correct-user" :pass "correct-pass"} + (fn [req] + (reset! req* req) + {:status 200 :body (:uri req)}) + (let [client (client/hato-client-make + {:eca.client-http/proxy-https + {:host *proxy-host* + :port *proxy-port* + :username "wrong-user" + :password "wrong-pass"}})] + ;; it throws a very intrinsic exception when proxy authentication fails using https BASIC authentication: + ;; Caused by: java.lang.NullPointerException: Cannot invoke "jdk.internal.net.http.ExchangeImpl.cancel(java.io.IOException)" because "exch.exchImpl" is null ... + (is (thrown? ;; expected as we only testing rerouting through proxy + Exception + (hato/post "https://localhost/fail-auth" {:http-client client}))) + (is (nil? @req*)))))) + + (testing "uses shared proxy credentials when both HTTP and HTTPS credentials match" + (let [reqs* (atom [])] + (with-proxy {:user "u" :pass "p"} + (fn [req] + (swap! reqs* conj req) + {:status 200 :body (:uri req)}) + + (let [client (client/hato-client-make + {:eca.client-http/proxy-http + {:host *proxy-host* :port *proxy-port* :username "u" :password "p"} + :eca.client-http/proxy-https + {:host *proxy-host* :port *proxy-port* :username "u" :password "p"}})] + + ;; HTTP request uses proxy + creds + (let [resp (hato/post "http://localhost:99/http" {:http-client client})] + (is (= 200 (:status resp))) + (is (= "/http" (:body resp)))) + + ;; HTTPS request routes via CONNECT using same creds + (is (thrown-with-msg? + Exception + #"Unrecognized SSL message, plaintext connection?" + (hato/post "https://localhost/https" {:http-client client}))) + + (is (= {:method "CONNECT" :uri "localhost:443"} + (select-keys (second @reqs*) [:method :uri]))))))) + + (testing "rejects configuration when HTTP and HTTPS proxy credentials differ" + (with-proxy {:user "u1" :pass "p1"} + (fn [_req] {:status 200 :body "ok"}) + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"must be identical" + (client/hato-client-make + {:eca.client-http/proxy-http + {:host *proxy-host* :port *proxy-port* :username "u1" :password "p1"} + :eca.client-http/proxy-https + {:host *proxy-host* :port *proxy-port* :username "u2" :password "p2"}})))))) + +(deftest hato-client-global-setup-tests + (testing "Hato uses a system proxy through *hato-http-client*" + (with-proxy {} + (fn [req] + {:status 200 + :body (str "proxied:" (:uri req))}) + + ;; Override get-env to simulate HTTP/HTTPS proxy variables + (with-redefs [config/get-env (fn [env] + (case env + "http_proxy" (str "http://" *proxy-host* ":" *proxy-port*) + nil))] + (try + (client/hato-client-global-setup! {:timeout 1000}) + + ;; Make a request using the global client + (let [resp (hato/get "http://localhost:99/test" {:http-client client/*hato-http-client*})] + (is (= 200 (:status resp))) + (is (= "proxied:/test" (:body resp)))) + + (finally + (alter-var-root #'client/*hato-http-client* (constantly nil))))))) + + (testing "Hato uses an HTTPS system proxy through *hato-http-client*" + (let [req* (atom nil)] + (with-proxy {} + (fn [req] + (reset! req* req) + {:status 200 + :body (:uri req)}) + + (with-redefs [config/get-env (fn [env] + (case env + "https_proxy" (str "http://" *proxy-host* ":" *proxy-port*) + nil))] + (try + (client/hato-client-global-setup! {:timeout 1000}) + + (is (thrown-with-msg? ;; expected as we only testing rerouting through proxy + Exception + #"Unrecognized SSL message, plaintext connection?" (hato/get "https://localhost/test" {:http-client client/*hato-http-client*}))) + (is (= {:method "CONNECT" :uri "localhost:443"} (select-keys @req* [:method :uri]))) + + (finally + (alter-var-root #'client/*hato-http-client* (constantly nil))))))))) +#_(hato-client-global-setup-tests) diff --git a/test/eca/client_test_helpers.clj b/test/eca/client_test_helpers.clj new file mode 100644 index 000000000..2fad98f2c --- /dev/null +++ b/test/eca/client_test_helpers.clj @@ -0,0 +1,215 @@ +(ns eca.client-test-helpers + (:require + [cheshire.core :as json] + [eca.client-http :as client]) + (:import [io.netty.buffer Unpooled] + [io.netty.handler.codec.http DefaultFullHttpResponse FullHttpRequest HttpHeaders HttpResponseStatus HttpResponseStatus HttpVersion] + [java.nio.charset StandardCharsets] + [org.littleshoot.proxy HttpFiltersAdapter HttpFiltersSource ProxyAuthenticator] + [org.littleshoot.proxy.impl DefaultHttpProxyServer])) + +(defn ^HttpFiltersSource proxy-filters-handler-make + "Creates a LittleProxy `HttpFiltersSource` that intercepts HTTP requests, + normalizes them into a simple map, passes them to HANDLER-FN, and converts + the handler's return value into an HTTP response. + + Request map passed to HANDLER-FN includes: + :method - HTTP method as a string + :uri - request URI + :protocol - protocol version + :headers - request headers as a map + :body - request body; JSON bodies are parsed into maps + + HANDLER-FN should return a map with: + :status - HTTP response status code (default 200) + :body - response body; maps are automatically encoded as JSON + :headers - optional map of response headers + + Buffer limits are set so the proxy can accept request bodies up to + 32 KB and does not buffer responses." + [handler-fn] + (proxy [HttpFiltersSource] [] + (filterRequest [original-request ctx] + (proxy [HttpFiltersAdapter] [original-request] + (proxyToServerRequest [http-obj] + (let [{:keys [response ^String error]} + (try + (if (instance? FullHttpRequest http-obj) + (let [req ^FullHttpRequest http-obj + headers-map (into {} + (for [^java.util.Map$Entry h (.entries (.headers req))] + [(.getKey h) (.getValue h)])) + content-type (get headers-map "Content-Type") + body-str (.toString (.content req) StandardCharsets/UTF_8) + parsed-body (if (and content-type + (.contains ^String content-type "application/json")) + (json/parse-string body-str true) + body-str) + body-map {:method (.name (.getMethod req)) + :uri (.getUri req) + :protocol (str (.getProtocolVersion req)) + :headers headers-map + :body parsed-body} + response-data (handler-fn body-map) + status (get response-data :status) + response-body (get response-data :body) + response-headers (get response-data :headers) + response-body-map? (map? response-body) + response-body-str (if response-body-map? + (json/generate-string response-body) + (str response-body)) + response-headers (cond-> response-headers + response-body-map? + (assoc "Content-Type" "application/json")) + content (Unpooled/copiedBuffer response-body-str StandardCharsets/UTF_8) + response (DefaultFullHttpResponse. + HttpVersion/HTTP_1_1 + (HttpResponseStatus/valueOf status) + content)] + ;; Set headers + (doseq [[^String k v] response-headers] + (.set ^HttpHeaders (.headers response) k v)) + (.set (.headers response) "Content-Length" (str (.readableBytes content))) + {:response response}) + + ;; error + {:error "Eca test proxy handler error: expected FullHttpRequest, check (getMaximumRequestBufferSizeInBytes) value"}) + (catch Exception e + {:error (str "Eca test proxy handler error: " (str e))}))] + + (if error + (let [content (Unpooled/copiedBuffer error StandardCharsets/UTF_8) + response (DefaultFullHttpResponse. + HttpVersion/HTTP_1_1 + HttpResponseStatus/BAD_REQUEST ;; 400 + content)] + (.set (.headers response) "Content-Length" (str (.readableBytes content))) + (.set (.headers response) "Connection" "close") + response) + + ;; success + response))))) + + (getMaximumRequestBufferSizeInBytes [] (* 1024 32)) + (getMaximumResponseBufferSizeInBytes [] 0))) + +(defn proxy-authenticator-make + "Creates a LittleProxy ProxyAuthenticator that returns true if the + provided USERNAME and PASSWORD match those of the incoming request, + allowing it to authenticate." + [username password] + (proxy [ProxyAuthenticator] [] + (authenticate [user pass] + (and (= user username) + (= pass password))) + (getRealm [] "EcaLittleProxyTestRealm") + (authScheme [_] "Basic"))) + +(defn little-proxy-interceptor-make + "Creates and starts a LittleProxy server that routes all intercepted requests to HANDLER-FN. + + The HANDLER-FN input and output format is defined by `proxy-filters-handler-make`. + + Optional authentication can be enforced by providing USERNAME and PASSWORD. + + Returns a map with: + :px - the running proxy server instance + :host - the hostname the proxy is listening on + :port - the port assigned to the proxy" + ([handler-fn] + (little-proxy-interceptor-make handler-fn nil nil)) + ([handler-fn username password] + (let [px (-> (DefaultHttpProxyServer/bootstrap) + (.withPort 0) ;; 0 pick random port + (.withFiltersSource (proxy-filters-handler-make handler-fn)) + (cond-> username (.withProxyAuthenticator (proxy-authenticator-make username password))) + (.start))] + {:px px + :host (.getHostString (.getListenAddress px)) + :port (.getPort (.getListenAddress px))}))) + +(def ^:dynamic ^String *proxy-host* + "Dynamic var for the host of a temporary proxy from `with-proxy`." + nil) + +(def ^:dynamic *proxy-port* + "Dynamic var for the port of a temporary proxy from `with-proxy`." + nil) + +(defmacro with-proxy + "Runs BODY with a temporary LittleProxy server active at a random local port. + + Starts a proxy using the provided request HANDLER-FN and optional + OPTS, making its host and port available via the dynamic vars + `*proxy-host*` and `*proxy-port*`. Ensures the proxy is shut down + after BODY executes. + + HANDLER-FN is a function that receives a normalized request map and + returns a response map. The request map includes: + :method - HTTP method as a string + :uri - request URI + :protocol - protocol version + :headers - request headers as a map + :body - request body, JSON is automatically parsed into a map if applicable + + OPTS may include: + :user - if set, the proxy requires this username for authentication + :pass - if set, the proxy requires this password for authentication" + [opts handler-fn & body] + (let [{:keys [user pass]} opts] + `(let [prx# (little-proxy-interceptor-make ~handler-fn ~user ~pass) + prx-host# (:host prx#) + prx-port# (:port prx#)] + (try + (binding [*proxy-host* prx-host# + *proxy-port* prx-port#] + ~@body) + (finally + (.abort ^DefaultHttpProxyServer (:px prx#))))))) + +(def ^:dynamic *http-client-captures* + "A record of all `eca.client-http/merge-with-global-http-client` merge + requests results done during the call to `with-client-proxied` + call." + nil) + +(defmacro with-client-proxied + "Runs BODY with a temporary LittleProxy server on a random local port, + configuring `eca.client-http/*hato-http-client*` so all Hato + requests in BODY are routed through the proxy. The proxy is started + using the provided HANDLER-FN and optional OPTS, and is always shut + down afterward, with `eca.client-http/*hato-http-client*` reset to + nil. + + During execution, any calls to `eca.client-http/merge-with-global-http-client` + are recorded in `*http-client-captures*` as a sequence of merged options maps. + + Accepts all OPTS supported by `hato.client-http/build-http-client`. + + HANDLER-FN is a function that receives a normalized request map and + returns a response map. The request map includes: + :method - HTTP method as a string + :uri - request URI + :protocol - protocol version + :headers - request headers as a map + :body - request body, JSON is automatically parsed into a map if applicable" + [opts handler-fn & body] + + `(with-proxy ~opts + ~handler-fn + + (let [client# (client/hato-client-make (assoc ~opts :eca.client-http/proxy-http {:host *proxy-host* :port *proxy-port*}))] + (try + (alter-var-root #'client/*hato-http-client* (constantly client#)) + (let [merges*# (atom []) + merge-fn# client/merge-with-global-http-client] + (with-redefs [client/merge-with-global-http-client + (fn [& args#] + (let [result# (apply merge-fn# args#)] + (swap! merges*# conj result#) + result#))] + (binding [*http-client-captures* merges*#] + ~@body))) + (finally + (alter-var-root #'client/*hato-http-client* (constantly nil))))))) + diff --git a/test/eca/client_test_helpers_test.clj b/test/eca/client_test_helpers_test.clj new file mode 100644 index 000000000..8272ca1b0 --- /dev/null +++ b/test/eca/client_test_helpers_test.clj @@ -0,0 +1,443 @@ +(ns eca.client-test-helpers-test + (:require + [cheshire.core :as json] + [clojure.string :as str] + [clojure.test :refer [deftest is testing]] + [eca.client-http :as client] + [eca.client-test-helpers :as cth] + [hato.client :as hato]) + (:import [io.netty.buffer Unpooled] + [io.netty.handler.codec.http DefaultFullHttpRequest DefaultFullHttpResponse DefaultHttpRequest HttpMethod HttpVersion] + [java.io IOException] + [java.net Authenticator InetSocketAddress PasswordAuthentication ProxySelector URI] + [java.net.http HttpClient HttpRequest HttpRequest$BodyPublishers HttpResponse$BodyHandlers] + [java.nio.charset StandardCharsets] + [org.littleshoot.proxy.impl DefaultHttpProxyServer])) + +(deftest source-handler-tests + (testing "successful plain text request with plain text response" + (let [handler-called (atom nil) + handler-fn (fn [req] + (reset! handler-called req) + {:status 201 + :headers {"X-Reply" "yes"} + :body "plain-response"}) + filters (.filterRequest (cth/proxy-filters-handler-make handler-fn) + nil nil) + content (Unpooled/copiedBuffer "hello world" StandardCharsets/UTF_8) + req (DefaultFullHttpRequest. + HttpVersion/HTTP_1_1 HttpMethod/POST "/t" content)] + + (.set (.headers req) "X-Test" "value") + + (let [^DefaultFullHttpResponse resp (.proxyToServerRequest filters req) + body (.toString (.content resp) StandardCharsets/UTF_8)] + (is (= 201 (.code (.getStatus resp)))) + (is (= "plain-response" body)) + (is (= "yes" (.get (.headers resp) "X-Reply"))) + (is (= {:method "POST" + :uri "/t" + :protocol "HTTP/1.1" + :headers {"X-Test" "value"} + :body "hello world"} + @handler-called))))) + + (testing "successful plain text request with map body as application/json response" + (let [handler-fn (fn [_] {:status 202 + :headers {"X-Type" "json"} + :body {:msg "ok"}}) + filters (.filterRequest (cth/proxy-filters-handler-make handler-fn) + nil nil) + content (Unpooled/copiedBuffer "ignored" StandardCharsets/UTF_8) + req (DefaultFullHttpRequest. + HttpVersion/HTTP_1_1 HttpMethod/GET "/json" content) + + ^DefaultFullHttpResponse resp (.proxyToServerRequest filters req) + body (.toString (.content resp) StandardCharsets/UTF_8)] + + (is (= 202 (.code (.getStatus resp)))) + (is (= {:msg "ok"} (json/parse-string body true))) + ;; handler returned a map -> Content-Type should be json + (is (= "application/json" (.get (.headers resp) "Content-Type"))))) + + (testing "successful application/json request with map boyd as application/json response" + (let [handler-called (atom nil) + handler-fn (fn [req] + (reset! handler-called req) + {:status 200 + :body {:echo (:body req)}}) + filters (.filterRequest (cth/proxy-filters-handler-make handler-fn) + nil nil) + payload "{\"a\":1}" + content (Unpooled/copiedBuffer payload StandardCharsets/UTF_8) + req (DefaultFullHttpRequest. + HttpVersion/HTTP_1_1 HttpMethod/POST "/echo" content)] + + (.set (.headers req) "Content-Type" "application/json") + + (let [^DefaultFullHttpResponse resp (.proxyToServerRequest filters req) + body (.toString (.content resp) StandardCharsets/UTF_8)] + + (is (= 200 (.code (.getStatus resp)))) + (is (= {:echo {:a 1}} (json/parse-string body true))) + (is (= {:method "POST" + :uri "/echo" + :protocol "HTTP/1.1" + :headers {"Content-Type" "application/json"} + :body {:a 1}} + @handler-called))))) + + (testing "successful plain text request with handler throwing an exception" + (let [handler-fn (fn [_] (throw (ex-info "handler failure" {}))) + filters (.filterRequest (cth/proxy-filters-handler-make handler-fn) + nil nil) + content (Unpooled/copiedBuffer "hello" StandardCharsets/UTF_8) + req (DefaultFullHttpRequest. + HttpVersion/HTTP_1_1 HttpMethod/POST "/fail" content) + ^DefaultFullHttpResponse resp (.proxyToServerRequest filters req) + body (.toString (.content resp) StandardCharsets/UTF_8)] + + ;; The proxy should return 400 and include the error message + (is (= 400 (.code (.getStatus resp)))) + (is (clojure.string/includes? body "handler failure")))) + + (testing "request is not a FullHttpRequest" + (let [handler-fn (fn [_] {:status 200 :body "should not be called"}) + proxy-src (cth/proxy-filters-handler-make handler-fn) + filters (.filterRequest proxy-src nil nil) + ;; create a non-FullHttpRequest (DefaultHttpRequest without aggregated content) + req (DefaultHttpRequest. HttpVersion/HTTP_1_1 HttpMethod/POST "/not-full") + ^DefaultFullHttpResponse resp (.proxyToServerRequest filters req) + body (.toString (.content resp) StandardCharsets/UTF_8)] + + ;; The proxy should return 400 with the dev error message + (is (= 400 (.code (.getStatus resp)))) + (is (clojure.string/includes? body "expected FullHttpRequest"))))) + +(deftest little-proxy-interceptor-make-test + (testing "intercepting simple GET request via LittleProxy" + (let [handler-called (atom nil) + handler-fn (fn [req] + (reset! handler-called req) + {:status 200 :body {:msg "ok"}}) + ;; start the proxy + prx (cth/little-proxy-interceptor-make handler-fn) + prx-host (:host prx) + prx-port (:port prx)] + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. ^String prx-host ^long prx-port))) + (.build)) + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/test")) + (.GET) + (.build)) + response (.send client request (HttpResponse$BodyHandlers/ofString)) + body-str (.body response) + body-map (json/parse-string body-str true)] + (is (= 200 (.statusCode response))) + (is (= {:msg "ok"} body-map)) + (is (= "/test" (:uri @handler-called))) + (is (= "GET" (:method @handler-called)))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "intercepting simple POST request via LittleProxy" + (let [handler-called (atom nil) + handler-fn (fn [req] + (reset! handler-called req) + {:status 201 + :body {:msg "created"}}) + prx (cth/little-proxy-interceptor-make handler-fn) + prx-host (:host prx) + prx-port (:port prx)] + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. ^String prx-host ^long prx-port))) + (.build)) + payload "{\"foo\":\"bar\"}" + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/create")) + (.POST (HttpRequest$BodyPublishers/ofString payload)) + (.header "Content-Type" "application/json") + (.build)) + response (.send client request (HttpResponse$BodyHandlers/ofString)) + body-str (.body response) + body-map (json/parse-string body-str true)] + (is (= 201 (.statusCode response))) + (is (= {:msg "created"} body-map)) + (is (= "/create" (:uri @handler-called))) + (is (= "POST" (:method @handler-called))) + (is (= {:foo "bar"} (:body @handler-called)))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "intercepting simple POST request with error response via LittleProxy" + (let [handler-fn (fn [_] (throw (ex-info "handler failure" {}))) + prx (cth/little-proxy-interceptor-make handler-fn) + ^String prx-host (:host prx) + ^long prx-port (:port prx)] + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. prx-host prx-port))) + (.build)) + payload "{\"bad\":true}" + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/fail")) + (.POST (HttpRequest$BodyPublishers/ofString payload)) + (.header "Content-Type" "application/json") + (.build)) + response (.send client request (HttpResponse$BodyHandlers/ofString)) + body-str (.body response)] + (is (= 400 (.statusCode response))) + (is (clojure.string/includes? body-str "handler failure"))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "HTTPS CONNECT request hits proxy" + (let [req* (atom nil) + handler-fn (fn [req] + (reset! req* req) + {:status 200 :body ""}) + prx (cth/little-proxy-interceptor-make handler-fn) + ^String host (:host prx) + ^long port (:port prx)] + + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of + (InetSocketAddress. host port))) + (.build)) + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "https://localhost/")) + (.GET) + (.build))] + + ;; We don't care if HTTPS fails after tunneling, + ;; we only care that CONNECT hit the proxy. + (try + (.send client request (HttpResponse$BodyHandlers/ofString)) + (catch Exception _)) + + ;; Confirm CONNECT request observed + (is (= "CONNECT" (:method @req*)))) + + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "intercepting simple GET request with via LittleProxy with authentication" + (let [handler-called (atom nil) + handler-fn (fn [req] + (reset! handler-called req) + {:status 200 :body {:msg "ok"}}) + username "user1" + password "pass1" + prx (cth/little-proxy-interceptor-make handler-fn username password) + ^String prx-host (:host prx) + ^long prx-port (:port prx)] + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. prx-host prx-port))) + (.authenticator (proxy [Authenticator] [] + (getPasswordAuthentication [] + (PasswordAuthentication. username (char-array password))))) + (.build)) + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/auth")) + (.GET) + (.build)) + response (.send client request (HttpResponse$BodyHandlers/ofString)) + body-str (.body response) + body-map (json/parse-string body-str true)] + (is (= 200 (.statusCode response))) + (is (= {:msg "ok"} body-map)) + (is (= "/auth" (:uri @handler-called)))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "intercepting simple GET request via LittleProxy with wrong authentication" + (let [handler-fn (fn [_] {:status 200 :body {:msg "ok"}}) + username "user1" + password "pass1" + prx (cth/little-proxy-interceptor-make handler-fn username password) + ^String prx-host (:host prx) + ^long prx-port (:port prx)] + (try + ;; HttpClient supplies wrong creds + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. prx-host prx-port))) + (.authenticator (proxy [Authenticator] [] + (getPasswordAuthentication [] + (PasswordAuthentication. "wrong" (char-array "creds"))))) + (.build)) + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/auth-fail")) + (.GET) + (.build))] + (is (thrown-with-msg? + IOException + #"too many authentication attempts" + (.send client request (HttpResponse$BodyHandlers/ofString))))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "proxy requires authentication but client provides none" + (let [handler-fn (fn [_] {:status 200 :body {:msg "ok"}}) + username "user1" + password "pass1" + prx (cth/little-proxy-interceptor-make handler-fn username password) + ^String prx-host (:host prx) + ^long prx-port (:port prx)] + (try + ;; client has no authenticator at all + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. prx-host prx-port))) + (.build)) + request (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:13/no-auth")) + (.GET) + (.build)) + response (.send client request (HttpResponse$BodyHandlers/ofString))] + + ;; Proxy Authentication Required (407) + (is (= 407 (.statusCode response)))) + + (finally + (.abort ^DefaultHttpProxyServer (:px prx)))))) + + (testing "intercepting simple HTTPS GET via proxy with authentication" + ;; requires 'jdk.http.auth.tunneling.disabledSchemes=', because + ;; we're going to make an https requests to an http proxy with + ;; BASIC authentications, which is otherwise in th disabled scheme. + (let [req* (atom nil) + handler-fn (fn [req] + (reset! req* req) + {:status 200 :body {:msg "ok"}}) + user "u1" + pass "p1" + prx (cth/little-proxy-interceptor-make handler-fn user pass) + ^String host (:host prx) + ^long port (:port prx)] + (try + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. host port))) + (.authenticator + (proxy [Authenticator] [] + (getPasswordAuthentication [] + (PasswordAuthentication. user (char-array pass))))) + (.build)) + req (-> (HttpRequest/newBuilder) + (.uri (URI/create "https://localhost/x")) + (.GET) + (.build))] + ;; We don't care if HTTPS fails after tunneling, + ;; we only care that CONNECT hit the proxy. + (try + (.send client req (HttpResponse$BodyHandlers/ofString)) + (catch Exception _)) + + ;; Confirm CONNECT request observed + (is (= "CONNECT" (:method @req*)))) + (finally + (.abort ^DefaultHttpProxyServer (:px prx))))))) + +(deftest with-proxy-test + (testing "can http post to the proxy and get a successful response" + (cth/with-proxy {} + (fn [_] {:status 201 :body "ok"}) + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of + (InetSocketAddress. cth/*proxy-host* ^long cth/*proxy-port*))) + (.build)) + + req (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/test")) + (.POST (HttpRequest$BodyPublishers/ofString "hello")) + (.build)) + + resp (.send client req (HttpResponse$BodyHandlers/ofString))] + + (is (= 201 (.statusCode resp))) + (is (= "ok" (.body resp)))))) + + (testing "can http post to the proxy but handler returns an exception" + (cth/with-proxy {} + (fn [_] (throw (ex-info "boom" {}))) + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of + (InetSocketAddress. cth/*proxy-host* ^long cth/*proxy-port*))) + (.build)) + + req (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/fail")) + (.POST (HttpRequest$BodyPublishers/ofString "x")) + (.build)) + + resp (.send client req (HttpResponse$BodyHandlers/ofString))] + + (is (= 400 (.statusCode resp))) + (is (clojure.string/includes? (.body resp) "boom")))))) + +(deftest with-proxy-authentication-tests + (testing "Requires username and password when proxy authentication is enabled" + (cth/with-proxy {:user "user1" :pass "secret"} + (fn [_] {:status 200 :body "ok"}) + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. cth/*proxy-host* ^long cth/*proxy-port*))) + (.authenticator + (proxy [Authenticator] [] + (getPasswordAuthentication [] + (PasswordAuthentication. "user1" (char-array "secret"))))) + (.build)) + req (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/test")) + (.GET) + (.build)) + resp (.send client req (HttpResponse$BodyHandlers/ofString))] + (is (= 200 (.statusCode resp))) + (is (= "ok" (.body resp)))))) + + (testing "Rejects requests without credentials if authentication is set" + (cth/with-proxy {:user "user1" :pass "secret"} + (fn [_] {:status 200 :body "ok"}) + (let [client (-> (HttpClient/newBuilder) + (.proxy (ProxySelector/of (InetSocketAddress. cth/*proxy-host* ^long cth/*proxy-port*))) + (.build)) + req (-> (HttpRequest/newBuilder) + (.uri (URI/create "http://localhost:99/test")) + (.GET) + (.build)) + resp (.send client req (HttpResponse$BodyHandlers/ofString))] + (is (not= 200 (.statusCode resp))) + (is (clojure.string/includes? (.body resp) "Proxy Authentication")))))) + +(deftest with-client-proxied-test + (testing "Restores `eca.client-http/*hato-http-client*` to nil after BODY executes" + (is (nil? client/*hato-http-client*)) + (cth/with-client-proxied {} + (fn [_] {:status 200 :body "ok"}) + ;; Inside BODY, the client is set + (is (some? client/*hato-http-client*))) + ;; After BODY, the client should be reset + (is (nil? client/*hato-http-client*))) + + (testing "Routes HTTP requests made through Hato to the temporary proxy" + (cth/with-client-proxied {} + (fn [req] {:status 200 :body (:uri req)}) + (let [resp (hato/get "http://localhost:99/test" + {:http-client client/*hato-http-client*})] + (is (= 200 (:status resp))) + (is (= "/test" (:body resp)))))) + + (testing "Captures all calls to `merge-with-global-http-client` during BODY" + (cth/with-client-proxied {:abc 52} + (fn [_] {:status 200 :body "ok"}) + ;; Make some merge calls inside BODY + (client/merge-with-global-http-client {:foo "bar"}) + (client/merge-with-global-http-client {:baz 42}) + ;; Assert that all merged results were captured + (let [captures @cth/*http-client-captures*] + (is (= 2 (count captures))) + (is (= {:foo "bar" :abc 52} (dissoc (first captures) :proxy))) + (is (= {:baz 42 :abc 52} (dissoc (second captures) :proxy))))))) + diff --git a/test/eca/llm_api_test.clj b/test/eca/llm_api_test.clj index c6994e4dc..0e4ad3b46 100644 --- a/test/eca/llm_api_test.clj +++ b/test/eca/llm_api_test.clj @@ -1,6 +1,7 @@ (ns eca.llm-api-test (:require [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied *http-client-captures*]] [eca.config :as config] [eca.llm-api :as llm-api] [eca.secrets :as secrets] @@ -60,3 +61,43 @@ (let [db {:models {}} config {}] (is (= "anthropic/claude-sonnet-4.5" (llm-api/default-model db config))))))) + +(deftest prompt-test + (testing "Custom OpenAI provider behavior and proper passing of httpClient options to the Hato client" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:usage {:prompt_tokens 5 :completion_tokens 2} + :choices [{:message {:content "hi" + :reasoning_content "think more"}}]}}) + + (let [response (#'eca.llm-api/prompt! + {:config {:providers {"lmstudio" + {:api "openai-chat", + :url "http://localhost:1234", + :completionUrlRelativePath "/v1/chat/completions", + :httpClient {:version :http-1.1}, + :models {"ibm/granite-4-h-tiny" {}}}}} + + :provider "lmstudio" + :model "ibm/granite-4-h-tiny" + + :model-capabilities {:tools false, + :reason? false, + :web-search false, + :model-name "ibm/granite-4-h-tiny"} + :sync? true})] + (is (= {:method "POST", + :uri "/v1/chat/completions"} + (select-keys @req* [:method :uri]))) + ;; Verify that a single Hato HTTP client request occurred and used HTTP/1.1 + (is (= [{:version :http-1.1}] (map #(dissoc % :proxy) @*http-client-captures*))) + (is (= {:usage {:input-tokens 5, :output-tokens 2, :input-cache-read-tokens nil}, + :tools-to-call (), + :reason-text "think more", + :reasoning-content "think more", + :output-text "hi"} + (select-keys response [:usage :tools-to-call :reason-text :reasoning-content :output-text])) response)))))) diff --git a/test/eca/llm_providers/anthropic_test.clj b/test/eca/llm_providers/anthropic_test.clj index e831d19be..f1d660b1e 100644 --- a/test/eca/llm_providers/anthropic_test.clj +++ b/test/eca/llm_providers/anthropic_test.clj @@ -1,9 +1,143 @@ (ns eca.llm-providers.anthropic-test (:require + [clojure.string :as string] [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied]] [eca.llm-providers.anthropic :as llm-providers.anthropic] [matcher-combinators.test :refer [match?]])) +(deftest base-request-test + (testing "constructs an Anthropics API request and extracts completion text" + (let [req* (atom nil) + fake-response {:content [{:text "Hello from Anthropics proxy!"}]}] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body fake-response}) + + (let [body {:model "claude-v1" + :input "hi" + :stream false} + response (#'llm-providers.anthropic/base-request! + {:rid "r1" + :api-key "fake-key" + :api-url "http://localhost:1" + :body body + :url-relative-path "/v1/messages" + :auth-type :auth/key})] + + (is (= {:method "POST" + :uri "/v1/messages" + :body body} + (select-keys @req* [:method :uri :body]))) + + (is (= {:output-text "Hello from Anthropics proxy!"} + (select-keys response [:output-text])))))))) + +(deftest oauth-authorize-test + (testing "exchanges an OAuth code for tokens and returns refresh/access tokens with expiry" + (let [req* (atom nil) + now-seconds (quot (System/currentTimeMillis) 1000)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:refresh_token "r-token" + :access_token "a-token" + :expires_in 3600}}) + + (let [raw-code "abc123#stateXYZ" + verifier "verifierXYZ" + [code state] (string/split raw-code #"#") + result (with-redefs [llm-providers.anthropic/oauth-token-url + "http://localhost:99/v1/oauth/token"] + (#'llm-providers.anthropic/oauth-authorize + raw-code verifier))] + + (is (= {:method "POST" + :uri "/v1/oauth/token"} + (select-keys @req* [:method :uri]))) + + (is (= {:grant_type "authorization_code" + :code code + :state state + :client_id @#'llm-providers.anthropic/client-id + :redirect_uri "https://console.anthropic.com/oauth/code/callback" + :code_verifier verifier} + (:body @req*)) + "Outgoing payload should match token-exchange fields") + + (is (= "r-token" (:refresh-token result))) + (is (= "a-token" (:access-token result))) + + ;; expires-at should be > now + (is (> (:expires-at result) now-seconds) + "expires-at should be computed relative to current time")))))) + +(deftest oauth-refresh-test + (testing "refreshes an OAuth token and returns new refresh/access tokens with expiry" + (let [req* (atom nil) + now-seconds (quot (System/currentTimeMillis) 1000)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:refresh_token "new-r-token" + :access_token "new-a-token" + :expires_in 3600}}) + + (let [refresh-token "old-r-token" + result (with-redefs [llm-providers.anthropic/oauth-token-url + "http://localhost:99/v1/oauth/token"] + (#'llm-providers.anthropic/oauth-refresh refresh-token))] + + (is (= {:method "POST" + :uri "/v1/oauth/token"} + (select-keys @req* [:method :uri]))) + + (is (= {:grant_type "refresh_token" + :refresh_token refresh-token + :client_id @#'llm-providers.anthropic/client-id} + (:body @req*)) + "Outgoing payload should match refresh-token fields") + + (is (= "new-r-token" (:refresh-token result))) + (is (= "new-a-token" (:access-token result))) + + ;; expires-at should be > now + (is (> (:expires-at result) now-seconds) + "expires-at should be computed relative to current time")))))) + +(deftest create-api-key-test + (testing "creates a new API key and sets the appropriate authorization headers" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:raw_key "sk-ant-test-key"}}) + + (let [access-token "access-123" + result (with-redefs [llm-providers.anthropic/create-api-key-url + "http://localhost:99/api/oauth/claude_cli/create_api_key"] + (#'llm-providers.anthropic/create-api-key access-token))] + + (is (= {:method "POST" + :uri "/api/oauth/claude_cli/create_api_key"} + (select-keys @req* [:method :uri]))) + + (is (= {"Authorization" "Bearer access-123" + "Content-Type" "application/x-www-form-urlencoded" + "Accept" "application/json, text/plain, */*"} + (select-keys (:headers @req*) ["Authorization" "Content-Type" "Accept"])) + "Authorization and content headers should be set") + (is (= "sk-ant-test-key" result))))))) + (deftest ->normalize-messages-test (testing "no previous history" (is (match? diff --git a/test/eca/llm_providers/copilot_test.clj b/test/eca/llm_providers/copilot_test.clj new file mode 100644 index 000000000..8d150c906 --- /dev/null +++ b/test/eca/llm_providers/copilot_test.clj @@ -0,0 +1,102 @@ +(ns eca.llm-providers.copilot-test + (:require + [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied]] + [eca.llm-providers.copilot :as llm-providers.copilot])) + +(deftest oauth-url-test + (testing "constructs GitHub device OAuth request and parses key response fields" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:user_code "USER-CODE" + :device_code "DEVICE-CODE" + :verification_uri "https://github.com/login/device"}}) + + (let [result + (with-redefs [llm-providers.copilot/oauth-login-device-url + "http://localhost:99/login/device/code"] + (#'llm-providers.copilot/oauth-url))] + + ;; request validation + (is (= {:method "POST" + :uri "/login/device/code"} + (select-keys @req* [:method :uri]))) + + (is (= {:client_id @#'llm-providers.copilot/client-id + :scope "read:user"} + (:body @req*)) + "Outgoing payload should match device-code request") + + ;; response parsing + (is (= {:user-code "USER-CODE" + :device-code "DEVICE-CODE" + :url "https://github.com/login/device"} + result))))))) + +(deftest oauth-access-token-test + (testing "builds device access-token request and parses access token" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:access_token "gh-access-token"}}) + + (let [device-code "device-code-123" + result + (with-redefs [llm-providers.copilot/oauth-login-access-token-url + "http://localhost:99/login/oauth/access_token"] + (#'llm-providers.copilot/oauth-access-token device-code))] + + ;; request validation + (is (= {:method "POST" + :uri "/login/oauth/access_token"} + (select-keys @req* [:method :uri]))) + + (is (= {:client_id @#'llm-providers.copilot/client-id + :device_code device-code + :grant_type "urn:ietf:params:oauth:grant-type:device_code"} + (:body @req*)) + "Outgoing payload should match access-token exchange") + + ;; response parsing + (is (= "gh-access-token" result))))))) + +(deftest oauth-renew-token-test + (testing "sends token renewal request and extracts API key and expiry" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:token "copilot-api-key" + :expires_at 9999999999}}) + + (let [access-token "gh-access-123" + result + (with-redefs [llm-providers.copilot/oauth-copilot-token-url + "http://localhost:99/copilot_internal/v2/token"] + (#'llm-providers.copilot/oauth-renew-token access-token))] + + ;; request validation + (is (= {:method "GET" + :uri "/copilot_internal/v2/token"} + (select-keys @req* [:method :uri]))) + + (is (= {"authorization" (str "token " access-token) + "Content-Type" "application/json" + "Accept" "application/json" + "editor-plugin-version" "eca/*"} + (select-keys (:headers @req*) ["authorization" "Content-Type" "Accept" "editor-plugin-version"])) + (str "Headers should include auth headers and access-token: " (:headers @req*))) + + ;; response parsing + (is (= {:api-key "copilot-api-key" + :expires-at 9999999999} + result))))))) diff --git a/test/eca/llm_providers/ollama_test.clj b/test/eca/llm_providers/ollama_test.clj index b08731a77..5bb6a541b 100644 --- a/test/eca/llm_providers/ollama_test.clj +++ b/test/eca/llm_providers/ollama_test.clj @@ -1,9 +1,85 @@ (ns eca.llm-providers.ollama-test (:require + [cheshire.core :as json] [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied]] [eca.llm-providers.ollama :as llm-providers.ollama] [matcher-combinators.test :refer [match?]])) +(deftest list-models-test + (testing "fetches available Ollama models" + (let [req* (atom nil) + fake-api-url "http://localhost:99" + fake-response {:status 200 + :body {:models [{:name "model-a"} + {:name "model-b"}]}}] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + fake-response) + + (let [result (#'eca.llm-providers.ollama/list-models {:api-url fake-api-url})] + (is (= {:method "GET" + :uri "/api/tags"} ;; matches list-models-url "%s/api/tags" + (select-keys @req* [:method :uri]))) + + ;; response parsing + (is (= [{:name "model-a"} {:name "model-b"}] result))))))) + +(deftest model-capabilities-test + (testing "fetches capabilities for a specific Ollama model" + (let [req* (atom nil) + fake-api-url "http://localhost:99" + fake-model "test-model" + fake-response {:status 200 + :body {:capabilities [:chat :completion]}}] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + fake-response) + + (let [result (#'eca.llm-providers.ollama/model-capabilities + {:model fake-model :api-url fake-api-url})] + + ;; request validation + (is (= {:method "POST" + :uri "/api/show"} ;; matches show-model-url "%s/api/show" + (select-keys @req* [:method :uri]))) + + (is (= (json/generate-string {:model fake-model}) + (:body @req*)) + "Outgoing payload should contain the model") + + (is (= ["chat" "completion"] result))))))) + +(deftest base-chat-request-test + (testing "sends Ollama chat request and extracts output text" + (let [req* (atom nil) + fake-url "http://localhost:99/api/chat" + rid "test-rid" + body {:model "test-model" :input "Hello"} + fake-response {:status 200 + :body {:message {:content "Hello world"}}}] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + fake-response) + + (let [result (#'eca.llm-providers.ollama/base-chat-request! + {:rid rid + :url fake-url + :body body})] + + ;; request validation + (is (= {:method "POST" + :uri "/api/chat"} + (select-keys @req* [:method :uri]))) + + (is (= {:output-text "Hello world"} result))))))) + (deftest ->normalize-messages-test (testing "no previous history" (is (match? diff --git a/test/eca/llm_providers/openai_chat_test.clj b/test/eca/llm_providers/openai_chat_test.clj index 03b1f8278..c07bce459 100644 --- a/test/eca/llm_providers/openai_chat_test.clj +++ b/test/eca/llm_providers/openai_chat_test.clj @@ -1,12 +1,55 @@ (ns eca.llm-providers.openai-chat-test (:require [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied]] [eca.llm-providers.openai-chat :as llm-providers.openai-chat] [matcher-combinators.test :refer [match?]])) (def thinking-start-tag "") (def thinking-end-tag "") +(deftest base-chat-req-test + (testing "builds a chat request and extracts assistant output text" + (let [req* (atom nil)] + (with-client-proxied {} + + (fn handler [req] + (reset! req* req) + {:status 200 + :body {:id "chatcmpl-p7ezf7cu8pbcg5e20p6er6", + :object "chat.completion", + :created 1763927678, + :model "ibm/granite-4-h-tiny", + :choices [{:index 0, + :message {:role "assistant", + :content "Hello there!" + :tool_calls []}, + :logprobs nil, + :finish_reason "stop"}] + + :usage {:prompt_tokens 17, + :completion_tokens 32, + :ytotal_tokens 49}, + :stats {}, + :system_fingerprint "ibm/granite-4-h-tiny"}}) + + (let [body {:model "ibm/granite-4-h-tiny" + :messages [{:role "system" :content "# title generator"} + {:role "user" :content "hi"}] + :stream false + :max_completion_tokens 32000} + response (#'llm-providers.openai-chat/base-chat-request! + {:api-key "username:password" + :api-url "http://localhost:1" + :body body + :url-relative-path "/v1/chat/completions"})] + (is (= {:method "POST" + :uri "/v1/chat/completions" + :body body} + (select-keys @req* [:method :uri :body]))) + (is (= {:output-text "Hello there!"} + (select-keys response [:output-text])))))))) + (deftest normalize-messages-test (testing "With tool_call history - assistant text and tool calls are merged" (is (match? diff --git a/test/eca/llm_providers/openai_test.clj b/test/eca/llm_providers/openai_test.clj index 0cd114b52..199105447 100644 --- a/test/eca/llm_providers/openai_test.clj +++ b/test/eca/llm_providers/openai_test.clj @@ -1,9 +1,81 @@ (ns eca.llm-providers.openai-test (:require [clojure.test :refer [deftest is testing]] + [eca.client-test-helpers :refer [with-client-proxied]] [eca.llm-providers.openai :as llm-providers.openai] [matcher-combinators.test :refer [match?]])) +(deftest base-responses-req-test + (testing "sends a responses request and extracts output text" + (let [req* (atom nil)] + (with-client-proxied {:version :http-2} + (fn [req] + (reset! req* req) + ;; fake a successful non-stream JSON response + {:status 200 + :body {:output [{:content [{:text "Hello from responses!"}]}]}}) + + (let [body {:model "mymodel" + :input "hi" + :stream false} + response (#'llm-providers.openai/base-responses-request! + {:rid "r1" + :api-key "fake-key" + :api-url "http://localhost:1" + :body body + :url-relative-path "/v1/responses"})] + + (is (= {:method "POST" + :uri "/v1/responses" + :body body} + (select-keys @req* [:method :uri :body]))) + + ;; parsed response + (is (= {:output-text "Hello from responses!"} + (select-keys response [:output-text])))))))) + +(deftest oauth-authorize-test + (testing "that OAuth token exchange is routed through the http proxy" + (let [req* (atom nil) + now-seconds (quot (System/currentTimeMillis) 1000)] + (with-client-proxied {} + + (fn handler [req] + ;; capture the outgoing request + (reset! req* req) + ;; fake token endpoint response + {:status 200 + :body {:refresh_token "r-token" + :access_token "a-token" + :expires_in 3600}}) + + (let [server-url "http://localhost/callback" + code "abc123" + verifier "verifierXYZ" + result (with-redefs [llm-providers.openai/oauth-token-url "http://localhost:99/oauth/token"] + (#'llm-providers.openai/oauth-authorize + server-url code verifier))] + + ;; request validation + (is (= {:method "POST" + :uri "/oauth/token"} + (select-keys @req* [:method :uri]))) + + (is (= {:grant_type "authorization_code" + :client_id @#'llm-providers.openai/client-id + :code code + :code_verifier verifier + :redirect_uri server-url} + (:body @req*)) + "Outgoing payload should match token-exchange fields") + + ;; response parsing + (is (= "r-token" (:refresh-token result))) + (is (= "a-token" (:access-token result))) + ;; expires-at should be > now + (is (> (:expires-at result) now-seconds) + "expires-at should be computed relative to current time")))))) + (deftest ->normalize-messages-test (testing "no previous history" (is (match? diff --git a/test/eca/proxy_test.clj b/test/eca/proxy_test.clj new file mode 100644 index 000000000..22e64f476 --- /dev/null +++ b/test/eca/proxy_test.clj @@ -0,0 +1,117 @@ +(ns eca.proxy-test + (:require [clojure.test :refer [deftest is testing]] + [eca.config :as config] + [eca.proxy :as p])) + +(deftest proxy-urls-system-env-get-tests + (testing "Returns correct HTTP and HTTPS proxy values when only lowercase env vars are set" + (with-redefs [config/get-env (fn [k] (case k "http_proxy" "http://lc-http" "https_proxy" "https://lc-https"))] + (is (= (p/proxy-urls-system-env-get) {:http "http://lc-http" :https "https://lc-https"})))) + + (testing "Returns correct HTTP and HTTPS proxy values when only uppercase env vars are set" + (with-redefs [config/get-env (fn [k] (case k "HTTP_PROXY" "http://uc-http" "HTTPS_PROXY" "https://uc-https" nil))] + (is (= (p/proxy-urls-system-env-get) {:http "http://uc-http" :https "https://uc-https"})))) + + (testing "Lowercase env vars take precedence over uppercase" + (with-redefs [config/get-env (fn [k] (case k "http_proxy" "http://lc-http" "HTTP_PROXY" "http://uc-http" + "https_proxy" "https://lc-https" "HTTPS_PROXY" "https://uc-https"))] + (is (= (p/proxy-urls-system-env-get) {:http "http://lc-http" :https "https://lc-https"})))) + + (testing "Returns nil when no proxy environment variables are set" + (with-redefs [config/get-env (fn [_] nil)] + (is (= (p/proxy-urls-system-env-get) {:http nil :https nil})))) + + (testing "Handles mixed case: lowercase and uppercase env vars present" + (with-redefs [config/get-env (fn [k] (case k "HTTP_PROXY" "http://uc-http" + "https_proxy" "https://lc-https" "HTTPS_PROXY" "https://uc-https" + nil))] + (is (= (p/proxy-urls-system-env-get) {:http "http://uc-http" :https "https://lc-https"}))))) + +(deftest parse-proxy-url-tests + (testing "Parses full URL with scheme, host, port, username, and password" + (is (= (p/parse-proxy-url "http://user:pass@example.com:8080") + {:host "example.com" :port 8080 :username "user" :password "pass"}))) + + (testing "Defaults port to 80 for http when not provided" + (is (= (p/parse-proxy-url "http://example.com") + {:host "example.com" :port 80 :username nil :password nil}))) + + (testing "Defaults port to 443 for https when not provided" + (is (= (p/parse-proxy-url "https://example.com") + {:host "example.com" :port 443 :username nil :password nil}))) + + (testing "Returns nil for blank or nil input" + (is (nil? (p/parse-proxy-url nil))) + (is (nil? (p/parse-proxy-url ""))) + (is (nil? (p/parse-proxy-url " ")))) + + (testing "Parses URL with no user info" + (is (= (p/parse-proxy-url "http://example.com:8080") + {:host "example.com" :port 8080 :username nil :password nil}))) + + (testing "Decodes URL-encoded username and password" + (is (= (p/parse-proxy-url "http://user%20name:pa%24s@example.com:8080") + {:host "example.com" :port 8080 :username "user name" :password "pa$s"}))) + + (testing "Throws IllegalArgumentException if scheme is unknown and port not provided" + (is (thrown-with-msg? IllegalArgumentException + #"Unsupported scheme: ftp" + (p/parse-proxy-url "ftp://example.com")))) + + (testing "Handles URLs with only host and scheme" + (is (= (p/parse-proxy-url "https://example.com") + {:host "example.com" :port 443 :username nil :password nil}))) + + (testing "Handles URLs with unusual but valid characters in host or user info" + (is (= (p/parse-proxy-url "http://u_ser:pa-ss@sub.example.com:8080") + {:host "sub.example.com" :port 8080 :username "u_ser" :password "pa-ss"})))) + +(deftest proxy-urls-parse-tests + (testing "Parses both :http and :https URLs correctly" + (let [urls {:http "http://user:pass@http.com:8080" + :https "https://https.com:8443"}] + (is (= (p/proxy-urls-parse urls) + {:http {:host "http.com" :port 8080 :username "user" :password "pass"} + :https {:host "https.com" :port 8443 :username nil :password nil}})))) + + (testing "Returns nil for missing or nil URL values" + (let [urls {:http nil + :https ""}] + (is (= (p/proxy-urls-parse urls) + {:http nil :https nil})))) + + (testing "Handles URLs with only one of :http or :https present" + (let [urls {:http "http://only-http.com"}] + (is (= (p/proxy-urls-parse urls) + {:http {:host "only-http.com" :port 80 :username nil :password nil}})))) + + (testing "Ignores unrelated keys in the input map" + (let [urls {:http "http://http.com" + :https "https://https.com" + :ftp "ftp://ftp.com"}] + (is (= (p/proxy-urls-parse urls) + {:http {:host "http.com" :port 80 :username nil :password nil} + :https {:host "https.com" :port 443 :username nil :password nil}}))))) + +(deftest env-proxy-urls-parse-tests + (testing "Parses both HTTP and HTTPS proxy environment variables correctly" + (with-redefs [config/get-env (fn [env] (case env + "http_proxy" "http://user:pass@http.com:8080" + "https_proxy" "https://https.com:8443"))] + (is (= (p/env-proxy-urls-parse) + {:http {:host "http.com" :port 8080 :username "user" :password "pass"} + :https {:host "https.com" :port 8443 :username nil :password nil}})))) + + (testing "Returns nil for proxies not set in the environment" + (with-redefs [config/get-env (fn [_] nil)] + (is (= (p/env-proxy-urls-parse) + {:http nil :https nil})))) + + (testing "Handles cases where only one of :http or :https is set" + (with-redefs [config/get-env (fn [env] (case env + "http_proxy" "http://only-http.com" + nil))] + (is (= (p/env-proxy-urls-parse) + {:http {:host "only-http.com" :port 80 :username nil :password nil} + :https nil}))))) +