Enable prom metrics for clock and deployment_updater (#600)

johha · web-flow · commit 69e6b63421b0 · 2026-01-19T15:14:28.000+01:00
Adds configuration flags to allow operators to enable prom metrics for
cloud_controller_clock and cc_deployment_updater.
Communication is secured via mTLS.
diff --git a/jobs/cc_deployment_updater/spec b/jobs/cc_deployment_updater/spec
@@ -27,6 +27,10 @@ templates:
   resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem
   droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem
   buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+  scrape.crt.erb: config/certs/scrape.crt
+  scrape.key.erb: config/certs/scrape.key
+  scrape_ca.crt.erb: config/certs/scrape_ca.crt
   storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json
   storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json
   storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json
@@ -226,3 +230,23 @@ properties:
   cc.locket.port:
     default: 8891
     description: "Port of the Locket server"
+
+  cc.publish_metrics:
+    default: false
+    description: "When set to true a small webserver will be started in a separate thread within the first worker's process.
+                  This webserver will publish prometheus metrics of the workers under '/metrics'. The webserver will listen on the port
+                  defined in 'cc.prometheus_port'."
+  cc.prometheus_port:
+    default: 9395
+    description: "When 'cc.publish_metrics' is set to true, the webserver, which publishes the metrics, will listen on this port."
+
+  cc.prom_scraper.disabled:
+    default: false
+    description: "When 'cc.publish_metrics' is enabled, a prom_scraper_config will be automatically generated. If you want to use another component for scraping, you can disable scraping by prom_scraper for cc-worker metrics with this."
+  cc.prom_scraper_tls.ca_cert:
+    description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.public_cert:
+    description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.private_key:
+    description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper"
+
diff --git a/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb b/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb
@@ -300,6 +300,8 @@ cpu_weight_max_memory: <%= link("cloud_controller_internal").p("cc.cpu_weight_ma
 custom_metric_tag_prefix_list: <%= link("cloud_controller_internal").p("cc.custom_metric_tag_prefix_list") %>
 
 app_log_revision: <%= link("cloud_controller_internal").p("cc.app_log_revision") %>
+publish_metrics: <%= p("cc.publish_metrics") %>
+prometheus_port: <%= p("cc.prometheus_port") %>
 
 <% link("cloud_controller_internal").if_p("cc.feature_flag_overrides") do |feature_flag_overrides| %>
 feature_flag_overrides: <%= feature_flag_overrides.to_json %>
diff --git a/jobs/cc_deployment_updater/templates/prom_scraper_config.yml.erb b/jobs/cc_deployment_updater/templates/prom_scraper_config.yml.erb
@@ -0,0 +1,10 @@
+<% if p("cc.publish_metrics") && (p("cc.prom_scraper.disabled") != true) -%>
+port: <%= p("cc.prometheus_port") %>
+source_id: "cc_deployment_updater"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: https
+server_name: "cc_deployment_updater_metrics"
+path: /metrics
+labels:
+  origin: cc_deployment_updater
+<% end -%>
diff --git a/jobs/cc_deployment_updater/templates/scrape.crt.erb b/jobs/cc_deployment_updater/templates/scrape.crt.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.public_cert', '') %>
diff --git a/jobs/cc_deployment_updater/templates/scrape.key.erb b/jobs/cc_deployment_updater/templates/scrape.key.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.private_key', '') %>
diff --git a/jobs/cc_deployment_updater/templates/scrape_ca.crt.erb b/jobs/cc_deployment_updater/templates/scrape_ca.crt.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.ca_cert', '') %>
diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec
@@ -25,6 +25,10 @@ templates:
   uaa_ca.crt.erb: config/certs/uaa_ca.crt
   db_ca.crt.erb: config/certs/db_ca.crt
   credhub_ca.crt.erb: config/certs/credhub_ca.crt
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+  scrape.crt.erb: config/certs/scrape.crt
+  scrape.key.erb: config/certs/scrape.key
+  scrape_ca.crt.erb: config/certs/scrape_ca.crt
   storage_cli_config_droplets.json.erb: config/storage_cli_config_droplets.json
   storage_cli_config_packages.json.erb: config/storage_cli_config_packages.json
   storage_cli_config_buildpacks.json.erb: config/storage_cli_config_buildpacks.json
@@ -584,3 +588,22 @@ properties:
   cc.credential_references.interpolate_service_bindings:
     description: "Controls whether CredHub credentials are automatically interpolated in VCAP_SERVICES"
     default: true
+
+  cc.publish_metrics:
+    default: false
+    description: "When set to true a small webserver will be started in a separate thread within the first worker's process.
+                  This webserver will publish prometheus metrics of the workers under '/metrics'. The webserver will listen on the port
+                  defined in 'cc.prometheus_port'."
+  cc.prometheus_port:
+    default: 9394
+    description: "When 'cc.publish_metrics' is set to true, the webserver, which publishes the metrics, will listen on this port."
+
+  cc.prom_scraper.disabled:
+    default: false
+    description: "When 'cc.publish_metrics' is enabled, a prom_scraper_config will be automatically generated. If you want to use another component for scraping, you can disable scraping by prom_scraper for cc-worker metrics with this."
+  cc.prom_scraper_tls.ca_cert:
+    description: "PEM-encoded CA certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.public_cert:
+    description: "PEM-encoded certificate for secure, mutually authenticated TLS communication with prom_scraper"
+  cc.prom_scraper_tls.private_key:
+    description: "PEM-encoded key for secure, mutually authenticated TLS communication with prom_scraper"
diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb
@@ -373,6 +373,8 @@ cpu_weight_max_memory: <%= link("cloud_controller_internal").p("cc.cpu_weight_ma
 custom_metric_tag_prefix_list: <%= link("cloud_controller_internal").p("cc.custom_metric_tag_prefix_list") %>
 
 app_log_revision: <%= link("cloud_controller_internal").p("cc.app_log_revision") %>
+publish_metrics: <%= p("cc.publish_metrics") %>
+prometheus_port: <%= p("cc.prometheus_port") %>
 
 <% link("cloud_controller_internal").if_p("cc.feature_flag_overrides") do |feature_flag_overrides| %>
 feature_flag_overrides: <%= feature_flag_overrides.to_json %>
diff --git a/jobs/cloud_controller_clock/templates/prom_scraper_config.yml.erb b/jobs/cloud_controller_clock/templates/prom_scraper_config.yml.erb
@@ -0,0 +1,10 @@
+<% if p("cc.publish_metrics") && (p("cc.prom_scraper.disabled") != true) -%>
+port: <%= p("cc.prometheus_port") %>
+source_id: "cloud_controller_clock"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: https
+server_name: "cc_clock_metrics"
+path: /metrics
+labels:
+  origin: cc_clock
+<% end -%>
diff --git a/jobs/cloud_controller_clock/templates/scrape.crt.erb b/jobs/cloud_controller_clock/templates/scrape.crt.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.public_cert', '') %>
diff --git a/jobs/cloud_controller_clock/templates/scrape.key.erb b/jobs/cloud_controller_clock/templates/scrape.key.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.private_key', '') %>
diff --git a/jobs/cloud_controller_clock/templates/scrape_ca.crt.erb b/jobs/cloud_controller_clock/templates/scrape_ca.crt.erb
@@ -0,0 +1 @@
+<%= p('cc.prom_scraper_tls.ca_cert', '') %>
diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec
@@ -466,7 +466,7 @@ properties:
 
   cc.publish_metrics:
     default: false
-    description: "When set to true a small webserver will be started in a seperate thread within the first worker's process.
+    description: "When set to true a small webserver will be started in a separate thread within the first worker's process.
                   This webserver will publish prometheus metrics of the workers under '/metrics'. The webserver will listen on the port
                   defined in 'cc.prometheus_port'."
   cc.prometheus_port:
diff --git a/spec/cc_deployment_updater/cc_deployment_updater_spec.rb b/spec/cc_deployment_updater/cc_deployment_updater_spec.rb
@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+require 'yaml'
+require 'json'
+
+module Bosh
+  module Template
+    module Test
+      describe 'cloud_controller_ng job template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('cc_deployment_updater') }
+
+        let(:manifest_properties) do
+          {
+            'system_domain' => 'brook-sentry.capi.land',
+
+            'cc' => {
+
+              'db_logging_level' => 100,
+              'staging_upload_user' => 'staging_user',
+              'staging_upload_password' => 'hunter2',
+              'database_encryption' => {
+                'experimental_pbkdf2_hmac_iterations' => 123,
+                'skip_validation' => false,
+                'current_key_label' => 'encryption_key_0',
+                :keys => { 'encryption_key_0' => '((cc_db_encryption_key))' }
+              }
+            },
+            'ccdb' => {
+              'db_scheme' => 'mysql',
+              'max_connections' => 'foo2',
+              'databases' => [{ 'tag' => 'cc' }],
+              'roles' => [{
+                'tag' => 'admin',
+                'name' => 'alex',
+                'password' => 'pass'
+              }],
+              'address' => 'foo5',
+              'port' => 'foo7',
+              'pool_timeout' => 'foo11',
+              'ssl_verify_hostname' => 'foo12',
+              'read_timeout' => 'foo13',
+              'connection_validation_timeout' => 'foo14',
+              'ca_cert' => 'foo15'
+            }
+          }
+        end
+
+        let(:webdav_config) do
+          { 'blobstore_timeout' => 5,
+            'ca_cert' => '((service_cf_internal_ca.certificate))',
+            'password' => '((blobstore_admin_users_password))',
+            'private_endpoint' => 'https://blobstore.service.cf.internal:4443',
+            'public_endpoint' => 'https://blobstore.brook-sentry.capi.land',
+            'username' => 'blobstore-user' }
+        end
+
+        let(:properties) do
+          {
+            'router' => { 'route_services_secret' => '((router_route_services_secret))' },
+            'system_domain' => 'system',
+            'ssl' => {
+              'skip_cert_verify' => false
+            },
+            'routing_api' => { 'enabled' => false },
+            'cc' => {
+              'external_port' => 9022,
+              'tls_port' => 9023,
+              'internal_service_hostname' => 'cloud-controller-ng.service.cf.internal',
+              'external_protocol' => 'https',
+              'external_host' => 'api',
+              'staging_timeout_in_seconds' => 900,
+              'staging_upload_user' => 'staging_user',
+              'staging_upload_password' => 'staging_user_password',
+              'default_health_check_timeout' => 60,
+              'maximum_health_check_timeout' => 180,
+              'resource_pool' => {
+                'minimum_size' => 65_536,
+                'maximum_size' => 536_870_912,
+                'resource_directory_key' => 'cc-resources',
+                'blobstore_type' => 'webdav',
+                'webdav_config' => webdav_config
+              },
+              'packages' => {
+                'min_package_size' => 65_536,
+                'max_package_size' => 536_870_912,
+                'app_package_directory_key' => 'cc-packages',
+                'blobstore_type' => 'webdav',
+                'webdav_config' => webdav_config
+              },
+              'droplets' => {
+                'droplet_directory_key' => 'cc-droplets',
+                'blobstore_type' => 'webdav',
+                'webdav_config' => webdav_config
+              },
+              'buildpacks' => {
+                'buildpack_directory_key' => 'cc-buildpacks',
+                'blobstore_type' => 'webdav',
+                'webdav_config' => webdav_config
+
+              },
+              'credential_references' => {
+                'interpolate_service_bindings' => true
+              },
+              'system_hostnames' => '',
+              'logging_max_retries' => 'bar1',
+              'default_app_ssh_access' => 'something',
+              'logging_level' => 'other thing',
+              'logging' => { 'format' => { 'timestamp' => 'rfc3339' } },
+              'db_logging_level' => 'bar2',
+              'db_encryption_key' => 'bar3',
+              'database_encryption' => {
+                'experimental_pbkdf2_hmac_iterations' => 123,
+                'skip_validation' => false,
+                'current_key_label' => 'encryption_key_0',
+                :keys => { 'encryption_key_0' => '((cc_db_encryption_key))' }
+              },
+              'statsd_host' => '127.0.0.1',
+              'statsd_port' => 8125,
+              'max_labels_per_resource' => true,
+              'max_annotations_per_resource' => 'yus',
+              'cpu_weight_min_memory' => 128,
+              'cpu_weight_max_memory' => 8192,
+              'custom_metric_tag_prefix_list' => ['heck.yes.example.com'],
+              'app_log_revision' => true
+            }
+          }
+        end
+        let(:cloud_controller_internal_link) do
+          Link.new(name: 'cloud_controller_internal', properties:, instances: [LinkInstance.new(address: 'default_app_ssh_access')])
+        end
+
+        let(:cloud_controller_db_link) do
+          properties = {
+            'ccdb' => {
+              'db_scheme' => 'mysql',
+              'max_connections' => 'foo2',
+              'databases' => [{ 'tag' => 'cc' }],
+              'roles' => [{
+                'tag' => 'admin',
+                'name' => 'alex',
+                'password' => 'pass'
+              }],
+              'address' => 'foo5',
+              'port' => 'foo7',
+              'pool_timeout' => 'foo11',
+              'ssl_verify_hostname' => 'foo12',
+              'read_timeout' => 'foo13',
+              'connection_validation_timeout' => 'foo14',
+              'ca_cert' => 'foo15'
+            }
+          }
+          Link.new(name: 'database', properties:, instances: [LinkInstance.new(address: 'cloud_controller_db')])
+        end
+
+        let(:links) { [cloud_controller_internal_link, cloud_controller_db_link] }
+
+        let(:template) { job.template('config/cloud_controller_ng.yml') }
+
+        it 'creates the cloud_controller_ng.yml config file' do
+          expect do
+            YAML.safe_load(template.render(manifest_properties, consumes: links))
+          end.not_to raise_error
+        end
+
+        describe 'statsd' do
+          it 'renders statsd_host and statsd_port from cloud_controller_internal link' do
+            template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+            expect(template_hash['statsd_host']).to eq(properties['cc']['statsd_host'])
+            expect(template_hash['statsd_port']).to eq(properties['cc']['statsd_port'])
+          end
+        end
+
+        describe 'metrics' do
+          context 'when cc.publish_metrics not set' do
+            it 'is set to false (default)' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['publish_metrics']).to be(false)
+            end
+          end
+
+          context 'when cc.publish_metrics set to true' do
+            before { manifest_properties['cc']['publish_metrics'] = true }
+
+            it 'is set to true' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['publish_metrics']).to be(true)
+            end
+          end
+
+          context 'when cc.prometheus_port not set' do
+            it 'uses default' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['prometheus_port']).to eq(9395)
+            end
+          end
+
+          context 'when cc.prometheus_port is set' do
+            before do
+              manifest_properties['cc']['prometheus_port'] = 9397
+            end
+
+            it 'uses the default' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['prometheus_port']).to eq(9397)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/cc_deployment_updater/prom_scraper_spec.rb b/spec/cc_deployment_updater/prom_scraper_spec.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+
+module Bosh
+  module Template
+    module Test
+      describe 'prom_scraper config template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('cc_deployment_updater') }
+        let(:rendered_file) { template.render(manifest_properties, consumes: {}) }
+
+        describe 'prom_scraper_config.yml' do
+          let(:template) { job.template('config/prom_scraper_config.yml') }
+          let(:manifest_properties) { {} }
+
+          it('renders an empty file') { expect(rendered_file).to eq('') }
+
+          context 'when cc.publish_metrics is enabled' do
+            before do
+              manifest_properties['cc'] = {}
+              manifest_properties['cc']['publish_metrics'] = true
+            end
+
+            it('renders default values') do
+              expect(rendered_file).to include('port: 9395')
+              expect(rendered_file).to include('source_id: "cc_deployment_updater"')
+              expect(rendered_file).to include('path: /metrics')
+              expect(rendered_file).to include('server_name: "cc_deployment_updater_metrics"')
+              expect(rendered_file).to include('origin: cc_deployment_updater')
+            end
+
+            context 'when different port is given' do
+              before { manifest_properties['cc']['prometheus_port'] = 9397 }
+
+              it('renders custom port') { expect(rendered_file).to include('port: 9397') }
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/cloud_controller_clock/cloud_controller_clock_spec.rb b/spec/cloud_controller_clock/cloud_controller_clock_spec.rb
diff --git a/spec/cloud_controller_clock/prom_scraper_spec.rb b/spec/cloud_controller_clock/prom_scraper_spec.rb
diff --git a/spec/cloud_controller_ng/prom_scraper_spec.rb b/spec/cloud_controller_ng/prom_scraper_spec.rb
diff --git a/spec/cloud_controller_worker/prom_scraper_spec.rb b/spec/cloud_controller_worker/prom_scraper_spec.rb

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+<%= p('cc.prom_scraper_tls.public_cert', '') %>`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+<%= p('cc.prom_scraper_tls.private_key', '') %>`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+<%= p('cc.prom_scraper_tls.ca_cert', '') %>`