diff --git a/.gitignore b/.gitignore index 3f15512..2803e56 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ /spec/fixtures/modules/* /tmp/ /vendor/ +/.vendor/ /convert_report.txt /update_report.txt .DS_Store @@ -26,3 +27,9 @@ .envrc /inventory.yaml /spec/fixtures/litmus_inventory.yaml +.resource_types +.modules +.task_cache.json +.plan_cache.json +.rerun.json +bolt-debug.log diff --git a/.pdkignore b/.pdkignore index 584438f..84684be 100644 --- a/.pdkignore +++ b/.pdkignore @@ -19,6 +19,7 @@ /spec/fixtures/modules/* /tmp/ /vendor/ +/.vendor/ /convert_report.txt /update_report.txt .DS_Store @@ -26,9 +27,16 @@ .envrc /inventory.yaml /spec/fixtures/litmus_inventory.yaml +.resource_types +.modules +.task_cache.json +.plan_cache.json +.rerun.json +bolt-debug.log /.fixtures.yml /Gemfile /.gitattributes +/.github/ /.gitignore /.pdkignore /.puppet-lint.rc diff --git a/.puppet-lint.rc b/.puppet-lint.rc index cc96ece..9e15c6e 100644 --- a/.puppet-lint.rc +++ b/.puppet-lint.rc @@ -1 +1,9 @@ +--fail-on-warnings --relative +--no-80chars-check +--no-140chars-check +--no-class_inherits_from_params_class-check +--no-autoloader_layout-check +--no-documentation-check +--no-single_quote_string_with_variables-check +--ignore-paths=.vendor/**/*.pp,.bundle/**/*.pp,pkg/**/*.pp,spec/**/*.pp,tests/**/*.pp,types/**/*.pp,vendor/**/*.pp diff --git a/.rubocop.yml b/.rubocop.yml index 5be1f9f..21b82b9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,6 +3,7 @@ require: - rubocop-performance - rubocop-rspec AllCops: + NewCops: enable DisplayCopNames: true TargetRubyVersion: '2.6' Include: @@ -527,6 +528,8 @@ Lint/DuplicateBranch: Enabled: false Lint/DuplicateMagicComment: Enabled: false +Lint/DuplicateMatchPattern: + Enabled: false Lint/DuplicateRegexpCharacterClassElement: Enabled: false Lint/EmptyBlock: @@ -643,6 +646,8 @@ Style/ComparableClamp: Enabled: false Style/ConcatArrayLiterals: Enabled: false +Style/DataInheritance: + Enabled: false Style/DirEmpty: Enabled: false Style/DocumentDynamicEvalDefinition: @@ -711,6 +716,8 @@ Style/RedundantHeredocDelimiterQuotes: Enabled: false Style/RedundantInitialize: Enabled: false +Style/RedundantLineContinuation: + Enabled: false Style/RedundantSelfAssignmentBranch: Enabled: false Style/RedundantStringEscape: diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 2f1e4f7..6da8d47 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,6 @@ { "recommendations": [ "puppet.puppet-vscode", - "rebornix.Ruby" + "Shopify.ruby-lsp" ] } diff --git a/Gemfile b/Gemfile index 418fe61..fa682a6 100644 --- a/Gemfile +++ b/Gemfile @@ -20,27 +20,33 @@ group :development do gem "json", '= 2.6.1', require: false if Gem::Requirement.create(['>= 3.1.0', '< 3.1.3']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) gem "json", '= 2.6.3', require: false if Gem::Requirement.create(['>= 3.2.0', '< 4.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) gem "racc", '~> 1.4.0', require: false if Gem::Requirement.create(['>= 2.7.0', '< 3.0.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "deep_merge", '~> 1.2.2', require: false gem "voxpupuli-puppet-lint-plugins", '~> 5.0', require: false - gem "facterdb", '~> 1.18', require: false - gem "metadata-json-lint", '~> 3.0', require: false - gem "puppetlabs_spec_helper", '~> 8.0', require: false - gem "rspec-puppet-facts", '~> 2.0', require: false - gem "codecov", '~> 0.2', require: false + gem "facterdb", '~> 2.1', require: false + gem "metadata-json-lint", '~> 4.0', require: false + gem "rspec-puppet-facts", '~> 4.0', require: false gem "dependency_checker", '~> 1.0.0', require: false gem "parallel_tests", '= 3.12.1', require: false gem "pry", '~> 0.10', require: false - gem "simplecov-console", '~> 0.5', require: false + gem "simplecov-console", '~> 0.9', require: false gem "puppet-debugger", '~> 1.0', require: false - gem "rubocop", '= 1.48.1', require: false + gem "rubocop", '~> 1.50.0', require: false gem "rubocop-performance", '= 1.16.0', require: false gem "rubocop-rspec", '= 2.19.0', require: false gem "rb-readline", '= 0.5.5', require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "rexml", '>= 3.0.0', '< 3.2.7', require: false gem "nokogiri", require: false gem "rspec-puppet-utils", require: false end +group :development, :release_prep do + gem "puppet-strings", '~> 4.0', require: false + gem "puppetlabs_spec_helper", '~> 8.0', require: false + gem "puppet-blacksmith", '~> 7.0', require: false +end group :system_tests do - gem "puppet_litmus", '~> 1.0', require: false, platforms: [:ruby, :x64_mingw] - gem "serverspec", '~> 2.41', require: false + gem "puppet_litmus", '~> 1.0', require: false, platforms: [:ruby, :x64_mingw] + gem "CFPropertyList", '< 3.0.7', require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "serverspec", '~> 2.41', require: false end puppet_version = ENV['PUPPET_GEM_VERSION'] diff --git a/Rakefile b/Rakefile index 5a951b0..51a66b8 100644 --- a/Rakefile +++ b/Rakefile @@ -4,86 +4,16 @@ require 'bundler' require 'puppet_litmus/rake_tasks' if Gem.loaded_specs.key? 'puppet_litmus' require 'puppetlabs_spec_helper/rake_tasks' require 'puppet-syntax/tasks/puppet-syntax' -require 'github_changelog_generator/task' if Gem.loaded_specs.key? 'github_changelog_generator' require 'puppet-strings/tasks' if Gem.loaded_specs.key? 'puppet-strings' -def changelog_user - return unless Rake.application.top_level_tasks.include? "changelog" - returnVal = nil || JSON.load(File.read('metadata.json'))['author'] - raise "unable to find the changelog_user in .sync.yml, or the author in metadata.json" if returnVal.nil? - puts "GitHubChangelogGenerator user:#{returnVal}" - returnVal -end - -def changelog_project - return unless Rake.application.top_level_tasks.include? "changelog" - - returnVal = nil - returnVal ||= begin - metadata_source = JSON.load(File.read('metadata.json'))['source'] - metadata_source_match = metadata_source && metadata_source.match(%r{.*\/([^\/]*?)(?:\.git)?\Z}) - - metadata_source_match && metadata_source_match[1] - end - - raise "unable to find the changelog_project in .sync.yml or calculate it from the source in metadata.json" if returnVal.nil? - - puts "GitHubChangelogGenerator project:#{returnVal}" - returnVal -end - -def changelog_future_release - return unless Rake.application.top_level_tasks.include? "changelog" - returnVal = "v%s" % JSON.load(File.read('metadata.json'))['version'] - raise "unable to find the future_release (version) in metadata.json" if returnVal.nil? - puts "GitHubChangelogGenerator future_release:#{returnVal}" - returnVal -end - PuppetLint.configuration.send('disable_relative') - - -if Gem.loaded_specs.key? 'github_changelog_generator' - GitHubChangelogGenerator::RakeTask.new :changelog do |config| - raise "Set CHANGELOG_GITHUB_TOKEN environment variable eg 'export CHANGELOG_GITHUB_TOKEN=valid_token_here'" if Rake.application.top_level_tasks.include? "changelog" and ENV['CHANGELOG_GITHUB_TOKEN'].nil? - config.user = "#{changelog_user}" - config.project = "#{changelog_project}" - config.future_release = "#{changelog_future_release}" - config.exclude_labels = ['maintenance'] - config.header = "# Change log\n\nAll notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org)." - config.add_pr_wo_labels = true - config.issues = false - config.merge_prefix = "### UNCATEGORIZED PRS; LABEL THEM ON GITHUB" - config.configure_sections = { - "Changed" => { - "prefix" => "### Changed", - "labels" => ["backwards-incompatible"], - }, - "Added" => { - "prefix" => "### Added", - "labels" => ["enhancement", "feature"], - }, - "Fixed" => { - "prefix" => "### Fixed", - "labels" => ["bug", "documentation", "bugfix"], - }, - } - end -else - desc 'Generate a Changelog from GitHub' - task :changelog do - raise < 1.15' - condition: "Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.3.0')" -EOM - end -end +PuppetLint.configuration.send('disable_80chars') +PuppetLint.configuration.send('disable_140chars') +PuppetLint.configuration.send('disable_class_inherits_from_params_class') +PuppetLint.configuration.send('disable_autoloader_layout') +PuppetLint.configuration.send('disable_documentation') +PuppetLint.configuration.send('disable_single_quote_string_with_variables') +PuppetLint.configuration.fail_on_warnings = true +PuppetLint.configuration.ignore_paths = [".vendor/**/*.pp", ".bundle/**/*.pp", "pkg/**/*.pp", "spec/**/*.pp", "tests/**/*.pp", "types/**/*.pp", "vendor/**/*.pp"] diff --git a/data/backend-defaults.yaml b/data/backend-defaults.yaml index 5766177..18af77c 100644 --- a/data/backend-defaults.yaml +++ b/data/backend-defaults.yaml @@ -5,6 +5,15 @@ storm::backend::install_native_libs_gpfs: false # Db connection storm::backend::db_username: 'storm' storm::backend::db_password: 'storm' +storm::backend::db_port: 3306 +storm::backend::db_properties: 'useLegacyDatetimeCode=false&serverTimezone=UTC' +storm::backend::db_pool_max_wait_millis: -1 +storm::backend::db_pool_test_on_borrow: true +storm::backend::db_pool_test_while_idle: true +storm::backend::db_pool_stormdb_max_total: 500 +storm::backend::db_pool_stormdb_min_idle: 50 +storm::backend::db_pool_stormbeisam_max_total: 200 +storm::backend::db_pool_stormbeisam_min_idle: 10 ### Default values for Storage Areas # 1. xroot @@ -48,6 +57,7 @@ storm::backend::xmlrpc_security_token: 'secret' # Skip ACL setup for PTG requests storm::backend::ptg_skip_acl_setup: false +storm::backend::ptp_skip_acl_setup: false # pin lifetime storm::backend::pinlifetime_default: 259200 @@ -73,7 +83,8 @@ storm::backend::gc_purge_enabled: true storm::backend::gc_purge_interval: 600 storm::backend::gc_purge_size: 800 storm::backend::gc_expired_request_time: 21600 -storm::backend::gc_expired_inprogress_time: 2592000 +storm::backend::gc_expired_inprogress_bol_time: 2592000 +storm::backend::gc_expired_inprogress_ptp_time: 2592000 # Expired-Put-Requests-Agent parameters storm::backend::gc_ptp_transit_interval: 300 storm::backend::gc_ptp_transit_start_delay: 10 diff --git a/data/frontend-defaults.yaml b/data/frontend-defaults.yaml index 0b33dec..b45945d 100644 --- a/data/frontend-defaults.yaml +++ b/data/frontend-defaults.yaml @@ -1,5 +1,6 @@ --- storm::frontend::storm_frontend_server_conf_file: '' +storm::frontend::cgsi_trace: false storm::frontend::db_user: 'storm' storm::frontend::db_passwd: 'storm' storm::frontend::port: 8444 diff --git a/examples/storage.pp b/examples/storage.pp index 7dfda8e..be76a9d 100644 --- a/examples/storage.pp +++ b/examples/storage.pp @@ -10,14 +10,14 @@ users => { 'edguser' => { 'comment' => 'Edguser user', - 'groups' => [ 'edguser', 'storm', ], + 'groups' => ['edguser', 'storm'], 'uid' => '995', 'gid' => '995', 'home' => '/home/storm', }, 'storm' => { 'comment' => 'StoRM user', - 'groups' => [ 'storm', 'edguser', 'storm-SA-read', 'storm-SA-write' ], + 'groups' => ['storm', 'edguser', 'storm-SA-read', 'storm-SA-write'], 'uid' => '991', 'gid' => '991', 'home' => '/home/edguser', diff --git a/examples/users.pp b/examples/users.pp index 639e1a1..0d13b82 100644 --- a/examples/users.pp +++ b/examples/users.pp @@ -7,14 +7,14 @@ users => { edguser => { comment => 'Edguser user', - groups => [ edguser, infosys, storm, ], + groups => ['edguser', 'infosys', 'storm'], uid => '995', gid => '995', home => '/home/edguser', }, storm => { comment => 'StoRM user', - groups => [ storm, edguser, ], + groups => ['storm', 'edguser'], uid => '991', gid => '991', home => '/home/storm', diff --git a/manifests/backend.pp b/manifests/backend.pp index 91d53cc..3605c0c 100644 --- a/manifests/backend.pp +++ b/manifests/backend.pp @@ -37,7 +37,7 @@ # @param hostname # StoRM Backend Fully Qualified Domain Name # -# @param db_hostname +# @param db_host # Fully Qualified Domain Name of database hostname. Default value: `hostname`. # # @param db_username @@ -46,6 +46,41 @@ # @param db_password # Password for the user in `db_storm_username`. Default: bluemoon # +# @param db_port +# Port used to connect StoRM Backend to the database. Default: 3306 +# +# @param db_properties +# String of properties appended to the database connection URL. Default: 'useLegacyDatetimeCode=false' +# +# @param db_pool_max_wait_millis +#  The maximum number of milliseconds that the pool will wait (when there are no available connections) +# for a connection to be returned before throwing an exception, or -1 to wait indefinitely. Default: -1 +# +# @param db_pool_test_on_borrow +# The indication of whether objects will be validated before being borrowed from the pool. +#  If the object fails to validate, it will be dropped from the pool, and we will attempt to borrow another. +# Default: true +# +# @param db_pool_test_while_idle +# The indication of whether objects will be validated by the idle object evictor (if any). +# If an object fails to validate, it will be dropped from the pool. Default: true +# +# @param db_pool_stormdb_max_total +# The maximum number of active connections that can be allocated from this pool at the same time, +# or negative for no limit. Default: 500 +# +# @param db_pool_stormdb_min_idle +# The minimum number of connections that can remain idle in the pool, without extra ones being created, +# or zero to create none. Default: 50 +# +# @param db_pool_stormbeisam_max_total +# The maximum number of active connections that can be allocated from this pool at the same time, +# or negative for no limit. Default: 200 +# +# @param db_pool_stormbeisam_min_idle +# The minimum number of connections that can remain idle in the pool, without extra ones being created, +# or zero to create none. Default: 10 +# # @param xroot_hostname # Root server (default value for all Storage Areas). # Note: you may change the settings for each SA acting on its configuration. @@ -137,6 +172,9 @@ # @param ptg_skip_acl_setup # Skip ACL setup for PtG requests. Default: false. # +# @param ptp_skip_acl_setup +# Skip ACL setup for PtP requests. This means also the creation of an empty file used by GridFTP. Default: false. +# # @param pinlifetime_default # Default PinLifetime in seconds used for pinning files in case of srmPrepareToPut or srmPrepareToGet operation # without any pinLifetime specified. Default: 259200. @@ -180,7 +218,10 @@ # Time in seconds to consider a request expired after its submission. Default: 604800 seconds (1 week). # From StoRM 1.11.13 it is used also to identify how much time is needed to consider a completed recall task as cleanable. # -# @param gc_expired_inprogress_time +# @param gc_expired_inprogress_bol_time +# Time in seconds to consider an in-progress bol request as expired. Default: 2592000 seconds (1 month). +# +# @param gc_expired_inprogress_ptp_time # Time in seconds to consider an in-progress ptp request as expired. Default: 2592000 seconds (1 month). # # @param gc_ptp_transit_interval @@ -312,6 +353,15 @@ # Db connection String $db_username, String $db_password, + Integer $db_port, + String $db_properties, + Integer $db_pool_max_wait_millis, + Boolean $db_pool_test_on_borrow, + Boolean $db_pool_test_while_idle, + Integer $db_pool_stormdb_max_total, + Integer $db_pool_stormdb_min_idle, + Integer $db_pool_stormbeisam_max_total, + Integer $db_pool_stormbeisam_min_idle, ### Default values for Storage Areas # 1. xroot @@ -351,8 +401,9 @@ Boolean $xmlrpc_security_enabled, String $xmlrpc_security_token, - # Skip ACL setup for PTG requests + # Skip ACL setup for PtG/PtP requests Boolean $ptg_skip_acl_setup, + Boolean $ptp_skip_acl_setup, # Pin lifetime Integer $pinlifetime_default, @@ -378,7 +429,8 @@ Integer $gc_purge_interval, Integer $gc_purge_size, Integer $gc_expired_request_time, - Integer $gc_expired_inprogress_time, + Integer $gc_expired_inprogress_bol_time, + Integer $gc_expired_inprogress_ptp_time, Integer $gc_ptp_transit_interval, Integer $gc_ptp_transit_start_delay, @@ -450,7 +502,7 @@ # hostnames String $hostname = $fqdn, - String $db_hostname = $hostname, + String $db_host = $hostname, String $xroot_hostname = $hostname, String $frontend_public_host = $hostname, diff --git a/manifests/backend/config.pp b/manifests/backend/config.pp index c2a77a9..792f49f 100644 --- a/manifests/backend/config.pp +++ b/manifests/backend/config.pp @@ -46,6 +46,7 @@ $properties_file='/etc/storm/backend-server/storm.properties' if $storm::backend::manage_storm_properties { + notice("File ${properties_file} is initialized from ${storm::backend::path_storm_properties} ...") file { $properties_file: ensure => file, source => $storm::backend::path_storm_properties, @@ -54,6 +55,7 @@ notify => [Service['storm-backend-server']], } } else { + notice("File ${properties_file} is initialized from a template using class parameters ...") $properties_template_file='storm/etc/storm/backend-server/storm.properties.erb' file { $properties_file: ensure => file, diff --git a/manifests/backend/configdb.pp b/manifests/backend/configdb.pp index f1f0ac5..8768b4e 100644 --- a/manifests/backend/configdb.pp +++ b/manifests/backend/configdb.pp @@ -3,7 +3,7 @@ class storm::backend::configdb ( ) { - $db_hostname = $storm::backend::db_hostname + $db_hostname = $storm::backend::db_host $db_username = $storm::backend::db_username $db_password = $storm::backend::db_password @@ -24,7 +24,7 @@ command => 'mysql storm_db < /tmp/storm_db.sql', onlyif => $check_storm_db, logoutput => true, - environment => "HOME=${::root_home}", + environment => "HOME=${facts['root_home']}", path => $paths, provider => 'shell', require => [File['/tmp/storm_db.sql']], @@ -35,7 +35,7 @@ command => 'mysql storm_be_ISAM < /tmp/storm_be_ISAM.sql', onlyif => $check_storm_be_isam, logoutput => true, - environment => "HOME=${::root_home}", + environment => "HOME=${facts['root_home']}", path => $paths, provider => 'shell', require => [File['/tmp/storm_be_ISAM.sql']], diff --git a/manifests/backend/install.pp b/manifests/backend/install.pp index 8bce298..bed8e51 100644 --- a/manifests/backend/install.pp +++ b/manifests/backend/install.pp @@ -5,13 +5,13 @@ ) { ## StoRM Backend package { 'storm-backend-server': - ensure => '>=1.11.22', + ensure => '>=1.12.0', } if $storm::backend::fs_type == 'gpfs' { ## Native libs GPFS package { 'storm-native-libs-gpfs' : - ensure => '>1.0.7', + ensure => '>=2.0.0', require => [Package['storm-backend-server']], } } else { @@ -19,7 +19,7 @@ if $sa['fs_type'] == 'gpfs' { ## Native libs GPFS package { 'storm-native-libs-gpfs' : - ensure => '>1.0.7', + ensure => '>=2.0.0', require => [Package['storm-backend-server']], } break() @@ -29,7 +29,7 @@ ## StoRM Info Provider package { 'storm-dynamic-info-provider': - ensure => '>=1.8.3', + ensure => '>=2.0.0', require => [Package['storm-backend-server']], } } diff --git a/manifests/frontend.pp b/manifests/frontend.pp index ca46d34..c3aadf8 100644 --- a/manifests/frontend.pp +++ b/manifests/frontend.pp @@ -53,16 +53,6 @@ # @param gsoap_maxpending [Integer] # Size of the GSOAP queue used to maintain pending SRM requests. Default value: 1000. # -# @param check_user_blacklisting [Boolean] -# Enable/disable user blacklisting. Default value: false. -# -# @param argus_pepd_endpoint [String] -# The complete service endpoint of Argus PEP server. Mandatory if `check_user_blacklisting` is true. -# -# @param argus_resource_id -# The resource id is used to target a resource (or set of resources, if wildcards are used) under the control of Argus authorization. -# Mandatory if `check_user_blacklisting` is true. -# # @param monitoring_enabled [Boolean] # Enable/disable monitoring. Default value: true. # @@ -78,6 +68,9 @@ # @param log_debuglevel [String] # Logging level. Possible values are: ERROR, WARN, INFO, DEBUG, DEBUG2. Default value: INFO. # +# @param cgsi_trace [Boolean] +# Enable CGSI trace on '/tmp/tracefile'. Default: false +# class storm::frontend ( String $storm_frontend_server_conf_file, @@ -95,10 +88,6 @@ Integer $threadpool_maxpending, Integer $gsoap_maxpending, - Boolean $check_user_blacklisting, - String $argus_pepd_endpoint, - String $argus_resource_id, - Boolean $monitoring_enabled, Integer $monitoring_time_interval, Boolean $monitoring_detailed, @@ -107,9 +96,10 @@ String $log_debuglevel, + Boolean $cgsi_trace, + String $be_xmlrpc_host = $fqdn, String $db_host = $be_xmlrpc_host, - ) { contain storm::frontend::install contain storm::frontend::config diff --git a/manifests/frontend/config.pp b/manifests/frontend/config.pp index c5633ec..9706b55 100644 --- a/manifests/frontend/config.pp +++ b/manifests/frontend/config.pp @@ -34,12 +34,11 @@ $conf_file='/etc/storm/frontend-server/storm-frontend-server.conf' if (!empty($storm::frontend::storm_frontend_server_conf_file)) { file { $conf_file: - ensure => file, - owner => 'root', - group => 'storm', - source => $storm::frontend::storm_frontend_server_conf_file, - notify => Service['storm-frontend-server'], - require => Package['storm-frontend-mp'], + ensure => file, + owner => 'root', + group => 'storm', + source => $storm::frontend::storm_frontend_server_conf_file, + notify => Service['storm-frontend-server'], } } else { $conf_template_file='storm/etc/storm/frontend-server/storm-frontend-server.conf.erb' @@ -49,7 +48,6 @@ group => 'storm', content => template($conf_template_file), notify => Service['storm-frontend-server'], - require => Package['storm-frontend-mp'], } } @@ -64,6 +62,5 @@ ensure => file, content => template($sysconfig_template_file), notify => Service['storm-frontend-server'], - require => Package['storm-frontend-mp'], } } diff --git a/manifests/frontend/install.pp b/manifests/frontend/install.pp index 6a3c293..9f4c4dd 100644 --- a/manifests/frontend/install.pp +++ b/manifests/frontend/install.pp @@ -3,7 +3,8 @@ class storm::frontend::install ( ) { - package { 'storm-frontend-mp': - ensure => installed, + ## StoRM Frontend + package { 'storm-frontend-server': + ensure => '>=1.8.16', } } diff --git a/manifests/frontend/service.pp b/manifests/frontend/service.pp index 13ab1c8..edea709 100644 --- a/manifests/frontend/service.pp +++ b/manifests/frontend/service.pp @@ -4,6 +4,6 @@ service { 'storm-frontend-server': ensure => running, enable => true, - require => Package['storm-frontend-mp'], + require => Package['storm-frontend-server'], } } diff --git a/manifests/rootdir.pp b/manifests/rootdir.pp index ab256b7..bc60fab 100644 --- a/manifests/rootdir.pp +++ b/manifests/rootdir.pp @@ -1,4 +1,8 @@ # @summary StoRM main storage area root directory defined resource +# +# @param mode +# The permission mask. E.g. '0755' +# define storm::rootdir ( String $mode = '0755', ) { diff --git a/manifests/sarootdir.pp b/manifests/sarootdir.pp index 35347f4..d3540ae 100644 --- a/manifests/sarootdir.pp +++ b/manifests/sarootdir.pp @@ -1,4 +1,8 @@ # @summary StoRM Storage Area root directory defined resource +# +# @param mode +# The permission mask. E.g. '0750' +# define storm::sarootdir ( String $mode = '0750', ) { diff --git a/manifests/webdav/install.pp b/manifests/webdav/install.pp index 8238122..29c7ef7 100644 --- a/manifests/webdav/install.pp +++ b/manifests/webdav/install.pp @@ -4,7 +4,7 @@ ) { package { 'storm-webdav': - ensure => '>=1.4.2', + ensure => '>=1.4.3', } if $storm::webdav::scitags_enabled { $el = $facts['os']['distro']['release']['major'] diff --git a/metadata.json b/metadata.json index 45d2d4b..9e003e9 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "cnafsd-storm", - "version": "4.3.0", + "version": "5.0.0", "author": "storm", "summary": "StoRM Puppet module", "license": "Apache-2.0", @@ -22,12 +22,6 @@ } ], "operatingsystem_support": [ - { - "operatingsystem": "CentOS", - "operatingsystemrelease": [ - "7" - ] - }, { "operatingsystem": "RedHat", "operatingsystemrelease": [ @@ -39,12 +33,6 @@ "operatingsystemrelease": [ "9" ] - }, - { - "operatingsystem": "Scientific", - "operatingsystemrelease": [ - "7" - ] } ], "requirements": [ @@ -53,7 +41,7 @@ "version_requirement": ">= 7.0.0 < 8.0.0" } ], - "pdk-version": "3.0.0", - "template-url": "pdk-default#3.0.0", - "template-ref": "tags/3.0.0-0-g056e50d" + "pdk-version": "3.4.0", + "template-url": "pdk-default#3.4.0", + "template-ref": "tags/3.4.0-0-gd3cc13f" } diff --git a/spec/classes/storm/namespace.xml.erb_spec.rb b/spec/classes/storm/namespace.xml.erb_spec.rb index aa90f2c..70bee87 100644 --- a/spec/classes/storm/namespace.xml.erb_spec.rb +++ b/spec/classes/storm/namespace.xml.erb_spec.rb @@ -76,7 +76,6 @@ end it 'render the same file each time' do - rendered = harness.run xml_doc = Nokogiri::XML(rendered) diff --git a/spec/classes/storm/storm_backend_install_spec.rb b/spec/classes/storm/storm_backend_install_spec.rb index a0e7701..d4a4b9b 100644 --- a/spec/classes/storm/storm_backend_install_spec.rb +++ b/spec/classes/storm/storm_backend_install_spec.rb @@ -77,7 +77,6 @@ class { 'storm::backend': is_expected.to contain_package('storm-native-libs-gpfs') end end - end end end diff --git a/spec/classes/storm/storm_backend_spec.rb b/spec/classes/storm/storm_backend_spec.rb index de31e6c..c30edbd 100644 --- a/spec/classes/storm/storm_backend_spec.rb +++ b/spec/classes/storm/storm_backend_spec.rb @@ -124,9 +124,6 @@ is_expected.to contain_file(title).with(content: %r{storm.service.port=8444}) is_expected.to contain_file(title).with(content: %r{storm.service.SURL.endpoint=srm:\/\/frontend-0.example.org:8445\/srm\/managerv2,srm:\/\/frontend-1.example.org:8444\/srm\/managerv2}) is_expected.to contain_file(title).with(content: %r{storm.service.SURL.default-ports=8445,8444}) - is_expected.to contain_file(title).with(content: %r{storm.service.request-db.host=storm.example.org}) - is_expected.to contain_file(title).with(content: %r{storm.service.request-db.username=test}) - is_expected.to contain_file(title).with(content: %r{storm.service.request-db.passwd=secret}) is_expected.to contain_file(title).with(content: %r{directory.automatic-creation=false}) is_expected.to contain_file(title).with(content: %r{directory.writeperm=false}) is_expected.to contain_file(title).with(content: %r{ptg.skip-acl-setup=false}) @@ -148,11 +145,20 @@ is_expected.to contain_file(title).with(content: %r{extraslashes.file=}) is_expected.to contain_file(title).with(content: %r{extraslashes.root=\/}) is_expected.to contain_file(title).with(content: %r{extraslashes.gsiftp=\/}) - is_expected.to contain_file(title).with(content: %r{persistence.internal-db.connection-pool=true}) - is_expected.to contain_file(title).with(content: %r{persistence.internal-db.connection-pool.maxActive=200}) - is_expected.to contain_file(title).with(content: %r{persistence.internal-db.connection-pool.maxWait=50}) - is_expected.to contain_file(title).with(content: %r{asynch.db.ReconnectPeriod=18000}) - is_expected.to contain_file(title).with(content: %r{asynch.db.DelayPeriod=30}) + + is_expected.to contain_file(title).with(content: %r{storm.service.db.host=storm.example.org}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.username=test}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.password=secret}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.port=3306}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.properties=useLegacyDatetimeCode=false&serverTimezone=UTC}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.maxWaitMillis=-1}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.testOnBorrow=true}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.testWhileIdle=true}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.stormdb.maxTotal=500}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.stormdb.minIdle=50}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.stormbeisam.maxTotal=200}) + is_expected.to contain_file(title).with(content: %r{storm.service.db.pool.stormbeisam.minIdle=10}) + is_expected.to contain_file(title).with(content: %r{asynch.PickingInitialDelay=1}) is_expected.to contain_file(title).with(content: %r{asynch.PickingTimeInterval=2}) is_expected.to contain_file(title).with(content: %r{asynch.PickingMaxBatchSize=100}) @@ -172,7 +178,8 @@ is_expected.to contain_file(title).with(content: %r{purge.interval=600}) is_expected.to contain_file(title).with(content: %r{purge.size=800}) is_expected.to contain_file(title).with(content: %r{expired.request.time=21600}) - is_expected.to contain_file(title).with(content: %r{expired.inprogress.time=2592000}) + is_expected.to contain_file(title).with(content: %r{expired.inprogress.bol.time=2592000}) + is_expected.to contain_file(title).with(content: %r{expired.inprogress.ptp.time=2592000}) is_expected.to contain_file(title).with(content: %r{transit.interval=300}) is_expected.to contain_file(title).with(content: %r{transit.delay=10}) is_expected.to contain_file(title).with(content: %r{ptg.skip-acl-setup=false}) diff --git a/spec/classes/storm/storm_db_spec.rb b/spec/classes/storm/storm_db_spec.rb index 85b0be1..f30a9ad 100644 --- a/spec/classes/storm/storm_db_spec.rb +++ b/spec/classes/storm/storm_db_spec.rb @@ -29,7 +29,7 @@ 'mysqld' => { 'bind-address' => '0.0.0.0', 'max_connections' => 2048, - 'wait_timeout' => 86400, + 'wait_timeout' => 86_400, }, }, databases: { diff --git a/spec/classes/storm/storm_frontend_install_spec.rb b/spec/classes/storm/storm_frontend_install_spec.rb index ab6f1bb..db5f8e7 100644 --- a/spec/classes/storm/storm_frontend_install_spec.rb +++ b/spec/classes/storm/storm_frontend_install_spec.rb @@ -8,7 +8,7 @@ end it 'check storm frontend metapackage is installed' do - is_expected.to contain_package('storm-frontend-mp').with(ensure: 'installed') + is_expected.to contain_package('storm-frontend-server').with(ensure: '>=1.8.16') end end end diff --git a/spec/classes/storm/storm_frontend_spec.rb b/spec/classes/storm/storm_frontend_spec.rb index 09db123..3cc8148 100644 --- a/spec/classes/storm/storm_frontend_spec.rb +++ b/spec/classes/storm/storm_frontend_spec.rb @@ -76,9 +76,6 @@ 'db_host' => 'storm.example.org', 'db_user' => 'test', 'db_passwd' => 'password', - 'check_user_blacklisting' => true, - 'argus_pepd_endpoint' => 'storm.example.org', - 'argus_resource_id' => 'storm', 'monitoring_enabled' => false, 'monitoring_time_interval' => 50, 'monitoring_detailed' => true, @@ -105,9 +102,6 @@ is_expected.to contain_file(title).with(content: %r{db.host=storm.example.org}) is_expected.to contain_file(title).with(content: %r{db.user=test}) is_expected.to contain_file(title).with(content: %r{db.passwd=password}) - is_expected.to contain_file(title).with(content: %r{check.user.blacklisting=true}) - is_expected.to contain_file(title).with(content: %r{argus-pepd-endpoint=storm.example.org}) - is_expected.to contain_file(title).with(content: %r{argus.resource-id=storm}) is_expected.to contain_file(title).with(content: %r{monitoring.enabled=false}) is_expected.to contain_file(title).with(content: %r{#monitoring.timeInterval=60}) is_expected.to contain_file(title).with(content: %r{#monitoring.detailed=false}) @@ -144,9 +138,6 @@ is_expected.to contain_file(title).with(content: %r{db.host=storm.example.org}) is_expected.to contain_file(title).with(content: %r{db.user=storm}) is_expected.to contain_file(title).with(content: %r{db.passwd=storm}) - is_expected.to contain_file(title).with(content: %r{check.user.blacklisting=false}) - is_expected.to contain_file(title).with(content: %r{#argus-pepd-endpoint=}) - is_expected.to contain_file(title).with(content: %r{#argus.resource-id=}) is_expected.to contain_file(title).with(content: %r{monitoring.enabled=true}) is_expected.to contain_file(title).with(content: %r{monitoring.timeInterval=60}) is_expected.to contain_file(title).with(content: %r{monitoring.detailed=false}) diff --git a/spec/classes/storm/storm_webdav_service_spec.rb b/spec/classes/storm/storm_webdav_service_spec.rb index 788b386..a31221b 100644 --- a/spec/classes/storm/storm_webdav_service_spec.rb +++ b/spec/classes/storm/storm_webdav_service_spec.rb @@ -4,7 +4,6 @@ on_supported_os.each do |os, facts| context "on #{os}" do let(:pre_condition) do - 'include storm::webdav::install' <<-EOF class { 'storm::webdav': scitags_enabled => false, diff --git a/spec/classes/storm/storm_webdav_spec.rb b/spec/classes/storm/storm_webdav_spec.rb index 6be274d..fe43312 100644 --- a/spec/classes/storm/storm_webdav_spec.rb +++ b/spec/classes/storm/storm_webdav_spec.rb @@ -62,10 +62,10 @@ 'tls_protocol' => 'TLSv1.2', 'checksum_filter_enabled' => false, 'macaroon_filter_enabled' => false, - 'file_buffer_size' => 1048577, + 'file_buffer_size' => 1_048_577, 'voms_trust_store_dir' => '/vomsdir', - 'voms_trust_store_refresh_interval_sec' => 43201, + 'voms_trust_store_refresh_interval_sec' => 43_201, 'voms_cache_enabled' => false, 'voms_cache_entry_lifetime_sec' => 301, diff --git a/spec/default_facts.yml b/spec/default_facts.yml index f777abf..3346c39 100644 --- a/spec/default_facts.yml +++ b/spec/default_facts.yml @@ -2,7 +2,8 @@ # # Facts specified here will override the values provided by rspec-puppet-facts. --- -ipaddress: "172.16.254.254" -ipaddress6: "FE80:0000:0000:0000:AAAA:AAAA:AAAA" +networking: + ip: "172.16.254.254" + ip6: "FE80:0000:0000:0000:AAAA:AAAA:AAAA" + mac: "AA:AA:AA:AA:AA:AA" is_pe: false -macaddress: "AA:AA:AA:AA:AA:AA" diff --git a/spec/defines/storm_webdav_drop_in_file_spec.rb b/spec/defines/storm_webdav_drop_in_file_spec.rb index 61b41cc..86c8e3e 100644 --- a/spec/defines/storm_webdav_drop_in_file_spec.rb +++ b/spec/defines/storm_webdav_drop_in_file_spec.rb @@ -29,4 +29,4 @@ end end end -end \ No newline at end of file +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 08a97ec..0be710f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -25,7 +25,8 @@ next unless File.exist?(f) && File.readable?(f) && File.size?(f) begin - default_facts.merge!(YAML.safe_load(File.read(f), permitted_classes: [], permitted_symbols: [], aliases: true)) + require 'deep_merge' + default_facts.deep_merge!(YAML.safe_load(File.read(f), permitted_classes: [], permitted_symbols: [], aliases: true)) rescue StandardError => e RSpec.configuration.reporter.message "WARNING: Unable to load #{f}: #{e}" end @@ -33,7 +34,7 @@ # read default_facts and merge them over what is provided by facterdb default_facts.each do |fact, value| - add_custom_fact fact, value + add_custom_fact fact, value, merge_facts: true end RSpec.configure do |c| diff --git a/templates/etc/storm/backend-server/storm.properties.erb b/templates/etc/storm/backend-server/storm.properties.erb index 9b4eeeb..40d4f23 100644 --- a/templates/etc/storm/backend-server/storm.properties.erb +++ b/templates/etc/storm/backend-server/storm.properties.erb @@ -53,18 +53,19 @@ storm.service.SURL.default-ports=<%= @srm_ports.join(',') %> # ============================ # # Parameters to connect to the DB used as channel for the requests. -storm.service.request-db.host=<%= scope.lookupvar('storm::backend::db_hostname') %> -storm.service.request-db.username=<%= scope.lookupvar('storm::backend::db_username') %> -storm.service.request-db.passwd=<%= scope.lookupvar('storm::backend::db_password') %> - -############################################# -############ PROFILE PARAMETERS ############ -############################################# -## -## NOTE: -## The PROFILE PARAMETERS are not managed by YAIM in any case, -## so any change must be managed by directly editing this file. -## +storm.service.db.host=<%= scope.lookupvar('storm::backend::db_host') %> +storm.service.db.username=<%= scope.lookupvar('storm::backend::db_username') %> +storm.service.db.password=<%= scope.lookupvar('storm::backend::db_password') %> +storm.service.db.port=<%= scope.lookupvar('storm::backend::db_port') %> +storm.service.db.properties=<%= scope.lookupvar('storm::backend::db_properties') %> +storm.service.db.pool.maxWaitMillis=<%= scope.lookupvar('storm::backend::db_pool_max_wait_millis') %> +storm.service.db.pool.testOnBorrow=<%= scope.lookupvar('storm::backend::db_pool_test_on_borrow') %> +storm.service.db.pool.testWhileIdle=<%= scope.lookupvar('storm::backend::db_pool_test_while_idle') %> +storm.service.db.pool.stormdb.maxTotal=<%= scope.lookupvar('storm::backend::db_pool_stormdb_max_total') %> +storm.service.db.pool.stormdb.minIdle=<%= scope.lookupvar('storm::backend::db_pool_stormdb_min_idle') %> +storm.service.db.pool.stormbeisam.maxTotal=<%= scope.lookupvar('storm::backend::db_pool_stormbeisam_max_total') %> +storm.service.db.pool.stormbeisam.minIdle=<%= scope.lookupvar('storm::backend::db_pool_stormbeisam_min_idle') %> + # ============================ # StoRM Service Generic Behavior @@ -105,24 +106,6 @@ default.overwrite=A default.storagetype=P -############################################# -############ TUNING PARAMETERS ############# -############################################# -## -## NOTE: -## The TUNING PARAMETERS are not managed by YAIM in any case, -## so any change must be managed by directly editing this file. -## - - -# ============================ -# BE-private RDBMS -# ============================ -persistence.internal-db.connection-pool=<%= scope.lookupvar('storm::backend::db_connection_pool_enabled') %> -persistence.internal-db.connection-pool.maxActive=<%= scope.lookupvar('storm::backend::db_connection_pool_max_active') %> -persistence.internal-db.connection-pool.maxWait=<%= scope.lookupvar('storm::backend::db_connection_pool_max_wait') %> - - # ============================ # ASYNCH SCHEDULER Component parameters # ============================ @@ -209,9 +192,13 @@ purge.size=<%= scope.lookupvar('storm::backend::gc_purge_size') %> # Default: 21600s (6h) expired.request.time=<%= scope.lookupvar('storm::backend::gc_expired_request_time') %> # +# Time after that the GC consider an _in-progress_ bol request as garbage +# Default: 2592000 (1 month) +expired.inprogress.bol.time=<%= scope.lookupvar('storm::backend::gc_expired_inprogress_bol_time') %> +# # Time after that the GC consider an _in-progress_ ptp request as garbage # Default: 2592000 (1 month) -expired.inprogress.time=<%= scope.lookupvar('storm::backend::gc_expired_inprogress_time') %> +expired.inprogress.ptp.time=<%= scope.lookupvar('storm::backend::gc_expired_inprogress_ptp_time') %> # ========================================================== @@ -228,8 +215,9 @@ transit.interval=<%= scope.lookupvar('storm::backend::gc_ptp_transit_interval') transit.delay=<%= scope.lookupvar('storm::backend::gc_ptp_transit_start_delay') %> -# Skip ACL setup for PtG requests +# Skip ACL setup for PtG/PtP requests ptg.skip-acl-setup=<%= scope.lookupvar('storm::backend::ptg_skip_acl_setup') %> +ptp.skip-acl-setup=<%= scope.lookupvar('storm::backend::ptp_skip_acl_setup') %> # The caching policy for successful name lookups from the name service. # The value is specified as integer to indicate the number of seconds to cache the successful lookup. diff --git a/templates/etc/storm/frontend-server/storm-frontend-server.conf.erb b/templates/etc/storm/frontend-server/storm-frontend-server.conf.erb index 0930fa5..3aae942 100644 --- a/templates/etc/storm/frontend-server/storm-frontend-server.conf.erb +++ b/templates/etc/storm/frontend-server/storm-frontend-server.conf.erb @@ -31,16 +31,6 @@ db.host=<%=scope.lookupvar('storm::frontend::db_host')%> db.user=<%=scope.lookupvar('storm::frontend::db_user')%> db.passwd=<%=scope.lookupvar('storm::frontend::db_passwd')%> -# Argus Blacklisting -check.user.blacklisting=<%=scope.lookupvar('storm::frontend::check_user_blacklisting')%> -<% if scope.lookupvar('storm::frontend::check_user_blacklisting') %> -argus-pepd-endpoint=<%=scope.lookupvar('storm::frontend::argus_pepd_endpoint')%> -argus.resource-id=<%=scope.lookupvar('storm::frontend::argus_resource_id')%> -<% else %> -#argus-pepd-endpoint= -#argus.resource-id= -<% end %> - # Monitoring monitoring.enabled=<%=scope.lookupvar('storm::frontend::monitoring_enabled')%> <% if scope.lookupvar('storm::frontend::monitoring_enabled') %> diff --git a/templates/etc/sysconfig/storm-frontend-server.erb b/templates/etc/sysconfig/storm-frontend-server.erb index b565d1e..d812e07 100644 --- a/templates/etc/sysconfig/storm-frontend-server.erb +++ b/templates/etc/sysconfig/storm-frontend-server.erb @@ -7,4 +7,9 @@ X509_USER_KEY=/etc/grid-security/storm/hostkey.pem LD_LIBRARY_PATH=<%=@ld_library_path%> +<%- if scope.lookupvar('storm::frontend::cgsi_trace') -%> +CGSI_TRACE=1 +CGSI_TRACEFILE=/tmp/tracefile +<%- end -%> + CONFIGURATION_FILE=/etc/storm/frontend-server/storm-frontend-server.conf \ No newline at end of file diff --git a/types/customrepo.pp b/types/customrepo.pp index 043f63d..984d927 100644 --- a/types/customrepo.pp +++ b/types/customrepo.pp @@ -1,5 +1,7 @@ # @summary The storage area type for storm-webdav -type Storm::CustomRepo = Struct[{ - name => String, - baseurl => String, -}] +type Storm::CustomRepo = Struct[ + { + name => String, + baseurl => String, + } +]