Skip to content

GitLab

Commit 4284724d authored by Rémy Coutable's avatar Rémy Coutable
Browse files

Merge branch 'refactor/ci-config-move-job-entries' into 'master' 

Move CI job config entries from legacy to new config

## What does this MR do?

This MR extracts jobs configuration logic from legacy CI config processor to the new code.

## What are the relevant issue numbers?

#15060

## Does this MR meet the acceptance criteria?

- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if you do - rebase it please)

See merge request !5087
parents df60723e a42cce1b
Hide whitespace changes
Inline Side-by-side
Showing with 524 additions and 260 deletions
lib/ci/gitlab_ci_yaml_processor.rb
View file @ 4284724d
......@@ -4,21 +4,11 @@ module Ci
include Gitlab::Ci::Config::Node::LegacyValidationHelpers
DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
:allow_failure, :type, :stage, :when, :artifacts, :cache,
:dependencies, :before_script, :after_script, :variables,
:environment]
ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
attr_reader :path, :cache, :stages
def initialize(config, path = nil)
@ci_config = Gitlab::Ci::Config.new(config)
@config = @ci_config.to_hash
@path = path
unless @ci_config.valid?
......@@ -26,7 +16,6 @@ module Ci
end
initial_parsing
validate!
rescue Gitlab::Ci::Config::Loader::FormatError => e
raise ValidationError, e.message
end
......@@ -73,7 +62,7 @@ module Ci
# - before script should be a concatenated command
commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
tag_list: job[:tags] || [],
name: name,
name: job[:name],
allow_failure: job[:allow_failure] || false,
when: job[:when] || 'on_success',
environment: job[:environment],
......@@ -92,6 +81,9 @@ module Ci
private
def initial_parsing
##
# Global config
#
@before_script = @ci_config.before_script
@image = @ci_config.image
@after_script = @ci_config.after_script
......@@ -100,34 +92,28 @@ module Ci
@stages = @ci_config.stages
@cache = @ci_config.cache
@jobs = {}
@config.except!(*ALLOWED_YAML_KEYS)
@config.each { |name, param| add_job(name, param) }
raise ValidationError, "Please define at least one job" if @jobs.none?
end
def add_job(name, job)
return if name.to_s.start_with?('.')
##
# Jobs
#
@jobs = @ci_config.jobs
raise ValidationError, "Unknown parameter: #{name}" unless job.is_a?(Hash) && job.has_key?(:script)
@jobs.each do |name, job|
# logical validation for job
stage = job[:stage] || job[:type] || DEFAULT_STAGE
@jobs[name] = { stage: stage }.merge(job)
validate_job_stage!(name, job)
validate_job_dependencies!(name, job)
end
end
def yaml_variables(name)
variables = global_variables.merge(job_variables(name))
variables = (@variables || {})
.merge(job_variables(name))
variables.map do |key, value|
{ key: key, value: value, public: true }
end
end
def global_variables
@variables || {}
end
def job_variables(name)
job = @jobs[name.to_sym]
return {} unless job
......@@ -135,154 +121,16 @@ module Ci
job[:variables] || {}
end
def validate!
@jobs.each do |name, job|
validate_job!(name, job)
end
true
end
def validate_job!(name, job)
validate_job_name!(name)
validate_job_keys!(name, job)
validate_job_types!(name, job)
validate_job_script!(name, job)
validate_job_stage!(name, job) if job[:stage]
validate_job_variables!(name, job) if job[:variables]
validate_job_cache!(name, job) if job[:cache]
validate_job_artifacts!(name, job) if job[:artifacts]
validate_job_dependencies!(name, job) if job[:dependencies]
end
def validate_job_name!(name)
if name.blank? || !validate_string(name)
raise ValidationError, "job name should be non-empty string"
end
end
def validate_job_keys!(name, job)
job.keys.each do |key|
unless ALLOWED_JOB_KEYS.include? key
raise ValidationError, "#{name} job: unknown parameter #{key}"
end
end
end
def validate_job_types!(name, job)
if job[:image] && !validate_string(job[:image])
raise ValidationError, "#{name} job: image should be a string"
end
if job[:services] && !validate_array_of_strings(job[:services])
raise ValidationError, "#{name} job: services should be an array of strings"
end
if job[:tags] && !validate_array_of_strings(job[:tags])
raise ValidationError, "#{name} job: tags parameter should be an array of strings"
end
if job[:only] && !validate_array_of_strings_or_regexps(job[:only])
raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps"
end
if job[:except] && !validate_array_of_strings_or_regexps(job[:except])
raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps"
end
if job[:allow_failure] && !validate_boolean(job[:allow_failure])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
if job[:when] && !job[:when].in?(%w[on_success on_failure always manual])
raise ValidationError, "#{name} job: when parameter should be on_success, on_failure, always or manual"
end
if job[:environment] && !validate_environment(job[:environment])
raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}"
end
end
def validate_job_script!(name, job)
if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
raise ValidationError, "#{name} job: script should be a string or an array of a strings"
end
if job[:before_script] && !validate_array_of_strings(job[:before_script])
raise ValidationError, "#{name} job: before_script should be an array of strings"
end
if job[:after_script] && !validate_array_of_strings(job[:after_script])
raise ValidationError, "#{name} job: after_script should be an array of strings"
end
end
def validate_job_stage!(name, job)
return unless job[:stage]
unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
end
end
def validate_job_variables!(name, job)
unless validate_variables(job[:variables])
raise ValidationError,
"#{name} job: variables should be a map of key-value strings"
end
end
def validate_job_cache!(name, job)
job[:cache].keys.each do |key|
unless ALLOWED_CACHE_KEYS.include? key
raise ValidationError, "#{name} job: cache unknown parameter #{key}"
end
end
if job[:cache][:key] && !validate_string(job[:cache][:key])
raise ValidationError, "#{name} job: cache:key parameter should be a string"
end
if job[:cache][:untracked] && !validate_boolean(job[:cache][:untracked])
raise ValidationError, "#{name} job: cache:untracked parameter should be an boolean"
end
if job[:cache][:paths] && !validate_array_of_strings(job[:cache][:paths])
raise ValidationError, "#{name} job: cache:paths parameter should be an array of strings"
end
end
def validate_job_artifacts!(name, job)
job[:artifacts].keys.each do |key|
unless ALLOWED_ARTIFACTS_KEYS.include? key
raise ValidationError, "#{name} job: artifacts unknown parameter #{key}"
end
end
if job[:artifacts][:name] && !validate_string(job[:artifacts][:name])
raise ValidationError, "#{name} job: artifacts:name parameter should be a string"
end
if job[:artifacts][:untracked] && !validate_boolean(job[:artifacts][:untracked])
raise ValidationError, "#{name} job: artifacts:untracked parameter should be an boolean"
end
if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths])
raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings"
end
if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
end
if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in])
raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration"
end
end
def validate_job_dependencies!(name, job)
unless validate_array_of_strings(job[:dependencies])
raise ValidationError, "#{name} job: dependencies parameter should be an array of strings"
end
return unless job[:dependencies]
stage_index = @stages.index(job[:stage])
......
lib/gitlab/ci/config.rb
View file @ 4284724d
......@@ -8,7 +8,7 @@ module Gitlab
# Temporary delegations that should be removed after refactoring
#
delegate :before_script, :image, :services, :after_script, :variables,
:stages, :cache, to: :@global
:stages, :cache, :jobs, to: :@global
def initialize(config)
@config = Loader.new(config).load!
......
lib/gitlab/ci/config/node/artifacts.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a configuration of job artifacts.
#
class Artifacts < Entry
include Validatable
include Attributable
ALLOWED_KEYS = %i[name untracked paths when expire_in]
attributes ALLOWED_KEYS
validations do
validates :config, type: Hash
validates :config, allowed_keys: ALLOWED_KEYS
with_options allow_nil: true do
validates :name, type: String
validates :untracked, boolean: true
validates :paths, array_of_strings: true
validates :when,
inclusion: { in: %w[on_success on_failure always],
message: 'should be on_success, on_failure ' \
'or always' }
validates :expire_in, duration: true
end
end
end
end
end
end
end
lib/gitlab/ci/config/node/attributable.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
module Attributable
extend ActiveSupport::Concern
class_methods do
def attributes(*attributes)
attributes.flatten.each do |attribute|
define_method(attribute) do
return unless config.is_a?(Hash)
config[attribute]
end
end
end
end
end
end
end
end
end
lib/gitlab/ci/config/node/cache.rb
View file @ 4284724d
......@@ -8,6 +8,12 @@ module Gitlab
class Cache < Entry
include Configurable
ALLOWED_KEYS = %i[key untracked paths]
validations do
validates :config, allowed_keys: ALLOWED_KEYS
end
node :key, Node::Key,
description: 'Cache key used to define a cache affinity.'
......@@ -16,10 +22,6 @@ module Gitlab
node :paths, Node::Paths,
description: 'Specify which paths should be cached across builds.'
validations do
validates :config, allowed_keys: true
end
end
end
end
......
lib/gitlab/ci/config/node/commands.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a job script.
#
class Commands < Entry
include Validatable
validations do
include LegacyValidationHelpers
validate do
unless string_or_array_of_strings?(config)
errors.add(:config,
'should be a string or an array of strings')
end
end
def string_or_array_of_strings?(field)
validate_string(field) || validate_array_of_strings(field)
end
end
def value
Array(@config)
end
end
end
end
end
end
lib/gitlab/ci/config/node/configurable.rb
View file @ 4284724d
......@@ -25,10 +25,14 @@ module Gitlab
private
def create_node(key, factory)
factory.with(value: @config[key], key: key, parent: self)
def compose!
self.class.nodes.each do |key, factory|
factory
.value(@config[key])
.with(key: key, parent: self)
factory.create!
@entries[key] = factory.create!
end
end
class_methods do
......@@ -38,22 +42,23 @@ module Gitlab
private
def node(symbol, entry_class, metadata)
factory = Node::Factory.new(entry_class)
def node(key, node, metadata)
factory = Node::Factory.new(node)
.with(description: metadata[:description])
(@nodes ||= {}).merge!(symbol.to_sym => factory)
(@nodes ||= {}).merge!(key.to_sym => factory)
end
def helpers(*nodes)
nodes.each do |symbol|
define_method("#{symbol}_defined?") do
@nodes[symbol].try(:defined?)
@entries[symbol].specified? if @entries[symbol]
end
define_method("#{symbol}_value") do
raise Entry::InvalidError unless valid?
@nodes[symbol].try(:value)
return unless @entries[symbol] && @entries[symbol].valid?
@entries[symbol].value
end
alias_method symbol.to_sym, "#{symbol}_value".to_sym
......
lib/gitlab/ci/config/node/entry.rb
View file @ 4284724d
......@@ -8,30 +8,31 @@ module Gitlab
class Entry
class InvalidError < StandardError; end
attr_reader :config
attr_reader :config, :metadata
attr_accessor :key, :parent, :description
def initialize(config)
def initialize(config, **metadata)
@config = config
@nodes = {}
@metadata = metadata
@entries = {}
@validator = self.class.validator.new(self)
@validator.validate
@validator.validate(:new)
end
def process!
return if leaf?
return unless valid?
compose!
process_nodes!
descendants.each(&:process!)
end
def nodes
@nodes.values
def leaf?
@entries.none?
end
def leaf?
self.class.nodes.none?
def descendants
@entries.values
end
def ancestors
......@@ -43,27 +44,30 @@ module Gitlab
end
def errors
@validator.messages + nodes.flat_map(&:errors)
@validator.messages + descendants.flat_map(&:errors)
end
def value
if leaf?
@config
else
defined = @nodes.select { |_key, value| value.defined? }
Hash[defined.map { |key, node| [key, node.value] }]
meaningful = @entries.select do |_key, value|
value.specified? && value.relevant?
end
Hash[meaningful.map { |key, entry| [key, entry.value] }]
end
end
def defined?
def specified?
true
end
def self.default
def relevant?
true
end
def self.nodes
{}
def self.default
end
def self.validator
......@@ -73,17 +77,6 @@ module Gitlab
private
def compose!
self.class.nodes.each do |key, essence|
@nodes[key] = create_node(key, essence)
end
end
def process_nodes!
nodes.each(&:process!)
end
def create_node(key, essence)
raise NotImplementedError
end
end
end
......
lib/gitlab/ci/config/node/factory.rb
View file @ 4284724d
......@@ -10,35 +10,60 @@ module Gitlab
def initialize(node)
@node = node
@metadata = {}
@attributes = {}
end
def value(value)
@value = value
self
end
def metadata(metadata)
@metadata.merge!(metadata)
self
end
def with(attributes)
@attributes.merge!(attributes)
self
end
def create!
raise InvalidFactory unless @attributes.has_key?(:value)
raise InvalidFactory unless defined?(@value)
fabricate.tap do |entry|
entry.key = @attributes[:key]
entry.parent = @attributes[:parent]
entry.description = @attributes[:description]
##
# We assume that unspecified entry is undefined.
# See issue #18775.
#
if @value.nil?
Node::Undefined.new(
fabricate_undefined
)
else
fabricate(@node, @value)
end
end
private
def fabricate
def fabricate_undefined
##
# We assume that unspecified entry is undefined.
# See issue #18775.
# If node has a default value we fabricate concrete node
# with default value.
#
if @attributes[:value].nil?
Node::Undefined.new(@node)
if @node.default.nil?
fabricate(Node::Null)
else
@node.new(@attributes[:value])
fabricate(@node, @node.default)
end
end
def fabricate(node, value = nil)
node.new(value, @metadata).tap do |entry|
entry.key = @attributes[:key]
entry.parent = @attributes[:parent]
entry.description = @attributes[:description]
end
end
end
......
lib/gitlab/ci/config/node/global.rb
View file @ 4284724d
......@@ -34,10 +34,36 @@ module Gitlab
description: 'Configure caching between build jobs.'
helpers :before_script, :image, :services, :after_script,
:variables, :stages, :types, :cache
:variables, :stages, :types, :cache, :jobs
def stages
stages_defined? ? stages_value : types_value
private
def compose!
super
compose_jobs!
compose_deprecated_entries!
end
def compose_jobs!
factory = Node::Factory.new(Node::Jobs)
.value(@config.except(*self.class.nodes.keys))
.with(key: :jobs, parent: self,
description: 'Jobs definition for this pipeline')
@entries[:jobs] = factory.create!
end
def compose_deprecated_entries!
##
# Deprecated `:types` key workaround - if types are defined and
# stages are not defined we use types definition as stages.
#
if types_defined? && !stages_defined?
@entries[:stages] = @entries[:types]
end
@entries.delete(:types)
end
end
end
......
lib/gitlab/ci/config/node/hidden_job.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a hidden CI/CD job.
#
class HiddenJob < Entry
include Validatable
validations do
validates :config, type: Hash
validates :config, presence: true
end
def relevant?
false
end
end
end
end
end
end
lib/gitlab/ci/config/node/job.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a concrete CI/CD job.
#
class Job < Entry
include Configurable
include Attributable
ALLOWED_KEYS = %i[tags script only except type image services allow_failure
type stage when artifacts cache dependencies before_script
after_script variables environment]
attributes :tags, :allow_failure, :when, :environment, :dependencies
validations do
validates :config, allowed_keys: ALLOWED_KEYS
validates :config, presence: true
validates :name, presence: true
validates :name, type: Symbol
with_options allow_nil: true do
validates :tags, array_of_strings: true
validates :allow_failure, boolean: true
validates :when,
inclusion: { in: %w[on_success on_failure always manual],
message: 'should be on_success, on_failure, ' \
'always or manual' }
validates :environment,
type: {
with: String,
message: Gitlab::Regex.environment_name_regex_message }
validates :environment,
format: {
with: Gitlab::Regex.environment_name_regex,
message: Gitlab::Regex.environment_name_regex_message }
validates :dependencies, array_of_strings: true
end
end
node :before_script, Script,
description: 'Global before script overridden in this job.'
node :script, Commands,
description: 'Commands that will be executed in this job.'
node :stage, Stage,
description: 'Pipeline stage this job will be executed into.'
node :type, Stage,
description: 'Deprecated: stage this job will be executed into.'
node :after_script, Script,
description: 'Commands that will be executed when finishing job.'
node :cache, Cache,
description: 'Cache definition for this job.'
node :image, Image,
description: 'Image that will be used to execute this job.'
node :services, Services,
description: 'Services that will be used to execute this job.'
node :only, Trigger,
description: 'Refs policy this job will be executed for.'
node :except, Trigger,
description: 'Refs policy this job will be executed for.'
node :variables, Variables,
description: 'Environment variables available for this job.'
node :artifacts, Artifacts,
description: 'Artifacts configuration for this job.'
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
:artifacts
def name
@metadata[:name]
end
def value
@config.merge(to_hash.compact)
end
private
def to_hash
{ name: name,
before_script: before_script,
script: script,
image: image,
services: services,
stage: stage,
cache: cache,
only: only,
except: except,
variables: variables_defined? ? variables : nil,
artifacts: artifacts,
after_script: after_script }
end
def compose!
super
if type_defined? && !stage_defined?
@entries[:stage] = @entries[:type]
end
@entries.delete(:type)
end
end
end
end
end
end
lib/gitlab/ci/config/node/jobs.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a set of jobs.
#
class Jobs < Entry
include Validatable
validations do
validates :config, type: Hash
validate do
unless has_visible_job?
errors.add(:config, 'should contain at least one visible job')
end
end
def has_visible_job?
config.any? { |name, _| !hidden?(name) }
end
end
def hidden?(name)
name.to_s.start_with?('.')
end
private
def compose!
@config.each do |name, config|
node = hidden?(name) ? Node::HiddenJob : Node::Job
factory = Node::Factory.new(node)
.value(config || {})
.metadata(name: name)
.with(key: name, parent: self,
description: "#{name} job definition.")
@entries[name] = factory.create!
end
end
end
end
end
end
end
lib/gitlab/ci/config/node/legacy_validation_helpers.rb
View file @ 4284724d
......@@ -41,10 +41,6 @@ module Gitlab
false
end
def validate_environment(value)
value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex
end
def validate_boolean(value)
value.in?([true, false])
end
......
lib/gitlab/ci/config/node/null.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# This class represents an undefined node.
#
# Implements the Null Object pattern.
#
class Null < Entry
def value
nil
end
def valid?
true
end
def errors
[]
end
def specified?
false
end
def relevant?
false
end
end
end
end
end
end
lib/gitlab/ci/config/node/stage.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a stage for a job.
#
class Stage < Entry
include Validatable
validations do
validates :config, type: String
end
def self.default
'test'
end
end
end
end
end
end
lib/gitlab/ci/config/node/trigger.rb 0 → 100644
View file @ 4284724d
module Gitlab
module Ci
class Config
module Node
##
# Entry that represents a trigger policy for the job.
#
class Trigger < Entry
include Validatable
validations do
include LegacyValidationHelpers
validate :array_of_strings_or_regexps
def array_of_strings_or_regexps
unless validate_array_of_strings_or_regexps(config)
errors.add(:config, 'should be an array of strings or regexps')
end
end
end
end
end
end
end
end
lib/gitlab/ci/config/node/undefined.rb
View file @ 4284724d
......@@ -3,24 +3,13 @@ module Gitlab
class Config
module Node
##
# This class represents an undefined entry node.
# This class represents an unspecified entry node.
#
# It takes original entry class as configuration and returns default
# value of original entry as self value.
# It decorates original entry adding method that indicates it is
# unspecified.
#
#
class Undefined < Entry
include Validatable
validations do
validates :config, type: Class
end
def value
@config.default
end
def defined?
class Undefined < SimpleDelegator
def specified?
false
end
end
......
lib/gitlab/ci/config/node/validator.rb
View file @ 4284724d
......@@ -21,18 +21,19 @@ module Gitlab
'Validator'
end
def unknown_keys
return [] unless config.is_a?(Hash)
config.keys - @node.class.nodes.keys
end
private
def location
predecessors = ancestors.map(&:key).compact
current = key || @node.class.name.demodulize.underscore
predecessors.append(current).join(':')
predecessors.append(key_name).join(':')
end
def key_name
if key.blank?
@node.class.name.demodulize.underscore.humanize
else
key
end
end
end
end
......
lib/gitlab/ci/config/node/validators.rb
View file @ 4284724d
......@@ -5,10 +5,11 @@ module Gitlab
module Validators
class AllowedKeysValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
if record.unknown_keys.any?
unknown_list = record.unknown_keys.join(', ')
record.errors.add(:config,
"contains unknown keys: #{unknown_list}")
unknown_keys = record.config.try(:keys).to_a - options[:in]
if unknown_keys.any?
record.errors.add(:config, 'contains unknown keys: ' +
unknown_keys.join(', '))
end
end
end
......@@ -33,6 +34,16 @@ module Gitlab
end
end
class DurationValidator < ActiveModel::EachValidator
include LegacyValidationHelpers
def validate_each(record, attribute, value)
unless validate_duration(value)
record.errors.add(attribute, 'should be a duration')
end
end
end
class KeyValidator < ActiveModel::EachValidator
include LegacyValidationHelpers
......@@ -49,7 +60,8 @@ module Gitlab
raise unless type.is_a?(Class)
unless value.is_a?(type)
record.errors.add(attribute, "should be a #{type.name}")
message = options[:message] || "should be a #{type.name}"
record.errors.add(attribute, message)
end
end
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment