Parallelizing SparkleFormation execution

We generate each templates CloudFormation .json file and check them in to spot subtle diffs when refactoring. We also validate all configs on every PR. Doing this serially takes a lot of time, but it can be parallelized easily, while also avoiding ruby boot overhead.

This took us from ~5 minutes runtime to 30s, enjoy!

desc "Validate templates via cloudformation api"
task :validate do
  each_template do |template|
    execute_sfn_command :validate, template
  end
end

desc "Generates cloudformation json files"
task :generate, [:pattern] do |_t, args|
  pattern = /#{args[:pattern]}/ if args[:pattern]
  previous = Dir["generated/*.json"]

  used = each_template do |template|
    next if pattern && template !~ pattern
    output = execute_sfn_command :print, template

    generated = "generated/#{File.basename(template.sub('.rb', '.json'))}"
    File.write generated, output
    generated
  end

  (previous - used).each { |f| File.unlink(f) } unless pattern
end

...

require 'parallel'
require 'timeout'

def each_template(&block)
  # preload slow requires
  require 'bogo-cli'
  require 'sfn'

  # run in parallel, but isolated to avoid caches from being reused
  options = {in_processes: 10, progress: "Progress", isolation: true}
  Parallel.map(Dir["templates/**/*.rb"], options, &block)
end

private

def execute_sfn_command(command, template, *args)
  Timeout.timeout(20) do
    capture_stdout do
      Sfn::Command.const_get(command.capitalize).new({
        defaults: true, # do not ask questions about parameters
        file: template,
        retry: {type: :flat, interval: 1} # we will run into rate limits, ignore them quickly
      }, args).execute!
    end
  end
rescue StandardError
  # give users context when something failed
  warn "bundle exec sfn #{command} #{args.join(" ")} --defaults -f #{template}"
  raise
end

def capture_stdout
  old = $stdout
  $stdout = StringIO.new
  yield
  $stdout.string
ensure
  $stdout = old
end
Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s