AWS CloudFormation SNSTopic EventSourceMapping

Cloudformation refuses to set up an EventSourceMapping, but it works in the UI,
because under the hood the UI creates a push based setup, which can be re-created via
Cloudformation.

{
  "Resources": {
    "SNSTopic": {
      "Type": "AWS::SNS::Topic",
      "Properties": {
        "Subscription": [
          {
            "Endpoint": {
              "Fn::GetAtt": [
                "Function",
                "Arn"
              ]
            },
            "Protocol": "lambda"
          }
        ]
      }
    },
    "Function": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Handler": "index.handler",
        "Role": {
          "Fn::GetAtt": [
            "LambdaExecutionRole",
            "Arn"
          ]
        },
        "Code": {
          "ZipFile": {
            "Fn::Join": [
              "",
              [
                "exports.handler = function(event, context) {",
                "context.done(null,event)",
                "};"
              ]
            ]
          }
        },
        "Runtime": "nodejs",
        "Timeout": "25"
      }
    },
    "LambdaInvokePermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "FunctionName": {
          "Fn::GetAtt": [
            "Function",
            "Arn"
          ]
        },
        "Action": "lambda:InvokeFunction",
        "Principal": "sns.amazonaws.com",
        "SourceArn": {
          "Ref": "SNSTopic"
        }
      }
    }
  },
  "LambdaExecutionRole": {
    "Type": "AWS::IAM::Role",
    "Properties": {
      "AssumeRolePolicyDocument": {
        "Version": "2012-10-17",
        "Statement": [
          {
            "Sid": "",
            "Effect": "Allow",
            "Principal": {
              "Service": "lambda.amazonaws.com"
            },
            "Action": "sts:AssumeRole"
          }
        ]
      },
      "Policies": [
        {
          "PolicyName": "root",
          "PolicyDocument": {
            "Version": "2012-10-17",
            "Statement": [
              {
                "Effect": "Allow",
                "Action": [
                  "logs:*"
                ],
                "Resource": "arn:aws:logs:*:*:*"
              },
              {
                "Effect": "Allow",
                "Action": [
                  "sns:Subscribe"
                ],
                "Resource": [
                  "*"
                ]
              }
            ]
          }
        }
      ]
    }
  }
}

Sending configuration into a AWS Lambda created via Cloudformation

Lambdas can only have static code (see code upload via cloudformation), so passing in DynamoDB table names/SNS topic ARNs etc is not possible. But there is a neat workaround:

Make the lambda read the stacks output.

# my-stack.json
"Outputs": {
  "World": {
    "Value": {
      "Ref": "MySnsTopic"
    }
  },
  ....
}


var AWS = require('aws-sdk');
var stack = context.invokedFunctionArn.match(/:function:(.*)-.*-.*/)[1];

exports.handler = function(event, context) {
  var cf = new AWS.CloudFormation();
  cf.describeStacks({"StackName": stack}, function(err, data){
    if (err) context.done(err, 'Error!');
    else {
      var config = {}
      data.Stacks[0].Outputs.map(function(out){ config[out.OutputKey] = out.OutputValue });
      context.succeed('hello ' + config.World)
    }
  })
};


# output
"hello arn:aws:sns:ap-northeast-1:8132302344234:my-stack-MySnSTopic-211Z1K3GAGK9"

Automated lambda code upload to S3 with CloudFormation

Maintaining lambda code directly in CloudFormation only works with zipfile property on nodejs and even there it is limited to 2000 characters. For python and bigger lambdas, we now use this ruby script to generate the s3 object that is set in the CloudFormation template.

require 'aws-sdk-core'

def code_via_s3(file, handler)
  bucket = "my-lambda-staging-area"
  content = File.read(file)
  sha = Digest::MD5.hexdigest(content)
  key = "#{file.gsub('/', '-')}-#{sha}.zip"

  # zip up the content (gzip is not supported)
  # needs to be at bottom of zip to support inline editing
  # and match the handler name
  content = `cd #{File.dirname(file)} && zip --quiet - #{File.basename(file)}`
  raise "Zip failed" unless $?.success?

  # upload to s3 (overwriting it ... checking for existance takes the same time ...)
  c = Aws::S3::Client.new
  begin
    c.put_object(body: content, bucket: bucket, key: key)
  rescue Aws::S3::Errors::NoSuchBucket
    c.create_bucket(bucket: bucket)
    retry
  end

  {
    "Handler" => File.basename(file).sub(/\..*/, '') + '.' + handler,
    "Code" => {"S3Bucket" => bucket, "Key" => key}
  }
end

Stubbing Ruby AWS SDK XML with webmock

Stubbing the XML AWS expects is not easy (expects lists to have member keys) and has lots of repetitive elements like XyzResponse + XyzRequest … so I wanted to share a few useful helpers that make it dry.

(Alternative: use stub_requests, example PR)

  # turn ruby hashes into aws style xml
  def fake_xml(name, body={})
    xml = {"#{name}Result" => body}
      .to_xml(root: "#{name}Response", camelize: true).
      .gsub(/ type="array"/, '')
    loop do
      break unless xml.gsub!(%r{<(\S+)s>\s*<\1>(.*?)</\1>\s*</\1s>}m, "<\\1s><member>\\2</member></\\1s>")
    end
    xml
  end

  def expect_aws_request(method, url, action, response={})
    request = stub_request(method, url).
      with(:body => /Action=#{action}(&|$)/)
    request = if response.is_a?(Exception)
      request.to_raise(response)
    else
      request.to_return(:body => fake_xml(action, response))
    end
    requested << request
    request
  end

  def expect_upload_certificate
    expect_aws_request(
      :post, "https://iam.amazonaws.com/",
      "UploadServerCertificate",
      {server_certificate_metadata: {arn: 'FAKE-ARN'}}
    )
  end

  after { requested.each { |r| assert_requested r } }

  it "uploads a cert" do
    expect_upload_certificate
    manager.upload.must_equal 'FAKE-ARN'
  end