Example setup to test a gem or plugin against rails 2 and 3:

Version 1

# Gemfile
gem 'activerecord', ENV['AR']

# Rakefile
task :default do
  sh "rspec spec"
end

task :all do
  sh "AR=2.3.14 && (bundle || bundle install) && bundle exec rake"
  sh "AR=3.0.10 && (bundle || bundle install) && bundle exec rake"
  sh "AR=3.1.1 && (bundle || bundle install) && bundle exec rake"
end

Version 2 (if 1 is not possible…)

# Gemfile
...
gem 'rails', '~>3'

# spec/rails2/Gemfile
...
gem 'rails', '~>2'

# Rakefile
task :spec do
  sh "bundle exec rspec spec"
end

task :rails2 do
  sh "cd spec/rails2 && bundle exec rspec ../../spec"
end

task :default do
  sh "rake spec && rake rails2"
end