Skip to content

P
paperclip
Commit 25be0b25 by Tute Costa
Options

Paperclip::Storage::S3 simplification 

We don't support AWS SDK v1 any longer.
parent 4adcbeda
Showing with 35 additions and 168 deletions
features/step_definitions/s3_steps.rb
When /^I attach the file "([^"]*)" to "([^"]*)" on S3$/ do |file_path, field|
definition = Paperclip::AttachmentRegistry.definitions_for(User)[field.downcase.to_sym]
path = if defined?(::AWS)
"https://paperclip.s3.amazonaws.com#{definition[:path]}"
else
"https://paperclip.s3-us-west-2.amazonaws.com#{definition[:path]}"
end
path = "https://paperclip.s3-us-west-2.amazonaws.com#{definition[:path]}"
path.gsub!(':filename', File.basename(file_path))
path.gsub!(/:([^\/\.]+)/) do |match|
"([^\/\.]+)"
end
FakeWeb.register_uri(:put, Regexp.new(path), :body => defined?(::AWS) ? "OK" : "<xml></xml>")
FakeWeb.register_uri(:put, Regexp.new(path), :body => "<xml></xml>")
step "I attach the file \"#{file_path}\" to \"#{field}\""
end
......
lib/paperclip/storage/s3.rb
......@@ -113,33 +113,24 @@ module Paperclip
module S3
def self.extended base
unless defined?(AWS_CLASS)
begin
require 'aws-sdk'
const_set('AWS_CLASS', defined?(::Aws) ? ::Aws : ::AWS)
const_set('AWS_BASE_ERROR',
defined?(::Aws) ? Aws::Errors::ServiceError : AWS::Errors::Base)
const_set('DEFAULT_PERMISSION',
defined?(::AWS) ? :public_read : :'public-read')
const_set 'DEFAULT_PERMISSION', :"public-read"
rescue LoadError => e
e.message << " (You may need to install the aws-sdk gem)"
raise e
end
if Gem::Version.new(AWS_CLASS::VERSION) >= Gem::Version.new(2) && Gem::Version.new(AWS_CLASS::VERSION) <= Gem::Version.new("2.0.33")
raise LoadError, "paperclip does not support aws-sdk versions 2.0.0 - 2.0.33. Please upgrade aws-sdk to a newer version."
end
end
# Overriding log formatter to make sure it return a UTF-8 string
if defined?(AWS_CLASS::Core::LogFormatter)
AWS_CLASS::Core::LogFormatter.class_eval do
if defined?(::Aws::Core::LogFormatter)
::Aws::Core::LogFormatter.class_eval do
def summarize_hash(hash)
hash.map { |key, value| ":#{key}=>#{summarize_value(value)}".force_encoding('UTF-8') }.sort.join(',')
end
end
elsif defined?(AWS_CLASS::Core::ClientLogging)
AWS_CLASS::Core::ClientLogging.class_eval do
elsif defined?(::Aws::Core::ClientLogging)
::Aws::Core::ClientLogging.class_eval do
def sanitize_hash(hash)
hash.map { |key, value| "#{sanitize_value(key)}=>#{sanitize_value(value)}".force_encoding('UTF-8') }.sort.join(',')
end
......@@ -266,7 +257,7 @@ module Paperclip
def obtain_s3_instance_for(options)
instances = (Thread.current[:paperclip_s3_instances] ||= {})
instances[options] ||= AWS_CLASS::S3::Resource.new(options)
instances[options] ||= ::Aws::S3::Resource.new(options)
end
def s3_bucket
......@@ -323,7 +314,7 @@ module Paperclip
else
false
end
rescue AWS_BASE_ERROR => e
rescue Aws::Errors::ServiceError => e
false
end
......@@ -381,10 +372,10 @@ module Paperclip
write_options.merge!(@s3_headers)
s3_object(style).upload_file(file.path, write_options)
rescue AWS_CLASS::S3::Errors::NoSuchBucket
rescue ::Aws::S3::Errors::NoSuchBucket
create_bucket
retry
rescue AWS_CLASS::S3::Errors::SlowDown
rescue ::Aws::S3::Errors::SlowDown
retries += 1
if retries <= 5
sleep((2 ** retries) * 0.5)
......@@ -407,7 +398,7 @@ module Paperclip
begin
log("deleting #{path}")
s3_bucket.object(path.sub(%r{\A/}, "")).delete
rescue AWS_BASE_ERROR => e
rescue Aws::Errors::ServiceError => e
# Ignore this.
end
end
......@@ -421,7 +412,7 @@ module Paperclip
local_file.write(chunk)
end
end
rescue AWS_BASE_ERROR => e
rescue Aws::Errors::ServiceError => e
warn("#{e} - cannot copy #{path(style)} to local file #{local_dest_path}")
false
end
......
spec/paperclip/storage/s3_spec.rb
......@@ -3,15 +3,11 @@ require 'aws-sdk'
describe Paperclip::Storage::S3 do
before do
if defined?(::Aws)
Aws.config[:stub_responses] = true
else
AWS.stub!
end
end
def aws2_add_region
defined?(::Aws) ? { s3_region: 'us-east-1' } : {}
{ s3_region: 'us-east-1' }
end
context "Parsing S3 credentials" do
......@@ -244,20 +240,12 @@ describe Paperclip::Storage::S3 do
# if using aws-sdk-v2, the s3_host_name will be defined by the s3_region
context "s3_host_name" do
before do
if defined?(::Aws)
rebuild_model storage: :s3,
s3_credentials: {},
bucket: "bucket",
path: ":attachment/:basename:dotextension",
s3_host_name: "s3-ap-northeast-1.amazonaws.com",
s3_region: "ap-northeast-1"
else
rebuild_model storage: :s3,
s3_credentials: {},
bucket: "bucket",
path: ":attachment/:basename:dotextension",
s3_host_name: "s3-ap-northeast-1.amazonaws.com"
end
@dummy = Dummy.new
@dummy.avatar = stringy_file
@dummy.stubs(:new_record?).returns(false)
......@@ -269,9 +257,7 @@ describe Paperclip::Storage::S3 do
it "uses the S3 bucket with the correct host name" do
assert_equal "s3-ap-northeast-1.amazonaws.com",
(defined?(::Aws) ?
@dummy.avatar.s3_bucket.client.config.endpoint.host :
@dummy.avatar.s3_bucket.config.s3_endpoint)
@dummy.avatar.s3_bucket.client.config.endpoint.host
end
end
......@@ -361,11 +347,10 @@ describe Paperclip::Storage::S3 do
@dummy.avatar.stubs(:s3_object).with(:original).returns(object)
@dummy.avatar.stubs(:s3_object).with(:thumbnail).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
cache_control: 'max-age=31557600')
......@@ -554,11 +539,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
if defined?(::Aws)
object.expects(:presigned_url).with(:get, expires_in: 3600)
else
object.expects(:url_for).with(:read, expires: 3600, secure: true)
end
@dummy.avatar.expiring_url
end
end
......@@ -572,15 +553,9 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
if defined?(::Aws)
object.expects(:presigned_url)
.with(:get, expires_in: 3600,
response_content_disposition: "inline")
else
object.expects(:url_for)
.with(:read, expires: 3600, secure: true,
response_content_disposition: "inline")
end
@dummy.avatar.expiring_url
end
end
......@@ -601,14 +576,8 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
if defined?(::Aws)
object.expects(:presigned_url)
.with(:get, expires_in: 3600, response_content_type: "image/png")
else
object.expects(:url_for)
.with(:read, expires: 3600, secure: true,
response_content_type: "image/png")
end
@dummy.avatar.expiring_url
end
end
......@@ -655,22 +624,14 @@ describe Paperclip::Storage::S3 do
it "generates a url for the thumb" do
object = stub
@dummy.avatar.stubs(:s3_object).with(:thumb).returns(object)
if defined?(::Aws)
object.expects(:presigned_url).with(:get, expires_in: 1800)
else
object.expects(:url_for).with(:read, expires: 1800, secure: true)
end
@dummy.avatar.expiring_url(1800, :thumb)
end
it "generates a url for the default style" do
object = stub
@dummy.avatar.stubs(:s3_object).with(:original).returns(object)
if defined?(::Aws)
object.expects(:presigned_url).with(:get, expires_in: 1800)
else
object.expects(:url_for).with(:read, expires: 1800, secure: true)
end
@dummy.avatar.expiring_url(1800)
end
end
......@@ -703,7 +664,6 @@ describe Paperclip::Storage::S3 do
# for aws-sdk-v2 the bucket.name is determined by the :s3_region
context "Parsing S3 credentials with a s3_host_name in them" do
before do
if defined?(::Aws)
rebuild_model storage: :s3,
bucket: 'testing',
s3_credentials: {
......@@ -714,55 +674,31 @@ describe Paperclip::Storage::S3 do
s3_region: "ap-northeast-1",
s3_host_name: "s3-ap-northeast-1.amazonaws.com" }
}
else
rebuild_model storage: :s3,
bucket: 'testing',
s3_credentials: {
production: { s3_host_name: "s3-world-end.amazonaws.com" },
development: { s3_host_name: "s3-ap-northeast-1.amazonaws.com" }
}
end
@dummy = Dummy.new
end
it "gets the right s3_host_name in production" do
rails_env("production") do
assert_match %r{^s3-world-end.amazonaws.com}, @dummy.avatar.s3_host_name
if defined?(::Aws)
assert_match %r{^s3.world-end.amazonaws.com},
@dummy.avatar.s3_bucket.client.config.endpoint.host
else
assert_match %r{^s3-world-end.amazonaws.com},
@dummy.avatar.s3_bucket.config.s3_endpoint
end
end
end
it "gets the right s3_host_name in development" do
rails_env("development") do
assert_match %r{^s3-ap-northeast-1.amazonaws.com}, @dummy.avatar.s3_host_name
if defined?(::Aws)
assert_match %r{^s3-ap-northeast-1.amazonaws.com},
@dummy.avatar.s3_bucket.client.config.endpoint.host
else
assert_match %r{^s3-ap-northeast-1.amazonaws.com},
@dummy.avatar.s3_bucket.config.s3_endpoint
end
end
end
it "gets the right s3_host_name if the key does not exist" do
rails_env("test") do
assert_match %r{^s3.amazonaws.com}, @dummy.avatar.s3_host_name
if defined?(::Aws)
# :s3_region is *required* for aws-sdk-v2
assert_raises(Aws::Errors::MissingRegionError) do
@dummy.avatar.s3_bucket.client.config.endpoint.host
end
else
assert_match %r{^s3.amazonaws.com},
@dummy.avatar.s3_bucket.config.s3_endpoint
end
end
end
end
......@@ -804,22 +740,14 @@ describe Paperclip::Storage::S3 do
it "is rewound after flush_writes" do
@dummy.avatar.instance_eval "def after_flush_writes; end"
if defined?(::Aws)
@dummy.avatar.stubs(:s3_object).returns(stub(upload_file: true))
else
@dummy.avatar.stubs(:s3_object).returns(stub(write: true))
end
files = @dummy.avatar.queued_for_write.values.each(&:read)
@dummy.save
assert files.none?(&:eof?), "Expect all the files to be rewound."
end
it "is removed after after_flush_writes" do
if defined?(::Aws)
@dummy.avatar.stubs(:s3_object).returns(stub(upload_file: true))
else
@dummy.avatar.stubs(:s3_object).returns(stub(write: true))
end
paths = @dummy.avatar.queued_for_write.values.map(&:path)
@dummy.save
assert paths.none?{ |path| File.exist?(path) },
......@@ -828,17 +756,10 @@ describe Paperclip::Storage::S3 do
it "will retry to save again but back off on SlowDown" do
@dummy.avatar.stubs(:sleep)
if defined?(::Aws)
Aws::S3::Object.any_instance.stubs(:upload_file).
raises(Aws::S3::Errors::SlowDown.new(stub,
stub(status: 503, body: "")))
expect {@dummy.save}.to raise_error(Aws::S3::Errors::SlowDown)
else
AWS::S3::S3Object.any_instance.stubs(:write).
raises(AWS::S3::Errors::SlowDown.new(stub,
stub(status: 503, body: "")))
expect {@dummy.save}.to raise_error(AWS::S3::Errors::SlowDown)
end
expect(@dummy.avatar).to have_received(:sleep).with(1)
expect(@dummy.avatar).to have_received(:sleep).with(2)
expect(@dummy.avatar).to have_received(:sleep).with(4)
......@@ -850,7 +771,7 @@ describe Paperclip::Storage::S3 do
before do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION)
@dummy.save
......@@ -863,21 +784,11 @@ describe Paperclip::Storage::S3 do
context "and saved without a bucket" do
before do
if defined?(::Aws)
Aws::S3::Bucket.any_instance.expects(:create)
Aws::S3::Object.any_instance.stubs(:upload_file).
raises(Aws::S3::Errors::NoSuchBucket
.new(stub,
stub(status: 404, body: "<foo/>"))).then.returns(nil)
else
AWS::S3::BucketCollection.any_instance.expects(:create)
.with("testing")
AWS::S3::S3Object.any_instance.stubs(:write).
raises(AWS::S3::Errors::NoSuchBucket.new(stub,
stub(status: 404,
body: "<foo/>"))).
then.returns(nil)
end
@dummy.save
end
......@@ -888,13 +799,8 @@ describe Paperclip::Storage::S3 do
context "and remove" do
before do
if defined?(::Aws)
Aws::S3::Object.any_instance.stubs(:exists?).returns(true)
Aws::S3::Object.any_instance.stubs(:delete)
else
AWS::S3::S3Object.any_instance.stubs(:exists?).returns(true)
AWS::S3::S3Object.any_instance.stubs(:delete)
end
@dummy.destroy
end
......@@ -905,14 +811,9 @@ describe Paperclip::Storage::S3 do
context 'that the file were missing' do
before do
if defined?(::Aws)
Aws::S3::Object.any_instance.stubs(:exists?)
.raises(Aws::S3::Errors::ServiceError.new("rspec stub raises",
"object exists?"))
else
AWS::S3::S3Object.any_instance.stubs(:exists?)
.raises(AWS::Errors::Base)
end
end
it 'returns false on exists?' do
......@@ -956,28 +857,16 @@ describe Paperclip::Storage::S3 do
before do
class DummyCredentialProvider; end
if defined?(::Aws)
rebuild_model (aws2_add_region).merge storage: :s3,
bucket: "testing",
s3_credentials: {
credentials: DummyCredentialProvider.new
}
else
rebuild_model storage: :s3,
bucket: "testing",
s3_credentials: {
credential_provider: DummyCredentialProvider.new
}
end
@dummy = Dummy.new
end
it "sets the credential-provider" do
if defined?(::Aws)
expect(@dummy.avatar.s3_bucket.client.config.credentials).to be_a DummyCredentialProvider
else
expect(@dummy.avatar.s3_bucket.config.credential_provider).to be_a DummyCredentialProvider
end
end
end
......@@ -1032,7 +921,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1073,7 +962,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1114,7 +1003,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1156,7 +1045,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: 'image/png',
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1209,7 +1098,7 @@ describe Paperclip::Storage::S3 do
}
expected_options.merge!(:storage_class => :reduced_redundancy) if style == :thumb
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, expected_options)
end
@dummy.save
......@@ -1252,7 +1141,7 @@ describe Paperclip::Storage::S3 do
[:thumb, :original].each do |style|
@dummy.avatar.stubs(:s3_object).with(style).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, :content_type => "image/png",
:acl => Paperclip::Storage::S3::DEFAULT_PERMISSION,
:storage_class => :reduced_redundancy)
......@@ -1295,7 +1184,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, :content_type => "image/png",
:acl => Paperclip::Storage::S3::DEFAULT_PERMISSION)
@dummy.save
......@@ -1335,7 +1224,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, content_type: "image/png",
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
server_side_encryption: :aes256)
......@@ -1375,7 +1264,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: "image/png",
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1409,14 +1298,10 @@ describe Paperclip::Storage::S3 do
assert_equal 'pathname_bucket', @dummy.avatar.bucket_name
assert_equal 'pathname_key',
(defined?(::Aws) ?
@dummy.avatar.s3_bucket.client.config.access_key_id :
@dummy.avatar.s3_bucket.config.access_key_id)
@dummy.avatar.s3_bucket.client.config.access_key_id
assert_equal 'pathname_secret',
(defined?(::Aws) ?
@dummy.avatar.s3_bucket.client.config.secret_access_key :
@dummy.avatar.s3_bucket.config.secret_access_key)
@dummy.avatar.s3_bucket.client.config.secret_access_key
end
end
......@@ -1440,14 +1325,10 @@ describe Paperclip::Storage::S3 do
assert_equal 'env_bucket', @dummy.avatar.bucket_name
assert_equal 'env_key',
(defined?(::Aws) ?
@dummy.avatar.s3_bucket.client.config.access_key_id :
@dummy.avatar.s3_bucket.config.access_key_id)
@dummy.avatar.s3_bucket.client.config.access_key_id
assert_equal 'env_secret',
(defined?(::Aws) ?
@dummy.avatar.s3_bucket.client.config.secret_access_key :
@dummy.avatar.s3_bucket.config.secret_access_key)
@dummy.avatar.s3_bucket.client.config.secret_access_key
end
end
......@@ -1477,7 +1358,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: "image/png",
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION)
......@@ -1517,7 +1398,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything, content_type: "image/png", acl: :private)
@dummy.save
end
......@@ -1562,7 +1443,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).with(style).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: "image/png",
acl: style == :thumb ? :public_read : :private)
......@@ -1656,7 +1537,7 @@ describe Paperclip::Storage::S3 do
object = stub
@dummy.avatar.stubs(:s3_object).with(style).returns(object)
object.expects((defined?(::Aws) ? :upload_file : :write))
object.expects(:upload_file)
.with(anything,
content_type: "image/png",
acl: Paperclip::Storage::S3::DEFAULT_PERMISSION,
......@@ -1691,7 +1572,6 @@ describe Paperclip::Storage::S3 do
end
end
private
def rails_env(env)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment