#4483: added S3 Write ACL Rule override config

exp/services/ledgerexporter/main.go

@@ -10,6 +10,7 @@ import (
  "strings"
  "time"
 
+ "github.com/aws/aws-sdk-go/service/s3"
  "github.com/stellar/go/historyarchive"
  "github.com/stellar/go/ingest/ledgerbackend"
  "github.com/stellar/go/network"
@@ -61,7 +62,8 @@ func main() {
  target, err := historyarchive.ConnectBackend(
  *targetUrl,
  storage.ConnectOptions{
- Context: context.Background(),
+ Context: context.Background(),
+ S3WriteACL: s3.ObjectCannedACLBucketOwnerFullControl,
  },
  )
  logFatalIf(err, "Could not connect to target")

support/storage/main.go

@@ -31,6 +31,9 @@ type ConnectOptions struct {
 
  // Wrap the Storage after connection. For example, to add a caching or introspection layer.
  Wrap func(Storage) (Storage, error)
+
+ // When putting file object to s3 bucket, specify the ACL for the object.
+ S3WriteACL string
 }
 
 func ConnectBackend(u string, opts ConnectOptions) (Storage, error) {
@@ -60,6 +63,7 @@ func ConnectBackend(u string, opts ConnectOptions) (Storage, error) {
  opts.S3Region,
  opts.S3Endpoint,
  opts.UnsignedRequests,
+ opts.S3WriteACL,
  )
 
  case "gcs":

support/storage/s3.go

@@ -12,15 +12,17 @@ import (
  "github.com/aws/aws-sdk-go/aws"
  "github.com/aws/aws-sdk-go/aws/session"
  "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3iface"
  "github.com/stellar/go/support/errors"
 )
 
 type S3Storage struct {
  ctx context.Context
- svc *s3.S3
+ svc s3iface.S3API
  bucket string
  prefix string
  unsignedRequests bool
+ writeACLrule string
 }
 
 func NewS3Storage(
@@ -30,6 +32,7 @@ func NewS3Storage(
  region string,
  endpoint string,
  unsignedRequests bool,
+ writeACLrule string,
 ) (Storage, error) {
  log.WithFields(log.Fields{"bucket": bucket,
  "prefix": prefix,
@@ -52,6 +55,7 @@ func NewS3Storage(
  bucket: bucket,
  prefix: prefix,
  unsignedRequests: unsignedRequests,
+ writeACLrule: writeACLrule,
  }
  return &backend, nil
 }
@@ -139,6 +143,13 @@ func (b *S3Storage) Size(pth string) (int64, error) {
  }
 }
 
+func (b *S3Storage) GetACLWriteRule() string {
+ if b.writeACLrule == "" {
+ return s3.ObjectCannedACLPublicRead
+ }
+ return b.writeACLrule
+}
+
 func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error {
  var buf bytes.Buffer
  _, err := buf.ReadFrom(in)
@@ -150,6 +161,7 @@ func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error {
  params := &s3.PutObjectInput{
  Bucket: aws.String(b.bucket),
  Key: aws.String(key),
+ ACL: aws.String(b.GetACLWriteRule()),
  Body: bytes.NewReader(buf.Bytes()),
  }
  req, _ := b.svc.PutObjectRequest(params)

support/storage/s3_test.go

@@ -0,0 +1,52 @@
+// Copyright 2016 Stellar Development Foundation and contributors. Licensed
+// under the Apache License, Version 2.0. See the COPYING file at the root
+// of this distribution or at http:https://www.apache.org/licenses/LICENSE-2.0
+
+package storage
+
+import (
+ "context"
+ "testing"
+
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3iface"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+type MockS3 struct {
+ mock.Mock
+ s3iface.S3API
+}
+
+func TestWriteACLRuleOverride(t *testing.T) {
+
+ mockS3 := &MockS3{}
+ s3Storage := S3Storage{
+ ctx: context.Background(),
+ svc: mockS3,
+ bucket: "bucket",
+ prefix: "prefix",
+ unsignedRequests: false,
+ writeACLrule: s3.ObjectCannedACLBucketOwnerFullControl,
+ }
+
+ aclRule := s3Storage.GetACLWriteRule()
+ assert.Equal(t, aclRule, s3.ObjectCannedACLBucketOwnerFullControl)
+}
+
+func TestWriteACLRuleDefault(t *testing.T) {
+
+ mockS3 := &MockS3{}
+ s3Storage := S3Storage{
+ ctx: context.Background(),
+ svc: mockS3,
+ bucket: "bucket",
+ prefix: "prefix",
+ unsignedRequests: false,
+ writeACLrule: "",
+ }
+
+ aclRule := s3Storage.GetACLWriteRule()
+ assert.Equal(t, aclRule, s3.ObjectCannedACLPublicRead)
+}