508 lines
17 KiB
Go
508 lines
17 KiB
Go
// monorepo/cloud/maplefileapps-backend/pkg/storage/object/s3/s3.go
|
|
package s3
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"mime/multipart"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/aws/smithy-go"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// ACL constants for public and private objects
|
|
const (
|
|
ACLPrivate = "private"
|
|
ACLPublicRead = "public-read"
|
|
)
|
|
|
|
type S3ObjectStorage interface {
|
|
UploadContent(ctx context.Context, objectKey string, content []byte) error
|
|
UploadContentWithVisibility(ctx context.Context, objectKey string, content []byte, isPublic bool) error
|
|
UploadContentFromMulipart(ctx context.Context, objectKey string, file multipart.File) error
|
|
UploadContentFromMulipartWithVisibility(ctx context.Context, objectKey string, file multipart.File, isPublic bool) error
|
|
BucketExists(ctx context.Context, bucketName string) (bool, error)
|
|
DeleteByKeys(ctx context.Context, key []string) error
|
|
Cut(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error
|
|
CutWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error
|
|
Copy(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error
|
|
CopyWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error
|
|
GetBinaryData(ctx context.Context, objectKey string) (io.ReadCloser, error)
|
|
DownloadToLocalfile(ctx context.Context, objectKey string, filePath string) (string, error)
|
|
ListAllObjects(ctx context.Context) (*s3.ListObjectsOutput, error)
|
|
FindMatchingObjectKey(s3Objects *s3.ListObjectsOutput, partialKey string) string
|
|
IsPublicBucket() bool
|
|
// GeneratePresignedUploadURL creates a presigned URL for uploading objects
|
|
GeneratePresignedUploadURL(ctx context.Context, key string, duration time.Duration) (string, error)
|
|
GetDownloadablePresignedURL(ctx context.Context, key string, duration time.Duration) (string, error)
|
|
ObjectExists(ctx context.Context, key string) (bool, error)
|
|
GetObjectSize(ctx context.Context, key string) (int64, error)
|
|
}
|
|
|
|
type s3ObjectStorage struct {
|
|
S3Client *s3.Client
|
|
PresignClient *s3.PresignClient
|
|
Logger *zap.Logger
|
|
BucketName string
|
|
IsPublic bool
|
|
}
|
|
|
|
// NewObjectStorage connects to a specific S3 bucket instance and returns a connected
|
|
// instance structure.
|
|
func NewObjectStorage(s3Config S3ObjectStorageConfigurationProvider, logger *zap.Logger) S3ObjectStorage {
|
|
logger = logger.Named("s3-object-storage")
|
|
|
|
// DEVELOPERS NOTE:
|
|
// How can I use the AWS SDK v2 for Go with DigitalOcean Spaces? via https://stackoverflow.com/a/74284205
|
|
logger.Info("⏳ Connecting to S3-compatible storage...",
|
|
zap.String("endpoint", s3Config.GetEndpoint()),
|
|
zap.String("bucket", s3Config.GetBucketName()),
|
|
zap.String("region", s3Config.GetRegion()))
|
|
|
|
// STEP 1: initialize the custom `endpoint` we will connect to.
|
|
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) {
|
|
return aws.Endpoint{
|
|
URL: s3Config.GetEndpoint(),
|
|
}, nil
|
|
})
|
|
|
|
// STEP 2: Configure.
|
|
sdkConfig, err := config.LoadDefaultConfig(
|
|
context.TODO(), config.WithRegion(s3Config.GetRegion()),
|
|
config.WithEndpointResolverWithOptions(customResolver),
|
|
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3Config.GetAccessKey(), s3Config.GetSecretKey(), "")),
|
|
)
|
|
if err != nil {
|
|
logger.Fatal("S3ObjectStorage failed loading default config", zap.Error(err)) // We need to crash the program at start to satisfy google wire requirement of having no errors.
|
|
}
|
|
|
|
// STEP 3\: Load up s3 instance.
|
|
s3Client := s3.NewFromConfig(sdkConfig)
|
|
|
|
// Create our storage handler.
|
|
s3Storage := &s3ObjectStorage{
|
|
S3Client: s3Client,
|
|
PresignClient: s3.NewPresignClient(s3Client),
|
|
Logger: logger,
|
|
BucketName: s3Config.GetBucketName(),
|
|
IsPublic: s3Config.GetIsPublicBucket(),
|
|
}
|
|
|
|
logger.Debug("Verifying bucket exists...")
|
|
|
|
// STEP 4: Connect to the s3 bucket instance and confirm that bucket exists.
|
|
doesExist, err := s3Storage.BucketExists(context.TODO(), s3Config.GetBucketName())
|
|
if err != nil {
|
|
logger.Fatal("S3ObjectStorage failed checking if bucket exists",
|
|
zap.String("bucket", s3Config.GetBucketName()),
|
|
zap.Error(err)) // We need to crash the program at start to satisfy google wire requirement of having no errors.
|
|
}
|
|
if !doesExist {
|
|
logger.Fatal("S3ObjectStorage failed - bucket does not exist",
|
|
zap.String("bucket", s3Config.GetBucketName())) // We need to crash the program at start to satisfy google wire requirement of having no errors.
|
|
}
|
|
|
|
logger.Info("✓ S3-compatible storage connected",
|
|
zap.String("bucket", s3Config.GetBucketName()),
|
|
zap.Bool("public", s3Config.GetIsPublicBucket()))
|
|
|
|
// Return our s3 storage handler.
|
|
return s3Storage
|
|
}
|
|
|
|
// IsPublicBucket returns whether the bucket is configured as public by default
|
|
func (s *s3ObjectStorage) IsPublicBucket() bool {
|
|
return s.IsPublic
|
|
}
|
|
|
|
// UploadContent uploads content using the default bucket visibility setting
|
|
func (s *s3ObjectStorage) UploadContent(ctx context.Context, objectKey string, content []byte) error {
|
|
return s.UploadContentWithVisibility(ctx, objectKey, content, s.IsPublic)
|
|
}
|
|
|
|
// UploadContentWithVisibility uploads content with specified visibility (public or private)
|
|
func (s *s3ObjectStorage) UploadContentWithVisibility(ctx context.Context, objectKey string, content []byte, isPublic bool) error {
|
|
acl := ACLPrivate
|
|
if isPublic {
|
|
acl = ACLPublicRead
|
|
}
|
|
|
|
s.Logger.Debug("Uploading content with visibility",
|
|
zap.String("objectKey", objectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.String("acl", acl))
|
|
|
|
_, err := s.S3Client.PutObject(ctx, &s3.PutObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(objectKey),
|
|
Body: bytes.NewReader(content),
|
|
ACL: types.ObjectCannedACL(acl),
|
|
})
|
|
if err != nil {
|
|
s.Logger.Error("Failed to upload content",
|
|
zap.String("objectKey", objectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.Any("error", err))
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UploadContentFromMulipart uploads file using the default bucket visibility setting
|
|
func (s *s3ObjectStorage) UploadContentFromMulipart(ctx context.Context, objectKey string, file multipart.File) error {
|
|
return s.UploadContentFromMulipartWithVisibility(ctx, objectKey, file, s.IsPublic)
|
|
}
|
|
|
|
// UploadContentFromMulipartWithVisibility uploads a multipart file with specified visibility
|
|
func (s *s3ObjectStorage) UploadContentFromMulipartWithVisibility(ctx context.Context, objectKey string, file multipart.File, isPublic bool) error {
|
|
acl := ACLPrivate
|
|
if isPublic {
|
|
acl = ACLPublicRead
|
|
}
|
|
|
|
s.Logger.Debug("Uploading multipart file with visibility",
|
|
zap.String("objectKey", objectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.String("acl", acl))
|
|
|
|
// Create the S3 upload input parameters
|
|
params := &s3.PutObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(objectKey),
|
|
Body: file,
|
|
ACL: types.ObjectCannedACL(acl),
|
|
}
|
|
|
|
// Perform the file upload to S3
|
|
_, err := s.S3Client.PutObject(ctx, params)
|
|
if err != nil {
|
|
s.Logger.Error("Failed to upload multipart file",
|
|
zap.String("objectKey", objectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.Any("error", err))
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *s3ObjectStorage) BucketExists(ctx context.Context, bucketName string) (bool, error) {
|
|
// Note: https://docs.aws.amazon.com/code-library/latest/ug/go_2_s3_code_examples.html#actions
|
|
|
|
_, err := s.S3Client.HeadBucket(ctx, &s3.HeadBucketInput{
|
|
Bucket: aws.String(bucketName),
|
|
})
|
|
exists := true
|
|
if err != nil {
|
|
var apiError smithy.APIError
|
|
if errors.As(err, &apiError) {
|
|
switch apiError.(type) {
|
|
case *types.NotFound:
|
|
s.Logger.Debug("Bucket is available", zap.String("bucket", bucketName))
|
|
exists = false
|
|
err = nil
|
|
default:
|
|
s.Logger.Error("Either you don't have access to bucket or another error occurred",
|
|
zap.String("bucket", bucketName),
|
|
zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
|
|
return exists, err
|
|
}
|
|
|
|
func (s *s3ObjectStorage) GetDownloadablePresignedURL(ctx context.Context, key string, duration time.Duration) (string, error) {
|
|
// DEVELOPERS NOTE:
|
|
// AWS S3 Bucket — presigned URL APIs with Go (2022) via https://ronen-niv.medium.com/aws-s3-handling-presigned-urls-2718ab247d57
|
|
|
|
presignedUrl, err := s.PresignClient.PresignGetObject(context.Background(),
|
|
&s3.GetObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(key),
|
|
ResponseContentDisposition: aws.String("attachment"), // This field allows the file to download it directly from your browser
|
|
},
|
|
s3.WithPresignExpires(duration))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return presignedUrl.URL, nil
|
|
}
|
|
|
|
func (s *s3ObjectStorage) DeleteByKeys(ctx context.Context, objectKeys []string) error {
|
|
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
defer cancel()
|
|
|
|
var objectIds []types.ObjectIdentifier
|
|
for _, key := range objectKeys {
|
|
objectIds = append(objectIds, types.ObjectIdentifier{Key: aws.String(key)})
|
|
}
|
|
_, err := s.S3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Delete: &types.Delete{Objects: objectIds},
|
|
})
|
|
if err != nil {
|
|
s.Logger.Error("Couldn't delete objects from bucket",
|
|
zap.String("bucket", s.BucketName),
|
|
zap.Error(err))
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Cut moves a file using the default bucket visibility setting
|
|
func (s *s3ObjectStorage) Cut(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error {
|
|
return s.CutWithVisibility(ctx, sourceObjectKey, destinationObjectKey, s.IsPublic)
|
|
}
|
|
|
|
// CutWithVisibility moves a file with specified visibility
|
|
func (s *s3ObjectStorage) CutWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error {
|
|
ctx, cancel := context.WithTimeout(ctx, 60*time.Second) // Increase timout so it runs longer then usual to handle this unique case.
|
|
defer cancel()
|
|
|
|
// First copy the object with the desired visibility
|
|
if err := s.CopyWithVisibility(ctx, sourceObjectKey, destinationObjectKey, isPublic); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Delete the original object
|
|
_, deleteErr := s.S3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(sourceObjectKey),
|
|
})
|
|
if deleteErr != nil {
|
|
s.Logger.Error("Failed to delete original object:", zap.Any("deleteErr", deleteErr))
|
|
return deleteErr
|
|
}
|
|
|
|
s.Logger.Debug("Original object deleted.")
|
|
|
|
return nil
|
|
}
|
|
|
|
// Copy copies a file using the default bucket visibility setting
|
|
func (s *s3ObjectStorage) Copy(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error {
|
|
return s.CopyWithVisibility(ctx, sourceObjectKey, destinationObjectKey, s.IsPublic)
|
|
}
|
|
|
|
// CopyWithVisibility copies a file with specified visibility
|
|
func (s *s3ObjectStorage) CopyWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error {
|
|
ctx, cancel := context.WithTimeout(ctx, 60*time.Second) // Increase timout so it runs longer then usual to handle this unique case.
|
|
defer cancel()
|
|
|
|
acl := ACLPrivate
|
|
if isPublic {
|
|
acl = ACLPublicRead
|
|
}
|
|
|
|
s.Logger.Debug("Copying object with visibility",
|
|
zap.String("sourceKey", sourceObjectKey),
|
|
zap.String("destinationKey", destinationObjectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.String("acl", acl))
|
|
|
|
_, copyErr := s.S3Client.CopyObject(ctx, &s3.CopyObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
CopySource: aws.String(s.BucketName + "/" + sourceObjectKey),
|
|
Key: aws.String(destinationObjectKey),
|
|
ACL: types.ObjectCannedACL(acl),
|
|
})
|
|
if copyErr != nil {
|
|
s.Logger.Error("Failed to copy object:",
|
|
zap.String("sourceKey", sourceObjectKey),
|
|
zap.String("destinationKey", destinationObjectKey),
|
|
zap.Bool("isPublic", isPublic),
|
|
zap.Any("copyErr", copyErr))
|
|
return copyErr
|
|
}
|
|
|
|
s.Logger.Debug("Object copied successfully.")
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetBinaryData function will return the binary data for the particular key.
|
|
func (s *s3ObjectStorage) GetBinaryData(ctx context.Context, objectKey string) (io.ReadCloser, error) {
|
|
input := &s3.GetObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(objectKey),
|
|
}
|
|
|
|
s3object, err := s.S3Client.GetObject(ctx, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s3object.Body, nil
|
|
}
|
|
|
|
func (s *s3ObjectStorage) DownloadToLocalfile(ctx context.Context, objectKey string, filePath string) (string, error) {
|
|
responseBin, err := s.GetBinaryData(ctx, objectKey)
|
|
if err != nil {
|
|
return filePath, err
|
|
}
|
|
out, err := os.Create(filePath)
|
|
if err != nil {
|
|
return filePath, err
|
|
}
|
|
defer out.Close()
|
|
|
|
_, err = io.Copy(out, responseBin)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return filePath, err
|
|
}
|
|
|
|
func (s *s3ObjectStorage) ListAllObjects(ctx context.Context) (*s3.ListObjectsOutput, error) {
|
|
input := &s3.ListObjectsInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
}
|
|
|
|
objects, err := s.S3Client.ListObjects(ctx, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return objects, nil
|
|
}
|
|
|
|
// Function will iterate over all the s3 objects to match the partial key with
|
|
// the actual key found in the S3 bucket.
|
|
func (s *s3ObjectStorage) FindMatchingObjectKey(s3Objects *s3.ListObjectsOutput, partialKey string) string {
|
|
for _, obj := range s3Objects.Contents {
|
|
|
|
match := strings.Contains(*obj.Key, partialKey)
|
|
|
|
// If a match happens then it means we have found the ACTUAL KEY in the
|
|
// s3 objects inside the bucket.
|
|
if match == true {
|
|
return *obj.Key
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// GeneratePresignedUploadURL creates a presigned URL for uploading objects to S3
|
|
func (s *s3ObjectStorage) GeneratePresignedUploadURL(ctx context.Context, key string, duration time.Duration) (string, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Create PutObjectInput without ACL to avoid requiring x-amz-acl header
|
|
putObjectInput := &s3.PutObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(key),
|
|
// Removed ACL field - files inherit bucket's default privacy settings.
|
|
}
|
|
|
|
presignedUrl, err := s.PresignClient.PresignPutObject(ctx, putObjectInput, s3.WithPresignExpires(duration))
|
|
if err != nil {
|
|
s.Logger.Error("Failed to generate presigned upload URL",
|
|
zap.String("key", key),
|
|
zap.Duration("duration", duration),
|
|
zap.Error(err))
|
|
return "", err
|
|
}
|
|
|
|
s.Logger.Debug("Generated presigned upload URL",
|
|
zap.String("key", key),
|
|
zap.Duration("duration", duration))
|
|
|
|
return presignedUrl.URL, nil
|
|
}
|
|
|
|
// ObjectExists checks if an object exists at the given key using HeadObject
|
|
func (s *s3ObjectStorage) ObjectExists(ctx context.Context, key string) (bool, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
_, err := s.S3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(key),
|
|
})
|
|
|
|
if err != nil {
|
|
var apiError smithy.APIError
|
|
if errors.As(err, &apiError) {
|
|
switch apiError.(type) {
|
|
case *types.NotFound:
|
|
// Object doesn't exist
|
|
s.Logger.Debug("Object does not exist",
|
|
zap.String("key", key))
|
|
return false, nil
|
|
case *types.NoSuchKey:
|
|
// Object doesn't exist
|
|
s.Logger.Debug("Object does not exist (NoSuchKey)",
|
|
zap.String("key", key))
|
|
return false, nil
|
|
default:
|
|
// Some other error occurred
|
|
s.Logger.Error("Error checking object existence",
|
|
zap.String("key", key),
|
|
zap.Error(err))
|
|
return false, err
|
|
}
|
|
}
|
|
// Non-API error
|
|
s.Logger.Error("Error checking object existence",
|
|
zap.String("key", key),
|
|
zap.Error(err))
|
|
return false, err
|
|
}
|
|
|
|
s.Logger.Debug("Object exists",
|
|
zap.String("key", key))
|
|
return true, nil
|
|
}
|
|
|
|
// GetObjectSize returns the size of an object at the given key using HeadObject
|
|
func (s *s3ObjectStorage) GetObjectSize(ctx context.Context, key string) (int64, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
|
|
result, err := s.S3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(s.BucketName),
|
|
Key: aws.String(key),
|
|
})
|
|
|
|
if err != nil {
|
|
var apiError smithy.APIError
|
|
if errors.As(err, &apiError) {
|
|
switch apiError.(type) {
|
|
case *types.NotFound:
|
|
s.Logger.Debug("Object not found when getting size",
|
|
zap.String("key", key))
|
|
return 0, errors.New("object not found")
|
|
case *types.NoSuchKey:
|
|
s.Logger.Debug("Object not found when getting size (NoSuchKey)",
|
|
zap.String("key", key))
|
|
return 0, errors.New("object not found")
|
|
default:
|
|
s.Logger.Error("Error getting object size",
|
|
zap.String("key", key),
|
|
zap.Error(err))
|
|
return 0, err
|
|
}
|
|
}
|
|
s.Logger.Error("Error getting object size",
|
|
zap.String("key", key),
|
|
zap.Error(err))
|
|
return 0, err
|
|
}
|
|
|
|
// Let's use aws.ToInt64 which handles both pointer and non-pointer cases
|
|
size := aws.ToInt64(result.ContentLength)
|
|
|
|
s.Logger.Debug("Retrieved object size",
|
|
zap.String("key", key),
|
|
zap.Int64("size", size))
|
|
|
|
return size, nil
|
|
}
|