Initial commit: Open sourcing all of the Maple Open Technologies code.
This commit is contained in:
commit
755d54a99d
2010 changed files with 448675 additions and 0 deletions
|
|
@ -0,0 +1,442 @@
|
|||
// monorepo/cloud/backend/internal/maplefile/service/file/complete_file_upload.go
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config/constants"
|
||||
uc_user "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/usecase/user"
|
||||
dom_collection "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/collection"
|
||||
dom_file "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/file"
|
||||
uc_filemetadata "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/usecase/filemetadata"
|
||||
uc_fileobjectstorage "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/usecase/fileobjectstorage"
|
||||
uc_storagedailyusage "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/usecase/storagedailyusage"
|
||||
uc_storageusageevent "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/usecase/storageusageevent"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/httperror"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/transaction"
|
||||
)
|
||||
|
||||
type CompleteFileUploadRequestDTO struct {
|
||||
FileID gocql.UUID `json:"file_id"`
|
||||
// Optional: Client can provide actual file size for validation
|
||||
ActualFileSizeInBytes int64 `json:"actual_file_size_in_bytes,omitempty"`
|
||||
// Optional: Client can provide actual thumbnail size for validation
|
||||
ActualThumbnailSizeInBytes int64 `json:"actual_thumbnail_size_in_bytes,omitempty"`
|
||||
// Optional: Client can confirm successful upload
|
||||
UploadConfirmed bool `json:"upload_confirmed,omitempty"`
|
||||
ThumbnailUploadConfirmed bool `json:"thumbnail_upload_confirmed,omitempty"`
|
||||
}
|
||||
|
||||
type CompleteFileUploadResponseDTO struct {
|
||||
File *FileResponseDTO `json:"file"`
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message"`
|
||||
ActualFileSize int64 `json:"actual_file_size"`
|
||||
ActualThumbnailSize int64 `json:"actual_thumbnail_size"`
|
||||
UploadVerified bool `json:"upload_verified"`
|
||||
ThumbnailVerified bool `json:"thumbnail_verified"`
|
||||
StorageAdjustment int64 `json:"storage_adjustment"` // Positive if more space used, negative if less
|
||||
}
|
||||
|
||||
type CompleteFileUploadService interface {
|
||||
Execute(ctx context.Context, req *CompleteFileUploadRequestDTO) (*CompleteFileUploadResponseDTO, error)
|
||||
}
|
||||
|
||||
type completeFileUploadServiceImpl struct {
|
||||
config *config.Configuration
|
||||
logger *zap.Logger
|
||||
collectionRepo dom_collection.CollectionRepository
|
||||
getMetadataUseCase uc_filemetadata.GetFileMetadataUseCase
|
||||
updateMetadataUseCase uc_filemetadata.UpdateFileMetadataUseCase
|
||||
verifyObjectExistsUseCase uc_fileobjectstorage.VerifyObjectExistsUseCase
|
||||
getObjectSizeUseCase uc_fileobjectstorage.GetObjectSizeUseCase
|
||||
deleteDataUseCase uc_fileobjectstorage.DeleteEncryptedDataUseCase
|
||||
storageQuotaHelperUseCase uc_user.UserStorageQuotaHelperUseCase
|
||||
// Add storage usage tracking use cases
|
||||
createStorageUsageEventUseCase uc_storageusageevent.CreateStorageUsageEventUseCase
|
||||
updateStorageUsageUseCase uc_storagedailyusage.UpdateStorageUsageUseCase
|
||||
}
|
||||
|
||||
func NewCompleteFileUploadService(
|
||||
config *config.Configuration,
|
||||
logger *zap.Logger,
|
||||
collectionRepo dom_collection.CollectionRepository,
|
||||
getMetadataUseCase uc_filemetadata.GetFileMetadataUseCase,
|
||||
updateMetadataUseCase uc_filemetadata.UpdateFileMetadataUseCase,
|
||||
verifyObjectExistsUseCase uc_fileobjectstorage.VerifyObjectExistsUseCase,
|
||||
getObjectSizeUseCase uc_fileobjectstorage.GetObjectSizeUseCase,
|
||||
deleteDataUseCase uc_fileobjectstorage.DeleteEncryptedDataUseCase,
|
||||
storageQuotaHelperUseCase uc_user.UserStorageQuotaHelperUseCase,
|
||||
createStorageUsageEventUseCase uc_storageusageevent.CreateStorageUsageEventUseCase,
|
||||
updateStorageUsageUseCase uc_storagedailyusage.UpdateStorageUsageUseCase,
|
||||
) CompleteFileUploadService {
|
||||
logger = logger.Named("CompleteFileUploadService")
|
||||
return &completeFileUploadServiceImpl{
|
||||
config: config,
|
||||
logger: logger,
|
||||
collectionRepo: collectionRepo,
|
||||
getMetadataUseCase: getMetadataUseCase,
|
||||
updateMetadataUseCase: updateMetadataUseCase,
|
||||
verifyObjectExistsUseCase: verifyObjectExistsUseCase,
|
||||
getObjectSizeUseCase: getObjectSizeUseCase,
|
||||
deleteDataUseCase: deleteDataUseCase,
|
||||
storageQuotaHelperUseCase: storageQuotaHelperUseCase,
|
||||
createStorageUsageEventUseCase: createStorageUsageEventUseCase,
|
||||
updateStorageUsageUseCase: updateStorageUsageUseCase,
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *completeFileUploadServiceImpl) Execute(ctx context.Context, req *CompleteFileUploadRequestDTO) (*CompleteFileUploadResponseDTO, error) {
|
||||
//
|
||||
// STEP 1: Validation
|
||||
//
|
||||
if req == nil {
|
||||
svc.logger.Warn("⚠️ Failed validation with nil request")
|
||||
return nil, httperror.NewForBadRequestWithSingleField("non_field_error", "File completion details are required")
|
||||
}
|
||||
|
||||
if req.FileID.String() == "" {
|
||||
svc.logger.Warn("⚠️ Empty file ID provided")
|
||||
return nil, httperror.NewForBadRequestWithSingleField("file_id", "File ID is required")
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 2: Get user ID from context
|
||||
//
|
||||
userID, ok := ctx.Value(constants.SessionUserID).(gocql.UUID)
|
||||
if !ok {
|
||||
svc.logger.Error("🔴 Failed getting user ID from context")
|
||||
return nil, httperror.NewForInternalServerErrorWithSingleField("message", "Authentication context error")
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 3: Get file metadata
|
||||
//
|
||||
// Developers note: Use `ExecuteWithAnyState` because initially created `FileMetadata` object has state set to `pending`.
|
||||
file, err := svc.getMetadataUseCase.Execute(req.FileID)
|
||||
if err != nil {
|
||||
svc.logger.Error("🔴 Failed to get file metadata",
|
||||
zap.Any("error", err),
|
||||
zap.Any("file_id", req.FileID))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 4: Verify user has write access to the file's collection
|
||||
//
|
||||
hasAccess, err := svc.collectionRepo.CheckAccess(ctx, file.CollectionID, userID, dom_collection.CollectionPermissionReadWrite)
|
||||
if err != nil {
|
||||
svc.logger.Error("🔴 Failed to check collection access",
|
||||
zap.Any("error", err),
|
||||
zap.Any("collection_id", file.CollectionID),
|
||||
zap.Any("user_id", userID))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !hasAccess {
|
||||
svc.logger.Warn("⚠️ Unauthorized file completion attempt",
|
||||
zap.Any("user_id", userID),
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.Any("collection_id", file.CollectionID))
|
||||
return nil, httperror.NewForForbiddenWithSingleField("message", "You don't have permission to complete this file upload")
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 5: Verify file is in pending state
|
||||
//
|
||||
if file.State != dom_file.FileStatePending {
|
||||
svc.logger.Warn("⚠️ File is not in pending state",
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("current_state", file.State))
|
||||
return nil, httperror.NewForBadRequestWithSingleField("file_id", fmt.Sprintf("File is not in pending state (current state: %s)", file.State))
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 6: Verify file exists in object storage and get actual size
|
||||
//
|
||||
fileExists, err := svc.verifyObjectExistsUseCase.Execute(file.EncryptedFileObjectKey)
|
||||
if err != nil {
|
||||
svc.logger.Error("🔴 Failed to verify file exists in storage",
|
||||
zap.Any("error", err),
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("storage_path", file.EncryptedFileObjectKey))
|
||||
return nil, httperror.NewForInternalServerErrorWithSingleField("message", "Failed to verify file upload")
|
||||
}
|
||||
|
||||
if !fileExists {
|
||||
svc.logger.Warn("⚠️ File does not exist in storage",
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("storage_path", file.EncryptedFileObjectKey))
|
||||
return nil, httperror.NewForBadRequestWithSingleField("file_id", "File has not been uploaded yet")
|
||||
}
|
||||
|
||||
// Get actual file size from storage
|
||||
actualFileSize, err := svc.getObjectSizeUseCase.Execute(file.EncryptedFileObjectKey)
|
||||
if err != nil {
|
||||
svc.logger.Error("🔴 Failed to get file size from storage",
|
||||
zap.Any("error", err),
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("storage_path", file.EncryptedFileObjectKey))
|
||||
return nil, httperror.NewForInternalServerErrorWithSingleField("message", "Failed to verify file size")
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 7: Verify thumbnail if expected
|
||||
//
|
||||
var actualThumbnailSize int64 = 0
|
||||
var thumbnailVerified bool = true
|
||||
|
||||
if file.EncryptedThumbnailObjectKey != "" {
|
||||
thumbnailExists, err := svc.verifyObjectExistsUseCase.Execute(file.EncryptedThumbnailObjectKey)
|
||||
if err != nil {
|
||||
svc.logger.Warn("⚠️ Failed to verify thumbnail exists, continuing without it",
|
||||
zap.Any("error", err),
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("thumbnail_storage_path", file.EncryptedThumbnailObjectKey))
|
||||
thumbnailVerified = false
|
||||
} else if thumbnailExists {
|
||||
actualThumbnailSize, err = svc.getObjectSizeUseCase.Execute(file.EncryptedThumbnailObjectKey)
|
||||
if err != nil {
|
||||
svc.logger.Warn("⚠️ Failed to get thumbnail size, continuing without it",
|
||||
zap.Any("error", err),
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.String("thumbnail_storage_path", file.EncryptedThumbnailObjectKey))
|
||||
thumbnailVerified = false
|
||||
}
|
||||
} else {
|
||||
// Thumbnail was expected but not uploaded - clear the path
|
||||
file.EncryptedThumbnailObjectKey = ""
|
||||
thumbnailVerified = false
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// SAGA: Initialize distributed transaction manager
|
||||
//
|
||||
saga := transaction.NewSaga("complete-file-upload", svc.logger)
|
||||
|
||||
//
|
||||
// STEP 8: Calculate storage adjustment and update quota
|
||||
//
|
||||
expectedTotalSize := file.EncryptedFileSizeInBytes + file.EncryptedThumbnailSizeInBytes
|
||||
actualTotalSize := actualFileSize + actualThumbnailSize
|
||||
storageAdjustment := actualTotalSize - expectedTotalSize
|
||||
|
||||
svc.logger.Info("Starting file upload completion with SAGA protection",
|
||||
zap.String("file_id", req.FileID.String()),
|
||||
zap.Int64("expected_file_size", file.EncryptedFileSizeInBytes),
|
||||
zap.Int64("actual_file_size", actualFileSize),
|
||||
zap.Int64("expected_thumbnail_size", file.EncryptedThumbnailSizeInBytes),
|
||||
zap.Int64("actual_thumbnail_size", actualThumbnailSize),
|
||||
zap.Int64("expected_total", expectedTotalSize),
|
||||
zap.Int64("actual_total", actualTotalSize),
|
||||
zap.Int64("adjustment", storageAdjustment))
|
||||
|
||||
// Handle storage quota adjustment (SAGA protected)
|
||||
if storageAdjustment != 0 {
|
||||
if storageAdjustment > 0 {
|
||||
// Need more quota than originally reserved
|
||||
err = svc.storageQuotaHelperUseCase.CheckAndReserveQuota(ctx, userID, storageAdjustment)
|
||||
if err != nil {
|
||||
svc.logger.Error("Failed to reserve additional storage quota",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Int64("additional_size", storageAdjustment),
|
||||
zap.Error(err))
|
||||
|
||||
// Clean up the uploaded file since we can't complete due to quota
|
||||
// Note: This is an exceptional case - quota exceeded before any SAGA operations
|
||||
if deleteErr := svc.deleteDataUseCase.Execute(file.EncryptedFileObjectKey); deleteErr != nil {
|
||||
svc.logger.Error("Failed to clean up file after quota exceeded", zap.Error(deleteErr))
|
||||
}
|
||||
if file.EncryptedThumbnailObjectKey != "" {
|
||||
if deleteErr := svc.deleteDataUseCase.Execute(file.EncryptedThumbnailObjectKey); deleteErr != nil {
|
||||
svc.logger.Error("Failed to clean up thumbnail after quota exceeded", zap.Error(deleteErr))
|
||||
}
|
||||
}
|
||||
|
||||
saga.Rollback(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Register compensation: release the additional quota if later steps fail
|
||||
storageAdjustmentCaptured := storageAdjustment
|
||||
userIDCaptured := userID
|
||||
saga.AddCompensation(func(ctx context.Context) error {
|
||||
svc.logger.Warn("SAGA compensation: releasing additional reserved quota",
|
||||
zap.Int64("size", storageAdjustmentCaptured))
|
||||
return svc.storageQuotaHelperUseCase.ReleaseQuota(ctx, userIDCaptured, storageAdjustmentCaptured)
|
||||
})
|
||||
} else {
|
||||
// Used less quota than originally reserved, release the difference
|
||||
err = svc.storageQuotaHelperUseCase.ReleaseQuota(ctx, userID, -storageAdjustment)
|
||||
if err != nil {
|
||||
svc.logger.Error("Failed to release excess quota",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Int64("excess_size", -storageAdjustment),
|
||||
zap.Error(err))
|
||||
saga.Rollback(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Register compensation: re-reserve the released quota if later steps fail
|
||||
excessQuotaCaptured := -storageAdjustment
|
||||
userIDCaptured := userID
|
||||
saga.AddCompensation(func(ctx context.Context) error {
|
||||
svc.logger.Warn("SAGA compensation: re-reserving released excess quota",
|
||||
zap.Int64("size", excessQuotaCaptured))
|
||||
return svc.storageQuotaHelperUseCase.CheckAndReserveQuota(ctx, userIDCaptured, excessQuotaCaptured)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 9: Validate file size if client provided it
|
||||
//
|
||||
if req.ActualFileSizeInBytes > 0 && req.ActualFileSizeInBytes != actualFileSize {
|
||||
svc.logger.Warn("⚠️ File size mismatch between client and storage",
|
||||
zap.Any("file_id", req.FileID),
|
||||
zap.Int64("client_reported_size", req.ActualFileSizeInBytes),
|
||||
zap.Int64("storage_actual_size", actualFileSize))
|
||||
// Continue with storage size as authoritative
|
||||
}
|
||||
|
||||
//
|
||||
// STEP 10: Update file metadata to active state (SAGA protected)
|
||||
//
|
||||
originalState := file.State
|
||||
originalFileSizeInBytes := file.EncryptedFileSizeInBytes
|
||||
originalThumbnailSizeInBytes := file.EncryptedThumbnailSizeInBytes
|
||||
originalVersion := file.Version
|
||||
|
||||
file.EncryptedFileSizeInBytes = actualFileSize
|
||||
file.EncryptedThumbnailSizeInBytes = actualThumbnailSize
|
||||
file.State = dom_file.FileStateActive
|
||||
file.ModifiedAt = time.Now()
|
||||
file.ModifiedByUserID = userID
|
||||
file.Version++ // Every mutation we need to keep a track of.
|
||||
|
||||
err = svc.updateMetadataUseCase.Execute(ctx, file)
|
||||
if err != nil {
|
||||
svc.logger.Error("Failed to update file metadata to active state",
|
||||
zap.Error(err),
|
||||
zap.String("file_id", req.FileID.String()))
|
||||
saga.Rollback(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Register compensation: restore original metadata state
|
||||
fileIDCaptured := file.ID
|
||||
originalStateCaptured := originalState
|
||||
originalFileSizeCaptured := originalFileSizeInBytes
|
||||
originalThumbnailSizeCaptured := originalThumbnailSizeInBytes
|
||||
originalVersionCaptured := originalVersion
|
||||
collectionIDCaptured := file.CollectionID
|
||||
saga.AddCompensation(func(ctx context.Context) error {
|
||||
svc.logger.Warn("SAGA compensation: restoring file metadata to pending state",
|
||||
zap.String("file_id", fileIDCaptured.String()))
|
||||
|
||||
restoredFile, err := svc.getMetadataUseCase.Execute(fileIDCaptured)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
restoredFile.State = originalStateCaptured
|
||||
restoredFile.EncryptedFileSizeInBytes = originalFileSizeCaptured
|
||||
restoredFile.EncryptedThumbnailSizeInBytes = originalThumbnailSizeCaptured
|
||||
restoredFile.Version = originalVersionCaptured
|
||||
restoredFile.ModifiedAt = time.Now()
|
||||
|
||||
// Note: The repository Update method handles file count adjustments based on state changes,
|
||||
// so restoring to pending state will automatically decrement the file count
|
||||
return svc.updateMetadataUseCase.Execute(ctx, restoredFile)
|
||||
})
|
||||
|
||||
// Note: File count increment is handled by the repository's Update method when state changes
|
||||
// from pending to active. No explicit increment needed here to avoid double counting.
|
||||
|
||||
//
|
||||
// STEP 11: Create storage usage event (SAGA protected)
|
||||
//
|
||||
_ = collectionIDCaptured // Keep variable for potential future use
|
||||
err = svc.createStorageUsageEventUseCase.Execute(ctx, file.OwnerID, actualTotalSize, "add")
|
||||
if err != nil {
|
||||
svc.logger.Error("Failed to create storage usage event",
|
||||
zap.String("owner_id", file.OwnerID.String()),
|
||||
zap.Int64("file_size", actualTotalSize),
|
||||
zap.Error(err))
|
||||
saga.Rollback(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Register compensation: create compensating "remove" event
|
||||
ownerIDCaptured := file.OwnerID
|
||||
actualTotalSizeCaptured := actualTotalSize
|
||||
saga.AddCompensation(func(ctx context.Context) error {
|
||||
svc.logger.Warn("SAGA compensation: creating compensating usage event")
|
||||
return svc.createStorageUsageEventUseCase.Execute(ctx, ownerIDCaptured, actualTotalSizeCaptured, "remove")
|
||||
})
|
||||
|
||||
//
|
||||
// STEP 13: Update daily storage usage (SAGA protected)
|
||||
//
|
||||
today := time.Now().Truncate(24 * time.Hour)
|
||||
updateReq := &uc_storagedailyusage.UpdateStorageUsageRequest{
|
||||
UserID: file.OwnerID,
|
||||
UsageDay: &today,
|
||||
TotalBytes: actualTotalSize,
|
||||
AddBytes: actualTotalSize,
|
||||
RemoveBytes: 0,
|
||||
IsIncrement: true, // Increment the existing values
|
||||
}
|
||||
err = svc.updateStorageUsageUseCase.Execute(ctx, updateReq)
|
||||
if err != nil {
|
||||
svc.logger.Error("Failed to update daily storage usage",
|
||||
zap.String("owner_id", file.OwnerID.String()),
|
||||
zap.Int64("file_size", actualTotalSize),
|
||||
zap.Error(err))
|
||||
saga.Rollback(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Register compensation: reverse the usage update
|
||||
saga.AddCompensation(func(ctx context.Context) error {
|
||||
svc.logger.Warn("SAGA compensation: reversing daily usage update")
|
||||
compensateReq := &uc_storagedailyusage.UpdateStorageUsageRequest{
|
||||
UserID: ownerIDCaptured,
|
||||
UsageDay: &today,
|
||||
TotalBytes: -actualTotalSizeCaptured, // Negative to reverse
|
||||
AddBytes: 0,
|
||||
RemoveBytes: actualTotalSizeCaptured,
|
||||
IsIncrement: true,
|
||||
}
|
||||
return svc.updateStorageUsageUseCase.Execute(ctx, compensateReq)
|
||||
})
|
||||
|
||||
//
|
||||
// SUCCESS: All operations completed with SAGA protection
|
||||
//
|
||||
svc.logger.Info("File upload completed successfully with SAGA protection",
|
||||
zap.String("file_id", req.FileID.String()),
|
||||
zap.String("collection_id", file.CollectionID.String()),
|
||||
zap.String("owner_id", file.OwnerID.String()),
|
||||
zap.Int64("actual_file_size", actualFileSize),
|
||||
zap.Int64("actual_thumbnail_size", actualThumbnailSize),
|
||||
zap.Int64("storage_adjustment", storageAdjustment))
|
||||
|
||||
return &CompleteFileUploadResponseDTO{
|
||||
File: mapFileToDTO(file),
|
||||
Success: true,
|
||||
Message: "File upload completed successfully with storage quota updated",
|
||||
ActualFileSize: actualFileSize,
|
||||
ActualThumbnailSize: actualThumbnailSize,
|
||||
UploadVerified: true,
|
||||
ThumbnailVerified: thumbnailVerified,
|
||||
StorageAdjustment: storageAdjustment,
|
||||
}, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue