Initial commit: Open sourcing all of the Maple Open Technologies code.

cloud/maplefile-backend/pkg/storage/cache/cassandracache/cassandracache.go

@@ -0,0 +1,108 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/cache/cassandracache/cassandaracache.go
+package cassandracache
+
+import (
+	"context"
+	"time"
+
+	"github.com/gocql/gocql"
+	"go.uber.org/zap"
+)
+
+type CassandraCacher interface {
+	Shutdown()
+	Get(ctx context.Context, key string) ([]byte, error)
+	Set(ctx context.Context, key string, val []byte) error
+	SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error
+	Delete(ctx context.Context, key string) error
+	PurgeExpired(ctx context.Context) error
+}
+
+type cache struct {
+	Session *gocql.Session
+	Logger  *zap.Logger
+}
+
+func NewCassandraCacher(session *gocql.Session, logger *zap.Logger) CassandraCacher {
+	logger = logger.Named("CassandraCache")
+	logger.Info("cassandra cache initialized")
+	return &cache{
+		Session: session,
+		Logger:  logger,
+	}
+}
+
+func (s *cache) Shutdown() {
+	s.Logger.Info("cassandra cache shutting down...")
+	s.Session.Close()
+}
+
+func (s *cache) Get(ctx context.Context, key string) ([]byte, error) {
+	var value []byte
+	var expiresAt time.Time
+
+	query := `SELECT value, expires_at FROM pkg_cache_by_key_with_asc_expire_at WHERE key=?`
+	err := s.Session.Query(query, key).WithContext(ctx).Consistency(gocql.LocalQuorum).Scan(&value, &expiresAt)
+
+	if err == gocql.ErrNotFound {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if expired in application code
+	if time.Now().After(expiresAt) {
+		// Entry is expired, delete it and return nil
+		_ = s.Delete(ctx, key) // Clean up expired entry
+		return nil, nil
+	}
+
+	return value, nil
+}
+
+func (s *cache) Set(ctx context.Context, key string, val []byte) error {
+	expiresAt := time.Now().Add(24 * time.Hour) // Default 24 hour expiry
+	return s.Session.Query(`INSERT INTO pkg_cache_by_key_with_asc_expire_at (key, expires_at, value) VALUES (?, ?, ?)`,
+		key, expiresAt, val).WithContext(ctx).Consistency(gocql.LocalQuorum).Exec()
+}
+
+func (s *cache) SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error {
+	expiresAt := time.Now().Add(expiry)
+	return s.Session.Query(`INSERT INTO pkg_cache_by_key_with_asc_expire_at (key, expires_at, value) VALUES (?, ?, ?)`,
+		key, expiresAt, val).WithContext(ctx).Consistency(gocql.LocalQuorum).Exec()
+}
+
+func (s *cache) Delete(ctx context.Context, key string) error {
+	return s.Session.Query(`DELETE FROM pkg_cache_by_key_with_asc_expire_at WHERE key=?`,
+		key).WithContext(ctx).Consistency(gocql.LocalQuorum).Exec()
+}
+
+func (s *cache) PurgeExpired(ctx context.Context) error {
+	now := time.Now()
+
+	// Thanks to the index on expires_at, this query is efficient
+	iter := s.Session.Query(`SELECT key FROM pkg_cache_by_key_with_asc_expire_at WHERE expires_at < ? ALLOW FILTERING`,
+		now).WithContext(ctx).Iter()
+
+	var expiredKeys []string
+	var key string
+	for iter.Scan(&key) {
+		expiredKeys = append(expiredKeys, key)
+	}
+
+	if err := iter.Close(); err != nil {
+		return err
+	}
+
+	// Delete expired keys in batch
+	if len(expiredKeys) > 0 {
+		batch := s.Session.NewBatch(gocql.LoggedBatch).WithContext(ctx)
+		for _, expiredKey := range expiredKeys {
+			batch.Query(`DELETE FROM pkg_cache_by_key_with_asc_expire_at WHERE key=?`, expiredKey)
+		}
+		return s.Session.ExecuteBatch(batch)
+	}
+
+	return nil
+}

cloud/maplefile-backend/pkg/storage/cache/cassandracache/provider.go

@@ -0,0 +1,11 @@
+package cassandracache
+
+import (
+	"github.com/gocql/gocql"
+	"go.uber.org/zap"
+)
+
+// ProvideCassandraCacher provides a Cassandra cache instance for Wire DI
+func ProvideCassandraCacher(session *gocql.Session, logger *zap.Logger) CassandraCacher {
+	return NewCassandraCacher(session, logger)
+}

cloud/maplefile-backend/pkg/storage/cache/twotiercache/provider.go

@@ -0,0 +1,17 @@
+package twotiercache
+
+import (
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage/cache/cassandracache"
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage/memory/redis"
+)
+
+// ProvideTwoTierCache provides a two-tier cache instance for Wire DI
+func ProvideTwoTierCache(
+	redisCache redis.Cacher,
+	cassandraCache cassandracache.CassandraCacher,
+	logger *zap.Logger,
+) TwoTierCacher {
+	return NewTwoTierCache(redisCache, cassandraCache, logger)
+}

cloud/maplefile-backend/pkg/storage/cache/twotiercache/twotiercache.go

@@ -0,0 +1,106 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/cache/twotiercache/twotiercache.go
+package twotiercache
+
+import (
+	"context"
+	"time"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage/cache/cassandracache"
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage/memory/redis"
+	"go.uber.org/zap"
+)
+
+type TwoTierCacher interface {
+	Shutdown(ctx context.Context)
+	Get(ctx context.Context, key string) ([]byte, error)
+	Set(ctx context.Context, key string, val []byte) error
+	SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error
+	Delete(ctx context.Context, key string) error
+	PurgeExpired(ctx context.Context) error
+}
+
+// twoTierCacheImpl: clean 2-layer (read-through write-through) cache
+//
+// L1: Redis (fast, in-memory)
+// L2: Cassandra (persistent)
+//
+// On Get: check Redis → then Cassandra → if found in Cassandra → populate Redis
+// On Set: write to both
+// On SetWithExpiry: write to both with expiry
+// On Delete: remove from both
+type twoTierCacheImpl struct {
+	RedisCache     redis.Cacher
+	CassandraCache cassandracache.CassandraCacher
+	Logger         *zap.Logger
+}
+
+func NewTwoTierCache(redisCache redis.Cacher, cassandraCache cassandracache.CassandraCacher, logger *zap.Logger) TwoTierCacher {
+	logger = logger.Named("TwoTierCache")
+	return &twoTierCacheImpl{
+		RedisCache:     redisCache,
+		CassandraCache: cassandraCache,
+		Logger:         logger,
+	}
+}
+
+func (c *twoTierCacheImpl) Get(ctx context.Context, key string) ([]byte, error) {
+	val, err := c.RedisCache.Get(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	if val != nil {
+		c.Logger.Debug("cache hit from Redis", zap.String("key", key))
+		return val, nil
+	}
+
+	val, err = c.CassandraCache.Get(ctx, key)
+	if err != nil {
+		return nil, err
+	}
+	if val != nil {
+		c.Logger.Debug("cache hit from Cassandra, writing back to Redis", zap.String("key", key))
+		_ = c.RedisCache.Set(ctx, key, val)
+	}
+	return val, nil
+}
+
+func (c *twoTierCacheImpl) Set(ctx context.Context, key string, val []byte) error {
+	if err := c.RedisCache.Set(ctx, key, val); err != nil {
+		return err
+	}
+	if err := c.CassandraCache.Set(ctx, key, val); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *twoTierCacheImpl) SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error {
+	if err := c.RedisCache.SetWithExpiry(ctx, key, val, expiry); err != nil {
+		return err
+	}
+	if err := c.CassandraCache.SetWithExpiry(ctx, key, val, expiry); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *twoTierCacheImpl) Delete(ctx context.Context, key string) error {
+	if err := c.RedisCache.Delete(ctx, key); err != nil {
+		return err
+	}
+	if err := c.CassandraCache.Delete(ctx, key); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *twoTierCacheImpl) PurgeExpired(ctx context.Context) error {
+	return c.CassandraCache.PurgeExpired(ctx)
+}
+
+func (c *twoTierCacheImpl) Shutdown(ctx context.Context) {
+	c.Logger.Info("two-tier cache shutting down...")
+	c.RedisCache.Shutdown(ctx)
+	c.CassandraCache.Shutdown()
+	c.Logger.Info("two-tier cache shutdown complete")
+}

cloud/maplefile-backend/pkg/storage/database/cassandradb/cassandradb.go

@@ -0,0 +1,159 @@
+// File Path: monorepo/cloud/maplefile-backend/pkg/storage/database/cassandradb/cassandradb.go
+package cassandradb
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gocql/gocql"
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// CassandraDB wraps the gocql session with additional functionality
+type CassandraDB struct {
+	Session *gocql.Session
+	config  config.DatabaseConfig
+}
+
+// gocqlLogger wraps zap logger to filter out noisy gocql warnings
+type gocqlLogger struct {
+	logger *zap.Logger
+}
+
+// Print implements gocql's Logger interface
+func (l *gocqlLogger) Print(v ...interface{}) {
+	msg := fmt.Sprint(v...)
+
+	// Filter out noisy "invalid peer" warnings from Cassandra gossip
+	// These are harmless and occur due to Docker networking
+	if strings.Contains(msg, "Found invalid peer") {
+		return
+	}
+
+	// Log other messages at debug level
+	l.logger.Debug(msg)
+}
+
+// Printf implements gocql's Logger interface
+func (l *gocqlLogger) Printf(format string, v ...interface{}) {
+	msg := fmt.Sprintf(format, v...)
+
+	// Filter out noisy "invalid peer" warnings from Cassandra gossip
+	if strings.Contains(msg, "Found invalid peer") {
+		return
+	}
+
+	// Log other messages at debug level
+	l.logger.Debug(msg)
+}
+
+// Println implements gocql's Logger interface
+func (l *gocqlLogger) Println(v ...interface{}) {
+	msg := fmt.Sprintln(v...)
+
+	// Filter out noisy "invalid peer" warnings from Cassandra gossip
+	if strings.Contains(msg, "Found invalid peer") {
+		return
+	}
+
+	// Log other messages at debug level
+	l.logger.Debug(msg)
+}
+
+// NewCassandraConnection establishes a connection to Cassandra cluster
+// Uses the simplified approach from MaplePress (working code)
+func NewCassandraConnection(cfg *config.Config, logger *zap.Logger) (*gocql.Session, error) {
+	dbConfig := cfg.Database
+
+	logger.Info("⏳ Connecting to Cassandra...",
+		zap.Strings("hosts", dbConfig.Hosts),
+		zap.String("keyspace", dbConfig.Keyspace))
+
+	// Create cluster configuration - let gocql handle DNS resolution
+	cluster := gocql.NewCluster(dbConfig.Hosts...)
+	cluster.Keyspace = dbConfig.Keyspace
+	cluster.Consistency = parseConsistency(dbConfig.Consistency)
+	cluster.ProtoVersion = 4
+	cluster.ConnectTimeout = dbConfig.ConnectTimeout
+	cluster.Timeout = dbConfig.RequestTimeout
+	cluster.NumConns = 2
+
+	// Set custom logger to filter out noisy warnings
+	cluster.Logger = &gocqlLogger{logger: logger.Named("gocql")}
+
+	// Retry policy
+	cluster.RetryPolicy = &gocql.ExponentialBackoffRetryPolicy{
+		NumRetries: int(dbConfig.MaxRetryAttempts),
+		Min:        dbConfig.RetryDelay,
+		Max:        10 * time.Second,
+	}
+
+	// Enable compression for better network efficiency
+	cluster.Compressor = &gocql.SnappyCompressor{}
+
+	// Create session
+	session, err := cluster.CreateSession()
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to Cassandra: %w", err)
+	}
+
+	logger.Info("✓ Cassandra connected",
+		zap.String("consistency", dbConfig.Consistency),
+		zap.Int("connections", cluster.NumConns))
+
+	return session, nil
+}
+
+// Close terminates the database connection
+func (db *CassandraDB) Close() {
+	if db.Session != nil {
+		db.Session.Close()
+	}
+}
+
+// Health checks if the database connection is still alive
+func (db *CassandraDB) Health() error {
+	// Quick health check using a simple query
+	var timestamp time.Time
+	err := db.Session.Query("SELECT now() FROM system.local").Scan(&timestamp)
+	if err != nil {
+		return fmt.Errorf("health check failed: %w", err)
+	}
+
+	// Validate that we got a reasonable timestamp (within last minute)
+	now := time.Now()
+	if timestamp.Before(now.Add(-time.Minute)) || timestamp.After(now.Add(time.Minute)) {
+		return fmt.Errorf("health check returned suspicious timestamp: %v (current: %v)", timestamp, now)
+	}
+
+	return nil
+}
+
+// parseConsistency converts string consistency level to gocql.Consistency
+func parseConsistency(consistency string) gocql.Consistency {
+	switch consistency {
+	case "ANY":
+		return gocql.Any
+	case "ONE":
+		return gocql.One
+	case "TWO":
+		return gocql.Two
+	case "THREE":
+		return gocql.Three
+	case "QUORUM":
+		return gocql.Quorum
+	case "ALL":
+		return gocql.All
+	case "LOCAL_QUORUM":
+		return gocql.LocalQuorum
+	case "EACH_QUORUM":
+		return gocql.EachQuorum
+	case "LOCAL_ONE":
+		return gocql.LocalOne
+	default:
+		return gocql.Quorum // Default to QUORUM
+	}
+}

cloud/maplefile-backend/pkg/storage/database/cassandradb/migration.go

@@ -0,0 +1,146 @@
+// File Path: monorepo/cloud/maplefile-backend/pkg/storage/database/cassandradb/migration.go
+package cassandradb
+
+import (
+	"fmt"
+
+	"go.uber.org/zap"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/cassandra"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// Migrator handles database schema migrations
+// This encapsulates all migration logic and makes it testable
+type Migrator struct {
+	config config.DatabaseConfig
+	logger *zap.Logger
+}
+
+// NewMigrator creates a new migration manager that works with fx dependency injection
+func NewMigrator(cfg *config.Configuration, logger *zap.Logger) *Migrator {
+	return &Migrator{
+		config: cfg.Database,
+		logger: logger.Named("Migrator"),
+	}
+}
+
+// Up runs all pending migrations with dirty state recovery
+func (m *Migrator) Up() error {
+	m.logger.Info("Creating migrator")
+	migrateInstance, err := m.createMigrate()
+	if err != nil {
+		return fmt.Errorf("failed to create migrator: %w", err)
+	}
+	defer migrateInstance.Close()
+
+	m.logger.Info("Checking migration version")
+	version, dirty, err := migrateInstance.Version()
+	if err != nil && err != migrate.ErrNilVersion {
+		return fmt.Errorf("failed to get migration version: %w", err)
+	}
+
+	if dirty {
+		m.logger.Warn("Database is in dirty state, attempting to force clean state",
+			zap.Uint("version", version))
+		if err := migrateInstance.Force(int(version)); err != nil {
+			return fmt.Errorf("failed to force clean migration state: %w", err)
+		}
+	}
+
+	// Run migrations
+	if err := migrateInstance.Up(); err != nil && err != migrate.ErrNoChange {
+		return fmt.Errorf("failed to run migrations: %w", err)
+	}
+
+	// Get final version
+	finalVersion, _, err := migrateInstance.Version()
+	if err != nil && err != migrate.ErrNilVersion {
+		m.logger.Warn("Could not get final migration version",
+			zap.Error(err))
+	} else if err != migrate.ErrNilVersion {
+		m.logger.Info("Database migrations completed successfully",
+			zap.Uint("version", finalVersion))
+	} else {
+		m.logger.Info("Database migrations completed successfully (no migrations applied)")
+	}
+
+	return nil
+}
+
+// Down rolls back the last migration
+// Useful for development and rollback scenarios
+func (m *Migrator) Down() error {
+	migrate, err := m.createMigrate()
+	if err != nil {
+		return fmt.Errorf("failed to create migrator: %w", err)
+	}
+	defer migrate.Close()
+
+	if err := migrate.Steps(-1); err != nil {
+		return fmt.Errorf("failed to rollback migration: %w", err)
+	}
+
+	return nil
+}
+
+// Version returns the current migration version
+func (m *Migrator) Version() (uint, bool, error) {
+	migrate, err := m.createMigrate()
+	if err != nil {
+		return 0, false, fmt.Errorf("failed to create migrator: %w", err)
+	}
+	defer migrate.Close()
+
+	return migrate.Version()
+}
+
+// ForceVersion forces the migration version (useful for fixing dirty states)
+func (m *Migrator) ForceVersion(version int) error {
+	migrateInstance, err := m.createMigrate()
+	if err != nil {
+		return fmt.Errorf("failed to create migrator: %w", err)
+	}
+	defer migrateInstance.Close()
+
+	if err := migrateInstance.Force(version); err != nil {
+		return fmt.Errorf("failed to force version %d: %w", version, err)
+	}
+
+	m.logger.Info("Successfully forced migration version",
+		zap.Int("version", version))
+	return nil
+}
+
+// createMigrate creates a migrate instance with proper configuration
+func (m *Migrator) createMigrate() (*migrate.Migrate, error) {
+	// Build Cassandra connection string
+	// Format: cassandra://host:port/keyspace?consistency=level
+	databaseURL := fmt.Sprintf("cassandra://%s/%s?consistency=%s",
+		m.config.Hosts[0], // Use first host for migrations
+		m.config.Keyspace,
+		m.config.Consistency,
+	)
+
+	// Add authentication if configured
+	if m.config.Username != "" && m.config.Password != "" {
+		databaseURL = fmt.Sprintf("cassandra://%s:%s@%s/%s?consistency=%s",
+			m.config.Username,
+			m.config.Password,
+			m.config.Hosts[0],
+			m.config.Keyspace,
+			m.config.Consistency,
+		)
+	}
+
+	// Create migrate instance
+	migrate, err := migrate.New(m.config.MigrationsPath, databaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize migrate: %w", err)
+	}
+
+	return migrate, nil
+}

cloud/maplefile-backend/pkg/storage/database/cassandradb/provider.go

@@ -0,0 +1,13 @@
+package cassandradb
+
+import (
+	"github.com/gocql/gocql"
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideCassandraConnection provides a Cassandra session for Wire DI
+func ProvideCassandraConnection(cfg *config.Config, logger *zap.Logger) (*gocql.Session, error) {
+	return NewCassandraConnection(cfg, logger)
+}

cloud/maplefile-backend/pkg/storage/interface.go

@@ -0,0 +1,29 @@
+package storage
+
+// Storage interface defines the methods that can be used to interact with a key-value database.
+type Storage interface {
+	// Get returns the value associated with the specified key, or an error if the key is not found.
+	Get(key string) ([]byte, error)
+
+	// Set sets the value associated with the specified key.
+	// If the key already exists, its value is updated.
+	Set(key string, val []byte) error
+
+	// Delete removes the value associated with the specified key from the database.
+	Delete(key string) error
+
+	// Iterate is similar to View, but allows the iteration to start from a specific key prefix.
+	// The seekThenIterateKey parameter can be used to specify a key to seek to before starting the iteration.
+	Iterate(processFunc func(key, value []byte) error) error
+
+	IterateWithFilterByKeys(ks []string, processFunc func(key, value []byte) error) error
+
+	// Close closes the database, releasing any system resources it holds.
+	Close() error
+
+	OpenTransaction() error
+
+	CommitTransaction() error
+
+	DiscardTransaction()
+}

cloud/maplefile-backend/pkg/storage/memory/inmemory/memory.go

@@ -0,0 +1,202 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/inmemory/memory.go
+package inmemory
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage"
+	"go.uber.org/zap"
+)
+
+type cacheValue struct {
+	value []byte
+}
+
+// keyValueStorerImpl implements the db.Database interface.
+// It uses a LevelDB database to store key-value pairs.
+type keyValueStorerImpl struct {
+	data   map[string]cacheValue
+	txData map[string]cacheValue
+	lock   sync.Mutex
+}
+
+// NewInMemoryStorage creates a new instance of the keyValueStorerImpl.
+func NewInMemoryStorage(logger *zap.Logger) storage.Storage {
+	logger = logger.Named("InMemoryStorage")
+	return &keyValueStorerImpl{
+		data:   make(map[string]cacheValue),
+		txData: nil,
+	}
+}
+
+// Get retrieves a value from the database by its key.
+// It returns an error if the key is not found.
+func (impl *keyValueStorerImpl) Get(k string) ([]byte, error) {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		cachedValue, ok := impl.txData[k]
+		if !ok {
+			return nil, fmt.Errorf("does not exist for: %v", k)
+		}
+		return cachedValue.value, nil
+	} else {
+		cachedValue, ok := impl.data[k]
+		if !ok {
+			return nil, fmt.Errorf("does not exist for: %v", k)
+		}
+		return cachedValue.value, nil
+	}
+}
+
+// Set sets a value in the database by its key.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Set(k string, val []byte) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		impl.txData[k] = cacheValue{
+			value: val,
+		}
+	} else {
+		impl.data[k] = cacheValue{
+			value: val,
+		}
+	}
+	return nil
+}
+
+// Delete deletes a value from the database by its key.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Delete(k string) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		delete(impl.txData, k)
+	} else {
+		delete(impl.data, k)
+	}
+	return nil
+}
+
+// Iterate iterates over the key-value pairs in the database, starting from the specified key prefix.
+// It calls the provided function for each pair.
+// It returns an error if the iteration fails.
+func (impl *keyValueStorerImpl) Iterate(processFunc func(key, value []byte) error) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.txData {
+			// Call the provided function for each pair
+			if err := processFunc([]byte(k), v.value); err != nil {
+				return err
+			}
+		}
+	} else {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.data {
+			// Call the provided function for each pair
+			if err := processFunc([]byte(k), v.value); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) IterateWithFilterByKeys(ks []string, processFunc func(key, value []byte) error) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.txData {
+			// Iterate over our keys to search by.
+			for _, searchK := range ks {
+				// If the item we currently have matches our keys then execute.
+				if k == searchK {
+					// Call the provided function for each pair
+					if err := processFunc([]byte(k), v.value); err != nil {
+						return err
+					}
+				}
+			}
+
+		}
+	} else {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.data {
+			// Iterate over our keys to search by.
+			for _, searchK := range ks {
+				// If the item we currently have matches our keys then execute.
+				if k == searchK {
+					// Call the provided function for each pair
+					if err := processFunc([]byte(k), v.value); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// Close closes the database.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Close() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Clear the data map
+	impl.data = make(map[string]cacheValue)
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) OpenTransaction() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Create a new transaction by creating a copy of the current data
+	impl.txData = make(map[string]cacheValue)
+	for k, v := range impl.data {
+		impl.txData[k] = v
+	}
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) CommitTransaction() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Check if a transaction is in progress
+	if impl.txData == nil {
+		return errors.New("no transaction in progress")
+	}
+
+	// Update the current data with the transaction data
+	impl.data = impl.txData
+	impl.txData = nil
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) DiscardTransaction() {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Check if a transaction is in progress
+	if impl.txData != nil {
+		impl.txData = nil
+	}
+}

cloud/maplefile-backend/pkg/storage/memory/inmemory/memory_test.go

@@ -0,0 +1,295 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/inmemory/memory_test.go
+package inmemory
+
+import (
+	"reflect"
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+// TestNewInMemoryStorage verifies that the NewInMemoryStorage function
+// correctly initializes a new storage instance
+func TestNewInMemoryStorage(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	if storage == nil {
+		t.Fatal("Expected non-nil storage instance")
+	}
+
+	// Type assertion to verify we get the correct implementation
+	_, ok := storage.(*keyValueStorerImpl)
+	if !ok {
+		t.Fatal("Expected keyValueStorerImpl instance")
+	}
+}
+
+// TestBasicOperations tests the basic Set/Get/Delete operations
+func TestBasicOperations(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Test Set and Get
+	t.Run("Set and Get", func(t *testing.T) {
+		key := "test-key"
+		value := []byte("test-value")
+
+		err := storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Set failed: %v", err)
+		}
+
+		retrieved, err := storage.Get(key)
+		if err != nil {
+			t.Fatalf("Get failed: %v", err)
+		}
+
+		if !reflect.DeepEqual(retrieved, value) {
+			t.Errorf("Retrieved value doesn't match: got %v, want %v", retrieved, value)
+		}
+	})
+
+	// Test Get with non-existent key
+	t.Run("Get Non-existent", func(t *testing.T) {
+		_, err := storage.Get("non-existent")
+		if err == nil {
+			t.Error("Expected error for non-existent key")
+		}
+	})
+
+	// Test Delete
+	t.Run("Delete", func(t *testing.T) {
+		key := "delete-test"
+		value := []byte("delete-value")
+
+		// First set a value
+		err := storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Set failed: %v", err)
+		}
+
+		// Delete it
+		err = storage.Delete(key)
+		if err != nil {
+			t.Fatalf("Delete failed: %v", err)
+		}
+
+		// Verify it's gone
+		_, err = storage.Get(key)
+		if err == nil {
+			t.Error("Expected error after deletion")
+		}
+	})
+}
+
+// TestIteration tests the Iterate functionality
+func TestIteration(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Prepare test data
+	testData := map[string][]byte{
+		"key1": []byte("value1"),
+		"key2": []byte("value2"),
+		"key3": []byte("value3"),
+	}
+
+	// Insert test data
+	for k, v := range testData {
+		if err := storage.Set(k, v); err != nil {
+			t.Fatalf("Failed to set test data: %v", err)
+		}
+	}
+
+	// Test basic iteration
+	t.Run("Basic Iteration", func(t *testing.T) {
+		found := make(map[string][]byte)
+
+		err := storage.Iterate(func(key, value []byte) error {
+			found[string(key)] = value
+			return nil
+		})
+
+		if err != nil {
+			t.Fatalf("Iteration failed: %v", err)
+		}
+
+		if !reflect.DeepEqual(testData, found) {
+			t.Errorf("Iteration results don't match: got %v, want %v", found, testData)
+		}
+	})
+
+	// Test filtered iteration
+	t.Run("Filtered Iteration", func(t *testing.T) {
+		filterKeys := []string{"key1", "key3"}
+		found := make(map[string][]byte)
+
+		err := storage.IterateWithFilterByKeys(filterKeys, func(key, value []byte) error {
+			found[string(key)] = value
+			return nil
+		})
+
+		if err != nil {
+			t.Fatalf("Filtered iteration failed: %v", err)
+		}
+
+		// Verify only requested keys were returned
+		if len(found) != len(filterKeys) {
+			t.Errorf("Expected %d items, got %d", len(filterKeys), len(found))
+		}
+
+		for _, k := range filterKeys {
+			if !reflect.DeepEqual(found[k], testData[k]) {
+				t.Errorf("Filtered data mismatch for key %s: got %v, want %v", k, found[k], testData[k])
+			}
+		}
+	})
+}
+
+// TestTransactions tests the transaction-related functionality
+func TestTransactions(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Test basic transaction commit
+	t.Run("Transaction Commit", func(t *testing.T) {
+		// Start transaction
+		err := storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open transaction: %v", err)
+		}
+
+		// Make changes in transaction
+		key := "tx-test"
+		value := []byte("tx-value")
+
+		err = storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Failed to set in transaction: %v", err)
+		}
+
+		// Commit transaction
+		err = storage.CommitTransaction()
+		if err != nil {
+			t.Fatalf("Failed to commit transaction: %v", err)
+		}
+
+		// Verify changes persisted
+		retrieved, err := storage.Get(key)
+		if err != nil {
+			t.Fatalf("Failed to get after commit: %v", err)
+		}
+
+		if !reflect.DeepEqual(retrieved, value) {
+			t.Errorf("Retrieved value doesn't match after commit: got %v, want %v", retrieved, value)
+		}
+	})
+
+	// Test transaction discard
+	t.Run("Transaction Discard", func(t *testing.T) {
+		// Start transaction
+		err := storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open transaction: %v", err)
+		}
+
+		// Make changes in transaction
+		key := "discard-test"
+		value := []byte("discard-value")
+
+		err = storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Failed to set in transaction: %v", err)
+		}
+
+		// Discard transaction
+		storage.DiscardTransaction()
+
+		// Verify changes were not persisted
+		_, err = storage.Get(key)
+		if err == nil {
+			t.Error("Expected error getting discarded value")
+		}
+	})
+
+	// Test transaction behavior with multiple opens
+	t.Run("Multiple Transaction Opens", func(t *testing.T) {
+		// Set initial value
+		err := storage.Set("tx-test", []byte("initial"))
+		if err != nil {
+			t.Fatalf("Failed to set initial value: %v", err)
+		}
+
+		// First transaction
+		err = storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open first transaction: %v", err)
+		}
+
+		// Modify value
+		err = storage.Set("tx-test", []byte("modified"))
+		if err != nil {
+			t.Fatalf("Failed to set value in transaction: %v", err)
+		}
+
+		// Opening another transaction while one is in progress overwrites the transaction data
+		err = storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open second transaction: %v", err)
+		}
+
+		// Modify value again
+		err = storage.Set("tx-test", []byte("final"))
+		if err != nil {
+			t.Fatalf("Failed to set value in second transaction: %v", err)
+		}
+
+		// Commit the transaction (only need to commit once as there's only one transaction state)
+		err = storage.CommitTransaction()
+		if err != nil {
+			t.Fatalf("Failed to commit transaction: %v", err)
+		}
+
+		// Verify attempting to commit again fails since transaction state is cleared
+		err = storage.CommitTransaction()
+		if err == nil {
+			t.Error("Expected error when committing with no transaction in progress")
+		}
+
+		// Verify final value
+		val, err := storage.Get("tx-test")
+		if err != nil {
+			t.Fatalf("Failed to get final value: %v", err)
+		}
+
+		if !reflect.DeepEqual(val, []byte("final")) {
+			t.Errorf("Unexpected final value: got %s, want %s", string(val), "final")
+		}
+	})
+}
+
+// TestClose verifies the Close functionality
+func TestClose(t *testing.T) {
+
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Add some data
+	err := storage.Set("test", []byte("value"))
+	if err != nil {
+		t.Fatalf("Failed to set test data: %v", err)
+	}
+
+	// Close storage
+	err = storage.Close()
+	if err != nil {
+		t.Fatalf("Close failed: %v", err)
+	}
+
+	// Verify data is cleared
+	_, err = storage.Get("test")
+	if err == nil {
+		t.Error("Expected error getting value after close")
+	}
+}

cloud/maplefile-backend/pkg/storage/memory/redis/client_provider.go

@@ -0,0 +1,41 @@
+package redis
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideRedisUniversalClient provides a Redis UniversalClient for Wire DI
+// This is needed for components like leader election that require the raw Redis client
+func ProvideRedisUniversalClient(cfg *config.Config, logger *zap.Logger) (redis.UniversalClient, error) {
+	logger = logger.Named("RedisClient")
+
+	// Create Redis client
+	client := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("%s:%d", cfg.Cache.Host, cfg.Cache.Port),
+		Password: cfg.Cache.Password,
+		DB:       cfg.Cache.DB,
+	})
+
+	// Test connection
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if _, err := client.Ping(ctx).Result(); err != nil {
+		logger.Error("Failed to connect to Redis", zap.Error(err))
+		return nil, fmt.Errorf("redis connection failed: %w", err)
+	}
+
+	logger.Info("✅ Redis client connected successfully",
+		zap.String("host", cfg.Cache.Host),
+		zap.Int("port", cfg.Cache.Port),
+		zap.Int("db", cfg.Cache.DB))
+
+	return client, nil
+}

cloud/maplefile-backend/pkg/storage/memory/redis/provider.go

@@ -0,0 +1,12 @@
+package redis
+
+import (
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideRedisCache provides a Redis cache instance for Wire DI
+func ProvideRedisCache(cfg *config.Config, logger *zap.Logger) Cacher {
+	return NewCache(cfg, logger)
+}

cloud/maplefile-backend/pkg/storage/memory/redis/redis.go

@@ -0,0 +1,73 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/redis/redis.go
+package redis
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	c "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+	"github.com/redis/go-redis/v9"
+	"go.uber.org/zap"
+)
+
+type Cacher interface {
+	Shutdown(ctx context.Context)
+	Get(ctx context.Context, key string) ([]byte, error)
+	Set(ctx context.Context, key string, val []byte) error
+	SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error
+	Delete(ctx context.Context, key string) error
+}
+
+type cache struct {
+	Client *redis.Client
+	Logger *zap.Logger
+}
+
+func NewCache(cfg *c.Configuration, logger *zap.Logger) Cacher {
+	logger = logger.Named("Redis Memory Storage")
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("%s:%d", cfg.Cache.Host, cfg.Cache.Port),
+		Password: cfg.Cache.Password,
+		DB:       cfg.Cache.DB,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if _, err := rdb.Ping(ctx).Result(); err != nil {
+		logger.Fatal("failed connecting to Redis", zap.Error(err))
+	}
+
+	return &cache{
+		Client: rdb,
+		Logger: logger,
+	}
+}
+
+func (s *cache) Shutdown(ctx context.Context) {
+	s.Logger.Info("shutting down Redis cache...")
+	s.Client.Close()
+}
+
+func (s *cache) Get(ctx context.Context, key string) ([]byte, error) {
+	val, err := s.Client.Get(ctx, key).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil, nil
+	}
+	return []byte(val), err
+}
+
+func (s *cache) Set(ctx context.Context, key string, val []byte) error {
+	return s.Client.Set(ctx, key, val, 0).Err()
+}
+
+func (s *cache) SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error {
+	return s.Client.Set(ctx, key, val, expiry).Err()
+}
+
+func (s *cache) Delete(ctx context.Context, key string) error {
+	return s.Client.Del(ctx, key).Err()
+}

cloud/maplefile-backend/pkg/storage/object/s3/config.go

@@ -0,0 +1,62 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/object/s3/config.go
+package s3
+
+type S3ObjectStorageConfigurationProvider interface {
+	GetAccessKey() string
+	GetSecretKey() string
+	GetEndpoint() string
+	GetRegion() string
+	GetBucketName() string
+	GetIsPublicBucket() bool
+	GetUsePathStyle() bool
+}
+
+type s3ObjectStorageConfigurationProviderImpl struct {
+	accessKey      string `env:"AWS_ACCESS_KEY,required"`
+	secretKey      string `env:"AWS_SECRET_KEY,required"`
+	endpoint       string `env:"AWS_ENDPOINT,required"`
+	region         string `env:"AWS_REGION,required"`
+	bucketName     string `env:"AWS_BUCKET_NAME,required"`
+	isPublicBucket bool   `env:"AWS_IS_PUBLIC_BUCKET"`
+	usePathStyle   bool   `env:"AWS_USE_PATH_STYLE"`
+}
+
+func NewS3ObjectStorageConfigurationProvider(accessKey, secretKey, endpoint, region, bucketName string, isPublicBucket, usePathStyle bool) S3ObjectStorageConfigurationProvider {
+	return &s3ObjectStorageConfigurationProviderImpl{
+		accessKey:      accessKey,
+		secretKey:      secretKey,
+		endpoint:       endpoint,
+		region:         region,
+		bucketName:     bucketName,
+		isPublicBucket: isPublicBucket,
+		usePathStyle:   usePathStyle,
+	}
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetAccessKey() string {
+	return me.accessKey
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetSecretKey() string {
+	return me.secretKey
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetEndpoint() string {
+	return me.endpoint
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetRegion() string {
+	return me.region
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetBucketName() string {
+	return me.bucketName
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetIsPublicBucket() bool {
+	return me.isPublicBucket
+}
+
+func (me *s3ObjectStorageConfigurationProviderImpl) GetUsePathStyle() bool {
+	return me.usePathStyle
+}

cloud/maplefile-backend/pkg/storage/object/s3/provider.go

@@ -0,0 +1,21 @@
+package s3
+
+import (
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideS3ObjectStorageProvider provides an S3 object storage provider for Wire DI
+func ProvideS3ObjectStorageProvider(cfg *config.Config, logger *zap.Logger) S3ObjectStorage {
+	s3Config := NewS3ObjectStorageConfigurationProvider(
+		cfg.S3.AccessKey,
+		cfg.S3.SecretKey,
+		cfg.S3.Endpoint,
+		cfg.S3.Region,
+		cfg.S3.BucketName,
+		false,               // isPublicBucket - set to false for security
+		cfg.S3.UsePathStyle, // true for SeaweedFS/MinIO, false for DO Spaces/AWS S3
+	)
+	return NewObjectStorage(s3Config, logger)
+}

cloud/maplefile-backend/pkg/storage/object/s3/s3.go

@@ -0,0 +1,520 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/object/s3/s3.go
+package s3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"log"
+	"mime/multipart"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"go.uber.org/zap"
+)
+
+// ACL constants for public and private objects
+const (
+	ACLPrivate    = "private"
+	ACLPublicRead = "public-read"
+)
+
+type S3ObjectStorage interface {
+	UploadContent(ctx context.Context, objectKey string, content []byte) error
+	UploadContentWithVisibility(ctx context.Context, objectKey string, content []byte, isPublic bool) error
+	UploadContentFromMulipart(ctx context.Context, objectKey string, file multipart.File) error
+	UploadContentFromMulipartWithVisibility(ctx context.Context, objectKey string, file multipart.File, isPublic bool) error
+	BucketExists(ctx context.Context, bucketName string) (bool, error)
+	DeleteByKeys(ctx context.Context, key []string) error
+	Cut(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error
+	CutWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error
+	Copy(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error
+	CopyWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error
+	GetBinaryData(ctx context.Context, objectKey string) (io.ReadCloser, error)
+	DownloadToLocalfile(ctx context.Context, objectKey string, filePath string) (string, error)
+	ListAllObjects(ctx context.Context) (*s3.ListObjectsOutput, error)
+	FindMatchingObjectKey(s3Objects *s3.ListObjectsOutput, partialKey string) string
+	IsPublicBucket() bool
+	// GeneratePresignedUploadURL creates a presigned URL for uploading objects
+	GeneratePresignedUploadURL(ctx context.Context, key string, duration time.Duration) (string, error)
+	GetDownloadablePresignedURL(ctx context.Context, key string, duration time.Duration) (string, error)
+	ObjectExists(ctx context.Context, key string) (bool, error)
+	GetObjectSize(ctx context.Context, key string) (int64, error)
+}
+
+type s3ObjectStorage struct {
+	S3Client      *s3.Client
+	PresignClient *s3.PresignClient
+	Logger        *zap.Logger
+	BucketName    string
+	IsPublic      bool
+}
+
+// NewObjectStorage connects to a specific S3 bucket instance and returns a connected
+// instance structure.
+func NewObjectStorage(s3Config S3ObjectStorageConfigurationProvider, logger *zap.Logger) S3ObjectStorage {
+	logger = logger.Named("S3ObjectStorage")
+
+	// DEVELOPERS NOTE:
+	// How can I use the AWS SDK v2 for Go with DigitalOcean Spaces? via https://stackoverflow.com/a/74284205
+	logger = logger.With(zap.String("component", "☁️🗄️ s3-object-storage"))
+	logger.Debug("s3 initializing...")
+
+	// STEP 1: initialize the custom `endpoint` we will connect to.
+	customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) {
+		return aws.Endpoint{
+			URL: s3Config.GetEndpoint(),
+		}, nil
+	})
+
+	// STEP 2: Configure.
+	sdkConfig, err := config.LoadDefaultConfig(
+		context.TODO(), config.WithRegion(s3Config.GetRegion()),
+		config.WithEndpointResolverWithOptions(customResolver),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3Config.GetAccessKey(), s3Config.GetSecretKey(), "")),
+	)
+	if err != nil {
+		log.Fatalf("S3ObjectStorage failed loading default config with error: %v", err) // We need to crash the program at start to satisfy google wire requirement of having no errors.
+	}
+
+	// STEP 3: Load up s3 instance with configurable path-style addressing.
+	// UsePathStyle = true for MinIO/SeaweedFS (development)
+	// UsePathStyle = false for AWS S3/DigitalOcean Spaces (production)
+	s3Client := s3.NewFromConfig(sdkConfig, func(o *s3.Options) {
+		o.UsePathStyle = s3Config.GetUsePathStyle()
+	})
+
+	// Create our storage handler.
+	s3Storage := &s3ObjectStorage{
+		S3Client:      s3Client,
+		PresignClient: s3.NewPresignClient(s3Client),
+		Logger:        logger,
+		BucketName:    s3Config.GetBucketName(),
+		IsPublic:      s3Config.GetIsPublicBucket(),
+	}
+
+	logger.Debug("s3 checking remote connection...")
+
+	// STEP 4: Connect to the s3 bucket instance and confirm that bucket exists or create it.
+	doesExist, err := s3Storage.BucketExists(context.TODO(), s3Config.GetBucketName())
+	if err != nil {
+		log.Fatalf("S3ObjectStorage failed checking if bucket `%v` exists: %v\n", s3Config.GetBucketName(), err) // We need to crash the program at start to satisfy google wire requirement of having no errors.
+	}
+	if !doesExist {
+		logger.Debug("s3 bucket does not exist, creating it...", zap.String("bucket", s3Config.GetBucketName()))
+		_, createErr := s3Storage.S3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+			Bucket: aws.String(s3Config.GetBucketName()),
+		})
+		if createErr != nil {
+			log.Fatalf("S3ObjectStorage failed to create bucket `%v`: %v\n", s3Config.GetBucketName(), createErr)
+		}
+		logger.Debug("s3 bucket created successfully", zap.String("bucket", s3Config.GetBucketName()))
+	}
+
+	logger.Debug("s3 initialized")
+
+	// Return our s3 storage handler.
+	return s3Storage
+}
+
+// IsPublicBucket returns whether the bucket is configured as public by default
+func (s *s3ObjectStorage) IsPublicBucket() bool {
+	return s.IsPublic
+}
+
+// UploadContent uploads content using the default bucket visibility setting
+func (s *s3ObjectStorage) UploadContent(ctx context.Context, objectKey string, content []byte) error {
+	return s.UploadContentWithVisibility(ctx, objectKey, content, s.IsPublic)
+}
+
+// UploadContentWithVisibility uploads content with specified visibility (public or private)
+func (s *s3ObjectStorage) UploadContentWithVisibility(ctx context.Context, objectKey string, content []byte, isPublic bool) error {
+	acl := ACLPrivate
+	if isPublic {
+		acl = ACLPublicRead
+	}
+
+	s.Logger.Debug("Uploading content with visibility",
+		zap.String("objectKey", objectKey),
+		zap.Bool("isPublic", isPublic),
+		zap.String("acl", acl))
+
+	_, err := s.S3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(content),
+		ACL:    types.ObjectCannedACL(acl),
+	})
+	if err != nil {
+		s.Logger.Error("Failed to upload content",
+			zap.String("objectKey", objectKey),
+			zap.Bool("isPublic", isPublic),
+			zap.Any("error", err))
+		return err
+	}
+	return nil
+}
+
+// UploadContentFromMulipart uploads file using the default bucket visibility setting
+func (s *s3ObjectStorage) UploadContentFromMulipart(ctx context.Context, objectKey string, file multipart.File) error {
+	return s.UploadContentFromMulipartWithVisibility(ctx, objectKey, file, s.IsPublic)
+}
+
+// UploadContentFromMulipartWithVisibility uploads a multipart file with specified visibility
+func (s *s3ObjectStorage) UploadContentFromMulipartWithVisibility(ctx context.Context, objectKey string, file multipart.File, isPublic bool) error {
+	acl := ACLPrivate
+	if isPublic {
+		acl = ACLPublicRead
+	}
+
+	s.Logger.Debug("Uploading multipart file with visibility",
+		zap.String("objectKey", objectKey),
+		zap.Bool("isPublic", isPublic),
+		zap.String("acl", acl))
+
+	// Create the S3 upload input parameters
+	params := &s3.PutObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(objectKey),
+		Body:   file,
+		ACL:    types.ObjectCannedACL(acl),
+	}
+
+	// Perform the file upload to S3
+	_, err := s.S3Client.PutObject(ctx, params)
+	if err != nil {
+		s.Logger.Error("Failed to upload multipart file",
+			zap.String("objectKey", objectKey),
+			zap.Bool("isPublic", isPublic),
+			zap.Any("error", err))
+		return err
+	}
+	return nil
+}
+
+func (s *s3ObjectStorage) BucketExists(ctx context.Context, bucketName string) (bool, error) {
+	// Note: https://docs.aws.amazon.com/code-library/latest/ug/go_2_s3_code_examples.html#actions
+
+	_, err := s.S3Client.HeadBucket(ctx, &s3.HeadBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	exists := true
+	if err != nil {
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) {
+			switch apiError.(type) {
+			case *types.NotFound:
+				log.Printf("Bucket %v is available.\n", bucketName)
+				exists = false
+				err = nil
+			default:
+				log.Printf("Either you don't have access to bucket %v or another error occurred. "+
+					"Here's what happened: %v\n", bucketName, err)
+			}
+		}
+	}
+
+	return exists, err
+}
+
+func (s *s3ObjectStorage) GetDownloadablePresignedURL(ctx context.Context, key string, duration time.Duration) (string, error) {
+	// DEVELOPERS NOTE:
+	// AWS S3 Bucket — presigned URL APIs with Go (2022) via https://ronen-niv.medium.com/aws-s3-handling-presigned-urls-2718ab247d57
+
+	presignedUrl, err := s.PresignClient.PresignGetObject(context.Background(),
+		&s3.GetObjectInput{
+			Bucket:                     aws.String(s.BucketName),
+			Key:                        aws.String(key),
+			ResponseContentDisposition: aws.String("attachment"), // This field allows the file to download it directly from your browser
+		},
+		s3.WithPresignExpires(duration))
+	if err != nil {
+		return "", err
+	}
+
+	// Note: The URL will contain the internal endpoint hostname by default.
+	// URL hostname replacement should be done at a higher level if needed
+	// (e.g., in the repository layer with PublicEndpoint config)
+	return presignedUrl.URL, nil
+}
+
+func (s *s3ObjectStorage) DeleteByKeys(ctx context.Context, objectKeys []string) error {
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+
+	var objectIds []types.ObjectIdentifier
+	for _, key := range objectKeys {
+		objectIds = append(objectIds, types.ObjectIdentifier{Key: aws.String(key)})
+	}
+	_, err := s.S3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
+		Bucket: aws.String(s.BucketName),
+		Delete: &types.Delete{Objects: objectIds},
+	})
+	if err != nil {
+		log.Printf("Couldn't delete objects from bucket %v. Here's why: %v\n", s.BucketName, err)
+	}
+	return err
+}
+
+// Cut moves a file using the default bucket visibility setting
+func (s *s3ObjectStorage) Cut(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error {
+	return s.CutWithVisibility(ctx, sourceObjectKey, destinationObjectKey, s.IsPublic)
+}
+
+// CutWithVisibility moves a file with specified visibility
+func (s *s3ObjectStorage) CutWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error {
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second) // Increase timout so it runs longer then usual to handle this unique case.
+	defer cancel()
+
+	// First copy the object with the desired visibility
+	if err := s.CopyWithVisibility(ctx, sourceObjectKey, destinationObjectKey, isPublic); err != nil {
+		return err
+	}
+
+	// Delete the original object
+	_, deleteErr := s.S3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(sourceObjectKey),
+	})
+	if deleteErr != nil {
+		s.Logger.Error("Failed to delete original object:", zap.Any("deleteErr", deleteErr))
+		return deleteErr
+	}
+
+	s.Logger.Debug("Original object deleted.")
+
+	return nil
+}
+
+// Copy copies a file using the default bucket visibility setting
+func (s *s3ObjectStorage) Copy(ctx context.Context, sourceObjectKey string, destinationObjectKey string) error {
+	return s.CopyWithVisibility(ctx, sourceObjectKey, destinationObjectKey, s.IsPublic)
+}
+
+// CopyWithVisibility copies a file with specified visibility
+func (s *s3ObjectStorage) CopyWithVisibility(ctx context.Context, sourceObjectKey string, destinationObjectKey string, isPublic bool) error {
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second) // Increase timout so it runs longer then usual to handle this unique case.
+	defer cancel()
+
+	acl := ACLPrivate
+	if isPublic {
+		acl = ACLPublicRead
+	}
+
+	s.Logger.Debug("Copying object with visibility",
+		zap.String("sourceKey", sourceObjectKey),
+		zap.String("destinationKey", destinationObjectKey),
+		zap.Bool("isPublic", isPublic),
+		zap.String("acl", acl))
+
+	_, copyErr := s.S3Client.CopyObject(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(s.BucketName),
+		CopySource: aws.String(s.BucketName + "/" + sourceObjectKey),
+		Key:        aws.String(destinationObjectKey),
+		ACL:        types.ObjectCannedACL(acl),
+	})
+	if copyErr != nil {
+		s.Logger.Error("Failed to copy object:",
+			zap.String("sourceKey", sourceObjectKey),
+			zap.String("destinationKey", destinationObjectKey),
+			zap.Bool("isPublic", isPublic),
+			zap.Any("copyErr", copyErr))
+		return copyErr
+	}
+
+	s.Logger.Debug("Object copied successfully.")
+
+	return nil
+}
+
+// GetBinaryData function will return the binary data for the particular key.
+func (s *s3ObjectStorage) GetBinaryData(ctx context.Context, objectKey string) (io.ReadCloser, error) {
+	input := &s3.GetObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(objectKey),
+	}
+
+	s3object, err := s.S3Client.GetObject(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+	return s3object.Body, nil
+}
+
+func (s *s3ObjectStorage) DownloadToLocalfile(ctx context.Context, objectKey string, filePath string) (string, error) {
+	responseBin, err := s.GetBinaryData(ctx, objectKey)
+	if err != nil {
+		return filePath, err
+	}
+	out, err := os.Create(filePath)
+	if err != nil {
+		return filePath, err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, responseBin)
+	if err != nil {
+		return "", err
+	}
+	return filePath, err
+}
+
+func (s *s3ObjectStorage) ListAllObjects(ctx context.Context) (*s3.ListObjectsOutput, error) {
+	input := &s3.ListObjectsInput{
+		Bucket: aws.String(s.BucketName),
+	}
+
+	objects, err := s.S3Client.ListObjects(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	return objects, nil
+}
+
+// Function will iterate over all the s3 objects to match the partial key with
+// the actual key found in the S3 bucket.
+func (s *s3ObjectStorage) FindMatchingObjectKey(s3Objects *s3.ListObjectsOutput, partialKey string) string {
+	for _, obj := range s3Objects.Contents {
+
+		match := strings.Contains(*obj.Key, partialKey)
+
+		// If a match happens then it means we have found the ACTUAL KEY in the
+		// s3 objects inside the bucket.
+		if match == true {
+			return *obj.Key
+		}
+	}
+	return ""
+}
+
+// GeneratePresignedUploadURL creates a presigned URL for uploading objects to S3
+func (s *s3ObjectStorage) GeneratePresignedUploadURL(ctx context.Context, key string, duration time.Duration) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	// Create PutObjectInput without ACL to avoid requiring x-amz-acl header
+	putObjectInput := &s3.PutObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(key),
+		// Removed ACL field - files inherit bucket's default privacy settings.
+	}
+
+	presignedUrl, err := s.PresignClient.PresignPutObject(ctx, putObjectInput, s3.WithPresignExpires(duration))
+	if err != nil {
+		s.Logger.Error("Failed to generate presigned upload URL",
+			zap.String("key", key),
+			zap.Duration("duration", duration),
+			zap.Error(err))
+		return "", err
+	}
+
+	s.Logger.Debug("Generated presigned upload URL",
+		zap.String("key", key),
+		zap.Duration("duration", duration))
+
+	// Replace internal Docker hostname with localhost for frontend access
+	// This allows the browser (outside Docker) to access the nginx proxy
+	url := presignedUrl.URL
+	url = strings.Replace(url, "http://nginx-s3-proxy:8334", "http://localhost:8334", 1)
+	url = strings.Replace(url, "http://seaweedfs:8333", "http://localhost:8333", 1)
+
+	return url, nil
+}
+
+// ObjectExists checks if an object exists at the given key using HeadObject
+func (s *s3ObjectStorage) ObjectExists(ctx context.Context, key string) (bool, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	_, err := s.S3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(key),
+	})
+
+	if err != nil {
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) {
+			switch apiError.(type) {
+			case *types.NotFound:
+				// Object doesn't exist
+				s.Logger.Debug("Object does not exist",
+					zap.String("key", key))
+				return false, nil
+			case *types.NoSuchKey:
+				// Object doesn't exist
+				s.Logger.Debug("Object does not exist (NoSuchKey)",
+					zap.String("key", key))
+				return false, nil
+			default:
+				// Some other error occurred
+				s.Logger.Error("Error checking object existence",
+					zap.String("key", key),
+					zap.Error(err))
+				return false, err
+			}
+		}
+		// Non-API error
+		s.Logger.Error("Error checking object existence",
+			zap.String("key", key),
+			zap.Error(err))
+		return false, err
+	}
+
+	s.Logger.Debug("Object exists",
+		zap.String("key", key))
+	return true, nil
+}
+
+// GetObjectSize returns the size of an object at the given key using HeadObject
+func (s *s3ObjectStorage) GetObjectSize(ctx context.Context, key string) (int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	result, err := s.S3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(s.BucketName),
+		Key:    aws.String(key),
+	})
+
+	if err != nil {
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) {
+			switch apiError.(type) {
+			case *types.NotFound:
+				s.Logger.Debug("Object not found when getting size",
+					zap.String("key", key))
+				return 0, errors.New("object not found")
+			case *types.NoSuchKey:
+				s.Logger.Debug("Object not found when getting size (NoSuchKey)",
+					zap.String("key", key))
+				return 0, errors.New("object not found")
+			default:
+				s.Logger.Error("Error getting object size",
+					zap.String("key", key),
+					zap.Error(err))
+				return 0, err
+			}
+		}
+		s.Logger.Error("Error getting object size",
+			zap.String("key", key),
+			zap.Error(err))
+		return 0, err
+	}
+
+	// Let's use aws.ToInt64 which handles both pointer and non-pointer cases
+	size := aws.ToInt64(result.ContentLength)
+
+	s.Logger.Debug("Retrieved object size",
+		zap.String("key", key),
+		zap.Int64("size", size))
+
+	return size, nil
+}

cloud/maplefile-backend/pkg/storage/utils/size_formatter.go

@@ -0,0 +1,112 @@
+// monorepo/cloud/maplefile-backend/pkg/storage/utils/size_formatter.go
+package utils
+
+import (
+	"fmt"
+	"math"
+)
+
+// StorageSizeUnit represents different storage size units
+type StorageSizeUnit string
+
+const (
+	UnitBytes     StorageSizeUnit = "B"
+	UnitKilobytes StorageSizeUnit = "KB"
+	UnitMegabytes StorageSizeUnit = "MB"
+	UnitGigabytes StorageSizeUnit = "GB"
+	UnitTerabytes StorageSizeUnit = "TB"
+	UnitPetabytes StorageSizeUnit = "PB"
+)
+
+// FormattedSize represents a storage size with value and unit
+type FormattedSize struct {
+	Value float64         `json:"value"`
+	Unit  StorageSizeUnit `json:"unit"`
+	Raw   int64           `json:"raw_bytes"`
+}
+
+// String returns a human-readable string representation
+func (fs FormattedSize) String() string {
+	if fs.Value == math.Trunc(fs.Value) {
+		return fmt.Sprintf("%.0f %s", fs.Value, fs.Unit)
+	}
+	return fmt.Sprintf("%.2f %s", fs.Value, fs.Unit)
+}
+
+// FormatBytes converts bytes to a human-readable format
+func FormatBytes(bytes int64) FormattedSize {
+	if bytes == 0 {
+		return FormattedSize{Value: 0, Unit: UnitBytes, Raw: 0}
+	}
+
+	const unit = 1024
+	if bytes < unit {
+		return FormattedSize{
+			Value: float64(bytes),
+			Unit:  UnitBytes,
+			Raw:   bytes,
+		}
+	}
+
+	div, exp := int64(unit), 0
+	for n := bytes / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+
+	units := []StorageSizeUnit{UnitKilobytes, UnitMegabytes, UnitGigabytes, UnitTerabytes, UnitPetabytes}
+
+	return FormattedSize{
+		Value: math.Round(float64(bytes)/float64(div)*100) / 100,
+		Unit:  units[exp],
+		Raw:   bytes,
+	}
+}
+
+// FormatBytesWithPrecision converts bytes to human-readable format with specified decimal places
+func FormatBytesWithPrecision(bytes int64, precision int) FormattedSize {
+	formatted := FormatBytes(bytes)
+
+	// Round to specified precision
+	multiplier := math.Pow(10, float64(precision))
+	formatted.Value = math.Round(formatted.Value*multiplier) / multiplier
+
+	return formatted
+}
+
+// Enhanced response types with formatted sizes
+type StorageSizeResponseFormatted struct {
+	TotalSizeBytes     int64         `json:"total_size_bytes"`
+	TotalSizeFormatted FormattedSize `json:"total_size_formatted"`
+}
+
+type StorageSizeBreakdownResponseFormatted struct {
+	OwnedSizeBytes               int64                    `json:"owned_size_bytes"`
+	OwnedSizeFormatted           FormattedSize            `json:"owned_size_formatted"`
+	SharedSizeBytes              int64                    `json:"shared_size_bytes"`
+	SharedSizeFormatted          FormattedSize            `json:"shared_size_formatted"`
+	TotalSizeBytes               int64                    `json:"total_size_bytes"`
+	TotalSizeFormatted           FormattedSize            `json:"total_size_formatted"`
+	CollectionBreakdownBytes     map[string]int64         `json:"collection_breakdown_bytes"`
+	CollectionBreakdownFormatted map[string]FormattedSize `json:"collection_breakdown_formatted"`
+	OwnedCollectionsCount        int                      `json:"owned_collections_count"`
+	SharedCollectionsCount       int                      `json:"shared_collections_count"`
+}
+
+// Example usage and outputs:
+/*
+FormatBytes(1024) -> {Value: 1, Unit: "KB", Raw: 1024} -> "1 KB"
+FormatBytes(1536) -> {Value: 1.5, Unit: "KB", Raw: 1536} -> "1.50 KB"
+FormatBytes(1073741824) -> {Value: 1, Unit: "GB", Raw: 1073741824} -> "1 GB"
+FormatBytes(2684354560) -> {Value: 2.5, Unit: "GB", Raw: 2684354560} -> "2.50 GB"
+
+Example formatted response:
+{
+  "total_size_bytes": 2684354560,
+  "total_size_formatted": {
+    "value": 2.5,
+    "unit": "GB",
+    "raw_bytes": 2684354560
+  }
+}
+*/