Initial commit: Open sourcing all of the Maple Open Technologies code.

cloud/maplefile-backend/pkg/storage/memory/inmemory/memory.go

@@ -0,0 +1,202 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/inmemory/memory.go
+package inmemory
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/pkg/storage"
+	"go.uber.org/zap"
+)
+
+type cacheValue struct {
+	value []byte
+}
+
+// keyValueStorerImpl implements the db.Database interface.
+// It uses a LevelDB database to store key-value pairs.
+type keyValueStorerImpl struct {
+	data   map[string]cacheValue
+	txData map[string]cacheValue
+	lock   sync.Mutex
+}
+
+// NewInMemoryStorage creates a new instance of the keyValueStorerImpl.
+func NewInMemoryStorage(logger *zap.Logger) storage.Storage {
+	logger = logger.Named("InMemoryStorage")
+	return &keyValueStorerImpl{
+		data:   make(map[string]cacheValue),
+		txData: nil,
+	}
+}
+
+// Get retrieves a value from the database by its key.
+// It returns an error if the key is not found.
+func (impl *keyValueStorerImpl) Get(k string) ([]byte, error) {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		cachedValue, ok := impl.txData[k]
+		if !ok {
+			return nil, fmt.Errorf("does not exist for: %v", k)
+		}
+		return cachedValue.value, nil
+	} else {
+		cachedValue, ok := impl.data[k]
+		if !ok {
+			return nil, fmt.Errorf("does not exist for: %v", k)
+		}
+		return cachedValue.value, nil
+	}
+}
+
+// Set sets a value in the database by its key.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Set(k string, val []byte) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		impl.txData[k] = cacheValue{
+			value: val,
+		}
+	} else {
+		impl.data[k] = cacheValue{
+			value: val,
+		}
+	}
+	return nil
+}
+
+// Delete deletes a value from the database by its key.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Delete(k string) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		delete(impl.txData, k)
+	} else {
+		delete(impl.data, k)
+	}
+	return nil
+}
+
+// Iterate iterates over the key-value pairs in the database, starting from the specified key prefix.
+// It calls the provided function for each pair.
+// It returns an error if the iteration fails.
+func (impl *keyValueStorerImpl) Iterate(processFunc func(key, value []byte) error) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.txData {
+			// Call the provided function for each pair
+			if err := processFunc([]byte(k), v.value); err != nil {
+				return err
+			}
+		}
+	} else {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.data {
+			// Call the provided function for each pair
+			if err := processFunc([]byte(k), v.value); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) IterateWithFilterByKeys(ks []string, processFunc func(key, value []byte) error) error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	if impl.txData != nil {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.txData {
+			// Iterate over our keys to search by.
+			for _, searchK := range ks {
+				// If the item we currently have matches our keys then execute.
+				if k == searchK {
+					// Call the provided function for each pair
+					if err := processFunc([]byte(k), v.value); err != nil {
+						return err
+					}
+				}
+			}
+
+		}
+	} else {
+		// Iterate over the key-value pairs in the database, starting from the starting point
+		for k, v := range impl.data {
+			// Iterate over our keys to search by.
+			for _, searchK := range ks {
+				// If the item we currently have matches our keys then execute.
+				if k == searchK {
+					// Call the provided function for each pair
+					if err := processFunc([]byte(k), v.value); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// Close closes the database.
+// It returns an error if the operation fails.
+func (impl *keyValueStorerImpl) Close() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Clear the data map
+	impl.data = make(map[string]cacheValue)
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) OpenTransaction() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Create a new transaction by creating a copy of the current data
+	impl.txData = make(map[string]cacheValue)
+	for k, v := range impl.data {
+		impl.txData[k] = v
+	}
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) CommitTransaction() error {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Check if a transaction is in progress
+	if impl.txData == nil {
+		return errors.New("no transaction in progress")
+	}
+
+	// Update the current data with the transaction data
+	impl.data = impl.txData
+	impl.txData = nil
+
+	return nil
+}
+
+func (impl *keyValueStorerImpl) DiscardTransaction() {
+	impl.lock.Lock()
+	defer impl.lock.Unlock()
+
+	// Check if a transaction is in progress
+	if impl.txData != nil {
+		impl.txData = nil
+	}
+}

cloud/maplefile-backend/pkg/storage/memory/inmemory/memory_test.go

@@ -0,0 +1,295 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/inmemory/memory_test.go
+package inmemory
+
+import (
+	"reflect"
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+// TestNewInMemoryStorage verifies that the NewInMemoryStorage function
+// correctly initializes a new storage instance
+func TestNewInMemoryStorage(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	if storage == nil {
+		t.Fatal("Expected non-nil storage instance")
+	}
+
+	// Type assertion to verify we get the correct implementation
+	_, ok := storage.(*keyValueStorerImpl)
+	if !ok {
+		t.Fatal("Expected keyValueStorerImpl instance")
+	}
+}
+
+// TestBasicOperations tests the basic Set/Get/Delete operations
+func TestBasicOperations(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Test Set and Get
+	t.Run("Set and Get", func(t *testing.T) {
+		key := "test-key"
+		value := []byte("test-value")
+
+		err := storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Set failed: %v", err)
+		}
+
+		retrieved, err := storage.Get(key)
+		if err != nil {
+			t.Fatalf("Get failed: %v", err)
+		}
+
+		if !reflect.DeepEqual(retrieved, value) {
+			t.Errorf("Retrieved value doesn't match: got %v, want %v", retrieved, value)
+		}
+	})
+
+	// Test Get with non-existent key
+	t.Run("Get Non-existent", func(t *testing.T) {
+		_, err := storage.Get("non-existent")
+		if err == nil {
+			t.Error("Expected error for non-existent key")
+		}
+	})
+
+	// Test Delete
+	t.Run("Delete", func(t *testing.T) {
+		key := "delete-test"
+		value := []byte("delete-value")
+
+		// First set a value
+		err := storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Set failed: %v", err)
+		}
+
+		// Delete it
+		err = storage.Delete(key)
+		if err != nil {
+			t.Fatalf("Delete failed: %v", err)
+		}
+
+		// Verify it's gone
+		_, err = storage.Get(key)
+		if err == nil {
+			t.Error("Expected error after deletion")
+		}
+	})
+}
+
+// TestIteration tests the Iterate functionality
+func TestIteration(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Prepare test data
+	testData := map[string][]byte{
+		"key1": []byte("value1"),
+		"key2": []byte("value2"),
+		"key3": []byte("value3"),
+	}
+
+	// Insert test data
+	for k, v := range testData {
+		if err := storage.Set(k, v); err != nil {
+			t.Fatalf("Failed to set test data: %v", err)
+		}
+	}
+
+	// Test basic iteration
+	t.Run("Basic Iteration", func(t *testing.T) {
+		found := make(map[string][]byte)
+
+		err := storage.Iterate(func(key, value []byte) error {
+			found[string(key)] = value
+			return nil
+		})
+
+		if err != nil {
+			t.Fatalf("Iteration failed: %v", err)
+		}
+
+		if !reflect.DeepEqual(testData, found) {
+			t.Errorf("Iteration results don't match: got %v, want %v", found, testData)
+		}
+	})
+
+	// Test filtered iteration
+	t.Run("Filtered Iteration", func(t *testing.T) {
+		filterKeys := []string{"key1", "key3"}
+		found := make(map[string][]byte)
+
+		err := storage.IterateWithFilterByKeys(filterKeys, func(key, value []byte) error {
+			found[string(key)] = value
+			return nil
+		})
+
+		if err != nil {
+			t.Fatalf("Filtered iteration failed: %v", err)
+		}
+
+		// Verify only requested keys were returned
+		if len(found) != len(filterKeys) {
+			t.Errorf("Expected %d items, got %d", len(filterKeys), len(found))
+		}
+
+		for _, k := range filterKeys {
+			if !reflect.DeepEqual(found[k], testData[k]) {
+				t.Errorf("Filtered data mismatch for key %s: got %v, want %v", k, found[k], testData[k])
+			}
+		}
+	})
+}
+
+// TestTransactions tests the transaction-related functionality
+func TestTransactions(t *testing.T) {
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Test basic transaction commit
+	t.Run("Transaction Commit", func(t *testing.T) {
+		// Start transaction
+		err := storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open transaction: %v", err)
+		}
+
+		// Make changes in transaction
+		key := "tx-test"
+		value := []byte("tx-value")
+
+		err = storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Failed to set in transaction: %v", err)
+		}
+
+		// Commit transaction
+		err = storage.CommitTransaction()
+		if err != nil {
+			t.Fatalf("Failed to commit transaction: %v", err)
+		}
+
+		// Verify changes persisted
+		retrieved, err := storage.Get(key)
+		if err != nil {
+			t.Fatalf("Failed to get after commit: %v", err)
+		}
+
+		if !reflect.DeepEqual(retrieved, value) {
+			t.Errorf("Retrieved value doesn't match after commit: got %v, want %v", retrieved, value)
+		}
+	})
+
+	// Test transaction discard
+	t.Run("Transaction Discard", func(t *testing.T) {
+		// Start transaction
+		err := storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open transaction: %v", err)
+		}
+
+		// Make changes in transaction
+		key := "discard-test"
+		value := []byte("discard-value")
+
+		err = storage.Set(key, value)
+		if err != nil {
+			t.Fatalf("Failed to set in transaction: %v", err)
+		}
+
+		// Discard transaction
+		storage.DiscardTransaction()
+
+		// Verify changes were not persisted
+		_, err = storage.Get(key)
+		if err == nil {
+			t.Error("Expected error getting discarded value")
+		}
+	})
+
+	// Test transaction behavior with multiple opens
+	t.Run("Multiple Transaction Opens", func(t *testing.T) {
+		// Set initial value
+		err := storage.Set("tx-test", []byte("initial"))
+		if err != nil {
+			t.Fatalf("Failed to set initial value: %v", err)
+		}
+
+		// First transaction
+		err = storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open first transaction: %v", err)
+		}
+
+		// Modify value
+		err = storage.Set("tx-test", []byte("modified"))
+		if err != nil {
+			t.Fatalf("Failed to set value in transaction: %v", err)
+		}
+
+		// Opening another transaction while one is in progress overwrites the transaction data
+		err = storage.OpenTransaction()
+		if err != nil {
+			t.Fatalf("Failed to open second transaction: %v", err)
+		}
+
+		// Modify value again
+		err = storage.Set("tx-test", []byte("final"))
+		if err != nil {
+			t.Fatalf("Failed to set value in second transaction: %v", err)
+		}
+
+		// Commit the transaction (only need to commit once as there's only one transaction state)
+		err = storage.CommitTransaction()
+		if err != nil {
+			t.Fatalf("Failed to commit transaction: %v", err)
+		}
+
+		// Verify attempting to commit again fails since transaction state is cleared
+		err = storage.CommitTransaction()
+		if err == nil {
+			t.Error("Expected error when committing with no transaction in progress")
+		}
+
+		// Verify final value
+		val, err := storage.Get("tx-test")
+		if err != nil {
+			t.Fatalf("Failed to get final value: %v", err)
+		}
+
+		if !reflect.DeepEqual(val, []byte("final")) {
+			t.Errorf("Unexpected final value: got %s, want %s", string(val), "final")
+		}
+	})
+}
+
+// TestClose verifies the Close functionality
+func TestClose(t *testing.T) {
+
+	logger, _ := zap.NewDevelopment()
+	storage := NewInMemoryStorage(logger)
+
+	// Add some data
+	err := storage.Set("test", []byte("value"))
+	if err != nil {
+		t.Fatalf("Failed to set test data: %v", err)
+	}
+
+	// Close storage
+	err = storage.Close()
+	if err != nil {
+		t.Fatalf("Close failed: %v", err)
+	}
+
+	// Verify data is cleared
+	_, err = storage.Get("test")
+	if err == nil {
+		t.Error("Expected error getting value after close")
+	}
+}

cloud/maplefile-backend/pkg/storage/memory/redis/client_provider.go

@@ -0,0 +1,41 @@
+package redis
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideRedisUniversalClient provides a Redis UniversalClient for Wire DI
+// This is needed for components like leader election that require the raw Redis client
+func ProvideRedisUniversalClient(cfg *config.Config, logger *zap.Logger) (redis.UniversalClient, error) {
+	logger = logger.Named("RedisClient")
+
+	// Create Redis client
+	client := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("%s:%d", cfg.Cache.Host, cfg.Cache.Port),
+		Password: cfg.Cache.Password,
+		DB:       cfg.Cache.DB,
+	})
+
+	// Test connection
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if _, err := client.Ping(ctx).Result(); err != nil {
+		logger.Error("Failed to connect to Redis", zap.Error(err))
+		return nil, fmt.Errorf("redis connection failed: %w", err)
+	}
+
+	logger.Info("✅ Redis client connected successfully",
+		zap.String("host", cfg.Cache.Host),
+		zap.Int("port", cfg.Cache.Port),
+		zap.Int("db", cfg.Cache.DB))
+
+	return client, nil
+}

cloud/maplefile-backend/pkg/storage/memory/redis/provider.go

@@ -0,0 +1,12 @@
+package redis
+
+import (
+	"go.uber.org/zap"
+
+	"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+)
+
+// ProvideRedisCache provides a Redis cache instance for Wire DI
+func ProvideRedisCache(cfg *config.Config, logger *zap.Logger) Cacher {
+	return NewCache(cfg, logger)
+}

cloud/maplefile-backend/pkg/storage/memory/redis/redis.go

@@ -0,0 +1,73 @@
+// monorepo/cloud/maplefileapps-backend/pkg/storage/memory/redis/redis.go
+package redis
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	c "codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
+	"github.com/redis/go-redis/v9"
+	"go.uber.org/zap"
+)
+
+type Cacher interface {
+	Shutdown(ctx context.Context)
+	Get(ctx context.Context, key string) ([]byte, error)
+	Set(ctx context.Context, key string, val []byte) error
+	SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error
+	Delete(ctx context.Context, key string) error
+}
+
+type cache struct {
+	Client *redis.Client
+	Logger *zap.Logger
+}
+
+func NewCache(cfg *c.Configuration, logger *zap.Logger) Cacher {
+	logger = logger.Named("Redis Memory Storage")
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("%s:%d", cfg.Cache.Host, cfg.Cache.Port),
+		Password: cfg.Cache.Password,
+		DB:       cfg.Cache.DB,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if _, err := rdb.Ping(ctx).Result(); err != nil {
+		logger.Fatal("failed connecting to Redis", zap.Error(err))
+	}
+
+	return &cache{
+		Client: rdb,
+		Logger: logger,
+	}
+}
+
+func (s *cache) Shutdown(ctx context.Context) {
+	s.Logger.Info("shutting down Redis cache...")
+	s.Client.Close()
+}
+
+func (s *cache) Get(ctx context.Context, key string) ([]byte, error) {
+	val, err := s.Client.Get(ctx, key).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil, nil
+	}
+	return []byte(val), err
+}
+
+func (s *cache) Set(ctx context.Context, key string, val []byte) error {
+	return s.Client.Set(ctx, key, val, 0).Err()
+}
+
+func (s *cache) SetWithExpiry(ctx context.Context, key string, val []byte, expiry time.Duration) error {
+	return s.Client.Set(ctx, key, val, expiry).Err()
+}
+
+func (s *cache) Delete(ctx context.Context, key string) error {
+	return s.Client.Del(ctx, key).Err()
+}