Initial commit: Open sourcing all of the Maple Open Technologies code.
This commit is contained in:
commit
755d54a99d
2010 changed files with 448675 additions and 0 deletions
|
|
@ -0,0 +1,88 @@
|
|||
// monorepo/cloud/maplefile-backend/internal/maplefile/repo/storageusageevent/create.go
|
||||
package storageusageevent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/storageusageevent"
|
||||
)
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) Create(ctx context.Context, event *storageusageevent.StorageUsageEvent) error {
|
||||
if event == nil {
|
||||
return fmt.Errorf("storage usage event cannot be nil")
|
||||
}
|
||||
|
||||
// Ensure event day is truncated to date only
|
||||
event.EventDay = event.EventDay.Truncate(24 * time.Hour)
|
||||
|
||||
// Set event time if not provided
|
||||
if event.EventTime.IsZero() {
|
||||
event.EventTime = time.Now()
|
||||
}
|
||||
|
||||
query := `INSERT INTO maplefile.storage_usage_events_by_user_id_and_event_day_with_asc_event_time
|
||||
(user_id, event_day, event_time, file_size, operation)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
|
||||
err := impl.Session.Query(query,
|
||||
event.UserID,
|
||||
event.EventDay,
|
||||
event.EventTime,
|
||||
event.FileSize,
|
||||
event.Operation).WithContext(ctx).Exec()
|
||||
|
||||
if err != nil {
|
||||
impl.Logger.Error("failed to create storage usage event",
|
||||
zap.String("user_id", event.UserID.String()),
|
||||
zap.String("operation", event.Operation),
|
||||
zap.Int64("file_size", event.FileSize),
|
||||
zap.Error(err))
|
||||
return fmt.Errorf("failed to create storage usage event: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) CreateMany(ctx context.Context, events []*storageusageevent.StorageUsageEvent) error {
|
||||
if len(events) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
batch := impl.Session.NewBatch(gocql.LoggedBatch).WithContext(ctx)
|
||||
|
||||
for _, event := range events {
|
||||
if event == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure event day is truncated to date only
|
||||
event.EventDay = event.EventDay.Truncate(24 * time.Hour)
|
||||
|
||||
// Set event time if not provided
|
||||
if event.EventTime.IsZero() {
|
||||
event.EventTime = time.Now()
|
||||
}
|
||||
|
||||
batch.Query(`INSERT INTO maplefile.storage_usage_events_by_user_id_and_event_day_with_asc_event_time
|
||||
(user_id, event_day, event_time, file_size, operation)
|
||||
VALUES (?, ?, ?, ?, ?)`,
|
||||
event.UserID,
|
||||
event.EventDay,
|
||||
event.EventTime,
|
||||
event.FileSize,
|
||||
event.Operation)
|
||||
}
|
||||
|
||||
err := impl.Session.ExecuteBatch(batch)
|
||||
if err != nil {
|
||||
impl.Logger.Error("failed to create multiple storage usage events", zap.Error(err))
|
||||
return fmt.Errorf("failed to create multiple storage usage events: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
// monorepo/cloud/maplefile-backend/internal/maplefile/repo/storageusageevent/delete.go
|
||||
package storageusageevent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) DeleteByUserAndDay(ctx context.Context, userID gocql.UUID, eventDay time.Time) error {
|
||||
// Ensure event day is truncated to date only
|
||||
eventDay = eventDay.Truncate(24 * time.Hour)
|
||||
|
||||
query := `DELETE FROM maplefile.storage_usage_events_by_user_id_and_event_day_with_asc_event_time
|
||||
WHERE user_id = ? AND event_day = ?`
|
||||
|
||||
err := impl.Session.Query(query, userID, eventDay).WithContext(ctx).Exec()
|
||||
if err != nil {
|
||||
impl.Logger.Error("failed to delete storage usage events by user and day", zap.Error(err))
|
||||
return fmt.Errorf("failed to delete storage usage events: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteByUserID deletes all storage usage events for a user (all days)
|
||||
// Used for GDPR right-to-be-forgotten implementation
|
||||
//
|
||||
// NOTE: Because storage_usage_events table is partitioned by (user_id, event_day),
|
||||
// we need to query to find all event_day values first, then delete each partition.
|
||||
// For efficiency, we'll delete up to 2 years of data (should cover most reasonable usage).
|
||||
func (impl *storageUsageEventRepositoryImpl) DeleteByUserID(ctx context.Context, userID gocql.UUID) error {
|
||||
// Delete events from the last 2 years (730 days)
|
||||
// This should cover all reasonable user data retention periods
|
||||
endDay := time.Now().Truncate(24 * time.Hour)
|
||||
startDay := endDay.Add(-730 * 24 * time.Hour) // 2 years ago
|
||||
|
||||
impl.Logger.Info("Deleting storage usage events for user",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Time("start_day", startDay),
|
||||
zap.Time("end_day", endDay))
|
||||
|
||||
// Use batch delete for efficiency
|
||||
batch := impl.Session.NewBatch(gocql.LoggedBatch).WithContext(ctx)
|
||||
deletedDays := 0
|
||||
|
||||
// Delete each day's partition
|
||||
for day := startDay; !day.After(endDay); day = day.Add(24 * time.Hour) {
|
||||
query := `DELETE FROM maplefile.storage_usage_events_by_user_id_and_event_day_with_asc_event_time
|
||||
WHERE user_id = ? AND event_day = ?`
|
||||
batch.Query(query, userID, day)
|
||||
deletedDays++
|
||||
|
||||
// Execute batch every 100 days to avoid batch size limits
|
||||
if deletedDays%100 == 0 {
|
||||
if err := impl.Session.ExecuteBatch(batch); err != nil {
|
||||
impl.Logger.Error("failed to execute batch delete for storage usage events",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Int("days_in_batch", 100),
|
||||
zap.Error(err))
|
||||
return fmt.Errorf("failed to delete storage usage events for user %s: %w", userID.String(), err)
|
||||
}
|
||||
// Create new batch for next set of days
|
||||
batch = impl.Session.NewBatch(gocql.LoggedBatch).WithContext(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute remaining batch
|
||||
if batch.Size() > 0 {
|
||||
if err := impl.Session.ExecuteBatch(batch); err != nil {
|
||||
impl.Logger.Error("failed to execute final batch delete for storage usage events",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Int("days_in_final_batch", batch.Size()),
|
||||
zap.Error(err))
|
||||
return fmt.Errorf("failed to delete storage usage events for user %s: %w", userID.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
impl.Logger.Info("✅ Deleted all storage usage events for user",
|
||||
zap.String("user_id", userID.String()),
|
||||
zap.Int("total_days_deleted", deletedDays))
|
||||
|
||||
return nil
|
||||
}
|
||||
148
cloud/maplefile-backend/internal/repo/storageusageevent/get.go
Normal file
148
cloud/maplefile-backend/internal/repo/storageusageevent/get.go
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
// monorepo/cloud/maplefile-backend/internal/maplefile/repo/storageusageevent/get.go
|
||||
package storageusageevent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/storageusageevent"
|
||||
)
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetByUserAndDay(ctx context.Context, userID gocql.UUID, eventDay time.Time) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
// Ensure event day is truncated to date only
|
||||
eventDay = eventDay.Truncate(24 * time.Hour)
|
||||
|
||||
query := `SELECT user_id, event_day, event_time, file_size, operation
|
||||
FROM maplefile.storage_usage_events_by_user_id_and_event_day_with_asc_event_time
|
||||
WHERE user_id = ? AND event_day = ?`
|
||||
|
||||
iter := impl.Session.Query(query, userID, eventDay).WithContext(ctx).Iter()
|
||||
|
||||
var events []*storageusageevent.StorageUsageEvent
|
||||
var (
|
||||
resultUserID gocql.UUID
|
||||
resultEventDay time.Time
|
||||
eventTime time.Time
|
||||
fileSize int64
|
||||
operation string
|
||||
)
|
||||
|
||||
for iter.Scan(&resultUserID, &resultEventDay, &eventTime, &fileSize, &operation) {
|
||||
event := &storageusageevent.StorageUsageEvent{
|
||||
UserID: resultUserID,
|
||||
EventDay: resultEventDay,
|
||||
EventTime: eventTime,
|
||||
FileSize: fileSize,
|
||||
Operation: operation,
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
|
||||
if err := iter.Close(); err != nil {
|
||||
impl.Logger.Error("failed to get storage usage events by user and day", zap.Error(err))
|
||||
return nil, fmt.Errorf("failed to get storage usage events: %w", err)
|
||||
}
|
||||
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetByUserDateRange(ctx context.Context, userID gocql.UUID, startDay, endDay time.Time) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
// Ensure dates are truncated to date only
|
||||
startDay = startDay.Truncate(24 * time.Hour)
|
||||
endDay = endDay.Truncate(24 * time.Hour)
|
||||
|
||||
// For better performance with large date ranges, we'll query in parallel
|
||||
var allEvents []*storageusageevent.StorageUsageEvent
|
||||
eventsChan := make(chan []*storageusageevent.StorageUsageEvent)
|
||||
errorsChan := make(chan error)
|
||||
|
||||
// Calculate number of days
|
||||
days := int(endDay.Sub(startDay).Hours()/24) + 1
|
||||
|
||||
// Query each day in parallel (limit concurrency to avoid overwhelming Cassandra)
|
||||
concurrency := 10
|
||||
if days < concurrency {
|
||||
concurrency = days
|
||||
}
|
||||
|
||||
semaphore := make(chan struct{}, concurrency)
|
||||
daysProcessed := 0
|
||||
|
||||
for day := startDay; !day.After(endDay); day = day.Add(24 * time.Hour) {
|
||||
semaphore <- struct{}{}
|
||||
daysProcessed++
|
||||
|
||||
go func(queryDay time.Time) {
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
events, err := impl.GetByUserAndDay(ctx, userID, queryDay)
|
||||
if err != nil {
|
||||
errorsChan <- err
|
||||
return
|
||||
}
|
||||
eventsChan <- events
|
||||
}(day)
|
||||
}
|
||||
|
||||
// Collect results
|
||||
var firstError error
|
||||
for i := 0; i < daysProcessed; i++ {
|
||||
select {
|
||||
case events := <-eventsChan:
|
||||
allEvents = append(allEvents, events...)
|
||||
case err := <-errorsChan:
|
||||
if firstError == nil {
|
||||
firstError = err
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
if firstError != nil {
|
||||
impl.Logger.Error("failed to get events for date range",
|
||||
zap.Error(firstError),
|
||||
zap.Int("days_requested", days))
|
||||
return allEvents, firstError // Return partial results
|
||||
}
|
||||
|
||||
return allEvents, nil
|
||||
}
|
||||
|
||||
// Convenience methods for trend analysis
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetLast7DaysEvents(ctx context.Context, userID gocql.UUID) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
endDay := time.Now().Truncate(24 * time.Hour)
|
||||
startDay := endDay.Add(-6 * 24 * time.Hour) // 7 days including today
|
||||
|
||||
return impl.GetByUserDateRange(ctx, userID, startDay, endDay)
|
||||
}
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetLastNDaysEvents(ctx context.Context, userID gocql.UUID, days int) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
if days <= 0 {
|
||||
return nil, fmt.Errorf("days must be positive")
|
||||
}
|
||||
|
||||
endDay := time.Now().Truncate(24 * time.Hour)
|
||||
startDay := endDay.Add(-time.Duration(days-1) * 24 * time.Hour)
|
||||
|
||||
return impl.GetByUserDateRange(ctx, userID, startDay, endDay)
|
||||
}
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetMonthlyEvents(ctx context.Context, userID gocql.UUID, year int, month time.Month) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
startDay := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC)
|
||||
endDay := startDay.AddDate(0, 1, -1) // Last day of the month
|
||||
|
||||
return impl.GetByUserDateRange(ctx, userID, startDay, endDay)
|
||||
}
|
||||
|
||||
func (impl *storageUsageEventRepositoryImpl) GetYearlyEvents(ctx context.Context, userID gocql.UUID, year int) ([]*storageusageevent.StorageUsageEvent, error) {
|
||||
startDay := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
endDay := time.Date(year, 12, 31, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
return impl.GetByUserDateRange(ctx, userID, startDay, endDay)
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
// monorepo/cloud/maplefile-backend/internal/maplefile/repo/storageusageevent/impl.go
|
||||
package storageusageevent
|
||||
|
||||
import (
|
||||
"github.com/gocql/gocql"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/storageusageevent"
|
||||
)
|
||||
|
||||
type storageUsageEventRepositoryImpl struct {
|
||||
Logger *zap.Logger
|
||||
Session *gocql.Session
|
||||
}
|
||||
|
||||
func NewRepository(appCfg *config.Configuration, session *gocql.Session, loggerp *zap.Logger) storageusageevent.StorageUsageEventRepository {
|
||||
loggerp = loggerp.Named("StorageUsageEventRepository")
|
||||
|
||||
return &storageUsageEventRepositoryImpl{
|
||||
Logger: loggerp,
|
||||
Session: session,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package storageusageevent
|
||||
|
||||
import (
|
||||
"github.com/gocql/gocql"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/config"
|
||||
"codeberg.org/mapleopentech/monorepo/cloud/maplefile-backend/internal/domain/storageusageevent"
|
||||
)
|
||||
|
||||
// ProvideRepository provides a storage usage event repository for Wire DI
|
||||
func ProvideRepository(cfg *config.Config, session *gocql.Session, logger *zap.Logger) storageusageevent.StorageUsageEventRepository {
|
||||
return NewRepository(cfg, session, logger)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue