diff --git a/pkg/debugcmd/deps.go b/pkg/debugcmd/deps.go index 0d62a252..e7e7552d 100644 --- a/pkg/debugcmd/deps.go +++ b/pkg/debugcmd/deps.go @@ -11,12 +11,13 @@ import ( "github.com/threefoldtech/zosbase/pkg/gridtypes/zos" ) -// Provision is the subset of the provision zbus interface used by debug commands. -type Provision interface { - ListTwins(ctx context.Context) ([]uint32, error) - List(ctx context.Context, twin uint32) ([]gridtypes.Deployment, error) - Get(ctx context.Context, twin uint32, contract uint64) (gridtypes.Deployment, error) - Changes(ctx context.Context, twin uint32, contract uint64) ([]gridtypes.Workload, error) +// Storage is the subset of the provision interface used by debug commands. +type Storage interface { + GetDeployment(ctx context.Context, twin uint32, contractID uint64) (gridtypes.Deployment, error) + GetDeployments(ctx context.Context, twin uint32) ([]gridtypes.Deployment, error) + GetTwins(ctx context.Context) ([]uint32, error) + Changes(ctx context.Context, twin uint32, contractID uint64) ([]gridtypes.Workload, error) + GetWorkload(ctx context.Context, twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error) } // VM is the subset of the vmd zbus interface used by debug commands. @@ -33,9 +34,9 @@ type Network interface { } type Deps struct { - Provision Provision - VM VM - Network Network + VM VM + Network Network + Storage Storage } // ParseDeploymentID parses a deployment identifier in the format "twin-id:contract-id" diff --git a/pkg/debugcmd/get.go b/pkg/debugcmd/get.go index 0146946f..a85a0e5b 100644 --- a/pkg/debugcmd/get.go +++ b/pkg/debugcmd/get.go @@ -29,8 +29,7 @@ func Get(ctx context.Context, deps Deps, req GetRequest) (GetResponse, error) { return GetResponse{}, err } - // TODO: only return active deployment. should return all - deployment, err := deps.Provision.Get(ctx, twinID, contractID) + deployment, err := deps.Storage.GetDeployment(ctx, twinID, contractID) if err != nil { return GetResponse{}, err } diff --git a/pkg/debugcmd/health.go b/pkg/debugcmd/health.go index 8dc337ba..8eef354b 100644 --- a/pkg/debugcmd/health.go +++ b/pkg/debugcmd/health.go @@ -69,7 +69,7 @@ func Health(ctx context.Context, deps Deps, req HealthRequest) (HealthResponse, } if req.Deployment != "" { - deployment, err := deps.Provision.Get(ctx, twinID, contractID) + deployment, err := deps.Storage.GetDeployment(ctx, twinID, contractID) if err != nil { return HealthResponse{}, fmt.Errorf("failed to get deployment: %w", err) } diff --git a/pkg/debugcmd/history.go b/pkg/debugcmd/history.go index 090d8850..5f243a31 100644 --- a/pkg/debugcmd/history.go +++ b/pkg/debugcmd/history.go @@ -39,8 +39,7 @@ func History(ctx context.Context, deps Deps, req HistoryRequest) (HistoryRespons return HistoryResponse{}, err } - // TODO: only return history for active deployment. - history, err := deps.Provision.Changes(ctx, twinID, contractID) + history, err := deps.Storage.Changes(ctx, twinID, contractID) if err != nil { return HistoryResponse{}, err } diff --git a/pkg/debugcmd/info.go b/pkg/debugcmd/info.go index 06c0fcc9..4272007d 100644 --- a/pkg/debugcmd/info.go +++ b/pkg/debugcmd/info.go @@ -41,20 +41,11 @@ func Info(ctx context.Context, deps Deps, req InfoRequest) (InfoResponse, error) return InfoResponse{}, err } - deployment, err := deps.Provision.Get(ctx, twinID, contractID) + workload, found, err := deps.Storage.GetWorkload(ctx, twinID, contractID, gridtypes.Name(req.Workload)) if err != nil { - return InfoResponse{}, fmt.Errorf("failed to get deployment: %w", err) + return InfoResponse{}, fmt.Errorf("failed to get workload: %w", err) } - - var workload *gridtypes.Workload - for i := range deployment.Workloads { - if string(deployment.Workloads[i].Name) == req.Workload { - workload = &deployment.Workloads[i] - break - } - } - - if workload == nil { + if !found { return InfoResponse{}, fmt.Errorf("workload '%s' not found in deployment", req.Workload) } @@ -70,7 +61,7 @@ func Info(ctx context.Context, deps Deps, req InfoRequest) (InfoResponse, error) case zos.ZMachineType, zos.ZMachineLightType: return handleZMachineInfo(ctx, deps, workloadID.String(), req.Verbose, resp) case zos.NetworkType, zos.NetworkLightType: - return handleNetworkInfo(ctx, deps, twinID, workload, resp) + return handleNetworkInfo(ctx, deps, twinID, &workload, resp) default: return InfoResponse{}, fmt.Errorf("workload type '%s' not supported for info command", workload.Type) } diff --git a/pkg/debugcmd/list.go b/pkg/debugcmd/list.go index 42f28ba8..0724f559 100644 --- a/pkg/debugcmd/list.go +++ b/pkg/debugcmd/list.go @@ -40,19 +40,16 @@ func ParseListRequest(payload []byte) (ListRequest, error) { func List(ctx context.Context, deps Deps, req ListRequest) (ListResponse, error) { twins := []uint32{req.TwinID} if req.TwinID == 0 { - allTwins, err := deps.Provision.ListTwins(ctx) + var err error + twins, err = deps.Storage.GetTwins(ctx) if err != nil { return ListResponse{}, err } - - twins = allTwins } deployments := make([]ListDeployment, 0) for _, twin := range twins { - // TODO: this is only returning active deployments, - // cause when deprovision the workload is removed from the key list. - deploymentList, err := deps.Provision.List(ctx, twin) + deploymentList, err := deps.Storage.GetDeployments(ctx, twin) if err != nil { return ListResponse{}, err } diff --git a/pkg/network/nr/net_resource.go b/pkg/network/nr/net_resource.go index c2bc6d9b..7d72d1c9 100644 --- a/pkg/network/nr/net_resource.go +++ b/pkg/network/nr/net_resource.go @@ -606,7 +606,7 @@ func (nr *NetResource) ConfigureWG(privateKey string) error { // Delete removes all the interfaces and namespaces created by the Create method func (nr *NetResource) Delete() error { log.Info().Str("network-id", nr.ID()).Str("subnet", nr.resource.Subnet.String()).Msg("deleting network resource") - + netnsName, err := nr.Namespace() if err != nil { return err diff --git a/pkg/provision.go b/pkg/provision.go index 8169aa84..0cc4e09f 100644 --- a/pkg/provision.go +++ b/pkg/provision.go @@ -21,6 +21,15 @@ type Provision interface { ListTwins() ([]uint32, error) ListPublicIPs() ([]string, error) ListPrivateIPs(twin uint32, network gridtypes.Name) ([]string, error) + // GetDeployment returns a deployment including soft-deleted ones. + GetDeployment(twin uint32, contractID uint64) (gridtypes.Deployment, error) + // GetDeployments returns all deployments for a twin including soft-deleted ones. + GetDeployments(twin uint32) ([]gridtypes.Deployment, error) + // GetTwins returns all twins including soft-deleted ones. + GetTwins() ([]uint32, error) + // GetWorkload returns the latest workload state by name including soft-deleted ones. + // Returns (workload, true, nil) if found, (zero, false, nil) if not found. + GetWorkload(twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error) } type Statistics interface { diff --git a/pkg/provision/debug.go b/pkg/provision/debug.go new file mode 100644 index 00000000..8e18aff3 --- /dev/null +++ b/pkg/provision/debug.go @@ -0,0 +1,44 @@ +package provision + +import "github.com/threefoldtech/zosbase/pkg/gridtypes" + +func (e *NativeEngine) GetDeployment(twin uint32, contractID uint64) (gridtypes.Deployment, error) { + return e.storage.Get(twin, contractID, WithDeleted()) +} + +func (e *NativeEngine) GetDeployments(twin uint32) ([]gridtypes.Deployment, error) { + ids, err := e.storage.ByTwin(twin, WithDeleted()) + if err != nil { + return nil, err + } + deployments := make([]gridtypes.Deployment, 0, len(ids)) + for _, id := range ids { + dep, err := e.storage.Get(twin, id, WithDeleted()) + if err != nil { + return nil, err + } + deployments = append(deployments, dep) + } + return deployments, nil +} + +func (e *NativeEngine) GetTwins() ([]uint32, error) { + return e.storage.Twins(WithDeleted()) +} + +func (e *NativeEngine) GetWorkload(twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error) { + dep, err := e.storage.Get(twin, contractID, WithDeleted()) + if err != nil { + return gridtypes.Workload{}, false, err + } + for i := range dep.Workloads { + if dep.Workloads[i].Name == name { + return dep.Workloads[i], true, nil + } + } + wl, err := e.storage.Current(twin, contractID, name, WithDeleted()) + if err != nil { + return gridtypes.Workload{}, false, nil // not found, no error + } + return wl, true, nil +} diff --git a/pkg/provision/interface.go b/pkg/provision/interface.go index 68fc2e44..34a5c8b4 100644 --- a/pkg/provision/interface.go +++ b/pkg/provision/interface.go @@ -110,6 +110,19 @@ type StorageCapacity struct { // and or workload that returns true from the capacity calculation. type Exclude = func(dl *gridtypes.Deployment, wl *gridtypes.Workload) bool +// QueryOpt is a functional option for Storage query methods +type QueryOpt func(*QueryOpts) + +// QueryOpts holds query options for Storage methods +type QueryOpts struct { + Deleted bool +} + +// WithDeleted returns a QueryOpt that includes soft-deleted items in query results +func WithDeleted() QueryOpt { + return func(o *QueryOpts) { o.Deleted = true } +} + // Storage interface type Storage interface { // Create a new deployment in storage, it sets the initial transactions @@ -120,7 +133,7 @@ type Storage interface { // Delete deletes a deployment from storage. Delete(twin uint32, deployment uint64) error // Get gets the current state of a deployment from storage - Get(twin uint32, deployment uint64) (gridtypes.Deployment, error) + Get(twin uint32, deployment uint64, opts ...QueryOpt) (gridtypes.Deployment, error) // Error sets global deployment error Error(twin uint32, deployment uint64, err error) error // Add workload to deployment, if no active deployment exists with same name @@ -132,11 +145,11 @@ type Storage interface { // Changes return all the historic transactions of a deployment Changes(twin uint32, deployment uint64) (changes []gridtypes.Workload, err error) // Current gets last state of a workload by name - Current(twin uint32, deployment uint64, name gridtypes.Name) (gridtypes.Workload, error) + Current(twin uint32, deployment uint64, name gridtypes.Name, opts ...QueryOpt) (gridtypes.Workload, error) // Twins list twins in storage - Twins() ([]uint32, error) + Twins(opts ...QueryOpt) ([]uint32, error) // ByTwin return list of deployments for a twin - ByTwin(twin uint32) ([]uint64, error) + ByTwin(twin uint32, opts ...QueryOpt) ([]uint64, error) // return total capacity and active deployments Capacity(exclude ...Exclude) (StorageCapacity, error) } diff --git a/pkg/provision/provisiner.go b/pkg/provision/provisiner.go index 07d2cf22..0472b347 100644 --- a/pkg/provision/provisiner.go +++ b/pkg/provision/provisiner.go @@ -122,7 +122,7 @@ func (p *mapProvisioner) Initialize(ctx context.Context) error { // Provision implements provision.Provisioner func (p *mapProvisioner) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (result gridtypes.Result, err error) { log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("provisioning workload") - + manager, ok := p.managers[wl.Type] if !ok { return result, fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID) @@ -139,7 +139,7 @@ func (p *mapProvisioner) Provision(ctx context.Context, wl *gridtypes.WorkloadWi // Decommission implementation for provision.Provisioner func (p *mapProvisioner) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error { log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("deprovisioning workload") - + manager, ok := p.managers[wl.Type] if !ok { return fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID) @@ -151,7 +151,7 @@ func (p *mapProvisioner) Deprovision(ctx context.Context, wl *gridtypes.Workload // Pause a workload func (p *mapProvisioner) Pause(ctx context.Context, wl *gridtypes.WorkloadWithID) (gridtypes.Result, error) { log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("pausing workload") - + if wl.Result.State != gridtypes.StateOk { return wl.Result, fmt.Errorf("can only pause workloads in ok state") } @@ -180,7 +180,7 @@ func (p *mapProvisioner) Pause(ctx context.Context, wl *gridtypes.WorkloadWithID // Resume a workload func (p *mapProvisioner) Resume(ctx context.Context, wl *gridtypes.WorkloadWithID) (gridtypes.Result, error) { log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("resuming workload") - + if wl.Result.State != gridtypes.StatePaused { return wl.Result, fmt.Errorf("can only resume workloads in paused state") } @@ -208,7 +208,7 @@ func (p *mapProvisioner) Resume(ctx context.Context, wl *gridtypes.WorkloadWithI // Provision implements provision.Provisioner func (p *mapProvisioner) Update(ctx context.Context, wl *gridtypes.WorkloadWithID) (result gridtypes.Result, err error) { log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("updating workload") - + manager, ok := p.managers[wl.Type] if !ok { return result, fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID) diff --git a/pkg/provision/storage/helpers.go b/pkg/provision/storage/helpers.go new file mode 100644 index 00000000..3d59b1cb --- /dev/null +++ b/pkg/provision/storage/helpers.go @@ -0,0 +1,195 @@ +package storage + +import ( + "encoding/binary" + "fmt" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zosbase/pkg/gridtypes" + "github.com/threefoldtech/zosbase/pkg/provision" + bolt "go.etcd.io/bbolt" +) + +// isWorkloadDeleted checks if a workload value indicates soft-deletion. +// Format: "type" (active) or "type|timestamp" (deleted) +func isWorkloadDeleted(value []byte) bool { + valueStr := string(value) + parts := strings.Split(valueStr, "|") + return len(parts) > 1 && parts[1] != "" +} + +// parseWorkloadType extracts the workload type from a value that may be soft-deleted. +// Returns the type and whether the workload is deleted. +func parseWorkloadType(value []byte) (gridtypes.WorkloadType, bool) { + valueStr := string(value) + parts := strings.Split(valueStr, "|") + deleted := len(parts) > 1 && parts[1] != "" + return gridtypes.WorkloadType(parts[0]), deleted +} + +// isDeploymentBucketDeleted checks if a deployment bucket has the deleted_at flag set. +func (b *BoltStorage) isDeploymentBucketDeleted(deployment *bolt.Bucket) bool { + if deployment == nil { + return false + } + deletedAt := deployment.Get([]byte(keyDeletedAt)) + return deletedAt != nil && b.l64(deletedAt) > 0 +} + +// isTwinBucketDeleted checks if a twin bucket has the deleted_at flag set. +func (b *BoltStorage) isTwinBucketDeleted(twin *bolt.Bucket) bool { + if twin == nil { + return false + } + meta := twin.Bucket([]byte(keyTwinMetadata)) + if meta == nil { + return false + } + deletedAt := meta.Get([]byte(keyDeletedAt)) + return deletedAt != nil && b.l64(deletedAt) > 0 +} + +// deletedTimestamp returns the soft-delete timestamp encoded in a workload value, +// and whether the workload is deleted at all. +func deletedTimestamp(value []byte) (uint64, bool) { + valueStr := string(value) + parts := strings.Split(valueStr, "|") + if len(parts) <= 1 || parts[1] == "" { + return 0, false + } + var ts uint64 + if _, err := fmt.Sscanf(parts[1], "%d", &ts); err != nil { + return 0, false + } + return ts, true +} + +// Binary encoding/decoding helpers + +func (b *BoltStorage) u32(u uint32) []byte { + var v [4]byte + binary.BigEndian.PutUint32(v[:], u) + return v[:] +} + +func (b *BoltStorage) l32(v []byte) uint32 { + return binary.BigEndian.Uint32(v) +} + +func (b *BoltStorage) u64(u uint64) []byte { + var v [8]byte + binary.BigEndian.PutUint64(v[:], u) + return v[:] +} + +func (b *BoltStorage) l64(v []byte) uint64 { + return binary.BigEndian.Uint64(v) +} + +// cleanWorkloads removes per-workload soft-delete entries older than beforeUnix +// from the workloads sub-bucket of a deployment. +func (b *BoltStorage) cleanWorkloads(workloadsBucket *bolt.Bucket, beforeUnix uint64) { + if workloadsBucket == nil { + return + } + + wlCursor := workloadsBucket.Cursor() + for wlKey, wlVal := wlCursor.First(); wlKey != nil; wlKey, wlVal = wlCursor.Next() { + if ts, deleted := deletedTimestamp(wlVal); deleted { + if ts < beforeUnix { + if err := workloadsBucket.Delete(wlKey); err != nil { + log.Error().Err(err).Str("workload", string(wlKey)). + Msg("failed to hard delete workload") + } + } + } + } +} + +// cleanTwin hard-deletes the twin bucket if deleted, old enough, and has no remaining deployments. +func (b *BoltStorage) cleanTwin(tx *bolt.Tx, k []byte, twinBucket *bolt.Bucket, beforeUnix uint64) { + if !b.isTwinBucketDeleted(twinBucket) { + return + } + + twinMeta := twinBucket.Bucket([]byte(keyTwinMetadata)) + if twinMeta == nil { + return + } + + deletedAt := twinMeta.Get([]byte(keyDeletedAt)) + if deletedAt == nil || b.l64(deletedAt) >= beforeUnix { + return + } + + // Check if twin has any deployments left + hasDeployments := false + checkCursor := twinBucket.Cursor() + for chkKey, chkVal := checkCursor.First(); chkKey != nil; chkKey, chkVal = checkCursor.Next() { + if chkVal != nil { + continue + } + if len(chkKey) == 8 && string(chkKey) != "global" && string(chkKey) != keyTwinMetadata { + hasDeployments = true + break + } + } + + if !hasDeployments { + twinID := b.l32(k) + if err := tx.DeleteBucket(k); err != nil { + log.Error().Err(err).Uint32("twin", twinID).Msg("failed to hard delete twin") + } + } +} + +// IsDeploymentDeleted checks if a deployment is soft-deleted. +func (b *BoltStorage) IsDeploymentDeleted(twin uint32, deployment uint64) (deleted bool, deletedAt gridtypes.Timestamp, err error) { + err = b.db.View(func(t *bolt.Tx) error { + twinBucket := t.Bucket(b.u32(twin)) + if twinBucket == nil { + return errors.Wrap(provision.ErrDeploymentNotExists, "twin not found") + } + deploymentBucket := twinBucket.Bucket(b.u64(deployment)) + if deploymentBucket == nil { + return errors.Wrap(provision.ErrDeploymentNotExists, "deployment not found") + } + + if b.isDeploymentBucketDeleted(deploymentBucket) { + if value := deploymentBucket.Get([]byte(keyDeletedAt)); value != nil { + deleted = true + deletedAt = gridtypes.Timestamp(b.l64(value)) + } + } + + return nil + }) + + return +} + +// IsTwinDeleted checks if a twin is soft-deleted. +func (b *BoltStorage) IsTwinDeleted(twin uint32) (deleted bool, deletedAt gridtypes.Timestamp, err error) { + err = b.db.View(func(t *bolt.Tx) error { + twinBucket := t.Bucket(b.u32(twin)) + if twinBucket == nil { + return fmt.Errorf("twin not found") + } + + if b.isTwinBucketDeleted(twinBucket) { + meta := twinBucket.Bucket([]byte(keyTwinMetadata)) + if meta != nil { + if value := meta.Get([]byte(keyDeletedAt)); value != nil { + deleted = true + deletedAt = gridtypes.Timestamp(b.l64(value)) + } + } + } + + return nil + }) + + return +} diff --git a/pkg/provision/storage/storage.go b/pkg/provision/storage/storage.go index 528ec705..26b33c90 100644 --- a/pkg/provision/storage/storage.go +++ b/pkg/provision/storage/storage.go @@ -1,7 +1,6 @@ package storage import ( - "encoding/binary" "encoding/json" "fmt" @@ -25,6 +24,8 @@ const ( keyWorkloads = "workloads" keyTransactions = "transactions" keyGlobal = "global" + keyDeletedAt = "deleted_at" // For deployment soft-delete + keyTwinMetadata = "twin_metadata" // For twin-level metadata ) type MigrationStorage struct { @@ -49,29 +50,20 @@ func New(path string) (*BoltStorage, error) { }, nil } -func (b BoltStorage) Migration() MigrationStorage { - b.unsafe = true - return MigrationStorage{unsafe: b} -} - -func (b *BoltStorage) u32(u uint32) []byte { - var v [4]byte - binary.BigEndian.PutUint32(v[:], u) - return v[:] -} - -func (b *BoltStorage) l32(v []byte) uint32 { - return binary.BigEndian.Uint32(v) -} +func NewReadOnly(path string) (*BoltStorage, error) { + db, err := bolt.Open(path, 0644, &bolt.Options{ReadOnly: true}) + if err != nil { + return nil, err + } -func (b *BoltStorage) u64(u uint64) []byte { - var v [8]byte - binary.BigEndian.PutUint64(v[:], u) - return v[:] + return &BoltStorage{ + db, false, + }, nil } -func (b *BoltStorage) l64(v []byte) uint64 { - return binary.BigEndian.Uint64(v) +func (b BoltStorage) Migration() MigrationStorage { + b.unsafe = true + return MigrationStorage{unsafe: b} } func (b *BoltStorage) Create(deployment gridtypes.Deployment) error { @@ -80,6 +72,14 @@ func (b *BoltStorage) Create(deployment gridtypes.Deployment) error { if err != nil { return errors.Wrap(err, "failed to create twin") } + + // If twin was soft-deleted, revive it by clearing the deletion flag + if b.isTwinBucketDeleted(twin) { + if err := twin.DeleteBucket([]byte(keyTwinMetadata)); err != nil && !errors.Is(err, bolt.ErrBucketNotFound) { + return errors.Wrap(err, "failed to revive twin") + } + } + dl, err := twin.CreateBucket(b.u64(deployment.ContractID)) if errors.Is(err, bolt.ErrBucketExists) { return provision.ErrDeploymentExists @@ -183,42 +183,71 @@ func (b *MigrationStorage) Migrate(dl gridtypes.Deployment) error { func (b *BoltStorage) Delete(twin uint32, deployment uint64) error { return b.db.Update(func(t *bolt.Tx) error { - bucket := t.Bucket(b.u32(twin)) - if bucket == nil { + twinBucket := t.Bucket(b.u32(twin)) + if twinBucket == nil { + return nil + } + + deploymentBucket := twinBucket.Bucket(b.u64(deployment)) + if deploymentBucket == nil { return nil } - if err := bucket.DeleteBucket(b.u64(deployment)); err != nil && !errors.Is(err, bolt.ErrBucketNotFound) { + // Soft-delete: Set deleted_at timestamp instead of deleting bucket + now := b.u64(uint64(gridtypes.Now())) + if err := deploymentBucket.Put([]byte(keyDeletedAt), now); err != nil { return err } - // if the twin now is empty then we can also delete the twin - curser := bucket.Cursor() - found := false - for k, v := curser.First(); k != nil; k, v = curser.Next() { + + // Check if all deployments in twin are deleted → mark twin as deleted + allDeleted := true + cursor := twinBucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { if v != nil { - // checking that it is a bucket + // Not a bucket continue } if len(k) != 8 || string(k) == "global" { - // sanity check it's a valid uint32 + // Skip non-deployment buckets continue } - found = true - break + dlBucket := twinBucket.Bucket(k) + if dlBucket == nil { + continue + } + + // Check if this deployment is deleted + deletedAt := dlBucket.Get([]byte(keyDeletedAt)) + if deletedAt == nil || b.l64(deletedAt) == 0 { + // Found non-deleted deployment + allDeleted = false + break + } } - if !found { - // empty bucket - return t.DeleteBucket(b.u32(twin)) + if allDeleted { + // Mark twin as deleted + twinMeta, err := twinBucket.CreateBucketIfNotExists([]byte(keyTwinMetadata)) + if err != nil { + return errors.Wrap(err, "failed to create twin metadata bucket") + } + if err := twinMeta.Put([]byte(keyDeletedAt), now); err != nil { + return err + } } return nil }) } -func (b *BoltStorage) Get(twin uint32, deployment uint64) (dl gridtypes.Deployment, err error) { +func (b *BoltStorage) Get(twin uint32, deployment uint64, opts ...provision.QueryOpt) (dl gridtypes.Deployment, err error) { + opts_ := &provision.QueryOpts{} + for _, opt := range opts { + opt(opts_) + } + dl.TwinID = twin dl.ContractID = deployment err = b.db.View(func(t *bolt.Tx) error { @@ -230,6 +259,12 @@ func (b *BoltStorage) Get(twin uint32, deployment uint64) (dl gridtypes.Deployme if deployment == nil { return errors.Wrap(provision.ErrDeploymentNotExists, "deployment not found") } + + // Check if deployment is soft-deleted + if !opts_.Deleted && b.isDeploymentBucketDeleted(deployment) { + return errors.Wrap(provision.ErrDeploymentNotExists, "deployment is deleted") + } + if value := deployment.Get([]byte(keyVersion)); value != nil { dl.Version = b.l32(value) } @@ -251,7 +286,7 @@ func (b *BoltStorage) Get(twin uint32, deployment uint64) (dl gridtypes.Deployme return dl, err } - dl.Workloads, err = b.workloads(twin, deployment) + dl.Workloads, err = b.workloads(twin, deployment, opts_) return } @@ -319,7 +354,11 @@ func (b *BoltStorage) add(tx *bolt.Tx, twinID uint32, dl uint64, workload gridty } if value := workloads.Get([]byte(workload.Name)); value != nil { - return errors.Wrap(provision.ErrWorkloadExists, "workload with same name already exists in deployment") + // Check if this is a soft-deleted workload + if !isWorkloadDeleted(value) { + // Active workload exists - cannot add duplicate + return errors.Wrap(provision.ErrWorkloadExists, "workload with same name already exists in deployment") + } } if err := workloads.Put([]byte(workload.Name), []byte(workload.Type.String())); err != nil { @@ -342,17 +381,17 @@ func (b *BoltStorage) Add(twin uint32, deployment uint64, workload gridtypes.Wor func (b *BoltStorage) Remove(twin uint32, deployment uint64, name gridtypes.Name) error { return b.db.Update(func(tx *bolt.Tx) error { - twin := tx.Bucket(b.u32(twin)) - if twin == nil { + twinBucket := tx.Bucket(b.u32(twin)) + if twinBucket == nil { return nil } - deployment := twin.Bucket(b.u64(deployment)) - if deployment == nil { + deploymentBucket := twinBucket.Bucket(b.u64(deployment)) + if deploymentBucket == nil { return nil } - workloads := deployment.Bucket([]byte(keyWorkloads)) + workloads := deploymentBucket.Bucket([]byte(keyWorkloads)) if workloads == nil { return nil } @@ -362,15 +401,19 @@ func (b *BoltStorage) Remove(twin uint32, deployment uint64, name gridtypes.Name return nil } + // Clean up global bucket for sharable types if gridtypes.IsSharable(gridtypes.WorkloadType(typ)) { - if shared := twin.Bucket([]byte(keyGlobal)); shared != nil { + if shared := twinBucket.Bucket([]byte(keyGlobal)); shared != nil { if err := shared.Delete([]byte(name)); err != nil { return err } } } - return workloads.Delete([]byte(name)) + // Soft-delete: Mark workload as deleted with timestamp + // Format: "type|deleted_at_unix_timestamp" + deletedValue := fmt.Sprintf("%s|%d", typ, gridtypes.Now()) + return workloads.Put([]byte(name), []byte(deletedValue)) }) } @@ -467,7 +510,7 @@ func (b *BoltStorage) Changes(twin uint32, deployment uint64) (changes []gridtyp return } -func (b *BoltStorage) workloads(twin uint32, deployment uint64) ([]gridtypes.Workload, error) { +func (b *BoltStorage) workloads(twin uint32, deployment uint64, opts *provision.QueryOpts) ([]gridtypes.Workload, error) { names := make(map[gridtypes.Name]gridtypes.WorkloadType) workloads := make(map[gridtypes.Name]gridtypes.Workload) @@ -488,7 +531,12 @@ func (b *BoltStorage) workloads(twin uint32, deployment uint64) ([]gridtypes.Wor } err := types.ForEach(func(k, v []byte) error { - names[gridtypes.Name(k)] = gridtypes.WorkloadType(v) + typ, deleted := parseWorkloadType(v) + if deleted && !opts.Deleted { + return nil + } + + names[gridtypes.Name(k)] = typ return nil }) @@ -561,7 +609,12 @@ func (b *BoltStorage) workloads(twin uint32, deployment uint64) ([]gridtypes.Wor return result, err } -func (b *BoltStorage) Current(twin uint32, deployment uint64, name gridtypes.Name) (gridtypes.Workload, error) { +func (b *BoltStorage) Current(twin uint32, deployment uint64, name gridtypes.Name, opts ...provision.QueryOpt) (gridtypes.Workload, error) { + opts_ := &provision.QueryOpts{} + for _, opt := range opts { + opt(opts_) + } + var workload gridtypes.Workload err := b.db.View(func(tx *bolt.Tx) error { twin := tx.Bucket(b.u32(twin)) @@ -586,7 +639,10 @@ func (b *BoltStorage) Current(twin uint32, deployment uint64, name gridtypes.Nam return errors.Wrap(provision.ErrWorkloadNotExist, "workload does not exist") } - typ := gridtypes.WorkloadType(typRaw) + typ, deleted := parseWorkloadType(typRaw) + if deleted && !opts_.Deleted { + return errors.Wrap(provision.ErrWorkloadNotExist, "workload is deleted") + } logs := deployment.Bucket([]byte(keyTransactions)) if logs == nil { @@ -627,11 +683,16 @@ func (b *BoltStorage) Current(twin uint32, deployment uint64, name gridtypes.Nam return workload, err } -func (b *BoltStorage) Twins() ([]uint32, error) { +func (b *BoltStorage) Twins(opts ...provision.QueryOpt) ([]uint32, error) { + opts_ := &provision.QueryOpts{} + for _, opt := range opts { + opt(opts_) + } + var twins []uint32 err := b.db.View(func(t *bolt.Tx) error { - curser := t.Cursor() - for k, v := curser.First(); k != nil; k, v = curser.Next() { + cursor := t.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { if v != nil { // checking that it is a bucket continue @@ -642,6 +703,15 @@ func (b *BoltStorage) Twins() ([]uint32, error) { continue } + twinBucket := t.Bucket(k) + if twinBucket == nil { + continue + } + + if !opts_.Deleted && b.isTwinBucketDeleted(twinBucket) { + continue + } + twins = append(twins, b.l32(k)) } @@ -651,7 +721,12 @@ func (b *BoltStorage) Twins() ([]uint32, error) { return twins, err } -func (b *BoltStorage) ByTwin(twin uint32) ([]uint64, error) { +func (b *BoltStorage) ByTwin(twin uint32, opts ...provision.QueryOpt) ([]uint64, error) { + opts_ := &provision.QueryOpts{} + for _, opt := range opts { + opt(opts_) + } + var deployments []uint64 err := b.db.View(func(t *bolt.Tx) error { bucket := t.Bucket(b.u32(twin)) @@ -659,15 +734,24 @@ func (b *BoltStorage) ByTwin(twin uint32) ([]uint64, error) { return nil } - curser := bucket.Cursor() - for k, v := curser.First(); k != nil; k, v = curser.Next() { + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { if v != nil { // checking that it is a bucket continue } - if len(k) != 8 || string(k) == "global" { - // sanity check it's a valid uint32 + if len(k) != 8 || string(k) == "global" || string(k) == keyTwinMetadata { + // sanity check it's a valid deployment bucket + continue + } + + deploymentBucket := bucket.Bucket(k) + if deploymentBucket == nil { + continue + } + + if !opts_.Deleted && b.isDeploymentBucketDeleted(deploymentBucket) { continue } @@ -731,53 +815,71 @@ func (b *BoltStorage) Capacity(exclude ...provision.Exclude) (storageCap provisi return storageCap, nil } -func (b *BoltStorage) Close() error { - return b.db.Close() -} +// CleanDeleted hard-deletes items that were soft-deleted before the given timestamp. +// This purges old deleted items from the database to reclaim space. +func (b *BoltStorage) CleanDeleted(before gridtypes.Timestamp) error { + beforeUnix := uint64(before) -// CleanDeleted is a cleaner method intended to clean up old "deleted" contracts -// that has no active workloads anymore. We used to always leave the entire history -// of all deployments that ever lived on the system. But we changed that so once -// a deployment is deleted, it's deleted forever. Hence this code is only needed -// temporary until it's available on all environments then can be dropped. -func (b *BoltStorage) CleanDeleted() error { - twins, err := b.Twins() - if err != nil { - return err - } + return b.db.Update(func(tx *bolt.Tx) error { + cursor := tx.Cursor() - for _, twin := range twins { - dls, err := b.ByTwin(twin) - if err != nil { - log.Error().Err(err).Uint32("twin", twin).Msg("failed to get twin deployments") - continue - } - for _, dl := range dls { - deployment, err := b.Get(twin, dl) - if err != nil { - log.Error().Err(err).Uint32("twin", twin).Uint64("deployment", dl).Msg("failed to get deployment") + // Iterate all twins + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + if v != nil { + continue + } + if len(k) != 4 { continue } - isActive := false - for _, wl := range deployment.Workloads { - if !wl.Result.State.IsOkay() { + twinBucket := tx.Bucket(k) + if twinBucket == nil { + continue + } + + // Iterate all deployments in this twin + dlCursor := twinBucket.Cursor() + for dlKey, dlVal := dlCursor.First(); dlKey != nil; dlKey, dlVal = dlCursor.Next() { + if dlVal != nil { + continue + } + if len(dlKey) != 8 || string(dlKey) == "global" || string(dlKey) == keyTwinMetadata { continue } - isActive = true - break - } + deploymentBucket := twinBucket.Bucket(dlKey) + if deploymentBucket == nil { + continue + } - if isActive { - continue - } + // Check if deployment is deleted and old enough + if b.isDeploymentBucketDeleted(deploymentBucket) { + deletedAt := deploymentBucket.Get([]byte(keyDeletedAt)) + if deletedAt != nil && b.l64(deletedAt) < beforeUnix { + // Hard delete this deployment bucket + deploymentID := b.l64(dlKey) + twinID := b.l32(k) + if err := twinBucket.DeleteBucket(dlKey); err != nil { + log.Error().Err(err).Uint32("twin", twinID).Uint64("deployment", deploymentID). + Msg("failed to hard delete deployment") + } + continue + } + } - if err := b.Delete(twin, dl); err != nil { - log.Error().Err(err).Uint32("twin", twin).Uint64("deployment", dl).Msg("failed to delete deployment") + // Clean up deleted workloads within this deployment + workloadsBucket := deploymentBucket.Bucket([]byte(keyWorkloads)) + b.cleanWorkloads(workloadsBucket, beforeUnix) } + + // Check if twin is deleted and old enough + b.cleanTwin(tx, k, twinBucket, beforeUnix) } - } - return nil + return nil + }) +} + +func (b *BoltStorage) Close() error { + return b.db.Close() } diff --git a/pkg/provision/storage/storage_test.go b/pkg/provision/storage/storage_test.go index 1180483e..f6302769 100644 --- a/pkg/provision/storage/storage_test.go +++ b/pkg/provision/storage/storage_test.go @@ -466,20 +466,29 @@ func TestDeleteDeployment(t *testing.T) { err = db.Delete(1, 10) require.NoError(err) + // Soft-delete: Get() should return error _, err = db.Get(1, 10) require.ErrorIs(err, provision.ErrDeploymentNotExists) + + // Soft-delete: ByTwin() should return empty (filtered) deployments, err := db.ByTwin(1) require.NoError(err) require.Empty(deployments) + // Soft-delete: Twin bucket should still exist but marked as deleted err = db.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(db.u32(1)) if bucket == nil { - return nil + return fmt.Errorf("twin bucket was deleted (should be soft-deleted)") } - return fmt.Errorf("twin bucket was not deleted") + return nil }) require.NoError(err) + + // Verify twin is marked as deleted + isDeleted, _, err := db.IsTwinDeleted(1) + require.NoError(err) + require.True(isDeleted) } func TestDeleteDeploymentMultiple(t *testing.T) { @@ -527,3 +536,1085 @@ func TestDeleteDeploymentMultiple(t *testing.T) { _, err = db.Get(1, 20) require.NoError(err) } + +// Soft-Delete Tests + +func TestSoftDeleteDeployment(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Description: "test deployment", + Workloads: []gridtypes.Workload{ + {Type: testType1, Name: "vm1"}, + }, + } + + err = db.Create(dl) + require.NoError(err) + + // Delete deployment (soft-delete) + err = db.Delete(1, 10) + require.NoError(err) + + // Get() should fail on soft-deleted deployment + _, err = db.Get(1, 10) + require.ErrorIs(err, provision.ErrDeploymentNotExists) + + // Get() with WithDeleted() should work + deleted, err := db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + require.Equal(uint32(1), deleted.TwinID) + require.Equal(uint64(10), deleted.ContractID) + + // Check deletion status + isDeleted, deletedAt, err := db.IsDeploymentDeleted(1, 10) + require.NoError(err) + require.True(isDeleted) + require.Greater(int64(deletedAt), int64(0)) +} + +func TestSoftDeleteTwin(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{{Type: testType1, Name: "vm1"}}, + } + err = db.Create(dl) + require.NoError(err) + + // Delete deployment - twin should be marked as deleted + err = db.Delete(1, 10) + require.NoError(err) + + // Twin should not appear in Twins() + twins, err := db.Twins() + require.NoError(err) + require.Empty(twins) + + // But should appear in Twins(WithDeleted()) + allTwins, err := db.Twins(provision.WithDeleted()) + require.NoError(err) + require.Len(allTwins, 1) + require.Equal(uint32(1), allTwins[0]) + + // Check twin deletion status + isDeleted, deletedAt, err := db.IsTwinDeleted(1) + require.NoError(err) + require.True(isDeleted) + require.Greater(int64(deletedAt), int64(0)) +} + +func TestSoftDeleteWorkload(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + } + err = db.Create(dl) + require.NoError(err) + + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + + // Remove workload (soft-delete) + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // Current() should fail on soft-deleted workload + _, err = db.Current(1, 10, "vm1") + require.ErrorIs(err, provision.ErrWorkloadNotExist) + + // Current() with WithDeleted() should work + wl, err := db.Current(1, 10, "vm1", provision.WithDeleted()) + require.NoError(err) + require.Equal(gridtypes.Name("vm1"), wl.Name) + require.Equal(testType1, wl.Type) +} + +func TestGetIgnoresDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{{Type: testType1, Name: "vm1"}}, + } + err = db.Create(dl) + require.NoError(err) + + // Verify we can get it before deletion + _, err = db.Get(1, 10) + require.NoError(err) + + // Delete it + err = db.Delete(1, 10) + require.NoError(err) + + // Get() should fail + _, err = db.Get(1, 10) + require.ErrorIs(err, provision.ErrDeploymentNotExists) +} + +func TestTwinsIgnoresDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create two twins + dl1 := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + } + dl2 := gridtypes.Deployment{ + Version: 1, + TwinID: 2, + ContractID: 20, + } + err = db.Create(dl1) + require.NoError(err) + err = db.Create(dl2) + require.NoError(err) + + // Should see both twins + twins, err := db.Twins() + require.NoError(err) + require.Len(twins, 2) + + // Delete twin 1 + err = db.Delete(1, 10) + require.NoError(err) + + // Should only see twin 2 + twins, err = db.Twins() + require.NoError(err) + require.Len(twins, 1) + require.Equal(uint32(2), twins[0]) +} + +func TestByTwinIgnoresDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create two deployments + dl1 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl1) + require.NoError(err) + err = db.Create(dl2) + require.NoError(err) + + // Should see both + deployments, err := db.ByTwin(1) + require.NoError(err) + require.Len(deployments, 2) + + // Delete one + err = db.Delete(1, 10) + require.NoError(err) + + // Should only see the other + deployments, err = db.ByTwin(1) + require.NoError(err) + require.Len(deployments, 1) + require.Equal(uint64(20), deployments[0]) +} + +func TestAllTwinsIncludesDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + } + err = db.Create(dl) + require.NoError(err) + + err = db.Delete(1, 10) + require.NoError(err) + + // Twins() should be empty + twins, err := db.Twins() + require.NoError(err) + require.Empty(twins) + + // Twins(WithDeleted()) should include deleted + allTwins, err := db.Twins(provision.WithDeleted()) + require.NoError(err) + require.Len(allTwins, 1) + require.Equal(uint32(1), allTwins[0]) +} + +func TestAllDeploymentsIncludesDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl1 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl1) + require.NoError(err) + err = db.Create(dl2) + require.NoError(err) + + err = db.Delete(1, 10) + require.NoError(err) + + // ByTwin() should only show active + deployments, err := db.ByTwin(1) + require.NoError(err) + require.Len(deployments, 1) + + // ByTwin(WithDeleted()) should show all + allDeployments, err := db.ByTwin(1, provision.WithDeleted()) + require.NoError(err) + require.Len(allDeployments, 2) +} + +func TestGetIncludingDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Description: "test", + Workloads: []gridtypes.Workload{{Type: testType1, Name: "vm1"}}, + } + err = db.Create(dl) + require.NoError(err) + + err = db.Delete(1, 10) + require.NoError(err) + + // Get() fails + _, err = db.Get(1, 10) + require.Error(err) + + // Get(WithDeleted()) works + deleted, err := db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + require.Equal("test", deleted.Description) + require.Len(deleted.Workloads, 1) +} + +func TestAllWorkloadsIncludesDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + err = db.Add(1, 10, gridtypes.Workload{Name: "vm2", Type: testType2}) + require.NoError(err) + + // Remove one workload + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // Get() should only show active workload + deployment, err := db.Get(1, 10) + require.NoError(err) + require.Len(deployment.Workloads, 1) + require.Equal(gridtypes.Name("vm2"), deployment.Workloads[0].Name) + + // Get(WithDeleted()) should show both + allWorkloads, err := db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + require.Len(allWorkloads.Workloads, 2) +} + +func TestCurrentIncludingDeletedFindsDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + + // Update to OK state + err = db.Transaction(1, 10, gridtypes.Workload{ + Name: "vm1", + Type: testType1, + Result: gridtypes.Result{ + Created: gridtypes.Now(), + State: gridtypes.StateOk, + }, + }) + require.NoError(err) + + // Remove workload + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // Current() should fail + _, err = db.Current(1, 10, "vm1") + require.Error(err) + + // Current(WithDeleted()) should work + wl, err := db.Current(1, 10, "vm1", provision.WithDeleted()) + require.NoError(err) + require.Equal(gridtypes.Name("vm1"), wl.Name) + require.Equal(gridtypes.StateOk, wl.Result.State) +} + +func TestIsDeploymentDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Not deleted initially + isDeleted, _, err := db.IsDeploymentDeleted(1, 10) + require.NoError(err) + require.False(isDeleted) + + // Delete it + err = db.Delete(1, 10) + require.NoError(err) + + // Should be deleted now + isDeleted, deletedAt, err := db.IsDeploymentDeleted(1, 10) + require.NoError(err) + require.True(isDeleted) + require.Greater(int64(deletedAt), int64(0)) +} + +func TestIsTwinDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Not deleted initially + isDeleted, _, err := db.IsTwinDeleted(1) + require.NoError(err) + require.False(isDeleted) + + // Delete deployment - twin should be marked deleted + err = db.Delete(1, 10) + require.NoError(err) + + // Should be deleted now + isDeleted, deletedAt, err := db.IsTwinDeleted(1) + require.NoError(err) + require.True(isDeleted) + require.Greater(int64(deletedAt), int64(0)) +} + +func TestCleanDeletedRemovesOld(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create and delete a deployment + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Delete(1, 10) + require.NoError(err) + + // Clean up old deleted items (far future timestamp) + future := gridtypes.Now() + 1000000 + err = db.CleanDeleted(future) + require.NoError(err) + + // Even Get(WithDeleted()) should fail now + _, err = db.Get(1, 10, provision.WithDeleted()) + require.Error(err) + + // Twins(WithDeleted()) should be empty + allTwins, err := db.Twins(provision.WithDeleted()) + require.NoError(err) + require.Empty(allTwins) +} + +func TestCleanDeletedPreservesRecent(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create and delete a deployment + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Delete(1, 10) + require.NoError(err) + + // Clean up items deleted before now (should preserve recent deletion) + before := gridtypes.Now() - 1000 + err = db.CleanDeleted(before) + require.NoError(err) + + // Get(WithDeleted()) should still work + _, err = db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + + // Twins(WithDeleted()) should still show the twin + allTwins, err := db.Twins(provision.WithDeleted()) + require.NoError(err) + require.Len(allTwins, 1) +} + +func TestCleanDeletedWorkloads(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + + // Remove workload + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // Verify it's soft-deleted + deployment, err := db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + require.Len(deployment.Workloads, 1) + + // Clean up old deletions + future := gridtypes.Now() + 1000000 + err = db.CleanDeleted(future) + require.NoError(err) + + // Get(WithDeleted()) should be empty now for workloads + deployment, err = db.Get(1, 10, provision.WithDeleted()) + require.NoError(err) + require.Empty(deployment.Workloads) +} + +func TestCreateDeploymentUnderDeletedTwinRevivesTwin(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create first deployment for twin 1 + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Delete the deployment - twin should be marked as deleted + err = db.Delete(1, 10) + require.NoError(err) + + // Verify twin is deleted + isDeleted, deletedAt1, err := db.IsTwinDeleted(1) + require.NoError(err) + require.True(isDeleted) + require.Greater(int64(deletedAt1), int64(0)) + + // Create a new deployment under the same twin - should revive the twin + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl2) + require.NoError(err) + + // Verify twin is no longer deleted + isDeleted, _, err = db.IsTwinDeleted(1) + require.NoError(err) + require.False(isDeleted) + + // Verify the new deployment exists + loaded, err := db.Get(1, 20) + require.NoError(err) + require.Equal(uint32(1), loaded.TwinID) + require.Equal(uint64(20), loaded.ContractID) + + // Verify the twin appears in Twins() now + twins, err := db.Twins() + require.NoError(err) + require.Len(twins, 1) + require.Equal(uint32(1), twins[0]) +} + +// Group 1: New Changes (Twin Revival & Soft-delete) + +func TestCreateAfterCleanDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create twin=1, contract=10 + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Delete it + err = db.Delete(1, 10) + require.NoError(err) + + // CleanDeleted with future timestamp (purges the bucket entirely) + future := gridtypes.Now() + 1000000 + err = db.CleanDeleted(future) + require.NoError(err) + + // Create again for same twin with new contract=20 + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl2) + require.NoError(err) + + // Verify no error and IsTwinDeleted returns false + isDeleted, _, err := db.IsTwinDeleted(1) + require.NoError(err) + require.False(isDeleted) + + // Verify we can Get the new deployment + loaded, err := db.Get(1, 20) + require.NoError(err) + require.Equal(uint64(20), loaded.ContractID) +} + +func TestDeleteNoOpOnMissingTwin(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Delete non-existent twin 999 + err = db.Delete(999, 10) + require.NoError(err) // Should be no-op, no error +} + +func TestDeleteNoOpOnMissingDeployment(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create twin 1 + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Delete non-existent deployment 999 under twin 1 + err = db.Delete(1, 999) + require.NoError(err) // Should be no-op, no error +} + +func TestDeleteWithSharableWorkloadMarksAllDeleted(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment with sharable workload + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{ + {Type: testSharableType1, Name: "net"}, + }, + } + err = db.Create(dl) + require.NoError(err) + + // Delete the deployment + err = db.Delete(1, 10) + require.NoError(err) + + // Twin should be marked as deleted (cursor correctly skipped "global" sub-bucket) + isDeleted, _, err := db.IsTwinDeleted(1) + require.NoError(err) + require.True(isDeleted) +} + +func TestResurrectSharableWorkload(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment with sharable workload + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{ + {Type: testSharableType1, Name: "net"}, + }, + } + err = db.Create(dl) + require.NoError(err) + + // Remove the workload (soft-delete, clears global bucket) + err = db.Remove(1, 10, "net") + require.NoError(err) + + // Add it back (re-registers in global bucket) + err = db.Add(1, 10, gridtypes.Workload{Type: testSharableType1, Name: "net"}) + require.NoError(err) + + // Verify workload is active + wl, err := db.Current(1, 10, "net") + require.NoError(err) + require.Equal(gridtypes.StateInit, wl.Result.State) + + // Verify global bucket has the entry: trying to add to a second deployment + // should trigger ErrDeploymentConflict + dl2 := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 20, + Workloads: []gridtypes.Workload{ + {Type: testSharableType1, Name: "net"}, + }, + } + err = db.Create(dl2) + require.ErrorIs(err, provision.ErrDeploymentConflict) +} + +func TestCleanDeletedPreservesTwinWithRemainingDeployments(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create two deployments for same twin + dl1 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl1) + require.NoError(err) + err = db.Create(dl2) + require.NoError(err) + + // Delete first deployment + err = db.Delete(1, 10) + require.NoError(err) + + // CleanDeleted with future timestamp + future := gridtypes.Now() + 1000000 + err = db.CleanDeleted(future) + require.NoError(err) + + // Deleted deployment should be purged + _, err = db.Get(1, 10, provision.WithDeleted()) + require.Error(err) + + // Active deployment should still be reachable + loaded, err := db.Get(1, 20) + require.NoError(err) + require.Equal(uint64(20), loaded.ContractID) + + // Twin should still exist and not be deleted (has remaining deployment) + isDeleted, _, err := db.IsTwinDeleted(1) + require.NoError(err) + require.False(isDeleted) +} + +func TestCleanDeletedLeavesActiveDeploymentBucketIntact(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment with active workload + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{ + {Type: testType1, Name: "vm1"}, + }, + } + err = db.Create(dl) + require.NoError(err) + + // Remove workload (soft-delete) + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // CleanDeleted with future timestamp (purges deleted workload but not deployment) + future := gridtypes.Now() + 1000000 + err = db.CleanDeleted(future) + require.NoError(err) + + // Deployment should still be reachable and intact + loaded, err := db.Get(1, 10) + require.NoError(err) + require.Equal(uint32(1), loaded.TwinID) + require.Equal(uint64(10), loaded.ContractID) +} + +// Group 2: Error & No-op Paths + +func TestRemoveNoOpOnMissingWorkload(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Remove non-existent workload + err = db.Remove(1, 10, "nonexistent") + require.NoError(err) // No-op, should return nil +} + +func TestRemoveNoOpOnMissingTwin(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Remove from non-existent twin + err = db.Remove(999, 10, "vm1") + require.NoError(err) // No-op, should return nil +} + +func TestRemoveSharableWorkloadCleansGlobalBucket(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment with sharable workload + dl1 := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Workloads: []gridtypes.Workload{ + {Type: testSharableType1, Name: "net"}, + }, + } + err = db.Create(dl1) + require.NoError(err) + + // Create second empty deployment + dl2 := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 20} + err = db.Create(dl2) + require.NoError(err) + + // Remove sharable workload from first deployment + err = db.Remove(1, 10, "net") + require.NoError(err) + + // Add same sharable name to second deployment - should succeed (global entry was cleared) + err = db.Add(1, 20, gridtypes.Workload{Type: testSharableType1, Name: "net"}) + require.NoError(err) +} + +func TestIsTwinDeletedNotFound(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // IsTwinDeleted on non-existent twin + _, _, err = db.IsTwinDeleted(999) + require.Error(err) +} + +func TestIsDeploymentDeletedNotFound(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create twin 1 + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // IsDeploymentDeleted on non-existent twin + _, _, err = db.IsDeploymentDeleted(999, 10) + require.ErrorIs(err, provision.ErrDeploymentNotExists) + + // IsDeploymentDeleted on non-existent deployment + _, _, err = db.IsDeploymentDeleted(1, 999) + require.ErrorIs(err, provision.ErrDeploymentNotExists) +} + +func TestTransactionOnSoftDeletedWorkloadFails(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment with workload + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + + // Remove workload (soft-delete) + err = db.Remove(1, 10, "vm1") + require.NoError(err) + + // Transaction on soft-deleted workload should fail + err = db.Transaction(1, 10, gridtypes.Workload{ + Type: testType1, + Name: "vm1", + Result: gridtypes.Result{ + Created: gridtypes.Now(), + State: gridtypes.StateOk, + }, + }) + require.ErrorIs(err, ErrInvalidWorkloadType) +} + +// Group 3: Completely Uncovered Public Methods + +func TestUpdate(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment + dl := gridtypes.Deployment{ + Version: 1, + TwinID: 1, + ContractID: 10, + Description: "original", + Metadata: "meta", + } + err = db.Create(dl) + require.NoError(err) + + // Update version and description + err = db.Update(1, 10, + provision.VersionField{Version: 2}, + provision.DescriptionField{Description: "updated"}, + ) + require.NoError(err) + + // Verify changes + loaded, err := db.Get(1, 10) + require.NoError(err) + require.Equal(uint32(2), loaded.Version) + require.Equal("updated", loaded.Description) + require.Equal("meta", loaded.Metadata) + + // Update metadata + err = db.Update(1, 10, + provision.MetadataField{Metadata: "new_meta"}, + ) + require.NoError(err) + + loaded, err = db.Get(1, 10) + require.NoError(err) + require.Equal("new_meta", loaded.Metadata) +} + +func TestUpdateTwinNotFound(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Update non-existent twin + err = db.Update(999, 10, provision.VersionField{Version: 2}) + require.ErrorIs(err, provision.ErrDeploymentNotExists) +} + +func TestUpdateDeploymentNotFound(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create twin 1 + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Update non-existent deployment + err = db.Update(1, 999, provision.VersionField{Version: 2}) + require.ErrorIs(err, provision.ErrDeploymentNotExists) +} + +func TestChanges(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Add workload + err = db.Add(1, 10, gridtypes.Workload{Name: "vm1", Type: testType1}) + require.NoError(err) + + // Get changes - should have one entry (StateInit from Add) + changes, err := db.Changes(1, 10) + require.NoError(err) + require.Len(changes, 1) + require.Equal(gridtypes.Name("vm1"), changes[0].Name) + require.Equal(gridtypes.StateInit, changes[0].Result.State) + + // Add a transaction + err = db.Transaction(1, 10, gridtypes.Workload{ + Type: testType1, + Name: "vm1", + Result: gridtypes.Result{ + Created: gridtypes.Now(), + State: gridtypes.StateOk, + }, + }) + require.NoError(err) + + // Get changes again - should have two entries + changes, err = db.Changes(1, 10) + require.NoError(err) + require.Len(changes, 2) + require.Equal(gridtypes.StateInit, changes[0].Result.State) + require.Equal(gridtypes.StateOk, changes[1].Result.State) +} + +func TestChangesNoWorkloads(t *testing.T) { + require := require.New(t) + path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63())) + defer os.RemoveAll(path) + + db, err := New(path) + require.NoError(err) + + // Create deployment without workloads + dl := gridtypes.Deployment{Version: 1, TwinID: 1, ContractID: 10} + err = db.Create(dl) + require.NoError(err) + + // Get changes - should be empty + changes, err := db.Changes(1, 10) + require.NoError(err) + require.Empty(changes) +} diff --git a/pkg/stubs/provision_stub.go b/pkg/stubs/provision_stub.go index cc24258f..8e47d53e 100644 --- a/pkg/stubs/provision_stub.go +++ b/pkg/stubs/provision_stub.go @@ -176,3 +176,72 @@ func (s *ProvisionStub) ListTwins(ctx context.Context) (ret0 []uint32, ret1 erro } return } + +func (s *ProvisionStub) GetDeployment(ctx context.Context, arg0 uint32, arg1 uint64) (ret0 gridtypes.Deployment, ret1 error) { + args := []interface{}{arg0, arg1} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetDeployment", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *ProvisionStub) GetDeployments(ctx context.Context, arg0 uint32) (ret0 []gridtypes.Deployment, ret1 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetDeployments", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *ProvisionStub) GetTwins(ctx context.Context) (ret0 []uint32, ret1 error) { + args := []interface{}{} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetTwins", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret1 = result.CallError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + +func (s *ProvisionStub) GetWorkload(ctx context.Context, arg0 uint32, arg1 uint64, arg2 gridtypes.Name) (ret0 gridtypes.Workload, ret1 bool, ret2 error) { + args := []interface{}{arg0, arg1, arg2} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetWorkload", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + ret2 = result.CallError() + loader := zbus.Loader{ + &ret0, + &ret1, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} diff --git a/pkg/zos_api/debug.go b/pkg/zos_api/debug.go index cea2097a..cf1ec3a0 100644 --- a/pkg/zos_api/debug.go +++ b/pkg/zos_api/debug.go @@ -48,8 +48,8 @@ func (g *ZosAPI) debugDeploymentHealthHandler(ctx context.Context, payload []byt func (g *ZosAPI) debugDeps() debugcmd.Deps { return debugcmd.Deps{ - Provision: g.provisionStub, - VM: g.vmStub, - Network: g.networkerStub, + VM: g.vmStub, + Network: g.networkerStub, + Storage: g.provisionStub, } } diff --git a/pkg/zos_api/zos_api.go b/pkg/zos_api/zos_api.go index d9287ce1..e4c1a655 100644 --- a/pkg/zos_api/zos_api.go +++ b/pkg/zos_api/zos_api.go @@ -46,6 +46,7 @@ func NewZosAPI(manager substrate.Manager, client zbus.Client, msgBrokerCon strin return ZosAPI{}, err } storageModuleStub := stubs.NewStorageModuleStub(client) + api := ZosAPI{ oracle: capacity.NewResourceOracle(storageModuleStub), versionMonitorStub: stubs.NewVersionMonitorStub(client),