From ca8268513691f52c5f47481e27174dcbea25ea57 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 14:00:26 +0100 Subject: [PATCH 001/191] Move old code out the way while creating the new data model --- internal/pkg/datamodel/session.go | 29 ++++++++ internal/pkg/datamodel/volume.go | 72 +++++++++++++++++++ .../dacctl/actions/actions.go | 0 .../dacctl/actions/actions_test.go | 0 internal/{pkg => pkg_old}/dacctl/buffer.go | 0 .../{pkg => pkg_old}/dacctl/buffer_test.go | 0 internal/{pkg => pkg_old}/dacctl/job.go | 0 internal/{pkg => pkg_old}/dacctl/job_test.go | 0 internal/{pkg => pkg_old}/dacctl/json.go | 0 .../{pkg => pkg_old}/dacctl/persistent.go | 0 internal/{pkg => pkg_old}/dacctl/pools.go | 0 .../{pkg => pkg_old}/dacctl/pools_test.go | 0 internal/{pkg => pkg_old}/dacctl/show.go | 0 internal/{pkg => pkg_old}/dacctl/show_test.go | 0 .../{pkg => pkg_old}/etcdregistry/keystore.go | 0 .../{pkg => pkg_old}/etcdregistry/watch.go | 0 .../etcdregistry/watch_test.go | 0 internal/{pkg => pkg_old}/fileio/disk.go | 0 .../keystoreregistry/keystore.go | 0 .../{pkg => pkg_old}/keystoreregistry/pool.go | 0 .../keystoreregistry/pool_test.go | 0 .../keystoreregistry/volume.go | 0 .../keystoreregistry/volume_test.go | 0 .../lifecycle/brickmanager/bricks.go | 0 .../lifecycle/brickmanager/manager.go | 0 .../lifecycle/brickmanager/manager_test.go | 0 internal/{pkg => pkg_old}/lifecycle/volume.go | 0 .../{pkg => pkg_old}/lifecycle/volume_test.go | 0 internal/{pkg => pkg_old}/mocks/disk_mock.go | 0 internal/{pkg => pkg_old}/mocks/job_mock.go | 0 .../mocks/pfsprovider_mock.go | 0 internal/{pkg => pkg_old}/mocks/pool_mock.go | 0 .../{pkg => pkg_old}/mocks/volume_mock.go | 0 .../pfsprovider/ansible/ansible.go | 0 .../pfsprovider/ansible/ansible_test.go | 0 .../pfsprovider/ansible/copy.go | 0 .../pfsprovider/ansible/copy_test.go | 0 .../pfsprovider/ansible/mount.go | 0 .../pfsprovider/ansible/mount_test.go | 0 .../pfsprovider/ansible/plugin.go | 0 .../pfsprovider/fake/plugin.go | 0 .../{pkg => pkg_old}/pfsprovider/interface.go | 0 internal/{pkg => pkg_old}/registry/pool.go | 0 internal/{pkg => pkg_old}/registry/volume.go | 0 44 files changed, 101 insertions(+) create mode 100644 internal/pkg/datamodel/session.go create mode 100644 internal/pkg/datamodel/volume.go rename internal/{pkg => pkg_old}/dacctl/actions/actions.go (100%) rename internal/{pkg => pkg_old}/dacctl/actions/actions_test.go (100%) rename internal/{pkg => pkg_old}/dacctl/buffer.go (100%) rename internal/{pkg => pkg_old}/dacctl/buffer_test.go (100%) rename internal/{pkg => pkg_old}/dacctl/job.go (100%) rename internal/{pkg => pkg_old}/dacctl/job_test.go (100%) rename internal/{pkg => pkg_old}/dacctl/json.go (100%) rename internal/{pkg => pkg_old}/dacctl/persistent.go (100%) rename internal/{pkg => pkg_old}/dacctl/pools.go (100%) rename internal/{pkg => pkg_old}/dacctl/pools_test.go (100%) rename internal/{pkg => pkg_old}/dacctl/show.go (100%) rename internal/{pkg => pkg_old}/dacctl/show_test.go (100%) rename internal/{pkg => pkg_old}/etcdregistry/keystore.go (100%) rename internal/{pkg => pkg_old}/etcdregistry/watch.go (100%) rename internal/{pkg => pkg_old}/etcdregistry/watch_test.go (100%) rename internal/{pkg => pkg_old}/fileio/disk.go (100%) rename internal/{pkg => pkg_old}/keystoreregistry/keystore.go (100%) rename internal/{pkg => pkg_old}/keystoreregistry/pool.go (100%) rename internal/{pkg => pkg_old}/keystoreregistry/pool_test.go (100%) rename internal/{pkg => pkg_old}/keystoreregistry/volume.go (100%) rename internal/{pkg => pkg_old}/keystoreregistry/volume_test.go (100%) rename internal/{pkg => pkg_old}/lifecycle/brickmanager/bricks.go (100%) rename internal/{pkg => pkg_old}/lifecycle/brickmanager/manager.go (100%) rename internal/{pkg => pkg_old}/lifecycle/brickmanager/manager_test.go (100%) rename internal/{pkg => pkg_old}/lifecycle/volume.go (100%) rename internal/{pkg => pkg_old}/lifecycle/volume_test.go (100%) rename internal/{pkg => pkg_old}/mocks/disk_mock.go (100%) rename internal/{pkg => pkg_old}/mocks/job_mock.go (100%) rename internal/{pkg => pkg_old}/mocks/pfsprovider_mock.go (100%) rename internal/{pkg => pkg_old}/mocks/pool_mock.go (100%) rename internal/{pkg => pkg_old}/mocks/volume_mock.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/ansible.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/ansible_test.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/copy.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/copy_test.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/mount.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/mount_test.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/ansible/plugin.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/fake/plugin.go (100%) rename internal/{pkg => pkg_old}/pfsprovider/interface.go (100%) rename internal/{pkg => pkg_old}/registry/pool.go (100%) rename internal/{pkg => pkg_old}/registry/volume.go (100%) diff --git a/internal/pkg/datamodel/session.go b/internal/pkg/datamodel/session.go new file mode 100644 index 00000000..6fa6edde --- /dev/null +++ b/internal/pkg/datamodel/session.go @@ -0,0 +1,29 @@ +package datamodel + +type SessionName string + +type Session struct { + // Job name or persistent buffer name + Name SessionName + + // unix uid and gid + Owner uint + Group uint + + // utc unix timestamp when buffer created + CreatedAt uint + + // The hosts that want to mount the storage + // Note: to allow for copy in/out the brick hosts are assumed to have an attachment + AttachHosts []string + + // If non-zero capacity requested, a volume is created for this job + // It may be exposed to the attach hosts in a variety of ways, as defined by the volume + JobVolume VolumeName + + // There maybe be attachments to multiple shared volumes + MultiJobVolumes []VolumeName + + // Environment variables for each volume associated with the job + Paths map[string]string +} \ No newline at end of file diff --git a/internal/pkg/datamodel/volume.go b/internal/pkg/datamodel/volume.go new file mode 100644 index 00000000..dc953477 --- /dev/null +++ b/internal/pkg/datamodel/volume.go @@ -0,0 +1,72 @@ +package datamodel + +type VolumeName string + +type Volume struct { + // e.g. job1 or Foo + Name VolumeName + // its 8 characters long, so works nicely with lustre + UUID string + // True if multiple jobs can attach to this volume + MultiJob bool + + // Message requested actions to primary brick host + // TODO: move mount and data copy actions to other parts of the volume state + State VolumeState + + // Requested pool of bricks for volume + Pool string // TODO: PoolName? + // Number of bricks requested, calculated from requested capacity + SizeBricks uint + // Actual size of the volume + SizeGB uint + + // Back reference to what job created this volume + JobName string + // e.g. 1001 + Owner uint + // If empty defaults to User + Group uint + // e.g. SLURM or Manila + CreatedBy string + // The unix (utc) timestamp of when this volume was created + CreatedAt uint + + // TODO: need to fill these in... + // They all related to how the volume is attached + + // All current attachments + Attachments []Attachment + // Attach all attachments to a shared global namespace + // Allowed for any volume type + AttachGlobalNamespace bool + // Have an attachment specific namespace mounted, only for non multi job + AttachPrivateNamespace bool + // If not zero, swap of the requested amount mounted for each attachment + // Not allowed for multi job + AttachAsSwapBytes uint + // Add attachment specific cache for each given filesystem path + // Not allowed for multi job + // Note: assumes the same path is cached for all attachments + AttachPrivateCache []string + + // TODO: maybe data copy should be a slice associated with the job? + // Request certain files to be staged in + // Not currently allowed for multi job volumes + StageIn DataCopyRequest + // Request certain files to be staged in + // Not currently allowed for multi job volumes + StageOut DataCopyRequest + + // BeeGFS wants each fs to be assigned a unique port number + ClientPort int + + // Track if we have had bricks assigned + // if we request delete, no bricks ever assigned, don't ait for dacd! + HadBricksAssigned bool + + // TODO: data model currently does not do these things well: + // 1. correctly track multiple jobs at the same time attach to the same persistent buffer + // 2. data in/out requests for persistent buffer + // 3. track amount of space used by swap and/or metadata +} diff --git a/internal/pkg/dacctl/actions/actions.go b/internal/pkg_old/dacctl/actions/actions.go similarity index 100% rename from internal/pkg/dacctl/actions/actions.go rename to internal/pkg_old/dacctl/actions/actions.go diff --git a/internal/pkg/dacctl/actions/actions_test.go b/internal/pkg_old/dacctl/actions/actions_test.go similarity index 100% rename from internal/pkg/dacctl/actions/actions_test.go rename to internal/pkg_old/dacctl/actions/actions_test.go diff --git a/internal/pkg/dacctl/buffer.go b/internal/pkg_old/dacctl/buffer.go similarity index 100% rename from internal/pkg/dacctl/buffer.go rename to internal/pkg_old/dacctl/buffer.go diff --git a/internal/pkg/dacctl/buffer_test.go b/internal/pkg_old/dacctl/buffer_test.go similarity index 100% rename from internal/pkg/dacctl/buffer_test.go rename to internal/pkg_old/dacctl/buffer_test.go diff --git a/internal/pkg/dacctl/job.go b/internal/pkg_old/dacctl/job.go similarity index 100% rename from internal/pkg/dacctl/job.go rename to internal/pkg_old/dacctl/job.go diff --git a/internal/pkg/dacctl/job_test.go b/internal/pkg_old/dacctl/job_test.go similarity index 100% rename from internal/pkg/dacctl/job_test.go rename to internal/pkg_old/dacctl/job_test.go diff --git a/internal/pkg/dacctl/json.go b/internal/pkg_old/dacctl/json.go similarity index 100% rename from internal/pkg/dacctl/json.go rename to internal/pkg_old/dacctl/json.go diff --git a/internal/pkg/dacctl/persistent.go b/internal/pkg_old/dacctl/persistent.go similarity index 100% rename from internal/pkg/dacctl/persistent.go rename to internal/pkg_old/dacctl/persistent.go diff --git a/internal/pkg/dacctl/pools.go b/internal/pkg_old/dacctl/pools.go similarity index 100% rename from internal/pkg/dacctl/pools.go rename to internal/pkg_old/dacctl/pools.go diff --git a/internal/pkg/dacctl/pools_test.go b/internal/pkg_old/dacctl/pools_test.go similarity index 100% rename from internal/pkg/dacctl/pools_test.go rename to internal/pkg_old/dacctl/pools_test.go diff --git a/internal/pkg/dacctl/show.go b/internal/pkg_old/dacctl/show.go similarity index 100% rename from internal/pkg/dacctl/show.go rename to internal/pkg_old/dacctl/show.go diff --git a/internal/pkg/dacctl/show_test.go b/internal/pkg_old/dacctl/show_test.go similarity index 100% rename from internal/pkg/dacctl/show_test.go rename to internal/pkg_old/dacctl/show_test.go diff --git a/internal/pkg/etcdregistry/keystore.go b/internal/pkg_old/etcdregistry/keystore.go similarity index 100% rename from internal/pkg/etcdregistry/keystore.go rename to internal/pkg_old/etcdregistry/keystore.go diff --git a/internal/pkg/etcdregistry/watch.go b/internal/pkg_old/etcdregistry/watch.go similarity index 100% rename from internal/pkg/etcdregistry/watch.go rename to internal/pkg_old/etcdregistry/watch.go diff --git a/internal/pkg/etcdregistry/watch_test.go b/internal/pkg_old/etcdregistry/watch_test.go similarity index 100% rename from internal/pkg/etcdregistry/watch_test.go rename to internal/pkg_old/etcdregistry/watch_test.go diff --git a/internal/pkg/fileio/disk.go b/internal/pkg_old/fileio/disk.go similarity index 100% rename from internal/pkg/fileio/disk.go rename to internal/pkg_old/fileio/disk.go diff --git a/internal/pkg/keystoreregistry/keystore.go b/internal/pkg_old/keystoreregistry/keystore.go similarity index 100% rename from internal/pkg/keystoreregistry/keystore.go rename to internal/pkg_old/keystoreregistry/keystore.go diff --git a/internal/pkg/keystoreregistry/pool.go b/internal/pkg_old/keystoreregistry/pool.go similarity index 100% rename from internal/pkg/keystoreregistry/pool.go rename to internal/pkg_old/keystoreregistry/pool.go diff --git a/internal/pkg/keystoreregistry/pool_test.go b/internal/pkg_old/keystoreregistry/pool_test.go similarity index 100% rename from internal/pkg/keystoreregistry/pool_test.go rename to internal/pkg_old/keystoreregistry/pool_test.go diff --git a/internal/pkg/keystoreregistry/volume.go b/internal/pkg_old/keystoreregistry/volume.go similarity index 100% rename from internal/pkg/keystoreregistry/volume.go rename to internal/pkg_old/keystoreregistry/volume.go diff --git a/internal/pkg/keystoreregistry/volume_test.go b/internal/pkg_old/keystoreregistry/volume_test.go similarity index 100% rename from internal/pkg/keystoreregistry/volume_test.go rename to internal/pkg_old/keystoreregistry/volume_test.go diff --git a/internal/pkg/lifecycle/brickmanager/bricks.go b/internal/pkg_old/lifecycle/brickmanager/bricks.go similarity index 100% rename from internal/pkg/lifecycle/brickmanager/bricks.go rename to internal/pkg_old/lifecycle/brickmanager/bricks.go diff --git a/internal/pkg/lifecycle/brickmanager/manager.go b/internal/pkg_old/lifecycle/brickmanager/manager.go similarity index 100% rename from internal/pkg/lifecycle/brickmanager/manager.go rename to internal/pkg_old/lifecycle/brickmanager/manager.go diff --git a/internal/pkg/lifecycle/brickmanager/manager_test.go b/internal/pkg_old/lifecycle/brickmanager/manager_test.go similarity index 100% rename from internal/pkg/lifecycle/brickmanager/manager_test.go rename to internal/pkg_old/lifecycle/brickmanager/manager_test.go diff --git a/internal/pkg/lifecycle/volume.go b/internal/pkg_old/lifecycle/volume.go similarity index 100% rename from internal/pkg/lifecycle/volume.go rename to internal/pkg_old/lifecycle/volume.go diff --git a/internal/pkg/lifecycle/volume_test.go b/internal/pkg_old/lifecycle/volume_test.go similarity index 100% rename from internal/pkg/lifecycle/volume_test.go rename to internal/pkg_old/lifecycle/volume_test.go diff --git a/internal/pkg/mocks/disk_mock.go b/internal/pkg_old/mocks/disk_mock.go similarity index 100% rename from internal/pkg/mocks/disk_mock.go rename to internal/pkg_old/mocks/disk_mock.go diff --git a/internal/pkg/mocks/job_mock.go b/internal/pkg_old/mocks/job_mock.go similarity index 100% rename from internal/pkg/mocks/job_mock.go rename to internal/pkg_old/mocks/job_mock.go diff --git a/internal/pkg/mocks/pfsprovider_mock.go b/internal/pkg_old/mocks/pfsprovider_mock.go similarity index 100% rename from internal/pkg/mocks/pfsprovider_mock.go rename to internal/pkg_old/mocks/pfsprovider_mock.go diff --git a/internal/pkg/mocks/pool_mock.go b/internal/pkg_old/mocks/pool_mock.go similarity index 100% rename from internal/pkg/mocks/pool_mock.go rename to internal/pkg_old/mocks/pool_mock.go diff --git a/internal/pkg/mocks/volume_mock.go b/internal/pkg_old/mocks/volume_mock.go similarity index 100% rename from internal/pkg/mocks/volume_mock.go rename to internal/pkg_old/mocks/volume_mock.go diff --git a/internal/pkg/pfsprovider/ansible/ansible.go b/internal/pkg_old/pfsprovider/ansible/ansible.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/ansible.go rename to internal/pkg_old/pfsprovider/ansible/ansible.go diff --git a/internal/pkg/pfsprovider/ansible/ansible_test.go b/internal/pkg_old/pfsprovider/ansible/ansible_test.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/ansible_test.go rename to internal/pkg_old/pfsprovider/ansible/ansible_test.go diff --git a/internal/pkg/pfsprovider/ansible/copy.go b/internal/pkg_old/pfsprovider/ansible/copy.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/copy.go rename to internal/pkg_old/pfsprovider/ansible/copy.go diff --git a/internal/pkg/pfsprovider/ansible/copy_test.go b/internal/pkg_old/pfsprovider/ansible/copy_test.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/copy_test.go rename to internal/pkg_old/pfsprovider/ansible/copy_test.go diff --git a/internal/pkg/pfsprovider/ansible/mount.go b/internal/pkg_old/pfsprovider/ansible/mount.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/mount.go rename to internal/pkg_old/pfsprovider/ansible/mount.go diff --git a/internal/pkg/pfsprovider/ansible/mount_test.go b/internal/pkg_old/pfsprovider/ansible/mount_test.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/mount_test.go rename to internal/pkg_old/pfsprovider/ansible/mount_test.go diff --git a/internal/pkg/pfsprovider/ansible/plugin.go b/internal/pkg_old/pfsprovider/ansible/plugin.go similarity index 100% rename from internal/pkg/pfsprovider/ansible/plugin.go rename to internal/pkg_old/pfsprovider/ansible/plugin.go diff --git a/internal/pkg/pfsprovider/fake/plugin.go b/internal/pkg_old/pfsprovider/fake/plugin.go similarity index 100% rename from internal/pkg/pfsprovider/fake/plugin.go rename to internal/pkg_old/pfsprovider/fake/plugin.go diff --git a/internal/pkg/pfsprovider/interface.go b/internal/pkg_old/pfsprovider/interface.go similarity index 100% rename from internal/pkg/pfsprovider/interface.go rename to internal/pkg_old/pfsprovider/interface.go diff --git a/internal/pkg/registry/pool.go b/internal/pkg_old/registry/pool.go similarity index 100% rename from internal/pkg/registry/pool.go rename to internal/pkg_old/registry/pool.go diff --git a/internal/pkg/registry/volume.go b/internal/pkg_old/registry/volume.go similarity index 100% rename from internal/pkg/registry/volume.go rename to internal/pkg_old/registry/volume.go From 81e7198fb615224dcbe57c81effa7e5f8bcc2615 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 16:38:59 +0100 Subject: [PATCH 002/191] First draft of new datamodel and interfaces --- internal/pkg/actions/session/session.go | 27 +++++++++++ internal/pkg/data/brickhost/bricks.go | 63 +++++++++++++++++++++++++ internal/pkg/data/brickhost/volume.go | 5 ++ internal/pkg/data/session/session.go | 25 ++++++++++ internal/pkg/datamodel/allocation.go | 24 ++++++++++ internal/pkg/datamodel/brickhost.go | 32 +++++++++++++ internal/pkg/datamodel/pool.go | 12 +++++ internal/pkg/datamodel/session.go | 60 ++++++++++++++++++++++- internal/pkg/datamodel/volume.go | 15 ++---- 9 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 internal/pkg/actions/session/session.go create mode 100644 internal/pkg/data/brickhost/bricks.go create mode 100644 internal/pkg/data/brickhost/volume.go create mode 100644 internal/pkg/data/session/session.go create mode 100644 internal/pkg/datamodel/allocation.go create mode 100644 internal/pkg/datamodel/brickhost.go create mode 100644 internal/pkg/datamodel/pool.go diff --git a/internal/pkg/actions/session/session.go b/internal/pkg/actions/session/session.go new file mode 100644 index 00000000..92d0081f --- /dev/null +++ b/internal/pkg/actions/session/session.go @@ -0,0 +1,27 @@ +package session + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + +// Each volume has an associated primary brick +// that is responsible for responding to any actions +// All the calls block until they are complete, or an error occurs +type Actions interface { + // Creates the requested volumes + // Error if Session has not had its bricks allocated + CreateSessionVolume(session datamodel.Session) error + + // Deletes the requested volume and session allocation + DeleteSession(session datamodel.Session) error + + // Update the session and trigger requested data copy in + DataIn(session datamodel.Session) error + + // Update session hosts and attach volumes as needed + AttachVolumes(session datamodel.Session) error + + // Attempt to detach volumes + DetachVolumes(session datamodel.Session) error + + // Update the session and trigger requested data copy out + DataOut(session datamodel.Session) error +} \ No newline at end of file diff --git a/internal/pkg/data/brickhost/bricks.go b/internal/pkg/data/brickhost/bricks.go new file mode 100644 index 00000000..49eaea5f --- /dev/null +++ b/internal/pkg/data/brickhost/bricks.go @@ -0,0 +1,63 @@ +package brickhost + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + +type BrickRegistry interface { + // Returns a summary of the current state of all pools, including the bricks in each pool + Pools() ([]datamodel.Pool, error) + + // BrickHosts need to check they match a pool + // which may involve creating the default pool + EnsureDefaultPoolCreated(granularityGB int) (datamodel.PoolName, error) + + // BrickHost updates bricks on startup + UpdateBrickHost(brickHostInfo datamodel.BrickHostInfo) error + + // Get information on the BrickHost + GetBrickHostInfo(name datamodel.BrickHostName) (datamodel.BrickHostInfo, error) + + // While the process is still running this notifies others the host is up + // + // When a host is dead non of its bricks will get new volumes assigned, + // and no bricks will get cleaned up until the next service start. + // Error will be returned if the host info has not yet been written. + KeepAliveHost(hostname string) error + + // Update a brick with allocation information. + // + // No update is made and an error is returned if: + // any brick already has an allocation, + // or any volume a brick is being assigned to already has an allocation, + // or if any of the volumes do not exist + // or if there is not exactly one primary brick. + // + // Note: you may assign multiple volumes in a single call, but all bricks + // for a particular volume must be set in a single call + AllocateBricksForVolume(volume Volume) ([]BrickAllocation, error) + + // Deallocate all bricks associated with the given volume + // + // No update is made and an error is returned if any of brick allocations don't match the current state. + // If any host associated with one of the bricks is down, an error is returned and the deallocate is + // recorded as requested and not executed. + // Note: this returns as soon as deallocate is requested, doesn't wait for cleanup completion + DeallocateBricks(volume VolumeName) error + + // This is called after DeallocateBricks has been processed + HardDeleteAllocations(allocations []BrickAllocation) error + + // Get all the allocations for bricks associated with the specified hostname + GetAllocationsForHost(hostname string) ([]BrickAllocation, error) + + // Get all the allocations for bricks associated with the specific volume + GetAllocationsForVolume(volume VolumeName) ([]BrickAllocation, error) + + // Get information on a specific brick + GetBrickInfo(hostname string, device string) (BrickInfo, error) + + // Returns a channel that reports all new brick allocations for given hostname + // + // The channel is closed when the context is cancelled or timeout. + // Any errors in the watching log the issue and panic + GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan BrickAllocation +} diff --git a/internal/pkg/data/brickhost/volume.go b/internal/pkg/data/brickhost/volume.go new file mode 100644 index 00000000..f906d181 --- /dev/null +++ b/internal/pkg/data/brickhost/volume.go @@ -0,0 +1,5 @@ +package brickhost + +type VolumeRegistry interface { + +} \ No newline at end of file diff --git a/internal/pkg/data/session/session.go b/internal/pkg/data/session/session.go new file mode 100644 index 00000000..f69defe3 --- /dev/null +++ b/internal/pkg/data/session/session.go @@ -0,0 +1,25 @@ +package session + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + +type Registry interface { + // Gets a session and its current allocations + // Returns an error if the session is not found + GetSession(token string) (datamodel.Session, error) + + // Any required allocations are created for the given session + // such that actions can now be sent to the given session + // Returns an error if the session already exists + // Note that deleting a session and its allocation is an action, as is any update + CreateSessionAllocations(s datamodel.Session) (datamodel.Session, error) + + // Checks it would be a valid call to CreateAllocations + // Error will describe any validation issues + ValidateSessionRequest(token string) (datamodel.Session, error) + + // Used for show instances and show sessions + GetAllSessions() ([]datamodel.Session, error) + + // Get all bricks listed by pools + GetAllPools() ([]datamodel.Pool, error) +} \ No newline at end of file diff --git a/internal/pkg/datamodel/allocation.go b/internal/pkg/datamodel/allocation.go new file mode 100644 index 00000000..6136212e --- /dev/null +++ b/internal/pkg/datamodel/allocation.go @@ -0,0 +1,24 @@ +package datamodel + +// You can only have zero or one allocations records for each Brick +type BrickAllocation struct { + // Brick that is allocated + Brick BrickInfo + + // Name of the volume that owns the brick + AllocatedVolume VolumeName + + // 0 index allocation is the primary brick, + // which is responsible for provisioning the associated volume + AllocatedIndex uint + + // One primary prick per volume + // this brick is responsible for watching for + // associated job and volume actions + IsPrimaryBrick bool + + // If any allocation sent to deallocate has a host that isn't + // alive, this flag is set rather than have allocations removed. + // A host should check for any allocations + DeallocateRequested bool +} \ No newline at end of file diff --git a/internal/pkg/datamodel/brickhost.go b/internal/pkg/datamodel/brickhost.go new file mode 100644 index 00000000..8e7c0652 --- /dev/null +++ b/internal/pkg/datamodel/brickhost.go @@ -0,0 +1,32 @@ +package datamodel + +type BrickHostName string + +type BrickHostInfo struct { + Name BrickHostName + + // Returns all bricks + Bricks []BrickInfo + + // True if dacd is detected as running + Alive bool + + // True if allowing new volumes to use bricks from this host + Enabled bool +} + +type BrickInfo struct { + // Bricks are identified by device and hostname + // It must only contain the characters A-Za-z0-9 + Device string + + // It must only contain the characters "A-Za-z0-9." + Hostname string + + // The bool a brick is associated with + // It must only contain the characters A-Za-z0-9 + PoolName string + + // Size of the brick, defines the pool granularity + CapacityGB uint +} \ No newline at end of file diff --git a/internal/pkg/datamodel/pool.go b/internal/pkg/datamodel/pool.go new file mode 100644 index 00000000..4e4d8e56 --- /dev/null +++ b/internal/pkg/datamodel/pool.go @@ -0,0 +1,12 @@ +package datamodel + +type PoolName string + +type Pool struct { + // The pool is derived from all the reported bricks + Name PoolName + + // This is the allocation unit for the pool + // It is the minimum size of any registered brick + GranularityGB uint +} \ No newline at end of file diff --git a/internal/pkg/datamodel/session.go b/internal/pkg/datamodel/session.go index 6fa6edde..304c8a7d 100644 --- a/internal/pkg/datamodel/session.go +++ b/internal/pkg/datamodel/session.go @@ -2,10 +2,17 @@ package datamodel type SessionName string +// This object is updated by dacctl +// And actions are sent relative to a Session +// and the primary brick waits for the session type Session struct { // Job name or persistent buffer name Name SessionName + // Currently stored revision + // this is checked when an update is requested + Revision int + // unix uid and gid Owner uint Group uint @@ -13,6 +20,20 @@ type Session struct { // utc unix timestamp when buffer created CreatedAt uint + // Details of what was requested + Request SessionRequest + + // Records if we have started trying to delete + DeleteRequested bool + + // Request certain files to be staged in + // Not currently allowed for multi job volumes + StageInRequests []DataCopyRequest + + // Request certain files to be staged in + // Not currently allowed for multi job volumes + StageOutRequests []DataCopyRequest + // The hosts that want to mount the storage // Note: to allow for copy in/out the brick hosts are assumed to have an attachment AttachHosts []string @@ -26,4 +47,41 @@ type Session struct { // Environment variables for each volume associated with the job Paths map[string]string -} \ No newline at end of file + + // List of the bricks allocated to implement the JobVolume + // One is the primary brick that should be watching for all actions + Allocations []BrickAllocation +} + +type SessionRequest struct { + + + // swap +} + +type DataCopyRequest struct { + // Source points to a File or a Directory, + // or a file that contains a list of source and destinations, + // with each pair on a new line + SourceType SourceType + // The path is either to a file or a directory or + // a list with source and destination file space separated pairs, each on a new line + Source string + // Must be empty string for type list, otherwise specified location + Destination string + // Used to notify if copy in has been requested + RequestCopyIn bool + // Report if the copy has completed + CopyCompleted bool + // if there was problem, record it + Error error +} + +type SourceType string + +const ( + File SourceType = "file" + Directory SourceType = "directory" + // Provided a file that has source and destination file space separated pairs, each on a new line + List SourceType = "list" +) \ No newline at end of file diff --git a/internal/pkg/datamodel/volume.go b/internal/pkg/datamodel/volume.go index dc953477..c37f3320 100644 --- a/internal/pkg/datamodel/volume.go +++ b/internal/pkg/datamodel/volume.go @@ -5,15 +5,13 @@ type VolumeName string type Volume struct { // e.g. job1 or Foo Name VolumeName + // its 8 characters long, so works nicely with lustre UUID string + // True if multiple jobs can attach to this volume MultiJob bool - // Message requested actions to primary brick host - // TODO: move mount and data copy actions to other parts of the volume state - State VolumeState - // Requested pool of bricks for volume Pool string // TODO: PoolName? // Number of bricks requested, calculated from requested capacity @@ -53,10 +51,10 @@ type Volume struct { // TODO: maybe data copy should be a slice associated with the job? // Request certain files to be staged in // Not currently allowed for multi job volumes - StageIn DataCopyRequest + StageInRequests []DataCopyRequest // Request certain files to be staged in // Not currently allowed for multi job volumes - StageOut DataCopyRequest + StageOutRequests []DataCopyRequest // BeeGFS wants each fs to be assigned a unique port number ClientPort int @@ -64,9 +62,4 @@ type Volume struct { // Track if we have had bricks assigned // if we request delete, no bricks ever assigned, don't ait for dacd! HadBricksAssigned bool - - // TODO: data model currently does not do these things well: - // 1. correctly track multiple jobs at the same time attach to the same persistent buffer - // 2. data in/out requests for persistent buffer - // 3. track amount of space used by swap and/or metadata } From a53ac39df786d8492a49b4ffd56c25dac1a5f8f4 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 16:46:59 +0100 Subject: [PATCH 003/191] Move old code back into place to fix build --- internal/{pkg_old => pkg}/dacctl/actions/actions.go | 0 internal/{pkg_old => pkg}/dacctl/actions/actions_test.go | 0 internal/{pkg_old => pkg}/dacctl/buffer.go | 0 internal/{pkg_old => pkg}/dacctl/buffer_test.go | 0 internal/{pkg_old => pkg}/dacctl/job.go | 0 internal/{pkg_old => pkg}/dacctl/job_test.go | 0 internal/{pkg_old => pkg}/dacctl/json.go | 0 internal/{pkg_old => pkg}/dacctl/persistent.go | 0 internal/{pkg_old => pkg}/dacctl/pools.go | 0 internal/{pkg_old => pkg}/dacctl/pools_test.go | 0 internal/{pkg_old => pkg}/dacctl/show.go | 0 internal/{pkg_old => pkg}/dacctl/show_test.go | 0 internal/{pkg_old => pkg}/etcdregistry/keystore.go | 0 internal/{pkg_old => pkg}/etcdregistry/watch.go | 0 internal/{pkg_old => pkg}/etcdregistry/watch_test.go | 0 internal/{pkg_old => pkg}/fileio/disk.go | 0 internal/{pkg_old => pkg}/keystoreregistry/keystore.go | 0 internal/{pkg_old => pkg}/keystoreregistry/pool.go | 0 internal/{pkg_old => pkg}/keystoreregistry/pool_test.go | 0 internal/{pkg_old => pkg}/keystoreregistry/volume.go | 0 internal/{pkg_old => pkg}/keystoreregistry/volume_test.go | 0 internal/{pkg_old => pkg}/lifecycle/brickmanager/bricks.go | 0 internal/{pkg_old => pkg}/lifecycle/brickmanager/manager.go | 0 internal/{pkg_old => pkg}/lifecycle/brickmanager/manager_test.go | 0 internal/{pkg_old => pkg}/lifecycle/volume.go | 0 internal/{pkg_old => pkg}/lifecycle/volume_test.go | 0 internal/{pkg_old => pkg}/mocks/disk_mock.go | 0 internal/{pkg_old => pkg}/mocks/job_mock.go | 0 internal/{pkg_old => pkg}/mocks/pfsprovider_mock.go | 0 internal/{pkg_old => pkg}/mocks/pool_mock.go | 0 internal/{pkg_old => pkg}/mocks/volume_mock.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/ansible.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/ansible_test.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/copy.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/copy_test.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/mount.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/mount_test.go | 0 internal/{pkg_old => pkg}/pfsprovider/ansible/plugin.go | 0 internal/{pkg_old => pkg}/pfsprovider/fake/plugin.go | 0 internal/{pkg_old => pkg}/pfsprovider/interface.go | 0 internal/{pkg_old => pkg}/registry/pool.go | 0 internal/{pkg_old => pkg}/registry/volume.go | 0 42 files changed, 0 insertions(+), 0 deletions(-) rename internal/{pkg_old => pkg}/dacctl/actions/actions.go (100%) rename internal/{pkg_old => pkg}/dacctl/actions/actions_test.go (100%) rename internal/{pkg_old => pkg}/dacctl/buffer.go (100%) rename internal/{pkg_old => pkg}/dacctl/buffer_test.go (100%) rename internal/{pkg_old => pkg}/dacctl/job.go (100%) rename internal/{pkg_old => pkg}/dacctl/job_test.go (100%) rename internal/{pkg_old => pkg}/dacctl/json.go (100%) rename internal/{pkg_old => pkg}/dacctl/persistent.go (100%) rename internal/{pkg_old => pkg}/dacctl/pools.go (100%) rename internal/{pkg_old => pkg}/dacctl/pools_test.go (100%) rename internal/{pkg_old => pkg}/dacctl/show.go (100%) rename internal/{pkg_old => pkg}/dacctl/show_test.go (100%) rename internal/{pkg_old => pkg}/etcdregistry/keystore.go (100%) rename internal/{pkg_old => pkg}/etcdregistry/watch.go (100%) rename internal/{pkg_old => pkg}/etcdregistry/watch_test.go (100%) rename internal/{pkg_old => pkg}/fileio/disk.go (100%) rename internal/{pkg_old => pkg}/keystoreregistry/keystore.go (100%) rename internal/{pkg_old => pkg}/keystoreregistry/pool.go (100%) rename internal/{pkg_old => pkg}/keystoreregistry/pool_test.go (100%) rename internal/{pkg_old => pkg}/keystoreregistry/volume.go (100%) rename internal/{pkg_old => pkg}/keystoreregistry/volume_test.go (100%) rename internal/{pkg_old => pkg}/lifecycle/brickmanager/bricks.go (100%) rename internal/{pkg_old => pkg}/lifecycle/brickmanager/manager.go (100%) rename internal/{pkg_old => pkg}/lifecycle/brickmanager/manager_test.go (100%) rename internal/{pkg_old => pkg}/lifecycle/volume.go (100%) rename internal/{pkg_old => pkg}/lifecycle/volume_test.go (100%) rename internal/{pkg_old => pkg}/mocks/disk_mock.go (100%) rename internal/{pkg_old => pkg}/mocks/job_mock.go (100%) rename internal/{pkg_old => pkg}/mocks/pfsprovider_mock.go (100%) rename internal/{pkg_old => pkg}/mocks/pool_mock.go (100%) rename internal/{pkg_old => pkg}/mocks/volume_mock.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/ansible.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/ansible_test.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/copy.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/copy_test.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/mount.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/mount_test.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/ansible/plugin.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/fake/plugin.go (100%) rename internal/{pkg_old => pkg}/pfsprovider/interface.go (100%) rename internal/{pkg_old => pkg}/registry/pool.go (100%) rename internal/{pkg_old => pkg}/registry/volume.go (100%) diff --git a/internal/pkg_old/dacctl/actions/actions.go b/internal/pkg/dacctl/actions/actions.go similarity index 100% rename from internal/pkg_old/dacctl/actions/actions.go rename to internal/pkg/dacctl/actions/actions.go diff --git a/internal/pkg_old/dacctl/actions/actions_test.go b/internal/pkg/dacctl/actions/actions_test.go similarity index 100% rename from internal/pkg_old/dacctl/actions/actions_test.go rename to internal/pkg/dacctl/actions/actions_test.go diff --git a/internal/pkg_old/dacctl/buffer.go b/internal/pkg/dacctl/buffer.go similarity index 100% rename from internal/pkg_old/dacctl/buffer.go rename to internal/pkg/dacctl/buffer.go diff --git a/internal/pkg_old/dacctl/buffer_test.go b/internal/pkg/dacctl/buffer_test.go similarity index 100% rename from internal/pkg_old/dacctl/buffer_test.go rename to internal/pkg/dacctl/buffer_test.go diff --git a/internal/pkg_old/dacctl/job.go b/internal/pkg/dacctl/job.go similarity index 100% rename from internal/pkg_old/dacctl/job.go rename to internal/pkg/dacctl/job.go diff --git a/internal/pkg_old/dacctl/job_test.go b/internal/pkg/dacctl/job_test.go similarity index 100% rename from internal/pkg_old/dacctl/job_test.go rename to internal/pkg/dacctl/job_test.go diff --git a/internal/pkg_old/dacctl/json.go b/internal/pkg/dacctl/json.go similarity index 100% rename from internal/pkg_old/dacctl/json.go rename to internal/pkg/dacctl/json.go diff --git a/internal/pkg_old/dacctl/persistent.go b/internal/pkg/dacctl/persistent.go similarity index 100% rename from internal/pkg_old/dacctl/persistent.go rename to internal/pkg/dacctl/persistent.go diff --git a/internal/pkg_old/dacctl/pools.go b/internal/pkg/dacctl/pools.go similarity index 100% rename from internal/pkg_old/dacctl/pools.go rename to internal/pkg/dacctl/pools.go diff --git a/internal/pkg_old/dacctl/pools_test.go b/internal/pkg/dacctl/pools_test.go similarity index 100% rename from internal/pkg_old/dacctl/pools_test.go rename to internal/pkg/dacctl/pools_test.go diff --git a/internal/pkg_old/dacctl/show.go b/internal/pkg/dacctl/show.go similarity index 100% rename from internal/pkg_old/dacctl/show.go rename to internal/pkg/dacctl/show.go diff --git a/internal/pkg_old/dacctl/show_test.go b/internal/pkg/dacctl/show_test.go similarity index 100% rename from internal/pkg_old/dacctl/show_test.go rename to internal/pkg/dacctl/show_test.go diff --git a/internal/pkg_old/etcdregistry/keystore.go b/internal/pkg/etcdregistry/keystore.go similarity index 100% rename from internal/pkg_old/etcdregistry/keystore.go rename to internal/pkg/etcdregistry/keystore.go diff --git a/internal/pkg_old/etcdregistry/watch.go b/internal/pkg/etcdregistry/watch.go similarity index 100% rename from internal/pkg_old/etcdregistry/watch.go rename to internal/pkg/etcdregistry/watch.go diff --git a/internal/pkg_old/etcdregistry/watch_test.go b/internal/pkg/etcdregistry/watch_test.go similarity index 100% rename from internal/pkg_old/etcdregistry/watch_test.go rename to internal/pkg/etcdregistry/watch_test.go diff --git a/internal/pkg_old/fileio/disk.go b/internal/pkg/fileio/disk.go similarity index 100% rename from internal/pkg_old/fileio/disk.go rename to internal/pkg/fileio/disk.go diff --git a/internal/pkg_old/keystoreregistry/keystore.go b/internal/pkg/keystoreregistry/keystore.go similarity index 100% rename from internal/pkg_old/keystoreregistry/keystore.go rename to internal/pkg/keystoreregistry/keystore.go diff --git a/internal/pkg_old/keystoreregistry/pool.go b/internal/pkg/keystoreregistry/pool.go similarity index 100% rename from internal/pkg_old/keystoreregistry/pool.go rename to internal/pkg/keystoreregistry/pool.go diff --git a/internal/pkg_old/keystoreregistry/pool_test.go b/internal/pkg/keystoreregistry/pool_test.go similarity index 100% rename from internal/pkg_old/keystoreregistry/pool_test.go rename to internal/pkg/keystoreregistry/pool_test.go diff --git a/internal/pkg_old/keystoreregistry/volume.go b/internal/pkg/keystoreregistry/volume.go similarity index 100% rename from internal/pkg_old/keystoreregistry/volume.go rename to internal/pkg/keystoreregistry/volume.go diff --git a/internal/pkg_old/keystoreregistry/volume_test.go b/internal/pkg/keystoreregistry/volume_test.go similarity index 100% rename from internal/pkg_old/keystoreregistry/volume_test.go rename to internal/pkg/keystoreregistry/volume_test.go diff --git a/internal/pkg_old/lifecycle/brickmanager/bricks.go b/internal/pkg/lifecycle/brickmanager/bricks.go similarity index 100% rename from internal/pkg_old/lifecycle/brickmanager/bricks.go rename to internal/pkg/lifecycle/brickmanager/bricks.go diff --git a/internal/pkg_old/lifecycle/brickmanager/manager.go b/internal/pkg/lifecycle/brickmanager/manager.go similarity index 100% rename from internal/pkg_old/lifecycle/brickmanager/manager.go rename to internal/pkg/lifecycle/brickmanager/manager.go diff --git a/internal/pkg_old/lifecycle/brickmanager/manager_test.go b/internal/pkg/lifecycle/brickmanager/manager_test.go similarity index 100% rename from internal/pkg_old/lifecycle/brickmanager/manager_test.go rename to internal/pkg/lifecycle/brickmanager/manager_test.go diff --git a/internal/pkg_old/lifecycle/volume.go b/internal/pkg/lifecycle/volume.go similarity index 100% rename from internal/pkg_old/lifecycle/volume.go rename to internal/pkg/lifecycle/volume.go diff --git a/internal/pkg_old/lifecycle/volume_test.go b/internal/pkg/lifecycle/volume_test.go similarity index 100% rename from internal/pkg_old/lifecycle/volume_test.go rename to internal/pkg/lifecycle/volume_test.go diff --git a/internal/pkg_old/mocks/disk_mock.go b/internal/pkg/mocks/disk_mock.go similarity index 100% rename from internal/pkg_old/mocks/disk_mock.go rename to internal/pkg/mocks/disk_mock.go diff --git a/internal/pkg_old/mocks/job_mock.go b/internal/pkg/mocks/job_mock.go similarity index 100% rename from internal/pkg_old/mocks/job_mock.go rename to internal/pkg/mocks/job_mock.go diff --git a/internal/pkg_old/mocks/pfsprovider_mock.go b/internal/pkg/mocks/pfsprovider_mock.go similarity index 100% rename from internal/pkg_old/mocks/pfsprovider_mock.go rename to internal/pkg/mocks/pfsprovider_mock.go diff --git a/internal/pkg_old/mocks/pool_mock.go b/internal/pkg/mocks/pool_mock.go similarity index 100% rename from internal/pkg_old/mocks/pool_mock.go rename to internal/pkg/mocks/pool_mock.go diff --git a/internal/pkg_old/mocks/volume_mock.go b/internal/pkg/mocks/volume_mock.go similarity index 100% rename from internal/pkg_old/mocks/volume_mock.go rename to internal/pkg/mocks/volume_mock.go diff --git a/internal/pkg_old/pfsprovider/ansible/ansible.go b/internal/pkg/pfsprovider/ansible/ansible.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/ansible.go rename to internal/pkg/pfsprovider/ansible/ansible.go diff --git a/internal/pkg_old/pfsprovider/ansible/ansible_test.go b/internal/pkg/pfsprovider/ansible/ansible_test.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/ansible_test.go rename to internal/pkg/pfsprovider/ansible/ansible_test.go diff --git a/internal/pkg_old/pfsprovider/ansible/copy.go b/internal/pkg/pfsprovider/ansible/copy.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/copy.go rename to internal/pkg/pfsprovider/ansible/copy.go diff --git a/internal/pkg_old/pfsprovider/ansible/copy_test.go b/internal/pkg/pfsprovider/ansible/copy_test.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/copy_test.go rename to internal/pkg/pfsprovider/ansible/copy_test.go diff --git a/internal/pkg_old/pfsprovider/ansible/mount.go b/internal/pkg/pfsprovider/ansible/mount.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/mount.go rename to internal/pkg/pfsprovider/ansible/mount.go diff --git a/internal/pkg_old/pfsprovider/ansible/mount_test.go b/internal/pkg/pfsprovider/ansible/mount_test.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/mount_test.go rename to internal/pkg/pfsprovider/ansible/mount_test.go diff --git a/internal/pkg_old/pfsprovider/ansible/plugin.go b/internal/pkg/pfsprovider/ansible/plugin.go similarity index 100% rename from internal/pkg_old/pfsprovider/ansible/plugin.go rename to internal/pkg/pfsprovider/ansible/plugin.go diff --git a/internal/pkg_old/pfsprovider/fake/plugin.go b/internal/pkg/pfsprovider/fake/plugin.go similarity index 100% rename from internal/pkg_old/pfsprovider/fake/plugin.go rename to internal/pkg/pfsprovider/fake/plugin.go diff --git a/internal/pkg_old/pfsprovider/interface.go b/internal/pkg/pfsprovider/interface.go similarity index 100% rename from internal/pkg_old/pfsprovider/interface.go rename to internal/pkg/pfsprovider/interface.go diff --git a/internal/pkg_old/registry/pool.go b/internal/pkg/registry/pool.go similarity index 100% rename from internal/pkg_old/registry/pool.go rename to internal/pkg/registry/pool.go diff --git a/internal/pkg_old/registry/volume.go b/internal/pkg/registry/volume.go similarity index 100% rename from internal/pkg_old/registry/volume.go rename to internal/pkg/registry/volume.go From f9203a41711f88dcca30fc8c5cbbd4575f17088e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 16:47:36 +0100 Subject: [PATCH 004/191] Add missing attachement details --- internal/pkg/datamodel/volume.go | 67 ++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/internal/pkg/datamodel/volume.go b/internal/pkg/datamodel/volume.go index c37f3320..bae87c5f 100644 --- a/internal/pkg/datamodel/volume.go +++ b/internal/pkg/datamodel/volume.go @@ -1,5 +1,10 @@ package datamodel +import ( + "bytes" + "encoding/json" +) + type VolumeName string type Volume struct { @@ -63,3 +68,65 @@ type Volume struct { // if we request delete, no bricks ever assigned, don't ait for dacd! HadBricksAssigned bool } + +type Attachment struct { + // Hostname, Job and Volume name uniquely identify an attachment + Hostname string + + // Associated jobName + Job string + + State AttachmentState + + // If any error happened, it is reported here + Error error +} + +type AttachmentState int + +const ( + UnknownAttachmentState AttachmentState = iota + RequestAttach + Attached + RequestDetach + Detached AttachmentState = 400 // all bricks correctly deprovisioned unless host down or gone to ERROR + AttachmentError AttachmentState = 500 +) + +var attachStateStrings = map[AttachmentState]string{ + UnknownAttachmentState: "", + RequestAttach: "RequestAttach", + Attached: "Attached", + RequestDetach: "RequestDetach", + Detached: "Detached", + AttachmentError: "AttachmentError", +} +var stringToAttachmentState = map[string]AttachmentState{ + "": UnknownAttachmentState, + "RequestAttach": RequestAttach, + "Attached": Attached, + "RequestDetach": RequestDetach, + "Detached": Detached, + "AttachmentError": AttachmentError, +} + +func (attachmentState AttachmentState) String() string { + return attachStateStrings[attachmentState] +} + +func (attachmentState AttachmentState) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(attachStateStrings[attachmentState]) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +func (attachmentState *AttachmentState) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + if err != nil { + return err + } + *attachmentState = stringToAttachmentState[str] + return nil +} From 29fc2c107cad885f4ee83db10cc9efc7cc85dcd8 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 17:01:24 +0100 Subject: [PATCH 005/191] Get things compiling again, split out jobfile parse --- internal/pkg/dacctlv2/parsers/jobfile/job.go | 309 ++++++++++++++++++ .../pkg/dacctlv2/parsers/jobfile/job_test.go | 52 +++ internal/pkg/data/brickhost/bricks.go | 14 +- 3 files changed, 368 insertions(+), 7 deletions(-) create mode 100644 internal/pkg/dacctlv2/parsers/jobfile/job.go create mode 100644 internal/pkg/dacctlv2/parsers/jobfile/job_test.go diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job.go b/internal/pkg/dacctlv2/parsers/jobfile/job.go new file mode 100644 index 00000000..dacf4e7f --- /dev/null +++ b/internal/pkg/dacctlv2/parsers/jobfile/job.go @@ -0,0 +1,309 @@ +package jobfile + +import ( + "encoding/json" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "log" + "strconv" + "strings" +) + +type jobSummary struct { + PerJobBuffer *cmdPerJobBuffer + Swap *cmdAttachPerJobSwap + Attachments []cmdAttachPersistent + DataIn *cmdStageInData + DataOut *cmdStageOutData + //createPersistent *cmdCreatePersistent + //destroyPersistent *cmdDestroyPersistent +} + +func (s jobSummary) String() string { + return toJson(s) +} + +func toJson(message interface{}) string { + b, error := json.Marshal(message) + if error != nil { + log.Fatal(error) + } + return string(b) +} + + +// Parse a given job file +func ParseJobFile(disk fileio.Disk, filename string) (jobSummary, error) { + lines, err := disk.Lines(filename) + if err != nil { + return jobSummary{}, err + } + return getJobSummary(lines) +} + +func getJobSummary(lines []string) (jobSummary, error) { + var summary jobSummary + jobCommands, err := parseJobRequest(lines) + if err != nil { + return summary, err + } + + for _, cmd := range jobCommands { + switch c := cmd.(type) { + case cmdPerJobBuffer: + if summary.PerJobBuffer == nil { + summary.PerJobBuffer = &c + } else { + return summary, fmt.Errorf("only one per job buffer allowed") + } + case cmdAttachPersistent: + summary.Attachments = append(summary.Attachments, c) + case cmdAttachPerJobSwap: + if summary.Swap != nil { + // TODO check amount isn't too big for per job buffer + return summary, fmt.Errorf("only one swap request allowed") + } + summary.Swap = &c + case cmdStageOutData: + if summary.DataOut != nil { + // TODO really should check if data out matches one of the requested buffers + return summary, fmt.Errorf("only one per data out requested allowed") + } + summary.DataOut = &c + case cmdStageInData: + if summary.DataIn != nil { + // TODO really should check if data in matches one of the requested buffers + return summary, fmt.Errorf("only one per data in requested allowed") + } + summary.DataIn = &c + default: + // do nothing + } + } + return summary, nil +} + +type jobCommand interface{} + +type AccessMode int + +const ( + striped AccessMode = 0 + private = 1 + privateAndStriped = 2 +) + +var stringToAccessMode = map[string]AccessMode{ + "": striped, + "striped": striped, + "private": private, + "private,striped": privateAndStriped, + "striped,private": privateAndStriped, +} + +func AccessModeFromString(raw string) AccessMode { + return stringToAccessMode[strings.ToLower(raw)] +} + +type BufferType int + +const ( + scratch BufferType = iota + cache +) + +var stringToBufferType = map[string]BufferType{ + "": scratch, + "scratch": scratch, + "cache": cache, +} + +type cmdCreatePersistent struct { + Name string + CapacityBytes int + AccessMode AccessMode + BufferType BufferType + GenericCmd bool +} + +func BufferTypeFromString(raw string) BufferType { + return stringToBufferType[strings.ToLower(raw)] +} + +type cmdDestroyPersistent struct { + Name string +} + +type cmdAttachPersistent struct { + Name string +} + +type cmdPerJobBuffer struct { + CapacityBytes int + AccessMode AccessMode + BufferType BufferType + GenericCmd bool +} + +type cmdAttachPerJobSwap struct { + SizeBytes int +} + +type StageType int + +const ( + directory StageType = iota + file // TODO there is also list, but we ignore that for now +) + +var stringToStageType = map[string]StageType{ + "": directory, + "directory": directory, + "file": file, +} + +func stageTypeFromString(raw string) StageType { + return stringToStageType[strings.ToLower(raw)] +} + +type cmdStageInData struct { + Source string + Destination string + StageType StageType +} + +type cmdStageOutData struct { + Source string + Destination string + StageType StageType +} + +var sizeSuffixMulitiplyer = map[string]int{ + "TiB": 1099511627776, + "TB": 1000000000000, + "GiB": 1073741824, + "GB": 1000000000, + "MiB": 1048576, + "MB": 1000000, +} + +func parseSize(raw string) (int, error) { + intVal, err := strconv.Atoi(raw) + if err == nil { + // specified raw bytes + return intVal, nil + } + for suffix, multiplyer := range sizeSuffixMulitiplyer { + if strings.HasSuffix(raw, suffix) { + rawInt := strings.TrimSuffix(raw, suffix) + intVal, err := strconv.Atoi(rawInt) + if err != nil { + return 0, err + } + return intVal * multiplyer, nil + } + } + return 0, fmt.Errorf("unable to parse size: %s", raw) +} + +func parseArgs(rawArgs []string) (map[string]string, error) { + args := make(map[string]string, len(rawArgs)) + for _, arg := range rawArgs { + parts := strings.Split(arg, "=") + if len(parts) != 2 { + return args, fmt.Errorf("unable to parse arg: %s", arg) + } + args[strings.ToLower(parts[0])] = parts[1] + } + return args, nil +} + +func parseJobRequest(lines []string) ([]jobCommand, error) { + var commands []jobCommand + for _, line := range lines { + tokens := strings.Split(line, " ") + if len(tokens) < 3 { + if line != "" && line != "#!/bin/bash" { + log.Println("Skip badly formatted line:", line) + } + continue + } + + cmdType := tokens[0] + cmd := tokens[1] + args := tokens[2:] + + var isGeneric bool + switch cmdType { + case "#DW": + isGeneric = false + case "#BB": + isGeneric = true + default: + log.Println("unrecognised command type:", cmdType) + continue + } + + argKeyPair, _ := parseArgs(args) // TODO deal with errors when not swap + + var command jobCommand + switch cmd { + case "create_persistent": + size, err := parseSize(argKeyPair["capacity"]) + if err != nil { + log.Println(err) + continue + } + command = cmdCreatePersistent{ + Name: argKeyPair["name"], + CapacityBytes: size, + GenericCmd: isGeneric, + AccessMode: AccessModeFromString(argKeyPair["access_mode"]), + BufferType: BufferTypeFromString(argKeyPair["type"]), + } + case "destroy_persistent": + command = cmdDestroyPersistent{Name: argKeyPair["name"]} + case "persistentdw": + command = cmdAttachPersistent{Name: argKeyPair["name"]} + case "jobdw": + size, err := parseSize(argKeyPair["capacity"]) + if err != nil { + log.Println(err) + continue + } + command = cmdPerJobBuffer{ + CapacityBytes: size, + GenericCmd: isGeneric, + AccessMode: AccessModeFromString(argKeyPair["access_mode"]), + BufferType: BufferTypeFromString(argKeyPair["type"]), + } + case "swap": + if len(args) != 1 { + log.Println("Unable to parse swap command:", line) + } + if size, err := parseSize(args[0]); err != nil { + log.Println(err) + continue + } else { + command = cmdAttachPerJobSwap{SizeBytes: size} + } + case "stage_in": + command = cmdStageInData{ + Source: argKeyPair["source"], + Destination: argKeyPair["destination"], + StageType: stageTypeFromString(argKeyPair["type"]), + } + case "stage_out": + command = cmdStageOutData{ + Source: argKeyPair["source"], + Destination: argKeyPair["destination"], + StageType: stageTypeFromString(argKeyPair["type"]), + } + default: + log.Println("unrecognised command:", cmd, "with argument length", len(args)) + continue + } + commands = append(commands, command) + } + return commands, nil +} diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job_test.go b/internal/pkg/dacctlv2/parsers/jobfile/job_test.go new file mode 100644 index 00000000..f486afcb --- /dev/null +++ b/internal/pkg/dacctlv2/parsers/jobfile/job_test.go @@ -0,0 +1,52 @@ +package jobfile + +import ( + "github.com/stretchr/testify/assert" + "log" + "testing" +) + +func TestParseJobRequest(t *testing.T) { + jobRequest := []string{ + `#BB create_persistent name=myBBname capacity=100GB access_mode=striped type=scratch`, + `#BB create_persistent name=myBBname capacity=1073741824 access_mode=striped type=cache`, + `#BB destroy_persistent name=myBBname`, + `#DW persistentdw name=myBBname1`, + `#DW persistentdw name=myBBname2`, + `#DW persistentdw name=myBBname2`, + `#DW jobdw capacity=10GB access_mode=striped type=scratch`, + `#DW jobdw capacity=2TB access_mode=private type=scratch`, + `#DW jobdw capacity=4TiB access_mode=striped,private type=scratch`, + `#BB jobdw capacity=42GiB access_mode=ldbalance type=cache pfs=/global/scratch1/john`, + `#DW swap 3TiB`, + `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, + `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, + } + if cmds, err := parseJobRequest(jobRequest); err != nil { + log.Fatal(err) + } else { + assert.Equal(t, 13, len(jobRequest)) // TODO should check returned values!! + for _, cmd := range cmds { + log.Printf("Cmd: %T Args: %s\n", cmd, cmd) + } + } +} + +func TestGetJobSummary(t *testing.T) { + lines := []string{ + `#DW persistentdw name=myBBname1`, + `#DW persistentdw name=myBBname2`, + `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, + `#DW swap 4MB`, + `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, + `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, + } + result, err := getJobSummary(lines) + + assert.Nil(t, err) + assert.EqualValues(t, "/global/cscratch1/filename1", result.DataIn.Source) + assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut.Source) + + assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) + assert.Equal(t, 4000000, result.Swap.SizeBytes) +} diff --git a/internal/pkg/data/brickhost/bricks.go b/internal/pkg/data/brickhost/bricks.go index 49eaea5f..ec04c3d3 100644 --- a/internal/pkg/data/brickhost/bricks.go +++ b/internal/pkg/data/brickhost/bricks.go @@ -33,7 +33,7 @@ type BrickRegistry interface { // // Note: you may assign multiple volumes in a single call, but all bricks // for a particular volume must be set in a single call - AllocateBricksForVolume(volume Volume) ([]BrickAllocation, error) + //AllocateBricksForVolume(volume Volume) ([]BrickAllocation, error) // Deallocate all bricks associated with the given volume // @@ -41,23 +41,23 @@ type BrickRegistry interface { // If any host associated with one of the bricks is down, an error is returned and the deallocate is // recorded as requested and not executed. // Note: this returns as soon as deallocate is requested, doesn't wait for cleanup completion - DeallocateBricks(volume VolumeName) error + //DeallocateBricks(volume VolumeName) error // This is called after DeallocateBricks has been processed - HardDeleteAllocations(allocations []BrickAllocation) error + //HardDeleteAllocations(allocations []BrickAllocation) error // Get all the allocations for bricks associated with the specified hostname - GetAllocationsForHost(hostname string) ([]BrickAllocation, error) + //GetAllocationsForHost(hostname string) ([]BrickAllocation, error) // Get all the allocations for bricks associated with the specific volume - GetAllocationsForVolume(volume VolumeName) ([]BrickAllocation, error) + //GetAllocationsForVolume(volume VolumeName) ([]BrickAllocation, error) // Get information on a specific brick - GetBrickInfo(hostname string, device string) (BrickInfo, error) + //GetBrickInfo(hostname string, device string) (BrickInfo, error) // Returns a channel that reports all new brick allocations for given hostname // // The channel is closed when the context is cancelled or timeout. // Any errors in the watching log the issue and panic - GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan BrickAllocation + //GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan BrickAllocation } From 39414d6226439f49caedc97506c424eed1f30d56 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 17:42:30 +0100 Subject: [PATCH 006/191] Put framework in place for unit tests on DacctlActions --- build/rebuild_mocks.sh | 7 +- internal/pkg/dacctlv2/actions/interface.go | 23 ++++ internal/pkg/dacctlv2/actionsImpl/actions.go | 80 ++++++++++++ .../pkg/dacctlv2/actionsImpl/actions_test.go | 61 +++++++++ internal/pkg/data/mock_session/actions.go | 118 ++++++++++++++++++ internal/pkg/data/mock_session/session.go | 109 ++++++++++++++++ .../session.go => data/session/actions.go} | 0 7 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 internal/pkg/dacctlv2/actions/interface.go create mode 100644 internal/pkg/dacctlv2/actionsImpl/actions.go create mode 100644 internal/pkg/dacctlv2/actionsImpl/actions_test.go create mode 100644 internal/pkg/data/mock_session/actions.go create mode 100644 internal/pkg/data/mock_session/session.go rename internal/pkg/{actions/session/session.go => data/session/actions.go} (100%) diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 54e21653..5a16c71f 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -24,6 +24,11 @@ for i in $items; do -package mocks >internal/pkg/mocks/${i}_mock.go done -items="interface" mockgen -source=internal/pkg/pfsprovider/interface.go \ -package mocks >internal/pkg/mocks/pfsprovider_mock.go + +items="actions session" +for i in $items; do + mockgen -source=internal/pkg/data/session/${i}.go \ + >internal/pkg/data/mock_session/${i}.go +done diff --git a/internal/pkg/dacctlv2/actions/interface.go b/internal/pkg/dacctlv2/actions/interface.go new file mode 100644 index 00000000..4337e0b7 --- /dev/null +++ b/internal/pkg/dacctlv2/actions/interface.go @@ -0,0 +1,23 @@ +package actions + +type CliContext interface { + String(name string) string + Int(name string) int +} + +type DacctlActions interface { + CreatePersistentBuffer(c CliContext) error + DeleteBuffer(c CliContext) error + CreatePerJobBuffer(c CliContext) error + ShowInstances() error + ShowSessions() error + ListPools() error + ShowConfigurations() error + ValidateJob(c CliContext) error + RealSize(c CliContext) error + DataIn(c CliContext) error + Paths(c CliContext) error + PreRun(c CliContext) error + PostRun(c CliContext) error + DataOut(c CliContext) error +} \ No newline at end of file diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go new file mode 100644 index 00000000..ed13ee4b --- /dev/null +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -0,0 +1,80 @@ +package actionsImpl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" +) + +func NewDacctlActions(registry session.Registry, actions session.Actions, disk fileio.Disk) actions.DacctlActions { + return &dacctlActions{ + registry: registry, + actions: actions, + disk: disk, + } +} + +type dacctlActions struct { + registry session.Registry + actions session.Actions + disk fileio.Disk +} + +func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { + bufferName := c.String("token") + d.registry.GetSession(bufferName) + return nil +} + +func (*dacctlActions) DeleteBuffer(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) ShowInstances() error { + panic("implement me") +} + +func (*dacctlActions) ShowSessions() error { + panic("implement me") +} + +func (*dacctlActions) ListPools() error { + panic("implement me") +} + +func (*dacctlActions) ShowConfigurations() error { + panic("implement me") +} + +func (*dacctlActions) ValidateJob(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) RealSize(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) DataIn(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) Paths(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) PreRun(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) PostRun(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) DataOut(c actions.CliContext) error { + panic("implement me") +} + diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actionsImpl/actions_test.go new file mode 100644 index 00000000..1a4987b4 --- /dev/null +++ b/internal/pkg/dacctlv2/actionsImpl/actions_test.go @@ -0,0 +1,61 @@ +package actionsImpl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +type mockCliContext struct { + capacity int +} + +func (c *mockCliContext) String(name string) string { + switch name { + case "capacity": + return fmt.Sprintf("pool1:%dGB", c.capacity) + case "token": + return "token" + case "caller": + return "caller" + case "user": + return "user" + case "access": + return "access" + case "type": + return "type" + case "job": + return "jobfile" + case "nodehostnamefile": + return "nodehostnamefile1" + case "pathfile": + return "pathfile1" + default: + return "" + } +} + +func (c *mockCliContext) Int(name string) int { + switch name { + case "user": + return 1001 + case "group": + return 1001 + default: + return 42 + len(name) + } +} + +func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + actions := NewDacctlActions(registry, session, nil) + err := actions.CreatePersistentBuffer(&mockCliContext{}) + + assert.Nil(t, err) +} \ No newline at end of file diff --git a/internal/pkg/data/mock_session/actions.go b/internal/pkg/data/mock_session/actions.go new file mode 100644 index 00000000..30335018 --- /dev/null +++ b/internal/pkg/data/mock_session/actions.go @@ -0,0 +1,118 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/data/session/actions.go + +// Package mock_session is a generated GoMock package. +package mock_session + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockActions is a mock of Actions interface +type MockActions struct { + ctrl *gomock.Controller + recorder *MockActionsMockRecorder +} + +// MockActionsMockRecorder is the mock recorder for MockActions +type MockActionsMockRecorder struct { + mock *MockActions +} + +// NewMockActions creates a new mock instance +func NewMockActions(ctrl *gomock.Controller) *MockActions { + mock := &MockActions{ctrl: ctrl} + mock.recorder = &MockActionsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockActions) EXPECT() *MockActionsMockRecorder { + return m.recorder +} + +// CreateSessionVolume mocks base method +func (m *MockActions) CreateSessionVolume(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSessionVolume", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateSessionVolume indicates an expected call of CreateSessionVolume +func (mr *MockActionsMockRecorder) CreateSessionVolume(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockActions)(nil).CreateSessionVolume), session) +} + +// DeleteSession mocks base method +func (m *MockActions) DeleteSession(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSession", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSession indicates an expected call of DeleteSession +func (mr *MockActionsMockRecorder) DeleteSession(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockActions)(nil).DeleteSession), session) +} + +// DataIn mocks base method +func (m *MockActions) DataIn(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataIn", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataIn indicates an expected call of DataIn +func (mr *MockActionsMockRecorder) DataIn(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataIn", reflect.TypeOf((*MockActions)(nil).DataIn), session) +} + +// AttachVolumes mocks base method +func (m *MockActions) AttachVolumes(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AttachVolumes", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// AttachVolumes indicates an expected call of AttachVolumes +func (mr *MockActionsMockRecorder) AttachVolumes(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockActions)(nil).AttachVolumes), session) +} + +// DetachVolumes mocks base method +func (m *MockActions) DetachVolumes(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DetachVolumes", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DetachVolumes indicates an expected call of DetachVolumes +func (mr *MockActionsMockRecorder) DetachVolumes(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachVolumes", reflect.TypeOf((*MockActions)(nil).DetachVolumes), session) +} + +// DataOut mocks base method +func (m *MockActions) DataOut(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataOut", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataOut indicates an expected call of DataOut +func (mr *MockActionsMockRecorder) DataOut(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataOut", reflect.TypeOf((*MockActions)(nil).DataOut), session) +} diff --git a/internal/pkg/data/mock_session/session.go b/internal/pkg/data/mock_session/session.go new file mode 100644 index 00000000..7c5e2d3c --- /dev/null +++ b/internal/pkg/data/mock_session/session.go @@ -0,0 +1,109 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/data/session/session.go + +// Package mock_session is a generated GoMock package. +package mock_session + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockRegistry is a mock of Registry interface +type MockRegistry struct { + ctrl *gomock.Controller + recorder *MockRegistryMockRecorder +} + +// MockRegistryMockRecorder is the mock recorder for MockRegistry +type MockRegistryMockRecorder struct { + mock *MockRegistry +} + +// NewMockRegistry creates a new mock instance +func NewMockRegistry(ctrl *gomock.Controller) *MockRegistry { + mock := &MockRegistry{ctrl: ctrl} + mock.recorder = &MockRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { + return m.recorder +} + +// GetSession mocks base method +func (m *MockRegistry) GetSession(token string) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSession", token) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSession indicates an expected call of GetSession +func (mr *MockRegistryMockRecorder) GetSession(token interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockRegistry)(nil).GetSession), token) +} + +// CreateSessionAllocations mocks base method +func (m *MockRegistry) CreateSessionAllocations(s datamodel.Session) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSessionAllocations", s) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSessionAllocations indicates an expected call of CreateSessionAllocations +func (mr *MockRegistryMockRecorder) CreateSessionAllocations(s interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionAllocations", reflect.TypeOf((*MockRegistry)(nil).CreateSessionAllocations), s) +} + +// ValidateSessionRequest mocks base method +func (m *MockRegistry) ValidateSessionRequest(token string) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateSessionRequest", token) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ValidateSessionRequest indicates an expected call of ValidateSessionRequest +func (mr *MockRegistryMockRecorder) ValidateSessionRequest(token interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateSessionRequest", reflect.TypeOf((*MockRegistry)(nil).ValidateSessionRequest), token) +} + +// GetAllSessions mocks base method +func (m *MockRegistry) GetAllSessions() ([]datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllSessions") + ret0, _ := ret[0].([]datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllSessions indicates an expected call of GetAllSessions +func (mr *MockRegistryMockRecorder) GetAllSessions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockRegistry)(nil).GetAllSessions)) +} + +// GetAllPools mocks base method +func (m *MockRegistry) GetAllPools() ([]datamodel.Pool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllPools") + ret0, _ := ret[0].([]datamodel.Pool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllPools indicates an expected call of GetAllPools +func (mr *MockRegistryMockRecorder) GetAllPools() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPools", reflect.TypeOf((*MockRegistry)(nil).GetAllPools)) +} diff --git a/internal/pkg/actions/session/session.go b/internal/pkg/data/session/actions.go similarity index 100% rename from internal/pkg/actions/session/session.go rename to internal/pkg/data/session/actions.go From 6f2d71e91f66007dfe6a786c35d01d409018e170 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 17:45:48 +0100 Subject: [PATCH 007/191] Tidy up with govet --- internal/pkg/dacctlv2/actions/interface.go | 2 +- internal/pkg/dacctlv2/actionsImpl/actions.go | 9 ++++----- internal/pkg/dacctlv2/actionsImpl/actions_test.go | 2 +- internal/pkg/dacctlv2/parsers/jobfile/job.go | 1 - internal/pkg/data/brickhost/volume.go | 3 +-- internal/pkg/data/session/actions.go | 2 +- internal/pkg/data/session/session.go | 2 +- internal/pkg/datamodel/allocation.go | 2 +- internal/pkg/datamodel/brickhost.go | 2 +- internal/pkg/datamodel/pool.go | 2 +- internal/pkg/datamodel/session.go | 7 +++---- 11 files changed, 15 insertions(+), 19 deletions(-) diff --git a/internal/pkg/dacctlv2/actions/interface.go b/internal/pkg/dacctlv2/actions/interface.go index 4337e0b7..3e7cb6b5 100644 --- a/internal/pkg/dacctlv2/actions/interface.go +++ b/internal/pkg/dacctlv2/actions/interface.go @@ -20,4 +20,4 @@ type DacctlActions interface { PreRun(c CliContext) error PostRun(c CliContext) error DataOut(c CliContext) error -} \ No newline at end of file +} diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go index ed13ee4b..80614b57 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -9,15 +9,15 @@ import ( func NewDacctlActions(registry session.Registry, actions session.Actions, disk fileio.Disk) actions.DacctlActions { return &dacctlActions{ registry: registry, - actions: actions, - disk: disk, + actions: actions, + disk: disk, } } type dacctlActions struct { registry session.Registry - actions session.Actions - disk fileio.Disk + actions session.Actions + disk fileio.Disk } func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { @@ -77,4 +77,3 @@ func (*dacctlActions) PostRun(c actions.CliContext) error { func (*dacctlActions) DataOut(c actions.CliContext) error { panic("implement me") } - diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actionsImpl/actions_test.go index 1a4987b4..a708c812 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions_test.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions_test.go @@ -58,4 +58,4 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { err := actions.CreatePersistentBuffer(&mockCliContext{}) assert.Nil(t, err) -} \ No newline at end of file +} diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job.go b/internal/pkg/dacctlv2/parsers/jobfile/job.go index dacf4e7f..7adef381 100644 --- a/internal/pkg/dacctlv2/parsers/jobfile/job.go +++ b/internal/pkg/dacctlv2/parsers/jobfile/job.go @@ -31,7 +31,6 @@ func toJson(message interface{}) string { return string(b) } - // Parse a given job file func ParseJobFile(disk fileio.Disk, filename string) (jobSummary, error) { lines, err := disk.Lines(filename) diff --git a/internal/pkg/data/brickhost/volume.go b/internal/pkg/data/brickhost/volume.go index f906d181..4c5834ef 100644 --- a/internal/pkg/data/brickhost/volume.go +++ b/internal/pkg/data/brickhost/volume.go @@ -1,5 +1,4 @@ package brickhost type VolumeRegistry interface { - -} \ No newline at end of file +} diff --git a/internal/pkg/data/session/actions.go b/internal/pkg/data/session/actions.go index 92d0081f..799abc71 100644 --- a/internal/pkg/data/session/actions.go +++ b/internal/pkg/data/session/actions.go @@ -24,4 +24,4 @@ type Actions interface { // Update the session and trigger requested data copy out DataOut(session datamodel.Session) error -} \ No newline at end of file +} diff --git a/internal/pkg/data/session/session.go b/internal/pkg/data/session/session.go index f69defe3..79d42566 100644 --- a/internal/pkg/data/session/session.go +++ b/internal/pkg/data/session/session.go @@ -22,4 +22,4 @@ type Registry interface { // Get all bricks listed by pools GetAllPools() ([]datamodel.Pool, error) -} \ No newline at end of file +} diff --git a/internal/pkg/datamodel/allocation.go b/internal/pkg/datamodel/allocation.go index 6136212e..dc6016e5 100644 --- a/internal/pkg/datamodel/allocation.go +++ b/internal/pkg/datamodel/allocation.go @@ -21,4 +21,4 @@ type BrickAllocation struct { // alive, this flag is set rather than have allocations removed. // A host should check for any allocations DeallocateRequested bool -} \ No newline at end of file +} diff --git a/internal/pkg/datamodel/brickhost.go b/internal/pkg/datamodel/brickhost.go index 8e7c0652..c29b3584 100644 --- a/internal/pkg/datamodel/brickhost.go +++ b/internal/pkg/datamodel/brickhost.go @@ -29,4 +29,4 @@ type BrickInfo struct { // Size of the brick, defines the pool granularity CapacityGB uint -} \ No newline at end of file +} diff --git a/internal/pkg/datamodel/pool.go b/internal/pkg/datamodel/pool.go index 4e4d8e56..d6aa327f 100644 --- a/internal/pkg/datamodel/pool.go +++ b/internal/pkg/datamodel/pool.go @@ -9,4 +9,4 @@ type Pool struct { // This is the allocation unit for the pool // It is the minimum size of any registered brick GranularityGB uint -} \ No newline at end of file +} diff --git a/internal/pkg/datamodel/session.go b/internal/pkg/datamodel/session.go index 304c8a7d..6baf9541 100644 --- a/internal/pkg/datamodel/session.go +++ b/internal/pkg/datamodel/session.go @@ -14,8 +14,8 @@ type Session struct { Revision int // unix uid and gid - Owner uint - Group uint + Owner uint + Group uint // utc unix timestamp when buffer created CreatedAt uint @@ -55,7 +55,6 @@ type Session struct { type SessionRequest struct { - // swap } @@ -84,4 +83,4 @@ const ( Directory SourceType = "directory" // Provided a file that has source and destination file space separated pairs, each on a new line List SourceType = "list" -) \ No newline at end of file +) From 1ef9e9defac82b288db459f04a452c8b7048dcf6 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 21:46:26 +0100 Subject: [PATCH 008/191] Get unit tests passing --- internal/pkg/dacctlv2/actionsImpl/actions.go | 78 ++++++++++++++++++- .../pkg/dacctlv2/actionsImpl/actions_test.go | 24 +++++- .../pkg/dacctlv2/parsers/capacity/capacity.go | 50 ++++++++++++ internal/pkg/datamodel/session.go | 26 ++++++- 4 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 internal/pkg/dacctlv2/parsers/capacity/capacity.go diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go index 80614b57..f0910801 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -2,15 +2,23 @@ package actionsImpl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "log" + "strings" + "time" ) +var fakeTime uint = 0 + func NewDacctlActions(registry session.Registry, actions session.Actions, disk fileio.Disk) actions.DacctlActions { return &dacctlActions{ registry: registry, actions: actions, disk: disk, + fakeTime: fakeTime, } } @@ -18,12 +26,76 @@ type dacctlActions struct { registry session.Registry actions session.Actions disk fileio.Disk + fakeTime uint +} + +func checkRequiredStrings(c actions.CliContext, flags ...string) { + errs := []string{} + for _, flag := range flags { + if str := c.String(flag); str == "" { + errs = append(errs, flag) + } + } + if len(errs) > 0 { + log.Fatalf("Please provide these required parameters: %s", strings.Join(errs, ", ")) + } +} + +var stringToAccessMode = map[string]datamodel.AccessMode{ + "": datamodel.Striped, + "striped": datamodel.Striped, + "private": datamodel.Private, + "private,striped": datamodel.PrivateAndStriped, + "striped,private": datamodel.PrivateAndStriped, +} + +func accessModeFromString(raw string) datamodel.AccessMode { + return stringToAccessMode[strings.ToLower(raw)] +} + +var stringToBufferType = map[string]datamodel.BufferType{ + "": datamodel.Scratch, + "scratch": datamodel.Scratch, + "cache": datamodel.Cache, +} + +func bufferTypeFromString(raw string) datamodel.BufferType { + return stringToBufferType[strings.ToLower(raw)] +} + +func getNow() uint { + return uint(time.Now().Unix()) } func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { - bufferName := c.String("token") - d.registry.GetSession(bufferName) - return nil + checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") + pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) + if err != nil { + return err + } + request := datamodel.PersistentVolumeRequest{ + Caller: c.String("caller"), + CapacityBytes: capacityBytes, + PoolName: pool, + Access: accessModeFromString(c.String("access")), + Type: bufferTypeFromString(c.String("type")), + } + createdAt := d.fakeTime + if createdAt == 0 { + createdAt = getNow() + } + session := datamodel.Session{ + Name: datamodel.SessionName(c.String("token")), + PersistentVolumeRequest: request, + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: createdAt, + } + session, err = d.registry.CreateSessionAllocations(session) + if err != nil { + return err + } + return d.actions.CreateSessionVolume(session) } func (*dacctlActions) DeleteBuffer(c actions.CliContext) error { diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actionsImpl/actions_test.go index a708c812..738a166a 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions_test.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions_test.go @@ -3,6 +3,7 @@ package actionsImpl import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -15,7 +16,7 @@ type mockCliContext struct { func (c *mockCliContext) String(name string) string { switch name { case "capacity": - return fmt.Sprintf("pool1:%dGB", c.capacity) + return fmt.Sprintf("pool1:%dGiB", c.capacity) case "token": return "token" case "caller": @@ -33,7 +34,7 @@ func (c *mockCliContext) String(name string) string { case "pathfile": return "pathfile1" default: - return "" + return "foobar1" } } @@ -42,7 +43,7 @@ func (c *mockCliContext) Int(name string) int { case "user": return 1001 case "group": - return 1001 + return 1002 default: return 42 + len(name) } @@ -54,8 +55,23 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().CreateSessionAllocations(datamodel.Session{ + Name: "token", + Owner: 1001, + Group: 1002, + CreatedAt: 123, + PersistentVolumeRequest: datamodel.PersistentVolumeRequest{ + Caller: "caller", + PoolName: "pool1", + CapacityBytes: 2147483648, + }, + }).Return(fakeSession, nil) + session.EXPECT().CreateSessionVolume(fakeSession) + fakeTime = 123 + actions := NewDacctlActions(registry, session, nil) - err := actions.CreatePersistentBuffer(&mockCliContext{}) + err := actions.CreatePersistentBuffer(&mockCliContext{capacity: 2}) assert.Nil(t, err) } diff --git a/internal/pkg/dacctlv2/parsers/capacity/capacity.go b/internal/pkg/dacctlv2/parsers/capacity/capacity.go new file mode 100644 index 00000000..f7fd38fc --- /dev/null +++ b/internal/pkg/dacctlv2/parsers/capacity/capacity.go @@ -0,0 +1,50 @@ +package capacity + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +var sizeSuffixMulitiplyer = map[string]int{ + "TiB": 1099511627776, + "TB": 1000000000000, + "GiB": 1073741824, + "GB": 1000000000, + "MiB": 1048576, + "MB": 1000000, +} + +func parseSize(raw string) (int, error) { + intVal, err := strconv.Atoi(raw) + if err == nil { + // specified raw bytes + return intVal, nil + } + for suffix, multiplyer := range sizeSuffixMulitiplyer { + if strings.HasSuffix(raw, suffix) { + rawInt := strings.TrimSuffix(raw, suffix) + intVal, err := strconv.Atoi(rawInt) + if err != nil { + return 0, err + } + return intVal * multiplyer, nil + } + } + return 0, fmt.Errorf("unable to parse size: %s", raw) +} + +func ParseCapacityBytes(raw string) (string, int, error) { + parts := strings.Split(raw, ":") + if len(parts) != 2 { + return "", 0, errors.New("must format capacity correctly and include pool") + } + pool := parts[0] + rawCapacity := parts[1] + sizeBytes, err := parseSize(rawCapacity) + if err != nil { + return "", 0, err + } + return pool, sizeBytes, nil +} diff --git a/internal/pkg/datamodel/session.go b/internal/pkg/datamodel/session.go index 6baf9541..d827eda5 100644 --- a/internal/pkg/datamodel/session.go +++ b/internal/pkg/datamodel/session.go @@ -21,7 +21,7 @@ type Session struct { CreatedAt uint // Details of what was requested - Request SessionRequest + PersistentVolumeRequest PersistentVolumeRequest // Records if we have started trying to delete DeleteRequested bool @@ -53,11 +53,29 @@ type Session struct { Allocations []BrickAllocation } -type SessionRequest struct { - - // swap +type PersistentVolumeRequest struct { + Caller string + CapacityBytes int + PoolName string + Access AccessMode + Type BufferType } +type AccessMode int + +const ( + Striped AccessMode = 0 + Private = 1 + PrivateAndStriped = 2 +) + +type BufferType int + +const ( + Scratch BufferType = iota + Cache +) + type DataCopyRequest struct { // Source points to a File or a Directory, // or a file that contains a list of source and destinations, From e0ebb0126ef7513023e263d5d090b75f4a0f2004 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 21:53:29 +0100 Subject: [PATCH 009/191] Split out create persistent --- internal/pkg/dacctlv2/actionsImpl/actions.go | 64 ------------------ .../pkg/dacctlv2/actionsImpl/persistent.go | 67 +++++++++++++++++++ 2 files changed, 67 insertions(+), 64 deletions(-) create mode 100644 internal/pkg/dacctlv2/actionsImpl/persistent.go diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go index f0910801..e33916e6 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -2,23 +2,17 @@ package actionsImpl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "log" "strings" - "time" ) -var fakeTime uint = 0 - func NewDacctlActions(registry session.Registry, actions session.Actions, disk fileio.Disk) actions.DacctlActions { return &dacctlActions{ registry: registry, actions: actions, disk: disk, - fakeTime: fakeTime, } } @@ -26,7 +20,6 @@ type dacctlActions struct { registry session.Registry actions session.Actions disk fileio.Disk - fakeTime uint } func checkRequiredStrings(c actions.CliContext, flags ...string) { @@ -41,63 +34,6 @@ func checkRequiredStrings(c actions.CliContext, flags ...string) { } } -var stringToAccessMode = map[string]datamodel.AccessMode{ - "": datamodel.Striped, - "striped": datamodel.Striped, - "private": datamodel.Private, - "private,striped": datamodel.PrivateAndStriped, - "striped,private": datamodel.PrivateAndStriped, -} - -func accessModeFromString(raw string) datamodel.AccessMode { - return stringToAccessMode[strings.ToLower(raw)] -} - -var stringToBufferType = map[string]datamodel.BufferType{ - "": datamodel.Scratch, - "scratch": datamodel.Scratch, - "cache": datamodel.Cache, -} - -func bufferTypeFromString(raw string) datamodel.BufferType { - return stringToBufferType[strings.ToLower(raw)] -} - -func getNow() uint { - return uint(time.Now().Unix()) -} - -func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { - checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") - pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) - if err != nil { - return err - } - request := datamodel.PersistentVolumeRequest{ - Caller: c.String("caller"), - CapacityBytes: capacityBytes, - PoolName: pool, - Access: accessModeFromString(c.String("access")), - Type: bufferTypeFromString(c.String("type")), - } - createdAt := d.fakeTime - if createdAt == 0 { - createdAt = getNow() - } - session := datamodel.Session{ - Name: datamodel.SessionName(c.String("token")), - PersistentVolumeRequest: request, - Owner: uint(c.Int("user")), - Group: uint(c.Int("group")), - CreatedAt: createdAt, - } - session, err = d.registry.CreateSessionAllocations(session) - if err != nil { - return err - } - return d.actions.CreateSessionVolume(session) -} - func (*dacctlActions) DeleteBuffer(c actions.CliContext) error { panic("implement me") } diff --git a/internal/pkg/dacctlv2/actionsImpl/persistent.go b/internal/pkg/dacctlv2/actionsImpl/persistent.go new file mode 100644 index 00000000..db78728e --- /dev/null +++ b/internal/pkg/dacctlv2/actionsImpl/persistent.go @@ -0,0 +1,67 @@ +package actionsImpl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "strings" + "time" +) + +var stringToAccessMode = map[string]datamodel.AccessMode{ + "": datamodel.Striped, + "striped": datamodel.Striped, + "private": datamodel.Private, + "private,striped": datamodel.PrivateAndStriped, + "striped,private": datamodel.PrivateAndStriped, +} + +func accessModeFromString(raw string) datamodel.AccessMode { + return stringToAccessMode[strings.ToLower(raw)] +} + +var stringToBufferType = map[string]datamodel.BufferType{ + "": datamodel.Scratch, + "scratch": datamodel.Scratch, + "cache": datamodel.Cache, +} + +func bufferTypeFromString(raw string) datamodel.BufferType { + return stringToBufferType[strings.ToLower(raw)] +} + +var fakeTime uint = 0 + +func getNow() uint { + if fakeTime != 0 { + return fakeTime + } + return uint(time.Now().Unix()) +} + +func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { + checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") + pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) + if err != nil { + return err + } + request := datamodel.PersistentVolumeRequest{ + Caller: c.String("caller"), + CapacityBytes: capacityBytes, + PoolName: pool, + Access: accessModeFromString(c.String("access")), + Type: bufferTypeFromString(c.String("type")), + } + session := datamodel.Session{ + Name: datamodel.SessionName(c.String("token")), + PersistentVolumeRequest: request, + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: getNow(), + } + session, err = d.registry.CreateSessionAllocations(session) + if err != nil { + return err + } + return d.actions.CreateSessionVolume(session) +} \ No newline at end of file From 5020448d7d7ede366c1ff1ed1f3dc3dd94871084 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 22:00:08 +0100 Subject: [PATCH 010/191] Add delete buffer --- internal/pkg/dacctlv2/actionsImpl/actions.go | 11 +++++++++-- .../pkg/dacctlv2/actionsImpl/actions_test.go | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go index e33916e6..6acd0443 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -1,6 +1,7 @@ package actionsImpl import ( + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" @@ -34,8 +35,14 @@ func checkRequiredStrings(c actions.CliContext, flags ...string) { } } -func (*dacctlActions) DeleteBuffer(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) DeleteBuffer(c actions.CliContext) error { + checkRequiredStrings(c, "token") + token := c.String("token") + session, err := d.registry.GetSession(token) + if err != nil { + return fmt.Errorf("unable to find session for token %s", token) + } + return d.actions.DeleteSession(session) } func (*dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actionsImpl/actions_test.go index 738a166a..4c6ca4ce 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions_test.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions_test.go @@ -1,6 +1,7 @@ package actionsImpl import ( + "errors" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" @@ -75,3 +76,20 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { assert.Nil(t, err) } + +func TestDacctlActions_DeleteBuffer(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().GetSession("token").Return(fakeSession, nil) + fakeError := errors.New("fake") + session.EXPECT().DeleteSession(fakeSession).Return(fakeError) + + actions := NewDacctlActions(registry, session, nil) + err := actions.DeleteBuffer(&mockCliContext{}) + + assert.Equal(t, fakeError, err) +} \ No newline at end of file From 7b1f35ace4fc39dc345b41ea9ab29b65cda5330c Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Aug 2019 22:13:39 +0100 Subject: [PATCH 011/191] Add data_in, data_out, pre/post_run --- internal/pkg/dacctlv2/actionsImpl/actions.go | 77 +++++++++---------- .../pkg/dacctlv2/actionsImpl/actions_test.go | 68 ++++++++++++++++ internal/pkg/dacctlv2/actionsImpl/job.go | 11 +++ internal/pkg/dacctlv2/actionsImpl/show.go | 28 +++++++ 4 files changed, 142 insertions(+), 42 deletions(-) create mode 100644 internal/pkg/dacctlv2/actionsImpl/job.go create mode 100644 internal/pkg/dacctlv2/actionsImpl/show.go diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actionsImpl/actions.go index 6acd0443..8786a040 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "log" "strings" @@ -35,60 +36,52 @@ func checkRequiredStrings(c actions.CliContext, flags ...string) { } } -func (d *dacctlActions) DeleteBuffer(c actions.CliContext) error { +func (d *dacctlActions) getSession(c actions.CliContext) (datamodel.Session, error) { checkRequiredStrings(c, "token") token := c.String("token") - session, err := d.registry.GetSession(token) + s, err := d.registry.GetSession(token) if err != nil { - return fmt.Errorf("unable to find session for token %s", token) + return s, fmt.Errorf("unable to find session for token %s", token) } - return d.actions.DeleteSession(session) -} - -func (*dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { - panic("implement me") -} - -func (*dacctlActions) ShowInstances() error { - panic("implement me") -} - -func (*dacctlActions) ShowSessions() error { - panic("implement me") -} - -func (*dacctlActions) ListPools() error { - panic("implement me") -} - -func (*dacctlActions) ShowConfigurations() error { - panic("implement me") -} - -func (*dacctlActions) ValidateJob(c actions.CliContext) error { - panic("implement me") + return s, nil } -func (*dacctlActions) RealSize(c actions.CliContext) error { - panic("implement me") -} - -func (*dacctlActions) DataIn(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) DeleteBuffer(c actions.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.actions.DeleteSession(s) } -func (*dacctlActions) Paths(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) DataIn(c actions.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.actions.DataIn(s) } -func (*dacctlActions) PreRun(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) PreRun(c actions.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.actions.AttachVolumes(s) } -func (*dacctlActions) PostRun(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) PostRun(c actions.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.actions.DetachVolumes(s) } -func (*dacctlActions) DataOut(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) DataOut(c actions.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.actions.DataOut(s) } diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actionsImpl/actions_test.go index 4c6ca4ce..88ebd3a2 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions_test.go +++ b/internal/pkg/dacctlv2/actionsImpl/actions_test.go @@ -91,5 +91,73 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { actions := NewDacctlActions(registry, session, nil) err := actions.DeleteBuffer(&mockCliContext{}) + assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_DataIn(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().GetSession("token").Return(fakeSession, nil) + fakeError := errors.New("fake") + session.EXPECT().DataIn(fakeSession).Return(fakeError) + + actions := NewDacctlActions(registry, session, nil) + err := actions.DataIn(&mockCliContext{}) + + assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_DataOut(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().GetSession("token").Return(fakeSession, nil) + fakeError := errors.New("fake") + session.EXPECT().DataOut(fakeSession).Return(fakeError) + + actions := NewDacctlActions(registry, session, nil) + err := actions.DataOut(&mockCliContext{}) + + assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_PreRun(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().GetSession("token").Return(fakeSession, nil) + fakeError := errors.New("fake") + session.EXPECT().AttachVolumes(fakeSession).Return(fakeError) + + actions := NewDacctlActions(registry, session, nil) + err := actions.PreRun(&mockCliContext{}) + + assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_PostRun(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().GetSession("token").Return(fakeSession, nil) + fakeError := errors.New("fake") + session.EXPECT().DetachVolumes(fakeSession).Return(fakeError) + + actions := NewDacctlActions(registry, session, nil) + err := actions.PostRun(&mockCliContext{}) + assert.Equal(t, fakeError, err) } \ No newline at end of file diff --git a/internal/pkg/dacctlv2/actionsImpl/job.go b/internal/pkg/dacctlv2/actionsImpl/job.go new file mode 100644 index 00000000..8330dd1e --- /dev/null +++ b/internal/pkg/dacctlv2/actionsImpl/job.go @@ -0,0 +1,11 @@ +package actionsImpl + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + +func (*dacctlActions) ValidateJob(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { + panic("implement me") +} diff --git a/internal/pkg/dacctlv2/actionsImpl/show.go b/internal/pkg/dacctlv2/actionsImpl/show.go new file mode 100644 index 00000000..2993cbf0 --- /dev/null +++ b/internal/pkg/dacctlv2/actionsImpl/show.go @@ -0,0 +1,28 @@ +package actionsImpl + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + +func (*dacctlActions) RealSize(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) Paths(c actions.CliContext) error { + panic("implement me") +} + +func (*dacctlActions) ShowInstances() error { + panic("implement me") +} + +func (*dacctlActions) ShowSessions() error { + panic("implement me") +} + +func (*dacctlActions) ListPools() error { + panic("implement me") +} + +func (*dacctlActions) ShowConfigurations() error { + panic("implement me") +} + From 644e8d2dcaa62b1d05b1b5e3da9d6895fab8d4fc Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 7 Aug 2019 09:26:04 +0100 Subject: [PATCH 012/191] Move datamodel under data --- .../{actionsImpl => actions_impl}/actions.go | 6 ++-- .../actions_test.go | 20 +++++------ .../{actionsImpl => actions_impl}/job.go | 2 +- .../persistent.go | 34 +++++++++---------- .../{actionsImpl => actions_impl}/show.go | 2 +- internal/pkg/data/brickhost/bricks.go | 10 +++--- internal/pkg/data/mock_session/actions.go | 18 +++++----- internal/pkg/data/mock_session/session.go | 26 +++++++------- .../{datamodel => data/model}/allocation.go | 2 +- .../{datamodel => data/model}/brickhost.go | 2 +- .../pkg/{datamodel => data/model}/pool.go | 2 +- .../pkg/{datamodel => data/model}/session.go | 2 +- .../pkg/{datamodel => data/model}/volume.go | 2 +- internal/pkg/data/session/actions.go | 14 ++++---- internal/pkg/data/session/session.go | 12 +++---- 15 files changed, 77 insertions(+), 77 deletions(-) rename internal/pkg/dacctlv2/{actionsImpl => actions_impl}/actions.go (91%) rename internal/pkg/dacctlv2/{actionsImpl => actions_impl}/actions_test.go (89%) rename internal/pkg/dacctlv2/{actionsImpl => actions_impl}/job.go (92%) rename internal/pkg/dacctlv2/{actionsImpl => actions_impl}/persistent.go (60%) rename internal/pkg/dacctlv2/{actionsImpl => actions_impl}/show.go (96%) rename internal/pkg/{datamodel => data/model}/allocation.go (97%) rename internal/pkg/{datamodel => data/model}/brickhost.go (97%) rename internal/pkg/{datamodel => data/model}/pool.go (92%) rename internal/pkg/{datamodel => data/model}/session.go (99%) rename internal/pkg/{datamodel => data/model}/volume.go (99%) diff --git a/internal/pkg/dacctlv2/actionsImpl/actions.go b/internal/pkg/dacctlv2/actions_impl/actions.go similarity index 91% rename from internal/pkg/dacctlv2/actionsImpl/actions.go rename to internal/pkg/dacctlv2/actions_impl/actions.go index 8786a040..743bf528 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions.go +++ b/internal/pkg/dacctlv2/actions_impl/actions.go @@ -1,10 +1,10 @@ -package actionsImpl +package actions_impl import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "log" "strings" @@ -36,7 +36,7 @@ func checkRequiredStrings(c actions.CliContext, flags ...string) { } } -func (d *dacctlActions) getSession(c actions.CliContext) (datamodel.Session, error) { +func (d *dacctlActions) getSession(c actions.CliContext) (model.Session, error) { checkRequiredStrings(c, "token") token := c.String("token") s, err := d.registry.GetSession(token) diff --git a/internal/pkg/dacctlv2/actionsImpl/actions_test.go b/internal/pkg/dacctlv2/actions_impl/actions_test.go similarity index 89% rename from internal/pkg/dacctlv2/actionsImpl/actions_test.go rename to internal/pkg/dacctlv2/actions_impl/actions_test.go index 88ebd3a2..12de0f65 100644 --- a/internal/pkg/dacctlv2/actionsImpl/actions_test.go +++ b/internal/pkg/dacctlv2/actions_impl/actions_test.go @@ -1,10 +1,10 @@ -package actionsImpl +package actions_impl import ( "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -56,13 +56,13 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().CreateSessionAllocations(datamodel.Session{ + fakeSession := model.Session{Name: "foo"} + registry.EXPECT().CreateSessionAllocations(model.Session{ Name: "token", Owner: 1001, Group: 1002, CreatedAt: 123, - PersistentVolumeRequest: datamodel.PersistentVolumeRequest{ + PersistentVolumeRequest: model.PersistentVolumeRequest{ Caller: "caller", PoolName: "pool1", CapacityBytes: 2147483648, @@ -83,7 +83,7 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} + fakeSession := model.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DeleteSession(fakeSession).Return(fakeError) @@ -100,7 +100,7 @@ func TestDacctlActions_DataIn(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} + fakeSession := model.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DataIn(fakeSession).Return(fakeError) @@ -117,7 +117,7 @@ func TestDacctlActions_DataOut(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} + fakeSession := model.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DataOut(fakeSession).Return(fakeError) @@ -134,7 +134,7 @@ func TestDacctlActions_PreRun(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} + fakeSession := model.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().AttachVolumes(fakeSession).Return(fakeError) @@ -151,7 +151,7 @@ func TestDacctlActions_PostRun(t *testing.T) { registry := mock_session.NewMockRegistry(mockCtrl) session := mock_session.NewMockActions(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} + fakeSession := model.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DetachVolumes(fakeSession).Return(fakeError) diff --git a/internal/pkg/dacctlv2/actionsImpl/job.go b/internal/pkg/dacctlv2/actions_impl/job.go similarity index 92% rename from internal/pkg/dacctlv2/actionsImpl/job.go rename to internal/pkg/dacctlv2/actions_impl/job.go index 8330dd1e..5dd917df 100644 --- a/internal/pkg/dacctlv2/actionsImpl/job.go +++ b/internal/pkg/dacctlv2/actions_impl/job.go @@ -1,4 +1,4 @@ -package actionsImpl +package actions_impl import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" diff --git a/internal/pkg/dacctlv2/actionsImpl/persistent.go b/internal/pkg/dacctlv2/actions_impl/persistent.go similarity index 60% rename from internal/pkg/dacctlv2/actionsImpl/persistent.go rename to internal/pkg/dacctlv2/actions_impl/persistent.go index db78728e..c81d5cc2 100644 --- a/internal/pkg/dacctlv2/actionsImpl/persistent.go +++ b/internal/pkg/dacctlv2/actions_impl/persistent.go @@ -1,32 +1,32 @@ -package actionsImpl +package actions_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" - "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "strings" "time" ) -var stringToAccessMode = map[string]datamodel.AccessMode{ - "": datamodel.Striped, - "striped": datamodel.Striped, - "private": datamodel.Private, - "private,striped": datamodel.PrivateAndStriped, - "striped,private": datamodel.PrivateAndStriped, +var stringToAccessMode = map[string]model.AccessMode{ + "": model.Striped, + "striped": model.Striped, + "private": model.Private, + "private,striped": model.PrivateAndStriped, + "striped,private": model.PrivateAndStriped, } -func accessModeFromString(raw string) datamodel.AccessMode { +func accessModeFromString(raw string) model.AccessMode { return stringToAccessMode[strings.ToLower(raw)] } -var stringToBufferType = map[string]datamodel.BufferType{ - "": datamodel.Scratch, - "scratch": datamodel.Scratch, - "cache": datamodel.Cache, +var stringToBufferType = map[string]model.BufferType{ + "": model.Scratch, + "scratch": model.Scratch, + "cache": model.Cache, } -func bufferTypeFromString(raw string) datamodel.BufferType { +func bufferTypeFromString(raw string) model.BufferType { return stringToBufferType[strings.ToLower(raw)] } @@ -45,15 +45,15 @@ func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { if err != nil { return err } - request := datamodel.PersistentVolumeRequest{ + request := model.PersistentVolumeRequest{ Caller: c.String("caller"), CapacityBytes: capacityBytes, PoolName: pool, Access: accessModeFromString(c.String("access")), Type: bufferTypeFromString(c.String("type")), } - session := datamodel.Session{ - Name: datamodel.SessionName(c.String("token")), + session := model.Session{ + Name: model.SessionName(c.String("token")), PersistentVolumeRequest: request, Owner: uint(c.Int("user")), Group: uint(c.Int("group")), diff --git a/internal/pkg/dacctlv2/actionsImpl/show.go b/internal/pkg/dacctlv2/actions_impl/show.go similarity index 96% rename from internal/pkg/dacctlv2/actionsImpl/show.go rename to internal/pkg/dacctlv2/actions_impl/show.go index 2993cbf0..b0b91dfb 100644 --- a/internal/pkg/dacctlv2/actionsImpl/show.go +++ b/internal/pkg/dacctlv2/actions_impl/show.go @@ -1,4 +1,4 @@ -package actionsImpl +package actions_impl import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" diff --git a/internal/pkg/data/brickhost/bricks.go b/internal/pkg/data/brickhost/bricks.go index ec04c3d3..73285fca 100644 --- a/internal/pkg/data/brickhost/bricks.go +++ b/internal/pkg/data/brickhost/bricks.go @@ -1,20 +1,20 @@ package brickhost -import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" type BrickRegistry interface { // Returns a summary of the current state of all pools, including the bricks in each pool - Pools() ([]datamodel.Pool, error) + Pools() ([]model.Pool, error) // BrickHosts need to check they match a pool // which may involve creating the default pool - EnsureDefaultPoolCreated(granularityGB int) (datamodel.PoolName, error) + EnsureDefaultPoolCreated(granularityGB int) (model.PoolName, error) // BrickHost updates bricks on startup - UpdateBrickHost(brickHostInfo datamodel.BrickHostInfo) error + UpdateBrickHost(brickHostInfo model.BrickHostInfo) error // Get information on the BrickHost - GetBrickHostInfo(name datamodel.BrickHostName) (datamodel.BrickHostInfo, error) + GetBrickHostInfo(name model.BrickHostName) (model.BrickHostInfo, error) // While the process is still running this notifies others the host is up // diff --git a/internal/pkg/data/mock_session/actions.go b/internal/pkg/data/mock_session/actions.go index 30335018..83f1cd2f 100644 --- a/internal/pkg/data/mock_session/actions.go +++ b/internal/pkg/data/mock_session/actions.go @@ -5,9 +5,9 @@ package mock_session import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - gomock "github.com/golang/mock/gomock" - reflect "reflect" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/golang/mock/gomock" + "reflect" ) // MockActions is a mock of Actions interface @@ -34,7 +34,7 @@ func (m *MockActions) EXPECT() *MockActionsMockRecorder { } // CreateSessionVolume mocks base method -func (m *MockActions) CreateSessionVolume(session datamodel.Session) error { +func (m *MockActions) CreateSessionVolume(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSessionVolume", session) ret0, _ := ret[0].(error) @@ -48,7 +48,7 @@ func (mr *MockActionsMockRecorder) CreateSessionVolume(session interface{}) *gom } // DeleteSession mocks base method -func (m *MockActions) DeleteSession(session datamodel.Session) error { +func (m *MockActions) DeleteSession(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteSession", session) ret0, _ := ret[0].(error) @@ -62,7 +62,7 @@ func (mr *MockActionsMockRecorder) DeleteSession(session interface{}) *gomock.Ca } // DataIn mocks base method -func (m *MockActions) DataIn(session datamodel.Session) error { +func (m *MockActions) DataIn(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DataIn", session) ret0, _ := ret[0].(error) @@ -76,7 +76,7 @@ func (mr *MockActionsMockRecorder) DataIn(session interface{}) *gomock.Call { } // AttachVolumes mocks base method -func (m *MockActions) AttachVolumes(session datamodel.Session) error { +func (m *MockActions) AttachVolumes(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AttachVolumes", session) ret0, _ := ret[0].(error) @@ -90,7 +90,7 @@ func (mr *MockActionsMockRecorder) AttachVolumes(session interface{}) *gomock.Ca } // DetachVolumes mocks base method -func (m *MockActions) DetachVolumes(session datamodel.Session) error { +func (m *MockActions) DetachVolumes(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DetachVolumes", session) ret0, _ := ret[0].(error) @@ -104,7 +104,7 @@ func (mr *MockActionsMockRecorder) DetachVolumes(session interface{}) *gomock.Ca } // DataOut mocks base method -func (m *MockActions) DataOut(session datamodel.Session) error { +func (m *MockActions) DataOut(session model.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DataOut", session) ret0, _ := ret[0].(error) diff --git a/internal/pkg/data/mock_session/session.go b/internal/pkg/data/mock_session/session.go index 7c5e2d3c..7579c6df 100644 --- a/internal/pkg/data/mock_session/session.go +++ b/internal/pkg/data/mock_session/session.go @@ -5,9 +5,9 @@ package mock_session import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - gomock "github.com/golang/mock/gomock" - reflect "reflect" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/golang/mock/gomock" + "reflect" ) // MockRegistry is a mock of Registry interface @@ -34,10 +34,10 @@ func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { } // GetSession mocks base method -func (m *MockRegistry) GetSession(token string) (datamodel.Session, error) { +func (m *MockRegistry) GetSession(token string) (model.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSession", token) - ret0, _ := ret[0].(datamodel.Session) + ret0, _ := ret[0].(model.Session) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -49,10 +49,10 @@ func (mr *MockRegistryMockRecorder) GetSession(token interface{}) *gomock.Call { } // CreateSessionAllocations mocks base method -func (m *MockRegistry) CreateSessionAllocations(s datamodel.Session) (datamodel.Session, error) { +func (m *MockRegistry) CreateSessionAllocations(s model.Session) (model.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSessionAllocations", s) - ret0, _ := ret[0].(datamodel.Session) + ret0, _ := ret[0].(model.Session) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -64,10 +64,10 @@ func (mr *MockRegistryMockRecorder) CreateSessionAllocations(s interface{}) *gom } // ValidateSessionRequest mocks base method -func (m *MockRegistry) ValidateSessionRequest(token string) (datamodel.Session, error) { +func (m *MockRegistry) ValidateSessionRequest(token string) (model.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ValidateSessionRequest", token) - ret0, _ := ret[0].(datamodel.Session) + ret0, _ := ret[0].(model.Session) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -79,10 +79,10 @@ func (mr *MockRegistryMockRecorder) ValidateSessionRequest(token interface{}) *g } // GetAllSessions mocks base method -func (m *MockRegistry) GetAllSessions() ([]datamodel.Session, error) { +func (m *MockRegistry) GetAllSessions() ([]model.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllSessions") - ret0, _ := ret[0].([]datamodel.Session) + ret0, _ := ret[0].([]model.Session) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -94,10 +94,10 @@ func (mr *MockRegistryMockRecorder) GetAllSessions() *gomock.Call { } // GetAllPools mocks base method -func (m *MockRegistry) GetAllPools() ([]datamodel.Pool, error) { +func (m *MockRegistry) GetAllPools() ([]model.Pool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllPools") - ret0, _ := ret[0].([]datamodel.Pool) + ret0, _ := ret[0].([]model.Pool) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/internal/pkg/datamodel/allocation.go b/internal/pkg/data/model/allocation.go similarity index 97% rename from internal/pkg/datamodel/allocation.go rename to internal/pkg/data/model/allocation.go index dc6016e5..6d2b465e 100644 --- a/internal/pkg/datamodel/allocation.go +++ b/internal/pkg/data/model/allocation.go @@ -1,4 +1,4 @@ -package datamodel +package model // You can only have zero or one allocations records for each Brick type BrickAllocation struct { diff --git a/internal/pkg/datamodel/brickhost.go b/internal/pkg/data/model/brickhost.go similarity index 97% rename from internal/pkg/datamodel/brickhost.go rename to internal/pkg/data/model/brickhost.go index c29b3584..abc716ac 100644 --- a/internal/pkg/datamodel/brickhost.go +++ b/internal/pkg/data/model/brickhost.go @@ -1,4 +1,4 @@ -package datamodel +package model type BrickHostName string diff --git a/internal/pkg/datamodel/pool.go b/internal/pkg/data/model/pool.go similarity index 92% rename from internal/pkg/datamodel/pool.go rename to internal/pkg/data/model/pool.go index d6aa327f..e6cf8ed2 100644 --- a/internal/pkg/datamodel/pool.go +++ b/internal/pkg/data/model/pool.go @@ -1,4 +1,4 @@ -package datamodel +package model type PoolName string diff --git a/internal/pkg/datamodel/session.go b/internal/pkg/data/model/session.go similarity index 99% rename from internal/pkg/datamodel/session.go rename to internal/pkg/data/model/session.go index d827eda5..5b272171 100644 --- a/internal/pkg/datamodel/session.go +++ b/internal/pkg/data/model/session.go @@ -1,4 +1,4 @@ -package datamodel +package model type SessionName string diff --git a/internal/pkg/datamodel/volume.go b/internal/pkg/data/model/volume.go similarity index 99% rename from internal/pkg/datamodel/volume.go rename to internal/pkg/data/model/volume.go index bae87c5f..2eac4d87 100644 --- a/internal/pkg/datamodel/volume.go +++ b/internal/pkg/data/model/volume.go @@ -1,4 +1,4 @@ -package datamodel +package model import ( "bytes" diff --git a/internal/pkg/data/session/actions.go b/internal/pkg/data/session/actions.go index 799abc71..7f559560 100644 --- a/internal/pkg/data/session/actions.go +++ b/internal/pkg/data/session/actions.go @@ -1,6 +1,6 @@ package session -import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" // Each volume has an associated primary brick // that is responsible for responding to any actions @@ -8,20 +8,20 @@ import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type Actions interface { // Creates the requested volumes // Error if Session has not had its bricks allocated - CreateSessionVolume(session datamodel.Session) error + CreateSessionVolume(session model.Session) error // Deletes the requested volume and session allocation - DeleteSession(session datamodel.Session) error + DeleteSession(session model.Session) error // Update the session and trigger requested data copy in - DataIn(session datamodel.Session) error + DataIn(session model.Session) error // Update session hosts and attach volumes as needed - AttachVolumes(session datamodel.Session) error + AttachVolumes(session model.Session) error // Attempt to detach volumes - DetachVolumes(session datamodel.Session) error + DetachVolumes(session model.Session) error // Update the session and trigger requested data copy out - DataOut(session datamodel.Session) error + DataOut(session model.Session) error } diff --git a/internal/pkg/data/session/session.go b/internal/pkg/data/session/session.go index 79d42566..fea44cb1 100644 --- a/internal/pkg/data/session/session.go +++ b/internal/pkg/data/session/session.go @@ -1,25 +1,25 @@ package session -import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" type Registry interface { // Gets a session and its current allocations // Returns an error if the session is not found - GetSession(token string) (datamodel.Session, error) + GetSession(token string) (model.Session, error) // Any required allocations are created for the given session // such that actions can now be sent to the given session // Returns an error if the session already exists // Note that deleting a session and its allocation is an action, as is any update - CreateSessionAllocations(s datamodel.Session) (datamodel.Session, error) + CreateSessionAllocations(s model.Session) (model.Session, error) // Checks it would be a valid call to CreateAllocations // Error will describe any validation issues - ValidateSessionRequest(token string) (datamodel.Session, error) + ValidateSessionRequest(token string) (model.Session, error) // Used for show instances and show sessions - GetAllSessions() ([]datamodel.Session, error) + GetAllSessions() ([]model.Session, error) // Get all bricks listed by pools - GetAllPools() ([]datamodel.Pool, error) + GetAllPools() ([]model.Pool, error) } From 9c4674d99a11118c01b3afaf41245ea138b7debc Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 7 Aug 2019 13:04:46 +0100 Subject: [PATCH 013/191] Add the start of a sendAction method --- go.mod | 1 + go.sum | 2 + .../pkg/dacctlv2/actions_impl/actions_test.go | 4 +- .../pkg/dacctlv2/actions_impl/persistent.go | 2 +- internal/pkg/dacctlv2/actions_impl/show.go | 1 - internal/pkg/data/mock_session/actions.go | 6 +- internal/pkg/data/mock_session/session.go | 6 +- internal/pkg/data/session_impl/actions.go | 103 ++++++++++++++++++ internal/pkg/data/store/interface.go | 99 +++++++++++++++++ 9 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 internal/pkg/data/session_impl/actions.go create mode 100644 internal/pkg/data/store/interface.go diff --git a/go.mod b/go.mod index b0e91f41..7020a68e 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/mock v1.3.1 github.com/google/btree v1.0.0 // indirect + github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect diff --git a/go.sum b/go.sum index cf6e3205..ad2328fa 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= diff --git a/internal/pkg/dacctlv2/actions_impl/actions_test.go b/internal/pkg/dacctlv2/actions_impl/actions_test.go index 12de0f65..15413b1e 100644 --- a/internal/pkg/dacctlv2/actions_impl/actions_test.go +++ b/internal/pkg/dacctlv2/actions_impl/actions_test.go @@ -3,8 +3,8 @@ package actions_impl import ( "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -160,4 +160,4 @@ func TestDacctlActions_PostRun(t *testing.T) { err := actions.PostRun(&mockCliContext{}) assert.Equal(t, fakeError, err) -} \ No newline at end of file +} diff --git a/internal/pkg/dacctlv2/actions_impl/persistent.go b/internal/pkg/dacctlv2/actions_impl/persistent.go index c81d5cc2..32f9c101 100644 --- a/internal/pkg/dacctlv2/actions_impl/persistent.go +++ b/internal/pkg/dacctlv2/actions_impl/persistent.go @@ -64,4 +64,4 @@ func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { return err } return d.actions.CreateSessionVolume(session) -} \ No newline at end of file +} diff --git a/internal/pkg/dacctlv2/actions_impl/show.go b/internal/pkg/dacctlv2/actions_impl/show.go index b0b91dfb..1b87a2ef 100644 --- a/internal/pkg/dacctlv2/actions_impl/show.go +++ b/internal/pkg/dacctlv2/actions_impl/show.go @@ -25,4 +25,3 @@ func (*dacctlActions) ListPools() error { func (*dacctlActions) ShowConfigurations() error { panic("implement me") } - diff --git a/internal/pkg/data/mock_session/actions.go b/internal/pkg/data/mock_session/actions.go index 83f1cd2f..6308a7ce 100644 --- a/internal/pkg/data/mock_session/actions.go +++ b/internal/pkg/data/mock_session/actions.go @@ -5,9 +5,9 @@ package mock_session import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/golang/mock/gomock" - "reflect" + model "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockActions is a mock of Actions interface diff --git a/internal/pkg/data/mock_session/session.go b/internal/pkg/data/mock_session/session.go index 7579c6df..6a45d512 100644 --- a/internal/pkg/data/mock_session/session.go +++ b/internal/pkg/data/mock_session/session.go @@ -5,9 +5,9 @@ package mock_session import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/golang/mock/gomock" - "reflect" + model "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockRegistry is a mock of Registry interface diff --git a/internal/pkg/data/session_impl/actions.go b/internal/pkg/data/session_impl/actions.go new file mode 100644 index 00000000..c8487842 --- /dev/null +++ b/internal/pkg/data/session_impl/actions.go @@ -0,0 +1,103 @@ +package session_impl + +import ( + "context" + "encoding/json" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/store" + "github.com/google/uuid" + "log" + "time" +) + +// TODO: this is the client side, need server side too +func NewSessionActions(keystore store.Keystore) session.Actions { + return &sessionActions{keystore: keystore} +} + +type sessionActions struct { + keystore store.Keystore +} + +func toJson(message interface{}) string { + b, error := json.Marshal(message) + if error != nil { + log.Fatal(error) + } + return string(b) +} + +func (s *sessionActions) sendAction(session model.Session, action string) error { + // TODO: update session? + + actionId := uuid.New().String() + sessionPrefix := fmt.Sprintf("/Session/Actions/%s", session.Name) + actionPrefix := fmt.Sprintf("%s/%s", sessionPrefix, actionId) + + mutex, err := s.keystore.NewMutex(sessionPrefix) + if err != nil { + return fmt.Errorf("unable to start action due to: %s", err.Error()) + } + mctxt, cancel := context.WithTimeout(context.Background(), time.Minute*20) + defer cancel() + err = mutex.Lock(mctxt) + if err != nil { + return fmt.Errorf("unable to start action due to: %s", err.Error()) + } + defer mutex.Unlock(context.Background()) + + ctxt, cancel := context.WithTimeout(context.Background(), time.Minute*20) + defer cancel() + responses := s.keystore.Watch(ctxt, fmt.Sprintf("%s/%s", actionPrefix, "output"), false) + + err = s.keystore.Add(store.KeyValue{ + Key: fmt.Sprintf("%s/%s", actionPrefix, "input"), + // TODO: need format request object + Value: action, + }) + if err != nil { + return err + } + + for response := range responses { + if response.Err != nil { + return response.Err + } + + if response.IsCreate { + // TODO: need formal response object? + if response.New.Value != "" { + return fmt.Errorf("error while sending action") + } + // TODO: who deletes the action key, if anyone? + } + return nil + } + return nil +} + +func (*sessionActions) CreateSessionVolume(session model.Session) error { + panic("implement me") +} + +func (*sessionActions) DeleteSession(session model.Session) error { + panic("implement me") +} + +func (*sessionActions) DataIn(session model.Session) error { + panic("implement me") +} + +func (*sessionActions) AttachVolumes(session model.Session) error { + panic("implement me") +} + +func (*sessionActions) DetachVolumes(session model.Session) error { + panic("implement me") +} + +func (*sessionActions) DataOut(session model.Session) error { + panic("implement me") +} diff --git a/internal/pkg/data/store/interface.go b/internal/pkg/data/store/interface.go new file mode 100644 index 00000000..39f5660b --- /dev/null +++ b/internal/pkg/data/store/interface.go @@ -0,0 +1,99 @@ +package store + +import ( + "context" + "encoding/json" + "log" +) + +type Keystore interface { + // Used to clean up any resources being used + // ... such as a connection to etcd. + Close() error + + // Removes any key starting with the given prefix. + // An error is returned if nothing was deleted, + // which some users may choose to safely ignore. + CleanPrefix(prefix string) error + + // Atomically add all the key value pairs + // + // If an error occurs no keyvalues are written. + // Error is returned if any key already exists. + Add(keyValue KeyValue) error + + // Update the specifed key values, atomically + // + // If ModRevision is 0, it is ignored. + // Otherwise if the revisions of any key doesn't + // match the current revision of that key, the update fails. + // When update fails an error is returned and no keyValues are updated + Update(keyValue KeyValueVersion) error + + // Delete the specifed key values, atomically + // + // Similar to update, checks ModRevision matches current key, + // ignores ModRevision if not zero. + // If any keys are not currently present, the request fails. + // Deletes no keys if an error is returned + DeleteAll(keyValues []KeyValueVersion) error + + // Get all key values for a given prefix. + GetAll(prefix string) ([]KeyValueVersion, error) + + // Get all keys for a given prefix. + Get(key string) (KeyValueVersion, error) + + // Get a channel containing all KeyValueUpdate events + // + // Use the context to control if you watch forever, or if you choose to cancel when a key + // is deleted, or you stop watching after some timeout. + Watch(ctxt context.Context, key string, withPrefix bool) KeyValueUpdateChan + + // Add a key, and remove it when calling process dies + // Error is returned if the key already exists + KeepAliveKey(key string) error + + // Get a new mutex associated with the specified key + NewMutex(lockKey string) (Mutex, error) +} + +type KeyValueUpdateChan <-chan KeyValueUpdate + +type KeyValue struct { + Key string + Value string // TODO: should this be []byte? Or have a json parsed version? +} + +type KeyValueVersion struct { + Key string + Value string + CreateRevision int64 + ModRevision int64 +} + +type KeyValueUpdate struct { + Old *KeyValueVersion + New *KeyValueVersion + IsCreate bool + IsModify bool + IsDelete bool + Err error +} + +func (kvv KeyValueVersion) String() string { + return toJson(kvv) +} + +func toJson(message interface{}) string { + b, error := json.Marshal(message) + if error != nil { + log.Fatal(error) + } + return string(b) +} + +type Mutex interface { + Lock(ctx context.Context) error + Unlock(ctx context.Context) error +} From 4ee70643043c1931746a430f76d34377769c2fea Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 7 Aug 2019 18:05:19 +0100 Subject: [PATCH 014/191] Add test for create job buffer --- .../pkg/dacctlv2/actions_impl/actions_test.go | 9 +- internal/pkg/dacctlv2/actions_impl/job.go | 106 +++++++++++- .../pkg/dacctlv2/actions_impl/job_test.go | 76 +++++++++ .../pkg/dacctlv2/actions_impl/persistent.go | 23 +-- .../pkg/dacctlv2/parsers/capacity/capacity.go | 5 +- internal/pkg/dacctlv2/parsers/jobfile/job.go | 158 ++++++------------ .../pkg/dacctlv2/parsers/jobfile/job_test.go | 13 +- internal/pkg/data/model/session.go | 23 +-- 8 files changed, 269 insertions(+), 144 deletions(-) create mode 100644 internal/pkg/dacctlv2/actions_impl/job_test.go diff --git a/internal/pkg/dacctlv2/actions_impl/actions_test.go b/internal/pkg/dacctlv2/actions_impl/actions_test.go index 15413b1e..a2f3cbf2 100644 --- a/internal/pkg/dacctlv2/actions_impl/actions_test.go +++ b/internal/pkg/dacctlv2/actions_impl/actions_test.go @@ -62,10 +62,11 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { Owner: 1001, Group: 1002, CreatedAt: 123, - PersistentVolumeRequest: model.PersistentVolumeRequest{ - Caller: "caller", - PoolName: "pool1", - CapacityBytes: 2147483648, + VolumeRequest: model.VolumeRequest{ + MultiJob: true, + Caller: "caller", + PoolName: "pool1", + TotalCapacityBytes: 2147483648, }, }).Return(fakeSession, nil) session.EXPECT().CreateSessionVolume(fakeSession) diff --git a/internal/pkg/dacctlv2/actions_impl/job.go b/internal/pkg/dacctlv2/actions_impl/job.go index 5dd917df..a3654d87 100644 --- a/internal/pkg/dacctlv2/actions_impl/job.go +++ b/internal/pkg/dacctlv2/actions_impl/job.go @@ -1,11 +1,107 @@ package actions_impl -import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/jobfile" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "log" +) -func (*dacctlActions) ValidateJob(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) ValidateJob(c actions.CliContext) error { + checkRequiredStrings(c, "job") + jobFile := c.String("job") + summary, err := jobfile.ParseJobFile(d.disk, jobFile) + if err != nil { + return err + } else { + // TODO check valid pools, etc, etc. + log.Println("Summary of job file:", summary) + } + return nil } -func (*dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { - panic("implement me") +func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { + checkRequiredStrings(c, "token", "job", "caller", "capacity") + // TODO: need to specify user and group too + + jobFile := c.String("job") + summary, err := jobfile.ParseJobFile(d.disk, jobFile) + if err != nil { + return err + } + + nodeFile := c.String("nodehostnamefile") + if nodeFile != "" { + // TODO we could add this into the volume as a scheduling hint, when its available? + log.Printf("Ignoring nodeFile in setup: %s", nodeFile) + } + + pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) + if err != nil { + return err + } + + // extract info from job file + swapBytes := 0 + if summary.Swap != nil { + swapBytes = summary.Swap.SizeBytes + } + access := model.NoAccess + bufferType := model.Scratch + if summary.PerJobBuffer != nil { + access = summary.PerJobBuffer.AccessMode + if summary.PerJobBuffer.BufferType == model.Cache { + return fmt.Errorf("cache is not supported") + } + } + var multiJobVolumes []model.VolumeName + for _, attachment := range summary.Attachments { + multiJobVolumes = append(multiJobVolumes, attachment) + } + + request := model.VolumeRequest{ + MultiJob: false, + Caller: c.String("caller"), + TotalCapacityBytes: capacityBytes, + PoolName: pool, + Access: access, + Type: bufferType, + SwapBytes: swapBytes, + } + session := model.Session{ + Name: model.SessionName(c.String("token")), + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: getNow(), + VolumeRequest: request, + MultiJobVolumes: multiJobVolumes, + StageInRequests: summary.DataIn, + StageOutRequests: summary.DataOut, + } + session.Paths = getPaths(session) + + session, err = d.registry.CreateSessionAllocations(session) + if err != nil { + return err + } + return d.actions.CreateSessionVolume(session) +} + +func getPaths(session model.Session) map[string]string { + paths := make(map[string]string) + if session.VolumeRequest.MultiJob == false { + if session.VolumeRequest.Access == model.Private || session.VolumeRequest.Access == model.PrivateAndStriped { + paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s_job_private", session.Name) + } + if session.VolumeRequest.Access == model.Striped || session.VolumeRequest.Access == model.PrivateAndStriped { + paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", session.Name) + } + } + for _, multiJobVolume := range session.MultiJobVolumes { + paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf( + "/dac/%s_persistent_%s", session.Name, multiJobVolume) + } + return paths } diff --git a/internal/pkg/dacctlv2/actions_impl/job_test.go b/internal/pkg/dacctlv2/actions_impl/job_test.go new file mode 100644 index 00000000..0eef705b --- /dev/null +++ b/internal/pkg/dacctlv2/actions_impl/job_test.go @@ -0,0 +1,76 @@ +package actions_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_session.NewMockRegistry(mockCtrl) + session := mock_session.NewMockActions(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + + lines := []string{ + `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, + `#DW persistentdw name=myBBname1`, + `#DW persistentdw name=myBBname2`, + `#DW swap 4MiB`, + `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, + `#DW stage_in source=/global/cscratch1/filelist type=list`, + `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, + } + disk.EXPECT().Lines("jobfile").Return(lines, nil) + fakeSession := model.Session{Name: "foo"} + registry.EXPECT().CreateSessionAllocations(model.Session{ + Name: "token", + Owner: 1001, + Group: 1002, + CreatedAt: 123, + MultiJobVolumes: []model.VolumeName{"myBBname1", "myBBname2"}, + StageInRequests: []model.DataCopyRequest{ + { + SourceType: model.File, + Source: "/global/cscratch1/filename1", + Destination: "$DW_JOB_STRIPED/filename1", + }, + { + SourceType: model.List, + Source: "/global/cscratch1/filelist", + }, + }, + StageOutRequests: []model.DataCopyRequest{ + { + SourceType: model.Directory, + Source: "$DW_JOB_STRIPED/outdir", + Destination: "/global/scratch1/outdir", + }, + }, + VolumeRequest: model.VolumeRequest{ + Caller: "caller", + PoolName: "pool1", + TotalCapacityBytes: 2147483648, + Access: model.PrivateAndStriped, + Type: model.Scratch, + SwapBytes: 4194304, + }, + Paths: map[string]string{ + "DW_JOB_PRIVATE": "/dac/token_job_private", + "DW_JOB_STRIPED": "/dac/token_job/global", + "DW_PERSISTENT_STRIPED_myBBname1": "/dac/token_persistent_myBBname1", + "DW_PERSISTENT_STRIPED_myBBname2": "/dac/token_persistent_myBBname2", + }, + }).Return(fakeSession, nil) + session.EXPECT().CreateSessionVolume(fakeSession) + fakeTime = 123 + + actions := NewDacctlActions(registry, session, disk) + err := actions.CreatePerJobBuffer(&mockCliContext{capacity: 2}) + + assert.Nil(t, err) +} diff --git a/internal/pkg/dacctlv2/actions_impl/persistent.go b/internal/pkg/dacctlv2/actions_impl/persistent.go index 32f9c101..c69cfaa6 100644 --- a/internal/pkg/dacctlv2/actions_impl/persistent.go +++ b/internal/pkg/dacctlv2/actions_impl/persistent.go @@ -45,19 +45,20 @@ func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { if err != nil { return err } - request := model.PersistentVolumeRequest{ - Caller: c.String("caller"), - CapacityBytes: capacityBytes, - PoolName: pool, - Access: accessModeFromString(c.String("access")), - Type: bufferTypeFromString(c.String("type")), + request := model.VolumeRequest{ + MultiJob: true, + Caller: c.String("caller"), + TotalCapacityBytes: capacityBytes, + PoolName: pool, + Access: accessModeFromString(c.String("access")), + Type: bufferTypeFromString(c.String("type")), } session := model.Session{ - Name: model.SessionName(c.String("token")), - PersistentVolumeRequest: request, - Owner: uint(c.Int("user")), - Group: uint(c.Int("group")), - CreatedAt: getNow(), + Name: model.SessionName(c.String("token")), + VolumeRequest: request, + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: getNow(), } session, err = d.registry.CreateSessionAllocations(session) if err != nil { diff --git a/internal/pkg/dacctlv2/parsers/capacity/capacity.go b/internal/pkg/dacctlv2/parsers/capacity/capacity.go index f7fd38fc..06f7b4a4 100644 --- a/internal/pkg/dacctlv2/parsers/capacity/capacity.go +++ b/internal/pkg/dacctlv2/parsers/capacity/capacity.go @@ -7,6 +7,7 @@ import ( "strings" ) +// TODO: missing a few? var sizeSuffixMulitiplyer = map[string]int{ "TiB": 1099511627776, "TB": 1000000000000, @@ -16,7 +17,7 @@ var sizeSuffixMulitiplyer = map[string]int{ "MB": 1000000, } -func parseSize(raw string) (int, error) { +func ParseSize(raw string) (int, error) { intVal, err := strconv.Atoi(raw) if err == nil { // specified raw bytes @@ -42,7 +43,7 @@ func ParseCapacityBytes(raw string) (string, int, error) { } pool := parts[0] rawCapacity := parts[1] - sizeBytes, err := parseSize(rawCapacity) + sizeBytes, err := ParseSize(rawCapacity) if err != nil { return "", 0, err } diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job.go b/internal/pkg/dacctlv2/parsers/jobfile/job.go index 7adef381..4147e83a 100644 --- a/internal/pkg/dacctlv2/parsers/jobfile/job.go +++ b/internal/pkg/dacctlv2/parsers/jobfile/job.go @@ -3,18 +3,19 @@ package jobfile import ( "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "log" - "strconv" "strings" ) type jobSummary struct { PerJobBuffer *cmdPerJobBuffer Swap *cmdAttachPerJobSwap - Attachments []cmdAttachPersistent - DataIn *cmdStageInData - DataOut *cmdStageOutData + Attachments []model.VolumeName + DataIn []model.DataCopyRequest + DataOut []model.DataCopyRequest //createPersistent *cmdCreatePersistent //destroyPersistent *cmdDestroyPersistent } @@ -56,25 +57,24 @@ func getJobSummary(lines []string) (jobSummary, error) { return summary, fmt.Errorf("only one per job buffer allowed") } case cmdAttachPersistent: - summary.Attachments = append(summary.Attachments, c) + summary.Attachments = append(summary.Attachments, model.VolumeName(c)) case cmdAttachPerJobSwap: if summary.Swap != nil { - // TODO check amount isn't too big for per job buffer return summary, fmt.Errorf("only one swap request allowed") } summary.Swap = &c case cmdStageOutData: - if summary.DataOut != nil { - // TODO really should check if data out matches one of the requested buffers - return summary, fmt.Errorf("only one per data out requested allowed") - } - summary.DataOut = &c + summary.DataOut = append(summary.DataOut, model.DataCopyRequest{ + SourceType: c.SourceType, + Source: c.Source, + Destination: c.Destination, + }) case cmdStageInData: - if summary.DataIn != nil { - // TODO really should check if data in matches one of the requested buffers - return summary, fmt.Errorf("only one per data in requested allowed") - } - summary.DataIn = &c + summary.DataIn = append(summary.DataIn, model.DataCopyRequest{ + SourceType: c.SourceType, + Source: c.Source, + Destination: c.Destination, + }) default: // do nothing } @@ -84,63 +84,44 @@ func getJobSummary(lines []string) (jobSummary, error) { type jobCommand interface{} -type AccessMode int - -const ( - striped AccessMode = 0 - private = 1 - privateAndStriped = 2 -) - -var stringToAccessMode = map[string]AccessMode{ - "": striped, - "striped": striped, - "private": private, - "private,striped": privateAndStriped, - "striped,private": privateAndStriped, +var stringToAccessMode = map[string]model.AccessMode{ + "": model.Striped, + "striped": model.Striped, + "private": model.Private, + "private,striped": model.PrivateAndStriped, + "striped,private": model.PrivateAndStriped, } -func AccessModeFromString(raw string) AccessMode { +func AccessModeFromString(raw string) model.AccessMode { return stringToAccessMode[strings.ToLower(raw)] } -type BufferType int - -const ( - scratch BufferType = iota - cache -) - -var stringToBufferType = map[string]BufferType{ - "": scratch, - "scratch": scratch, - "cache": cache, +var stringToBufferType = map[string]model.BufferType{ + "": model.Scratch, + "scratch": model.Scratch, + "cache": model.Cache, } type cmdCreatePersistent struct { Name string CapacityBytes int - AccessMode AccessMode - BufferType BufferType + AccessMode model.AccessMode + BufferType model.BufferType GenericCmd bool } -func BufferTypeFromString(raw string) BufferType { +func BufferTypeFromString(raw string) model.BufferType { return stringToBufferType[strings.ToLower(raw)] } -type cmdDestroyPersistent struct { - Name string -} +type cmdDestroyPersistent string -type cmdAttachPersistent struct { - Name string -} +type cmdAttachPersistent string type cmdPerJobBuffer struct { CapacityBytes int - AccessMode AccessMode - BufferType BufferType + AccessMode model.AccessMode + BufferType model.BufferType GenericCmd bool } @@ -148,62 +129,19 @@ type cmdAttachPerJobSwap struct { SizeBytes int } -type StageType int - -const ( - directory StageType = iota - file // TODO there is also list, but we ignore that for now -) - -var stringToStageType = map[string]StageType{ - "": directory, - "directory": directory, - "file": file, +var stringToStageType = map[string]model.SourceType{ + "directory": model.Directory, + "file": model.File, + "list": model.List, } -func stageTypeFromString(raw string) StageType { +func sourceTypeFromString(raw string) model.SourceType { return stringToStageType[strings.ToLower(raw)] } -type cmdStageInData struct { - Source string - Destination string - StageType StageType -} - -type cmdStageOutData struct { - Source string - Destination string - StageType StageType -} - -var sizeSuffixMulitiplyer = map[string]int{ - "TiB": 1099511627776, - "TB": 1000000000000, - "GiB": 1073741824, - "GB": 1000000000, - "MiB": 1048576, - "MB": 1000000, -} +type cmdStageInData model.DataCopyRequest -func parseSize(raw string) (int, error) { - intVal, err := strconv.Atoi(raw) - if err == nil { - // specified raw bytes - return intVal, nil - } - for suffix, multiplyer := range sizeSuffixMulitiplyer { - if strings.HasSuffix(raw, suffix) { - rawInt := strings.TrimSuffix(raw, suffix) - intVal, err := strconv.Atoi(rawInt) - if err != nil { - return 0, err - } - return intVal * multiplyer, nil - } - } - return 0, fmt.Errorf("unable to parse size: %s", raw) -} +type cmdStageOutData model.DataCopyRequest func parseArgs(rawArgs []string) (map[string]string, error) { args := make(map[string]string, len(rawArgs)) @@ -248,7 +186,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { var command jobCommand switch cmd { case "create_persistent": - size, err := parseSize(argKeyPair["capacity"]) + size, err := capacity.ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) continue @@ -261,11 +199,11 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { BufferType: BufferTypeFromString(argKeyPair["type"]), } case "destroy_persistent": - command = cmdDestroyPersistent{Name: argKeyPair["name"]} + command = cmdDestroyPersistent(argKeyPair["name"]) case "persistentdw": - command = cmdAttachPersistent{Name: argKeyPair["name"]} + command = cmdAttachPersistent(argKeyPair["name"]) case "jobdw": - size, err := parseSize(argKeyPair["capacity"]) + size, err := capacity.ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) continue @@ -280,7 +218,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { if len(args) != 1 { log.Println("Unable to parse swap command:", line) } - if size, err := parseSize(args[0]); err != nil { + if size, err := capacity.ParseSize(args[0]); err != nil { log.Println(err) continue } else { @@ -290,13 +228,13 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { command = cmdStageInData{ Source: argKeyPair["source"], Destination: argKeyPair["destination"], - StageType: stageTypeFromString(argKeyPair["type"]), + SourceType: sourceTypeFromString(argKeyPair["type"]), } case "stage_out": command = cmdStageOutData{ Source: argKeyPair["source"], Destination: argKeyPair["destination"], - StageType: stageTypeFromString(argKeyPair["type"]), + SourceType: sourceTypeFromString(argKeyPair["type"]), } default: log.Println("unrecognised command:", cmd, "with argument length", len(args)) diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job_test.go b/internal/pkg/dacctlv2/parsers/jobfile/job_test.go index f486afcb..c7133db5 100644 --- a/internal/pkg/dacctlv2/parsers/jobfile/job_test.go +++ b/internal/pkg/dacctlv2/parsers/jobfile/job_test.go @@ -1,6 +1,7 @@ package jobfile import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/stretchr/testify/assert" "log" "testing" @@ -39,13 +40,21 @@ func TestGetJobSummary(t *testing.T) { `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, `#DW swap 4MB`, `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, + `#DW stage_in source=/global/cscratch1/filename2 destination=$DW_JOB_STRIPED/filename2 type=file`, `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, } result, err := getJobSummary(lines) assert.Nil(t, err) - assert.EqualValues(t, "/global/cscratch1/filename1", result.DataIn.Source) - assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut.Source) + assert.Equal(t, 2, len(result.DataIn)) + assert.Equal(t, 1, len(result.DataOut)) + assert.EqualValues(t, "/global/cscratch1/filename1", result.DataIn[0].Source) + assert.EqualValues(t, "/global/cscratch1/filename2", result.DataIn[1].Source) + assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut[0].Source) + + assert.Equal(t, 2, len(result.Attachments)) + assert.Equal(t, model.VolumeName("myBBname1"), result.Attachments[0]) + assert.Equal(t, model.VolumeName("myBBname2"), result.Attachments[1]) assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) assert.Equal(t, 4000000, result.Swap.SizeBytes) diff --git a/internal/pkg/data/model/session.go b/internal/pkg/data/model/session.go index 5b272171..6cc8e83b 100644 --- a/internal/pkg/data/model/session.go +++ b/internal/pkg/data/model/session.go @@ -21,7 +21,7 @@ type Session struct { CreatedAt uint // Details of what was requested - PersistentVolumeRequest PersistentVolumeRequest + VolumeRequest VolumeRequest // Records if we have started trying to delete DeleteRequested bool @@ -53,20 +53,23 @@ type Session struct { Allocations []BrickAllocation } -type PersistentVolumeRequest struct { - Caller string - CapacityBytes int - PoolName string - Access AccessMode - Type BufferType +type VolumeRequest struct { + MultiJob bool + Caller string + TotalCapacityBytes int + PoolName string + Access AccessMode + Type BufferType + SwapBytes int } type AccessMode int const ( - Striped AccessMode = 0 - Private = 1 - PrivateAndStriped = 2 + NoAccess AccessMode = iota + Striped + Private + PrivateAndStriped ) type BufferType int From e2778a9bc1487cb26a83bcdbf68786f8911a7f4a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 12 Aug 2019 18:31:36 +0100 Subject: [PATCH 015/191] Make implementation notes around dacd interfaces --- .../pkg/dacctlv2/actions_impl/actions_test.go | 2 +- internal/pkg/dacctlv2/actions_impl/job.go | 2 +- .../pkg/dacctlv2/actions_impl/job_test.go | 2 +- .../pkg/dacctlv2/actions_impl/persistent.go | 2 +- .../pkg/dacdv2/brick_manager/interface.go | 16 +++++ .../dacdv2/brick_manager_impl/brick_manger.go | 47 ++++++++++++++ internal/pkg/data/brick_host/bricks.go | 53 ++++++++++++++++ internal/pkg/data/brickhost/bricks.go | 63 ------------------- internal/pkg/data/brickhost/volume.go | 4 -- internal/pkg/data/mock_session/session.go | 32 +++++++--- internal/pkg/data/model/brickhost.go | 10 ++- internal/pkg/data/model/pool.go | 10 +++ internal/pkg/data/model/session.go | 13 ++++ internal/pkg/data/pool/allocations.go | 21 +++++++ internal/pkg/data/session/session.go | 14 ++++- internal/pkg/data/session_impl/actions.go | 13 ++-- internal/pkg/data/utils/mutex.go | 8 +++ 17 files changed, 223 insertions(+), 89 deletions(-) create mode 100644 internal/pkg/dacdv2/brick_manager/interface.go create mode 100644 internal/pkg/dacdv2/brick_manager_impl/brick_manger.go create mode 100644 internal/pkg/data/brick_host/bricks.go delete mode 100644 internal/pkg/data/brickhost/bricks.go delete mode 100644 internal/pkg/data/brickhost/volume.go create mode 100644 internal/pkg/data/pool/allocations.go create mode 100644 internal/pkg/data/utils/mutex.go diff --git a/internal/pkg/dacctlv2/actions_impl/actions_test.go b/internal/pkg/dacctlv2/actions_impl/actions_test.go index a2f3cbf2..a7f92ff7 100644 --- a/internal/pkg/dacctlv2/actions_impl/actions_test.go +++ b/internal/pkg/dacctlv2/actions_impl/actions_test.go @@ -57,7 +57,7 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { session := mock_session.NewMockActions(mockCtrl) fakeSession := model.Session{Name: "foo"} - registry.EXPECT().CreateSessionAllocations(model.Session{ + registry.EXPECT().CreateSession(model.Session{ Name: "token", Owner: 1001, Group: 1002, diff --git a/internal/pkg/dacctlv2/actions_impl/job.go b/internal/pkg/dacctlv2/actions_impl/job.go index a3654d87..3869283d 100644 --- a/internal/pkg/dacctlv2/actions_impl/job.go +++ b/internal/pkg/dacctlv2/actions_impl/job.go @@ -82,7 +82,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { } session.Paths = getPaths(session) - session, err = d.registry.CreateSessionAllocations(session) + session, err = d.registry.CreateSession(session) if err != nil { return err } diff --git a/internal/pkg/dacctlv2/actions_impl/job_test.go b/internal/pkg/dacctlv2/actions_impl/job_test.go index 0eef705b..be132324 100644 --- a/internal/pkg/dacctlv2/actions_impl/job_test.go +++ b/internal/pkg/dacctlv2/actions_impl/job_test.go @@ -27,7 +27,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { } disk.EXPECT().Lines("jobfile").Return(lines, nil) fakeSession := model.Session{Name: "foo"} - registry.EXPECT().CreateSessionAllocations(model.Session{ + registry.EXPECT().CreateSession(model.Session{ Name: "token", Owner: 1001, Group: 1002, diff --git a/internal/pkg/dacctlv2/actions_impl/persistent.go b/internal/pkg/dacctlv2/actions_impl/persistent.go index c69cfaa6..f6a1ce03 100644 --- a/internal/pkg/dacctlv2/actions_impl/persistent.go +++ b/internal/pkg/dacctlv2/actions_impl/persistent.go @@ -60,7 +60,7 @@ func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { Group: uint(c.Int("group")), CreatedAt: getNow(), } - session, err = d.registry.CreateSessionAllocations(session) + session, err = d.registry.CreateSession(session) if err != nil { return err } diff --git a/internal/pkg/dacdv2/brick_manager/interface.go b/internal/pkg/dacdv2/brick_manager/interface.go new file mode 100644 index 00000000..72085e85 --- /dev/null +++ b/internal/pkg/dacdv2/brick_manager/interface.go @@ -0,0 +1,16 @@ +package brick_manager + +type BrickManager interface { + // Get the current hostname key that is being kept alive + Hostname() string + + // Tidy up from previous shutdowns + // , then start waiting for session actions + // notify dacctl we are listening via keep alive key + // if drainSessions = True, we don't allow new + Startup(drainSessions bool) error + + // Wait for any events to complete + // then do any tidy up required for a graceful shutdown + Shutdown() error +} diff --git a/internal/pkg/dacdv2/brick_manager_impl/brick_manger.go b/internal/pkg/dacdv2/brick_manager_impl/brick_manger.go new file mode 100644 index 00000000..da838600 --- /dev/null +++ b/internal/pkg/dacdv2/brick_manager_impl/brick_manger.go @@ -0,0 +1,47 @@ +package brick_manager_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacdv2/brick_manager" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/brick_host" + "log" + "os" +) + +func NewBrickManager(brickRegistry brick_host.BrickHostRegistry) brick_manager.BrickManager { + return &brickManager{getHostname(), brickRegistry} +} + +func getHostname() string { + hostname, err := os.Hostname() + if err != nil { + log.Fatal(err) + } + return hostname +} + +type brickManager struct { + hostname string + brickRegistry brick_host.BrickHostRegistry +} + +func (bm *brickManager) Hostname() string { + return bm.hostname +} + +func (bm *brickManager) Startup(drainSessions bool) error { + panic("implement me") + // * update current brick status + // ** error out if removing bricks with existing assignments? + // * start listening for create sessions (new primary bricks) and session actions + // * report we are listening with keep-alive + // * check all brick assignments + // ** ensure all brick hosts are up, warn if there are issues + // ** if primary brick, refresh ansible run (i.e. recover from host reboot) +} + +func (bm *brickManager) Shutdown() error { + // Delete the keepalive key, to stop new actions being sent + // Wait for existing actions by trying to get a lock on all + // sessions we for which we are the primary brick + panic("implement me") +} diff --git a/internal/pkg/data/brick_host/bricks.go b/internal/pkg/data/brick_host/bricks.go new file mode 100644 index 00000000..6012da5f --- /dev/null +++ b/internal/pkg/data/brick_host/bricks.go @@ -0,0 +1,53 @@ +package brick_host + +import ( + "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" +) + +type BrickHostRegistry interface { + // Creates the pool if it doesn't exist + // error if the granularity doesn't match and existing pool + EnsurePoolCreated(poolName model.PoolName, granularityGB int) (model.PoolName, error) + + // BrickHost updates bricks on startup + UpdateBrickHost(brickHostInfo model.BrickHostInfo) error + + // While the process is still running this notifies others the host is up + // + // When a host is dead non of its bricks will get new volumes assigned, + // and no bricks will get cleaned up until the next service start. + // Error will be returned if the host info has not yet been written. + KeepAliveHost(hostname string) error + + // Get sessions where given hostname is the primary brick + // Created by dacctl via CreateSessionAllocations + GetNewSessionRequests(ctxt context.Context, hostname model.BrickHostName) (<-chan model.Session, error) + + // Gets all new actions for the given Session + // This confirms that dacd is ready to service requests for this session + // and updates the session with an appropriate SessionActionPrefix + // The channel tracks all actions sent to that sub keys of the above prefix + // This is also called when dacd is restarted and you want to resume sending + // any actions that are not marked as complete + // New tasks are only sent if the keepalive key is active + GetNewSessionAction(ctxt context.Context, hostname model.BrickHostName) (<-chan model.SessionAction, error) + + // Update provided session + // Ensures there are no other updates to Session since the revision contained in it + UpdateSession(session model.Session) (model.Session, error) + + // This is called before confirming the Session delete request + // after all bricks have been de-allocated + DeleteSession(session model.Session) error + + // Mark the given SessionAction as completed + // Optionally returns an error to caller of the action + CompleteSessionAction(action model.SessionAction, error error) error + + // Check if a given action is complete + IsSessionActionComplete(action model.SessionAction) (bool, error) + + // Check if given brick host is alive + IsBrickHostAlive(hostname model.BrickHostName) (bool, error) +} diff --git a/internal/pkg/data/brickhost/bricks.go b/internal/pkg/data/brickhost/bricks.go deleted file mode 100644 index 73285fca..00000000 --- a/internal/pkg/data/brickhost/bricks.go +++ /dev/null @@ -1,63 +0,0 @@ -package brickhost - -import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - -type BrickRegistry interface { - // Returns a summary of the current state of all pools, including the bricks in each pool - Pools() ([]model.Pool, error) - - // BrickHosts need to check they match a pool - // which may involve creating the default pool - EnsureDefaultPoolCreated(granularityGB int) (model.PoolName, error) - - // BrickHost updates bricks on startup - UpdateBrickHost(brickHostInfo model.BrickHostInfo) error - - // Get information on the BrickHost - GetBrickHostInfo(name model.BrickHostName) (model.BrickHostInfo, error) - - // While the process is still running this notifies others the host is up - // - // When a host is dead non of its bricks will get new volumes assigned, - // and no bricks will get cleaned up until the next service start. - // Error will be returned if the host info has not yet been written. - KeepAliveHost(hostname string) error - - // Update a brick with allocation information. - // - // No update is made and an error is returned if: - // any brick already has an allocation, - // or any volume a brick is being assigned to already has an allocation, - // or if any of the volumes do not exist - // or if there is not exactly one primary brick. - // - // Note: you may assign multiple volumes in a single call, but all bricks - // for a particular volume must be set in a single call - //AllocateBricksForVolume(volume Volume) ([]BrickAllocation, error) - - // Deallocate all bricks associated with the given volume - // - // No update is made and an error is returned if any of brick allocations don't match the current state. - // If any host associated with one of the bricks is down, an error is returned and the deallocate is - // recorded as requested and not executed. - // Note: this returns as soon as deallocate is requested, doesn't wait for cleanup completion - //DeallocateBricks(volume VolumeName) error - - // This is called after DeallocateBricks has been processed - //HardDeleteAllocations(allocations []BrickAllocation) error - - // Get all the allocations for bricks associated with the specified hostname - //GetAllocationsForHost(hostname string) ([]BrickAllocation, error) - - // Get all the allocations for bricks associated with the specific volume - //GetAllocationsForVolume(volume VolumeName) ([]BrickAllocation, error) - - // Get information on a specific brick - //GetBrickInfo(hostname string, device string) (BrickInfo, error) - - // Returns a channel that reports all new brick allocations for given hostname - // - // The channel is closed when the context is cancelled or timeout. - // Any errors in the watching log the issue and panic - //GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan BrickAllocation -} diff --git a/internal/pkg/data/brickhost/volume.go b/internal/pkg/data/brickhost/volume.go deleted file mode 100644 index 4c5834ef..00000000 --- a/internal/pkg/data/brickhost/volume.go +++ /dev/null @@ -1,4 +0,0 @@ -package brickhost - -type VolumeRegistry interface { -} diff --git a/internal/pkg/data/mock_session/session.go b/internal/pkg/data/mock_session/session.go index 6a45d512..285d7f9b 100644 --- a/internal/pkg/data/mock_session/session.go +++ b/internal/pkg/data/mock_session/session.go @@ -6,6 +6,7 @@ package mock_session import ( model "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + utils "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -48,19 +49,19 @@ func (mr *MockRegistryMockRecorder) GetSession(token interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockRegistry)(nil).GetSession), token) } -// CreateSessionAllocations mocks base method -func (m *MockRegistry) CreateSessionAllocations(s model.Session) (model.Session, error) { +// CreateSession mocks base method +func (m *MockRegistry) CreateSession(s model.Session) (model.Session, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSessionAllocations", s) + ret := m.ctrl.Call(m, "CreateSession", s) ret0, _ := ret[0].(model.Session) ret1, _ := ret[1].(error) return ret0, ret1 } -// CreateSessionAllocations indicates an expected call of CreateSessionAllocations -func (mr *MockRegistryMockRecorder) CreateSessionAllocations(s interface{}) *gomock.Call { +// CreateSession indicates an expected call of CreateSession +func (mr *MockRegistryMockRecorder) CreateSession(s interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionAllocations", reflect.TypeOf((*MockRegistry)(nil).CreateSessionAllocations), s) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockRegistry)(nil).CreateSession), s) } // ValidateSessionRequest mocks base method @@ -94,10 +95,10 @@ func (mr *MockRegistryMockRecorder) GetAllSessions() *gomock.Call { } // GetAllPools mocks base method -func (m *MockRegistry) GetAllPools() ([]model.Pool, error) { +func (m *MockRegistry) GetAllPools() ([]model.PoolInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllPools") - ret0, _ := ret[0].([]model.Pool) + ret0, _ := ret[0].([]model.PoolInfo) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -107,3 +108,18 @@ func (mr *MockRegistryMockRecorder) GetAllPools() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPools", reflect.TypeOf((*MockRegistry)(nil).GetAllPools)) } + +// GetAllocationMutex mocks base method +func (m *MockRegistry) GetAllocationMutex() (utils.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllocationMutex") + ret0, _ := ret[0].(utils.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllocationMutex indicates an expected call of GetAllocationMutex +func (mr *MockRegistryMockRecorder) GetAllocationMutex() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockRegistry)(nil).GetAllocationMutex)) +} diff --git a/internal/pkg/data/model/brickhost.go b/internal/pkg/data/model/brickhost.go index abc716ac..e551c053 100644 --- a/internal/pkg/data/model/brickhost.go +++ b/internal/pkg/data/model/brickhost.go @@ -8,13 +8,17 @@ type BrickHostInfo struct { // Returns all bricks Bricks []BrickInfo - // True if dacd is detected as running - Alive bool - // True if allowing new volumes to use bricks from this host Enabled bool } +type BrickHostStatus struct { + Info BrickHostInfo + + // True is current keepalive key exists + Alive bool +} + type BrickInfo struct { // Bricks are identified by device and hostname // It must only contain the characters A-Za-z0-9 diff --git a/internal/pkg/data/model/pool.go b/internal/pkg/data/model/pool.go index e6cf8ed2..57707d7e 100644 --- a/internal/pkg/data/model/pool.go +++ b/internal/pkg/data/model/pool.go @@ -10,3 +10,13 @@ type Pool struct { // It is the minimum size of any registered brick GranularityGB uint } + +type PoolInfo struct { + Pool Pool + + // Bricks from alive hosts + AvailableBricks []BrickInfo + + // All currently active bricks + AllocatedBricks []BrickAllocation +} diff --git a/internal/pkg/data/model/session.go b/internal/pkg/data/model/session.go index 6cc8e83b..159bf5a4 100644 --- a/internal/pkg/data/model/session.go +++ b/internal/pkg/data/model/session.go @@ -51,8 +51,21 @@ type Session struct { // List of the bricks allocated to implement the JobVolume // One is the primary brick that should be watching for all actions Allocations []BrickAllocation + + // If not empty, says where to send actions too + // If empty the session has not yet been acknowledged by the dacd process + SessionActionPrefix string +} + +type SessionAction struct { + Uuid string + Session Session + Action SessionActionType } +// TODO: turn into enum +type SessionActionType string + type VolumeRequest struct { MultiJob bool Caller string diff --git a/internal/pkg/data/pool/allocations.go b/internal/pkg/data/pool/allocations.go new file mode 100644 index 00000000..3a03ea60 --- /dev/null +++ b/internal/pkg/data/pool/allocations.go @@ -0,0 +1,21 @@ +package pool + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" +) + +// Used by implementor of session to access brick_host things +type Registry interface { + // Get this mutex before calling GetAllPools + // choosing the required bricks, then calling + // CreateSession that creates the allocation records + GetAllocationMutex() (utils.Mutex, error) + + // Get all bricks listed by pools + GetAllPools() ([]model.PoolInfo, error) + + // Dacd is waiting for new allocations + // This call blocks until Dacd creates a key to what queue it is waiting for session actions on + CreateAllocations(name model.SessionName, allocations []model.BrickAllocation) +} diff --git a/internal/pkg/data/session/session.go b/internal/pkg/data/session/session.go index fea44cb1..bab86b55 100644 --- a/internal/pkg/data/session/session.go +++ b/internal/pkg/data/session/session.go @@ -1,6 +1,9 @@ package session -import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" +) type Registry interface { // Gets a session and its current allocations @@ -11,7 +14,7 @@ type Registry interface { // such that actions can now be sent to the given session // Returns an error if the session already exists // Note that deleting a session and its allocation is an action, as is any update - CreateSessionAllocations(s model.Session) (model.Session, error) + CreateSession(s model.Session) (model.Session, error) // Checks it would be a valid call to CreateAllocations // Error will describe any validation issues @@ -21,5 +24,10 @@ type Registry interface { GetAllSessions() ([]model.Session, error) // Get all bricks listed by pools - GetAllPools() ([]model.Pool, error) + GetAllPools() ([]model.PoolInfo, error) + + // Get this mutex before calling GetAllPools + // choosing the required bricks, then calling + // CreateSession that creates the allocation records + GetAllocationMutex() (utils.Mutex, error) } diff --git a/internal/pkg/data/session_impl/actions.go b/internal/pkg/data/session_impl/actions.go index c8487842..7fbd559f 100644 --- a/internal/pkg/data/session_impl/actions.go +++ b/internal/pkg/data/session_impl/actions.go @@ -14,11 +14,12 @@ import ( // TODO: this is the client side, need server side too func NewSessionActions(keystore store.Keystore) session.Actions { - return &sessionActions{keystore: keystore} + return &sessionActions{keystore: keystore, defaultTimeout: time.Minute * 20} } type sessionActions struct { - keystore store.Keystore + keystore store.Keystore + defaultTimeout time.Duration } func toJson(message interface{}) string { @@ -32,6 +33,9 @@ func toJson(message interface{}) string { func (s *sessionActions) sendAction(session model.Session, action string) error { // TODO: update session? + // TODO: check primary session host is alive before sending event + // TODO: If we timeout, cancel the event only if the host is now not alive + actionId := uuid.New().String() sessionPrefix := fmt.Sprintf("/Session/Actions/%s", session.Name) actionPrefix := fmt.Sprintf("%s/%s", sessionPrefix, actionId) @@ -40,7 +44,7 @@ func (s *sessionActions) sendAction(session model.Session, action string) error if err != nil { return fmt.Errorf("unable to start action due to: %s", err.Error()) } - mctxt, cancel := context.WithTimeout(context.Background(), time.Minute*20) + mctxt, cancel := context.WithTimeout(context.Background(), s.defaultTimeout) defer cancel() err = mutex.Lock(mctxt) if err != nil { @@ -48,7 +52,7 @@ func (s *sessionActions) sendAction(session model.Session, action string) error } defer mutex.Unlock(context.Background()) - ctxt, cancel := context.WithTimeout(context.Background(), time.Minute*20) + ctxt, cancel := context.WithTimeout(context.Background(), s.defaultTimeout) defer cancel() responses := s.keystore.Watch(ctxt, fmt.Sprintf("%s/%s", actionPrefix, "output"), false) @@ -61,6 +65,7 @@ func (s *sessionActions) sendAction(session model.Session, action string) error return err } + // TODO: how do we detect a timeout here? for response := range responses { if response.Err != nil { return response.Err diff --git a/internal/pkg/data/utils/mutex.go b/internal/pkg/data/utils/mutex.go new file mode 100644 index 00000000..5313e010 --- /dev/null +++ b/internal/pkg/data/utils/mutex.go @@ -0,0 +1,8 @@ +package utils + +import "context" + +type Mutex interface { + Lock(ctx context.Context) error + Unlock(ctx context.Context) error +} From 76b66dc68c5c644e09f546ce1d199d2b870f79a9 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 08:53:43 +0100 Subject: [PATCH 016/191] Add more nodes to brick_host --- .idea/dictionaries/john.xml | 8 ++++++++ .idea/inspectionProfiles/Project_Default.xml | 10 ++++++++++ internal/pkg/data/brick_host/bricks.go | 16 +++++++++------- internal/pkg/data/model/session.go | 5 +++++ 4 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 .idea/dictionaries/john.xml create mode 100644 .idea/inspectionProfiles/Project_Default.xml diff --git a/.idea/dictionaries/john.xml b/.idea/dictionaries/john.xml new file mode 100644 index 00000000..9b259619 --- /dev/null +++ b/.idea/dictionaries/john.xml @@ -0,0 +1,8 @@ + + + + dacctl + dacd + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..85fa229c --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/internal/pkg/data/brick_host/bricks.go b/internal/pkg/data/brick_host/bricks.go index 6012da5f..dad89016 100644 --- a/internal/pkg/data/brick_host/bricks.go +++ b/internal/pkg/data/brick_host/bricks.go @@ -11,15 +11,10 @@ type BrickHostRegistry interface { EnsurePoolCreated(poolName model.PoolName, granularityGB int) (model.PoolName, error) // BrickHost updates bricks on startup + // This will error if we remove a brick that has an allocation + // for a Session that isn't in an error state UpdateBrickHost(brickHostInfo model.BrickHostInfo) error - // While the process is still running this notifies others the host is up - // - // When a host is dead non of its bricks will get new volumes assigned, - // and no bricks will get cleaned up until the next service start. - // Error will be returned if the host info has not yet been written. - KeepAliveHost(hostname string) error - // Get sessions where given hostname is the primary brick // Created by dacctl via CreateSessionAllocations GetNewSessionRequests(ctxt context.Context, hostname model.BrickHostName) (<-chan model.Session, error) @@ -33,6 +28,13 @@ type BrickHostRegistry interface { // New tasks are only sent if the keepalive key is active GetNewSessionAction(ctxt context.Context, hostname model.BrickHostName) (<-chan model.SessionAction, error) + // While the process is still running this notifies others the host is up + // + // When a host is dead non of its bricks will get new volumes assigned, + // and no bricks will get cleaned up until the next service start. + // Error will be returned if the host info has not yet been written. + KeepAliveHost(hostname string) error + // Update provided session // Ensures there are no other updates to Session since the revision contained in it UpdateSession(session model.Session) (model.Session, error) diff --git a/internal/pkg/data/model/session.go b/internal/pkg/data/model/session.go index 159bf5a4..5a1129b8 100644 --- a/internal/pkg/data/model/session.go +++ b/internal/pkg/data/model/session.go @@ -55,6 +55,11 @@ type Session struct { // If not empty, says where to send actions too // If empty the session has not yet been acknowledged by the dacd process SessionActionPrefix string + + // If not nil, the session has an unresolved error + // and can't be mounted by any new sessions + // it can be deleted + Error error } type SessionAction struct { From 2af4bf3594397f700fa8d6d56de6cd8654084f45 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 08:54:15 +0100 Subject: [PATCH 017/191] Add missing golang file --- .idea/webResources.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .idea/webResources.xml diff --git a/.idea/webResources.xml b/.idea/webResources.xml new file mode 100644 index 00000000..717d9d66 --- /dev/null +++ b/.idea/webResources.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file From 438f4b3f78c8202ae8bbe8529851820735515768 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 17:18:49 +0100 Subject: [PATCH 018/191] Rework structure of new code into v2 directory --- .idea/dictionaries/john.xml | 2 + build/rebuild_mocks.sh | 12 +- internal/pkg/dacctlv2/actions_impl/actions.go | 87 ---------- .../pkg/dacctlv2/actions_impl/persistent.go | 68 -------- internal/pkg/data/brick_host/bricks.go | 55 ------ internal/pkg/data/mock_session/session.go | 125 -------------- internal/pkg/data/pool/allocations.go | 21 --- internal/pkg/data/session/actions.go | 27 --- internal/pkg/data/session/session.go | 33 ---- internal/pkg/data/utils/mutex.go | 8 - internal/pkg/registry/volume.go | 2 +- .../pkg/v2/dacctl/actions_impl/actions.go | 91 ++++++++++ .../dacctl}/actions_impl/actions_test.go | 44 ++--- .../dacctl}/actions_impl/job.go | 39 ++--- .../dacctl}/actions_impl/job_test.go | 30 ++-- .../dacctl/actions_impl/parsers}/capacity.go | 8 +- .../dacctl/actions_impl/parsers}/job.go | 69 ++++---- .../dacctl/actions_impl/parsers}/job_test.go | 8 +- .../pkg/v2/dacctl/actions_impl/persistent.go | 68 ++++++++ .../dacctl}/actions_impl/show.go | 8 +- .../actions => v2/dacctl}/interface.go | 2 +- .../dacd}/brick_manager/interface.go | 0 .../dacd}/brick_manager_impl/brick_manger.go | 13 +- .../datamodel/brick_allocation.go} | 15 +- .../datamodel/brick_host.go} | 15 +- internal/pkg/v2/datamodel/datamodel_test.go | 42 +++++ .../pkg/{data/model => v2/datamodel}/pool.go | 4 +- .../{data/model => v2/datamodel}/session.go | 5 +- .../{data/model => v2/datamodel}/volume.go | 2 +- .../mock_registry}/actions.go | 24 +-- internal/pkg/v2/mock_registry/allocation.go | 94 ++++++++++ internal/pkg/v2/mock_registry/brick.go | 93 ++++++++++ internal/pkg/v2/mock_registry/pool.go | 64 +++++++ internal/pkg/v2/mock_registry/session.go | 124 +++++++++++++ .../pkg/v2/mock_registry/session_actions.go | 109 ++++++++++++ internal/pkg/v2/mock_workflow/session.go | 163 ++++++++++++++++++ internal/pkg/v2/registry/allocation.go | 24 +++ internal/pkg/v2/registry/brick.go | 34 ++++ internal/pkg/v2/registry/pool.go | 14 ++ internal/pkg/v2/registry/session.go | 40 +++++ internal/pkg/v2/registry/session_actions.go | 44 +++++ .../registry_impl}/actions.go | 60 +++---- .../interface.go => v2/store/keystore.go} | 0 internal/pkg/v2/workflow/session.go | 37 ++++ 44 files changed, 1231 insertions(+), 596 deletions(-) delete mode 100644 internal/pkg/dacctlv2/actions_impl/actions.go delete mode 100644 internal/pkg/dacctlv2/actions_impl/persistent.go delete mode 100644 internal/pkg/data/brick_host/bricks.go delete mode 100644 internal/pkg/data/mock_session/session.go delete mode 100644 internal/pkg/data/pool/allocations.go delete mode 100644 internal/pkg/data/session/actions.go delete mode 100644 internal/pkg/data/session/session.go delete mode 100644 internal/pkg/data/utils/mutex.go create mode 100644 internal/pkg/v2/dacctl/actions_impl/actions.go rename internal/pkg/{dacctlv2 => v2/dacctl}/actions_impl/actions_test.go (74%) rename internal/pkg/{dacctlv2 => v2/dacctl}/actions_impl/job.go (63%) rename internal/pkg/{dacctlv2 => v2/dacctl}/actions_impl/job_test.go (71%) rename internal/pkg/{dacctlv2/parsers/capacity => v2/dacctl/actions_impl/parsers}/capacity.go (85%) rename internal/pkg/{dacctlv2/parsers/jobfile => v2/dacctl/actions_impl/parsers}/job.go (74%) rename internal/pkg/{dacctlv2/parsers/jobfile => v2/dacctl/actions_impl/parsers}/job_test.go (90%) create mode 100644 internal/pkg/v2/dacctl/actions_impl/persistent.go rename internal/pkg/{dacctlv2 => v2/dacctl}/actions_impl/show.go (65%) rename internal/pkg/{dacctlv2/actions => v2/dacctl}/interface.go (97%) rename internal/pkg/{dacdv2 => v2/dacd}/brick_manager/interface.go (100%) rename internal/pkg/{dacdv2 => v2/dacd}/brick_manager_impl/brick_manger.go (69%) rename internal/pkg/{data/model/allocation.go => v2/datamodel/brick_allocation.go} (68%) rename internal/pkg/{data/model/brickhost.go => v2/datamodel/brick_host.go} (76%) create mode 100644 internal/pkg/v2/datamodel/datamodel_test.go rename internal/pkg/{data/model => v2/datamodel}/pool.go (89%) rename internal/pkg/{data/model => v2/datamodel}/session.go (96%) rename internal/pkg/{data/model => v2/datamodel}/volume.go (99%) rename internal/pkg/{data/mock_session => v2/mock_registry}/actions.go (83%) create mode 100644 internal/pkg/v2/mock_registry/allocation.go create mode 100644 internal/pkg/v2/mock_registry/brick.go create mode 100644 internal/pkg/v2/mock_registry/pool.go create mode 100644 internal/pkg/v2/mock_registry/session.go create mode 100644 internal/pkg/v2/mock_registry/session_actions.go create mode 100644 internal/pkg/v2/mock_workflow/session.go create mode 100644 internal/pkg/v2/registry/allocation.go create mode 100644 internal/pkg/v2/registry/brick.go create mode 100644 internal/pkg/v2/registry/pool.go create mode 100644 internal/pkg/v2/registry/session.go create mode 100644 internal/pkg/v2/registry/session_actions.go rename internal/pkg/{data/session_impl => v2/registry_impl}/actions.go (65%) rename internal/pkg/{data/store/interface.go => v2/store/keystore.go} (100%) create mode 100644 internal/pkg/v2/workflow/session.go diff --git a/.idea/dictionaries/john.xml b/.idea/dictionaries/john.xml index 9b259619..8e65c41e 100644 --- a/.idea/dictionaries/john.xml +++ b/.idea/dictionaries/john.xml @@ -1,8 +1,10 @@ + ansible dacctl dacd + nodehostnamefile \ No newline at end of file diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 5a16c71f..6095c67d 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -27,8 +27,14 @@ done mockgen -source=internal/pkg/pfsprovider/interface.go \ -package mocks >internal/pkg/mocks/pfsprovider_mock.go -items="actions session" +items="allocation brick pool session session_actions" for i in $items; do - mockgen -source=internal/pkg/data/session/${i}.go \ - >internal/pkg/data/mock_session/${i}.go + mockgen -source=internal/pkg/v2/registry/${i}.go \ + >internal/pkg/v2/mock_registry/${i}.go +done + +items="session" +for i in $items; do + mockgen -source=internal/pkg/v2/workflow/${i}.go \ + >internal/pkg/v2/mock_workflow/${i}.go done diff --git a/internal/pkg/dacctlv2/actions_impl/actions.go b/internal/pkg/dacctlv2/actions_impl/actions.go deleted file mode 100644 index 743bf528..00000000 --- a/internal/pkg/dacctlv2/actions_impl/actions.go +++ /dev/null @@ -1,87 +0,0 @@ -package actions_impl - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "log" - "strings" -) - -func NewDacctlActions(registry session.Registry, actions session.Actions, disk fileio.Disk) actions.DacctlActions { - return &dacctlActions{ - registry: registry, - actions: actions, - disk: disk, - } -} - -type dacctlActions struct { - registry session.Registry - actions session.Actions - disk fileio.Disk -} - -func checkRequiredStrings(c actions.CliContext, flags ...string) { - errs := []string{} - for _, flag := range flags { - if str := c.String(flag); str == "" { - errs = append(errs, flag) - } - } - if len(errs) > 0 { - log.Fatalf("Please provide these required parameters: %s", strings.Join(errs, ", ")) - } -} - -func (d *dacctlActions) getSession(c actions.CliContext) (model.Session, error) { - checkRequiredStrings(c, "token") - token := c.String("token") - s, err := d.registry.GetSession(token) - if err != nil { - return s, fmt.Errorf("unable to find session for token %s", token) - } - return s, nil -} - -func (d *dacctlActions) DeleteBuffer(c actions.CliContext) error { - s, err := d.getSession(c) - if err != nil { - return err - } - return d.actions.DeleteSession(s) -} - -func (d *dacctlActions) DataIn(c actions.CliContext) error { - s, err := d.getSession(c) - if err != nil { - return err - } - return d.actions.DataIn(s) -} - -func (d *dacctlActions) PreRun(c actions.CliContext) error { - s, err := d.getSession(c) - if err != nil { - return err - } - return d.actions.AttachVolumes(s) -} - -func (d *dacctlActions) PostRun(c actions.CliContext) error { - s, err := d.getSession(c) - if err != nil { - return err - } - return d.actions.DetachVolumes(s) -} - -func (d *dacctlActions) DataOut(c actions.CliContext) error { - s, err := d.getSession(c) - if err != nil { - return err - } - return d.actions.DataOut(s) -} diff --git a/internal/pkg/dacctlv2/actions_impl/persistent.go b/internal/pkg/dacctlv2/actions_impl/persistent.go deleted file mode 100644 index f6a1ce03..00000000 --- a/internal/pkg/dacctlv2/actions_impl/persistent.go +++ /dev/null @@ -1,68 +0,0 @@ -package actions_impl - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "strings" - "time" -) - -var stringToAccessMode = map[string]model.AccessMode{ - "": model.Striped, - "striped": model.Striped, - "private": model.Private, - "private,striped": model.PrivateAndStriped, - "striped,private": model.PrivateAndStriped, -} - -func accessModeFromString(raw string) model.AccessMode { - return stringToAccessMode[strings.ToLower(raw)] -} - -var stringToBufferType = map[string]model.BufferType{ - "": model.Scratch, - "scratch": model.Scratch, - "cache": model.Cache, -} - -func bufferTypeFromString(raw string) model.BufferType { - return stringToBufferType[strings.ToLower(raw)] -} - -var fakeTime uint = 0 - -func getNow() uint { - if fakeTime != 0 { - return fakeTime - } - return uint(time.Now().Unix()) -} - -func (d *dacctlActions) CreatePersistentBuffer(c actions.CliContext) error { - checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") - pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) - if err != nil { - return err - } - request := model.VolumeRequest{ - MultiJob: true, - Caller: c.String("caller"), - TotalCapacityBytes: capacityBytes, - PoolName: pool, - Access: accessModeFromString(c.String("access")), - Type: bufferTypeFromString(c.String("type")), - } - session := model.Session{ - Name: model.SessionName(c.String("token")), - VolumeRequest: request, - Owner: uint(c.Int("user")), - Group: uint(c.Int("group")), - CreatedAt: getNow(), - } - session, err = d.registry.CreateSession(session) - if err != nil { - return err - } - return d.actions.CreateSessionVolume(session) -} diff --git a/internal/pkg/data/brick_host/bricks.go b/internal/pkg/data/brick_host/bricks.go deleted file mode 100644 index dad89016..00000000 --- a/internal/pkg/data/brick_host/bricks.go +++ /dev/null @@ -1,55 +0,0 @@ -package brick_host - -import ( - "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" -) - -type BrickHostRegistry interface { - // Creates the pool if it doesn't exist - // error if the granularity doesn't match and existing pool - EnsurePoolCreated(poolName model.PoolName, granularityGB int) (model.PoolName, error) - - // BrickHost updates bricks on startup - // This will error if we remove a brick that has an allocation - // for a Session that isn't in an error state - UpdateBrickHost(brickHostInfo model.BrickHostInfo) error - - // Get sessions where given hostname is the primary brick - // Created by dacctl via CreateSessionAllocations - GetNewSessionRequests(ctxt context.Context, hostname model.BrickHostName) (<-chan model.Session, error) - - // Gets all new actions for the given Session - // This confirms that dacd is ready to service requests for this session - // and updates the session with an appropriate SessionActionPrefix - // The channel tracks all actions sent to that sub keys of the above prefix - // This is also called when dacd is restarted and you want to resume sending - // any actions that are not marked as complete - // New tasks are only sent if the keepalive key is active - GetNewSessionAction(ctxt context.Context, hostname model.BrickHostName) (<-chan model.SessionAction, error) - - // While the process is still running this notifies others the host is up - // - // When a host is dead non of its bricks will get new volumes assigned, - // and no bricks will get cleaned up until the next service start. - // Error will be returned if the host info has not yet been written. - KeepAliveHost(hostname string) error - - // Update provided session - // Ensures there are no other updates to Session since the revision contained in it - UpdateSession(session model.Session) (model.Session, error) - - // This is called before confirming the Session delete request - // after all bricks have been de-allocated - DeleteSession(session model.Session) error - - // Mark the given SessionAction as completed - // Optionally returns an error to caller of the action - CompleteSessionAction(action model.SessionAction, error error) error - - // Check if a given action is complete - IsSessionActionComplete(action model.SessionAction) (bool, error) - - // Check if given brick host is alive - IsBrickHostAlive(hostname model.BrickHostName) (bool, error) -} diff --git a/internal/pkg/data/mock_session/session.go b/internal/pkg/data/mock_session/session.go deleted file mode 100644 index 285d7f9b..00000000 --- a/internal/pkg/data/mock_session/session.go +++ /dev/null @@ -1,125 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/data/session/session.go - -// Package mock_session is a generated GoMock package. -package mock_session - -import ( - model "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - utils "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockRegistry is a mock of Registry interface -type MockRegistry struct { - ctrl *gomock.Controller - recorder *MockRegistryMockRecorder -} - -// MockRegistryMockRecorder is the mock recorder for MockRegistry -type MockRegistryMockRecorder struct { - mock *MockRegistry -} - -// NewMockRegistry creates a new mock instance -func NewMockRegistry(ctrl *gomock.Controller) *MockRegistry { - mock := &MockRegistry{ctrl: ctrl} - mock.recorder = &MockRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { - return m.recorder -} - -// GetSession mocks base method -func (m *MockRegistry) GetSession(token string) (model.Session, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSession", token) - ret0, _ := ret[0].(model.Session) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSession indicates an expected call of GetSession -func (mr *MockRegistryMockRecorder) GetSession(token interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockRegistry)(nil).GetSession), token) -} - -// CreateSession mocks base method -func (m *MockRegistry) CreateSession(s model.Session) (model.Session, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSession", s) - ret0, _ := ret[0].(model.Session) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateSession indicates an expected call of CreateSession -func (mr *MockRegistryMockRecorder) CreateSession(s interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockRegistry)(nil).CreateSession), s) -} - -// ValidateSessionRequest mocks base method -func (m *MockRegistry) ValidateSessionRequest(token string) (model.Session, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateSessionRequest", token) - ret0, _ := ret[0].(model.Session) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ValidateSessionRequest indicates an expected call of ValidateSessionRequest -func (mr *MockRegistryMockRecorder) ValidateSessionRequest(token interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateSessionRequest", reflect.TypeOf((*MockRegistry)(nil).ValidateSessionRequest), token) -} - -// GetAllSessions mocks base method -func (m *MockRegistry) GetAllSessions() ([]model.Session, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllSessions") - ret0, _ := ret[0].([]model.Session) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllSessions indicates an expected call of GetAllSessions -func (mr *MockRegistryMockRecorder) GetAllSessions() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockRegistry)(nil).GetAllSessions)) -} - -// GetAllPools mocks base method -func (m *MockRegistry) GetAllPools() ([]model.PoolInfo, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllPools") - ret0, _ := ret[0].([]model.PoolInfo) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllPools indicates an expected call of GetAllPools -func (mr *MockRegistryMockRecorder) GetAllPools() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPools", reflect.TypeOf((*MockRegistry)(nil).GetAllPools)) -} - -// GetAllocationMutex mocks base method -func (m *MockRegistry) GetAllocationMutex() (utils.Mutex, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllocationMutex") - ret0, _ := ret[0].(utils.Mutex) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllocationMutex indicates an expected call of GetAllocationMutex -func (mr *MockRegistryMockRecorder) GetAllocationMutex() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockRegistry)(nil).GetAllocationMutex)) -} diff --git a/internal/pkg/data/pool/allocations.go b/internal/pkg/data/pool/allocations.go deleted file mode 100644 index 3a03ea60..00000000 --- a/internal/pkg/data/pool/allocations.go +++ /dev/null @@ -1,21 +0,0 @@ -package pool - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" -) - -// Used by implementor of session to access brick_host things -type Registry interface { - // Get this mutex before calling GetAllPools - // choosing the required bricks, then calling - // CreateSession that creates the allocation records - GetAllocationMutex() (utils.Mutex, error) - - // Get all bricks listed by pools - GetAllPools() ([]model.PoolInfo, error) - - // Dacd is waiting for new allocations - // This call blocks until Dacd creates a key to what queue it is waiting for session actions on - CreateAllocations(name model.SessionName, allocations []model.BrickAllocation) -} diff --git a/internal/pkg/data/session/actions.go b/internal/pkg/data/session/actions.go deleted file mode 100644 index 7f559560..00000000 --- a/internal/pkg/data/session/actions.go +++ /dev/null @@ -1,27 +0,0 @@ -package session - -import "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - -// Each volume has an associated primary brick -// that is responsible for responding to any actions -// All the calls block until they are complete, or an error occurs -type Actions interface { - // Creates the requested volumes - // Error if Session has not had its bricks allocated - CreateSessionVolume(session model.Session) error - - // Deletes the requested volume and session allocation - DeleteSession(session model.Session) error - - // Update the session and trigger requested data copy in - DataIn(session model.Session) error - - // Update session hosts and attach volumes as needed - AttachVolumes(session model.Session) error - - // Attempt to detach volumes - DetachVolumes(session model.Session) error - - // Update the session and trigger requested data copy out - DataOut(session model.Session) error -} diff --git a/internal/pkg/data/session/session.go b/internal/pkg/data/session/session.go deleted file mode 100644 index bab86b55..00000000 --- a/internal/pkg/data/session/session.go +++ /dev/null @@ -1,33 +0,0 @@ -package session - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/utils" -) - -type Registry interface { - // Gets a session and its current allocations - // Returns an error if the session is not found - GetSession(token string) (model.Session, error) - - // Any required allocations are created for the given session - // such that actions can now be sent to the given session - // Returns an error if the session already exists - // Note that deleting a session and its allocation is an action, as is any update - CreateSession(s model.Session) (model.Session, error) - - // Checks it would be a valid call to CreateAllocations - // Error will describe any validation issues - ValidateSessionRequest(token string) (model.Session, error) - - // Used for show instances and show sessions - GetAllSessions() ([]model.Session, error) - - // Get all bricks listed by pools - GetAllPools() ([]model.PoolInfo, error) - - // Get this mutex before calling GetAllPools - // choosing the required bricks, then calling - // CreateSession that creates the allocation records - GetAllocationMutex() (utils.Mutex, error) -} diff --git a/internal/pkg/data/utils/mutex.go b/internal/pkg/data/utils/mutex.go deleted file mode 100644 index 5313e010..00000000 --- a/internal/pkg/data/utils/mutex.go +++ /dev/null @@ -1,8 +0,0 @@ -package utils - -import "context" - -type Mutex interface { - Lock(ctx context.Context) error - Unlock(ctx context.Context) error -} diff --git a/internal/pkg/registry/volume.go b/internal/pkg/registry/volume.go index 4c6fbd60..c2c33753 100644 --- a/internal/pkg/registry/volume.go +++ b/internal/pkg/registry/volume.go @@ -168,7 +168,7 @@ type Volume struct { // if we request delete, no bricks ever assigned, don't ait for dacd! HadBricksAssigned bool - // TODO: data model currently does not do these things well: + // TODO: data datamodel currently does not do these things well: // 1. correctly track multiple jobs at the same time attach to the same persistent buffer // 2. data in/out requests for persistent buffer // 3. track amount of space used by swap and/or metadata diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go new file mode 100644 index 00000000..f89a63ab --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -0,0 +1,91 @@ +package actions_impl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "log" + "strings" +) + +func NewDacctlActions(registry registry.SessionRegistry, actions workflow.Session, disk fileio.Disk) dacctl.DacctlActions { + return &dacctlActions{ + registry: registry, + session: actions, + disk: disk, + } +} + +type dacctlActions struct { + registry registry.SessionRegistry + session workflow.Session + disk fileio.Disk +} + +func checkRequiredStrings(c dacctl.CliContext, flags ...string) { + var errs []string + for _, flag := range flags { + if str := c.String(flag); str == "" { + errs = append(errs, flag) + } + } + if len(errs) > 0 { + log.Fatalf("Please provide these required parameters: %s", strings.Join(errs, ", ")) + } +} + +func (d *dacctlActions) getSession(c dacctl.CliContext) (datamodel.Session, error) { + // TODO - not sure we need this now...? + checkRequiredStrings(c, "token") + token := c.String("token") + sessionName := datamodel.SessionName(token) + s, err := d.registry.GetSession(sessionName) + if err != nil { + return s, fmt.Errorf("unable to find session for token %s", token) + } + return s, nil +} + +func (d *dacctlActions) DeleteBuffer(c dacctl.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.session.DeleteSession(s.Name) +} + +func (d *dacctlActions) DataIn(c dacctl.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.session.DataIn(s.Name) +} + +func (d *dacctlActions) PreRun(c dacctl.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + // TODO - fix attach hosts + return d.session.AttachVolumes(s.Name, nil) +} + +func (d *dacctlActions) PostRun(c dacctl.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.session.DetachVolumes(s.Name) +} + +func (d *dacctlActions) DataOut(c dacctl.CliContext) error { + s, err := d.getSession(c) + if err != nil { + return err + } + return d.session.DataOut(s.Name) +} diff --git a/internal/pkg/dacctlv2/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go similarity index 74% rename from internal/pkg/dacctlv2/actions_impl/actions_test.go rename to internal/pkg/v2/dacctl/actions_impl/actions_test.go index a7f92ff7..6e98ab01 100644 --- a/internal/pkg/dacctlv2/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -3,8 +3,8 @@ package actions_impl import ( "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -53,16 +53,16 @@ func (c *mockCliContext) Int(name string) int { func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} - registry.EXPECT().CreateSession(model.Session{ + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().CreateSession(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, CreatedAt: 123, - VolumeRequest: model.VolumeRequest{ + VolumeRequest: datamodel.VolumeRequest{ MultiJob: true, Caller: "caller", PoolName: "pool1", @@ -81,10 +81,10 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { func TestDacctlActions_DeleteBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} + fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DeleteSession(fakeSession).Return(fakeError) @@ -98,10 +98,10 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { func TestDacctlActions_DataIn(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} + fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DataIn(fakeSession).Return(fakeError) @@ -115,10 +115,10 @@ func TestDacctlActions_DataIn(t *testing.T) { func TestDacctlActions_DataOut(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} + fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DataOut(fakeSession).Return(fakeError) @@ -132,10 +132,10 @@ func TestDacctlActions_DataOut(t *testing.T) { func TestDacctlActions_PreRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} + fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().AttachVolumes(fakeSession).Return(fakeError) @@ -149,10 +149,10 @@ func TestDacctlActions_PreRun(t *testing.T) { func TestDacctlActions_PostRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) - fakeSession := model.Session{Name: "foo"} + fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().GetSession("token").Return(fakeSession, nil) fakeError := errors.New("fake") session.EXPECT().DetachVolumes(fakeSession).Return(fakeError) diff --git a/internal/pkg/dacctlv2/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go similarity index 63% rename from internal/pkg/dacctlv2/actions_impl/job.go rename to internal/pkg/v2/dacctl/actions_impl/job.go index 3869283d..33d2e8a0 100644 --- a/internal/pkg/dacctlv2/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -2,17 +2,16 @@ package actions_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/jobfile" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" ) -func (d *dacctlActions) ValidateJob(c actions.CliContext) error { +func (d *dacctlActions) ValidateJob(c dacctl.CliContext) error { checkRequiredStrings(c, "job") jobFile := c.String("job") - summary, err := jobfile.ParseJobFile(d.disk, jobFile) + summary, err := parsers.ParseJobFile(d.disk, jobFile) if err != nil { return err } else { @@ -22,12 +21,12 @@ func (d *dacctlActions) ValidateJob(c actions.CliContext) error { return nil } -func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { +func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { checkRequiredStrings(c, "token", "job", "caller", "capacity") // TODO: need to specify user and group too jobFile := c.String("job") - summary, err := jobfile.ParseJobFile(d.disk, jobFile) + summary, err := parsers.ParseJobFile(d.disk, jobFile) if err != nil { return err } @@ -38,7 +37,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { log.Printf("Ignoring nodeFile in setup: %s", nodeFile) } - pool, capacityBytes, err := capacity.ParseCapacityBytes(c.String("capacity")) + pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) if err != nil { return err } @@ -48,20 +47,20 @@ func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { if summary.Swap != nil { swapBytes = summary.Swap.SizeBytes } - access := model.NoAccess - bufferType := model.Scratch + access := datamodel.NoAccess + bufferType := datamodel.Scratch if summary.PerJobBuffer != nil { access = summary.PerJobBuffer.AccessMode - if summary.PerJobBuffer.BufferType == model.Cache { + if summary.PerJobBuffer.BufferType == datamodel.Cache { return fmt.Errorf("cache is not supported") } } - var multiJobVolumes []model.VolumeName + var multiJobVolumes []datamodel.VolumeName for _, attachment := range summary.Attachments { multiJobVolumes = append(multiJobVolumes, attachment) } - request := model.VolumeRequest{ + request := datamodel.VolumeRequest{ MultiJob: false, Caller: c.String("caller"), TotalCapacityBytes: capacityBytes, @@ -70,8 +69,8 @@ func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { Type: bufferType, SwapBytes: swapBytes, } - session := model.Session{ - Name: model.SessionName(c.String("token")), + session := datamodel.Session{ + Name: datamodel.SessionName(c.String("token")), Owner: uint(c.Int("user")), Group: uint(c.Int("group")), CreatedAt: getNow(), @@ -86,16 +85,16 @@ func (d *dacctlActions) CreatePerJobBuffer(c actions.CliContext) error { if err != nil { return err } - return d.actions.CreateSessionVolume(session) + return d.session.CreateSessionVolume(session) } -func getPaths(session model.Session) map[string]string { +func getPaths(session datamodel.Session) map[string]string { paths := make(map[string]string) if session.VolumeRequest.MultiJob == false { - if session.VolumeRequest.Access == model.Private || session.VolumeRequest.Access == model.PrivateAndStriped { + if session.VolumeRequest.Access == datamodel.Private || session.VolumeRequest.Access == datamodel.PrivateAndStriped { paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s_job_private", session.Name) } - if session.VolumeRequest.Access == model.Striped || session.VolumeRequest.Access == model.PrivateAndStriped { + if session.VolumeRequest.Access == datamodel.Striped || session.VolumeRequest.Access == datamodel.PrivateAndStriped { paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", session.Name) } } diff --git a/internal/pkg/dacctlv2/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go similarity index 71% rename from internal/pkg/dacctlv2/actions_impl/job_test.go rename to internal/pkg/v2/dacctl/actions_impl/job_test.go index be132324..41756346 100644 --- a/internal/pkg/dacctlv2/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -1,9 +1,9 @@ package actions_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/mock_session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -12,8 +12,8 @@ import ( func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_session.NewMockRegistry(mockCtrl) - session := mock_session.NewMockActions(mockCtrl) + registry := mock_registry.NewMockRegistry(mockCtrl) + session := mock_registry.NewMockActions(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) lines := []string{ @@ -26,37 +26,37 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, } disk.EXPECT().Lines("jobfile").Return(lines, nil) - fakeSession := model.Session{Name: "foo"} - registry.EXPECT().CreateSession(model.Session{ + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().CreateSession(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, CreatedAt: 123, - MultiJobVolumes: []model.VolumeName{"myBBname1", "myBBname2"}, - StageInRequests: []model.DataCopyRequest{ + MultiJobVolumes: []datamodel.VolumeName{"myBBname1", "myBBname2"}, + StageInRequests: []datamodel.DataCopyRequest{ { - SourceType: model.File, + SourceType: datamodel.File, Source: "/global/cscratch1/filename1", Destination: "$DW_JOB_STRIPED/filename1", }, { - SourceType: model.List, + SourceType: datamodel.List, Source: "/global/cscratch1/filelist", }, }, - StageOutRequests: []model.DataCopyRequest{ + StageOutRequests: []datamodel.DataCopyRequest{ { - SourceType: model.Directory, + SourceType: datamodel.Directory, Source: "$DW_JOB_STRIPED/outdir", Destination: "/global/scratch1/outdir", }, }, - VolumeRequest: model.VolumeRequest{ + VolumeRequest: datamodel.VolumeRequest{ Caller: "caller", PoolName: "pool1", TotalCapacityBytes: 2147483648, - Access: model.PrivateAndStriped, - Type: model.Scratch, + Access: datamodel.PrivateAndStriped, + Type: datamodel.Scratch, SwapBytes: 4194304, }, Paths: map[string]string{ diff --git a/internal/pkg/dacctlv2/parsers/capacity/capacity.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go similarity index 85% rename from internal/pkg/dacctlv2/parsers/capacity/capacity.go rename to internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go index 06f7b4a4..0351a85c 100644 --- a/internal/pkg/dacctlv2/parsers/capacity/capacity.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go @@ -1,4 +1,4 @@ -package capacity +package parsers import ( "errors" @@ -8,7 +8,7 @@ import ( ) // TODO: missing a few? -var sizeSuffixMulitiplyer = map[string]int{ +var sizeSuffixMultiplier = map[string]int{ "TiB": 1099511627776, "TB": 1000000000000, "GiB": 1073741824, @@ -23,14 +23,14 @@ func ParseSize(raw string) (int, error) { // specified raw bytes return intVal, nil } - for suffix, multiplyer := range sizeSuffixMulitiplyer { + for suffix, multiplier := range sizeSuffixMultiplier { if strings.HasSuffix(raw, suffix) { rawInt := strings.TrimSuffix(raw, suffix) intVal, err := strconv.Atoi(rawInt) if err != nil { return 0, err } - return intVal * multiplyer, nil + return intVal * multiplier, nil } } return 0, fmt.Errorf("unable to parse size: %s", raw) diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go similarity index 74% rename from internal/pkg/dacctlv2/parsers/jobfile/job.go rename to internal/pkg/v2/dacctl/actions_impl/parsers/job.go index 4147e83a..ada535e4 100644 --- a/internal/pkg/dacctlv2/parsers/jobfile/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go @@ -1,11 +1,10 @@ -package jobfile +package parsers import ( "encoding/json" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/parsers/capacity" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "strings" ) @@ -13,9 +12,9 @@ import ( type jobSummary struct { PerJobBuffer *cmdPerJobBuffer Swap *cmdAttachPerJobSwap - Attachments []model.VolumeName - DataIn []model.DataCopyRequest - DataOut []model.DataCopyRequest + Attachments []datamodel.VolumeName + DataIn []datamodel.DataCopyRequest + DataOut []datamodel.DataCopyRequest //createPersistent *cmdCreatePersistent //destroyPersistent *cmdDestroyPersistent } @@ -57,20 +56,20 @@ func getJobSummary(lines []string) (jobSummary, error) { return summary, fmt.Errorf("only one per job buffer allowed") } case cmdAttachPersistent: - summary.Attachments = append(summary.Attachments, model.VolumeName(c)) + summary.Attachments = append(summary.Attachments, datamodel.VolumeName(c)) case cmdAttachPerJobSwap: if summary.Swap != nil { return summary, fmt.Errorf("only one swap request allowed") } summary.Swap = &c case cmdStageOutData: - summary.DataOut = append(summary.DataOut, model.DataCopyRequest{ + summary.DataOut = append(summary.DataOut, datamodel.DataCopyRequest{ SourceType: c.SourceType, Source: c.Source, Destination: c.Destination, }) case cmdStageInData: - summary.DataIn = append(summary.DataIn, model.DataCopyRequest{ + summary.DataIn = append(summary.DataIn, datamodel.DataCopyRequest{ SourceType: c.SourceType, Source: c.Source, Destination: c.Destination, @@ -84,33 +83,33 @@ func getJobSummary(lines []string) (jobSummary, error) { type jobCommand interface{} -var stringToAccessMode = map[string]model.AccessMode{ - "": model.Striped, - "striped": model.Striped, - "private": model.Private, - "private,striped": model.PrivateAndStriped, - "striped,private": model.PrivateAndStriped, +var stringToAccessMode = map[string]datamodel.AccessMode{ + "": datamodel.Striped, + "striped": datamodel.Striped, + "private": datamodel.Private, + "private,striped": datamodel.PrivateAndStriped, + "striped,private": datamodel.PrivateAndStriped, } -func AccessModeFromString(raw string) model.AccessMode { +func AccessModeFromString(raw string) datamodel.AccessMode { return stringToAccessMode[strings.ToLower(raw)] } -var stringToBufferType = map[string]model.BufferType{ - "": model.Scratch, - "scratch": model.Scratch, - "cache": model.Cache, +var stringToBufferType = map[string]datamodel.BufferType{ + "": datamodel.Scratch, + "scratch": datamodel.Scratch, + "cache": datamodel.Cache, } type cmdCreatePersistent struct { Name string CapacityBytes int - AccessMode model.AccessMode - BufferType model.BufferType + AccessMode datamodel.AccessMode + BufferType datamodel.BufferType GenericCmd bool } -func BufferTypeFromString(raw string) model.BufferType { +func BufferTypeFromString(raw string) datamodel.BufferType { return stringToBufferType[strings.ToLower(raw)] } @@ -120,8 +119,8 @@ type cmdAttachPersistent string type cmdPerJobBuffer struct { CapacityBytes int - AccessMode model.AccessMode - BufferType model.BufferType + AccessMode datamodel.AccessMode + BufferType datamodel.BufferType GenericCmd bool } @@ -129,19 +128,19 @@ type cmdAttachPerJobSwap struct { SizeBytes int } -var stringToStageType = map[string]model.SourceType{ - "directory": model.Directory, - "file": model.File, - "list": model.List, +var stringToStageType = map[string]datamodel.SourceType{ + "directory": datamodel.Directory, + "file": datamodel.File, + "list": datamodel.List, } -func sourceTypeFromString(raw string) model.SourceType { +func sourceTypeFromString(raw string) datamodel.SourceType { return stringToStageType[strings.ToLower(raw)] } -type cmdStageInData model.DataCopyRequest +type cmdStageInData datamodel.DataCopyRequest -type cmdStageOutData model.DataCopyRequest +type cmdStageOutData datamodel.DataCopyRequest func parseArgs(rawArgs []string) (map[string]string, error) { args := make(map[string]string, len(rawArgs)) @@ -186,7 +185,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { var command jobCommand switch cmd { case "create_persistent": - size, err := capacity.ParseSize(argKeyPair["capacity"]) + size, err := ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) continue @@ -203,7 +202,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { case "persistentdw": command = cmdAttachPersistent(argKeyPair["name"]) case "jobdw": - size, err := capacity.ParseSize(argKeyPair["capacity"]) + size, err := ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) continue @@ -218,7 +217,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { if len(args) != 1 { log.Println("Unable to parse swap command:", line) } - if size, err := capacity.ParseSize(args[0]); err != nil { + if size, err := ParseSize(args[0]); err != nil { log.Println(err) continue } else { diff --git a/internal/pkg/dacctlv2/parsers/jobfile/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go similarity index 90% rename from internal/pkg/dacctlv2/parsers/jobfile/job_test.go rename to internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index c7133db5..a2f3eec8 100644 --- a/internal/pkg/dacctlv2/parsers/jobfile/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -1,7 +1,7 @@ -package jobfile +package parsers import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/stretchr/testify/assert" "log" "testing" @@ -53,8 +53,8 @@ func TestGetJobSummary(t *testing.T) { assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut[0].Source) assert.Equal(t, 2, len(result.Attachments)) - assert.Equal(t, model.VolumeName("myBBname1"), result.Attachments[0]) - assert.Equal(t, model.VolumeName("myBBname2"), result.Attachments[1]) + assert.Equal(t, datamodel.VolumeName("myBBname1"), result.Attachments[0]) + assert.Equal(t, datamodel.VolumeName("myBBname2"), result.Attachments[1]) assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) assert.Equal(t, 4000000, result.Swap.SizeBytes) diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go new file mode 100644 index 00000000..91405105 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -0,0 +1,68 @@ +package actions_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "strings" + "time" +) + +var stringToAccessMode = map[string]datamodel.AccessMode{ + "": datamodel.Striped, + "striped": datamodel.Striped, + "private": datamodel.Private, + "private,striped": datamodel.PrivateAndStriped, + "striped,private": datamodel.PrivateAndStriped, +} + +func accessModeFromString(raw string) datamodel.AccessMode { + return stringToAccessMode[strings.ToLower(raw)] +} + +var stringToBufferType = map[string]datamodel.BufferType{ + "": datamodel.Scratch, + "scratch": datamodel.Scratch, + "cache": datamodel.Cache, +} + +func bufferTypeFromString(raw string) datamodel.BufferType { + return stringToBufferType[strings.ToLower(raw)] +} + +var fakeTime uint = 0 + +func getNow() uint { + if fakeTime != 0 { + return fakeTime + } + return uint(time.Now().Unix()) +} + +func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { + checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") + pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) + if err != nil { + return err + } + request := datamodel.VolumeRequest{ + MultiJob: true, + Caller: c.String("caller"), + TotalCapacityBytes: capacityBytes, + PoolName: pool, + Access: accessModeFromString(c.String("access")), + Type: bufferTypeFromString(c.String("type")), + } + session := datamodel.Session{ + Name: datamodel.SessionName(c.String("token")), + VolumeRequest: request, + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: getNow(), + } + session, err = d.registry.CreateSession(session) + if err != nil { + return err + } + return d.session.CreateSessionVolume(session) +} diff --git a/internal/pkg/dacctlv2/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go similarity index 65% rename from internal/pkg/dacctlv2/actions_impl/show.go rename to internal/pkg/v2/dacctl/actions_impl/show.go index 1b87a2ef..fa529beb 100644 --- a/internal/pkg/dacctlv2/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -1,12 +1,14 @@ package actions_impl -import "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctlv2/actions" +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" +) -func (*dacctlActions) RealSize(c actions.CliContext) error { +func (*dacctlActions) RealSize(c dacctl.CliContext) error { panic("implement me") } -func (*dacctlActions) Paths(c actions.CliContext) error { +func (*dacctlActions) Paths(c dacctl.CliContext) error { panic("implement me") } diff --git a/internal/pkg/dacctlv2/actions/interface.go b/internal/pkg/v2/dacctl/interface.go similarity index 97% rename from internal/pkg/dacctlv2/actions/interface.go rename to internal/pkg/v2/dacctl/interface.go index 3e7cb6b5..246ec2af 100644 --- a/internal/pkg/dacctlv2/actions/interface.go +++ b/internal/pkg/v2/dacctl/interface.go @@ -1,4 +1,4 @@ -package actions +package dacctl type CliContext interface { String(name string) string diff --git a/internal/pkg/dacdv2/brick_manager/interface.go b/internal/pkg/v2/dacd/brick_manager/interface.go similarity index 100% rename from internal/pkg/dacdv2/brick_manager/interface.go rename to internal/pkg/v2/dacd/brick_manager/interface.go diff --git a/internal/pkg/dacdv2/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go similarity index 69% rename from internal/pkg/dacdv2/brick_manager_impl/brick_manger.go rename to internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index da838600..483738c6 100644 --- a/internal/pkg/dacdv2/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -1,14 +1,14 @@ package brick_manager_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacdv2/brick_manager" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/brick_host" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" "os" ) -func NewBrickManager(brickRegistry brick_host.BrickHostRegistry) brick_manager.BrickManager { - return &brickManager{getHostname(), brickRegistry} +func NewBrickManager(store store.Keystore) brick_manager.BrickManager { + return &brickManager{getHostname()} } func getHostname() string { @@ -20,8 +20,7 @@ func getHostname() string { } type brickManager struct { - hostname string - brickRegistry brick_host.BrickHostRegistry + hostname string } func (bm *brickManager) Hostname() string { @@ -36,7 +35,7 @@ func (bm *brickManager) Startup(drainSessions bool) error { // * report we are listening with keep-alive // * check all brick assignments // ** ensure all brick hosts are up, warn if there are issues - // ** if primary brick, refresh ansible run (i.e. recover from host reboot) + // ** if primary brick, refresh Ansible run (i.e. recover from host reboot) } func (bm *brickManager) Shutdown() error { diff --git a/internal/pkg/data/model/allocation.go b/internal/pkg/v2/datamodel/brick_allocation.go similarity index 68% rename from internal/pkg/data/model/allocation.go rename to internal/pkg/v2/datamodel/brick_allocation.go index 6d2b465e..e459a3b3 100644 --- a/internal/pkg/data/model/allocation.go +++ b/internal/pkg/v2/datamodel/brick_allocation.go @@ -1,12 +1,16 @@ -package model +package datamodel // You can only have zero or one allocations records for each Brick type BrickAllocation struct { // Brick that is allocated - Brick BrickInfo + Brick Brick - // Name of the volume that owns the brick - AllocatedVolume VolumeName + // Name of the session that owns the brick + Session SessionName + + // Unique id for this allocation + // matched when delete allocation is called + UUID string // 0 index allocation is the primary brick, // which is responsible for provisioning the associated volume @@ -17,6 +21,9 @@ type BrickAllocation struct { // associated job and volume actions IsPrimaryBrick bool + // This is set when server has accepted the allocation + IsAllocationAccepted bool + // If any allocation sent to deallocate has a host that isn't // alive, this flag is set rather than have allocations removed. // A host should check for any allocations diff --git a/internal/pkg/data/model/brickhost.go b/internal/pkg/v2/datamodel/brick_host.go similarity index 76% rename from internal/pkg/data/model/brickhost.go rename to internal/pkg/v2/datamodel/brick_host.go index e551c053..5b0ca333 100644 --- a/internal/pkg/data/model/brickhost.go +++ b/internal/pkg/v2/datamodel/brick_host.go @@ -1,35 +1,36 @@ -package model +package datamodel type BrickHostName string -type BrickHostInfo struct { +type BrickHost struct { Name BrickHostName // Returns all bricks - Bricks []BrickInfo + Bricks []Brick // True if allowing new volumes to use bricks from this host Enabled bool } type BrickHostStatus struct { - Info BrickHostInfo + BrickHost BrickHost // True is current keepalive key exists Alive bool } -type BrickInfo struct { +type Brick struct { // Bricks are identified by device and hostname // It must only contain the characters A-Za-z0-9 + // e.g. sdb, not /dev/sdb Device string // It must only contain the characters "A-Za-z0-9." - Hostname string + BrickHostName BrickHostName // The bool a brick is associated with // It must only contain the characters A-Za-z0-9 - PoolName string + PoolName PoolName // Size of the brick, defines the pool granularity CapacityGB uint diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go new file mode 100644 index 00000000..872af464 --- /dev/null +++ b/internal/pkg/v2/datamodel/datamodel_test.go @@ -0,0 +1,42 @@ +package datamodel + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_Volume(t *testing.T) { + volume := Volume{} + + volumeAsString, err := json.Marshal(volume) + + assert.Nil(t, err) + expected := `{"Name":"","UUID":"","MultiJob":false,"Pool":"","SizeBricks":0,"SizeGB":0,"JobName":"","Owner":0,"Group":0,"CreatedBy":"","CreatedAt":0,"Attachments":null,"AttachGlobalNamespace":false,"AttachPrivateNamespace":false,"AttachAsSwapBytes":0,"AttachPrivateCache":null,"StageInRequests":null,"StageOutRequests":null,"ClientPort":0,"HadBricksAssigned":false}` + assert.Equal(t, expected, string(volumeAsString)) + + var volumeFromString Volume + data := []byte(expected) + err = json.Unmarshal(data, &volumeFromString) + + assert.Nil(t, err) + assert.Equal(t, volume, volumeFromString) +} + +func Test_Session(t *testing.T) { + session := Session{} + + sessionAsString, err := json.Marshal(session) + + assert.Nil(t, err) + // TODO: not very human readable for Type and Action + expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"DeleteRequested":false,"StageInRequests":null,"StageOutRequests":null,"AttachHosts":null,"JobVolume":"","MultiJobVolumes":null,"Paths":null,"Allocations":null,"SessionActionPrefix":"","ActualSizeBytes":0,"Error":null}` + assert.Equal(t, expected, string(sessionAsString)) + + var sessionFromString Session + data := []byte(expected) + err = json.Unmarshal(data, &sessionFromString) + + assert.Nil(t, err) + assert.Equal(t, session, sessionFromString) +} diff --git a/internal/pkg/data/model/pool.go b/internal/pkg/v2/datamodel/pool.go similarity index 89% rename from internal/pkg/data/model/pool.go rename to internal/pkg/v2/datamodel/pool.go index 57707d7e..53c18e1e 100644 --- a/internal/pkg/data/model/pool.go +++ b/internal/pkg/v2/datamodel/pool.go @@ -1,4 +1,4 @@ -package model +package datamodel type PoolName string @@ -15,7 +15,7 @@ type PoolInfo struct { Pool Pool // Bricks from alive hosts - AvailableBricks []BrickInfo + AvailableBricks []Brick // All currently active bricks AllocatedBricks []BrickAllocation diff --git a/internal/pkg/data/model/session.go b/internal/pkg/v2/datamodel/session.go similarity index 96% rename from internal/pkg/data/model/session.go rename to internal/pkg/v2/datamodel/session.go index 5a1129b8..deb9512e 100644 --- a/internal/pkg/data/model/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -1,4 +1,4 @@ -package model +package datamodel type SessionName string @@ -56,6 +56,9 @@ type Session struct { // If empty the session has not yet been acknowledged by the dacd process SessionActionPrefix string + // Resources used by session once pool granularity is taken into account + ActualSizeBytes int + // If not nil, the session has an unresolved error // and can't be mounted by any new sessions // it can be deleted diff --git a/internal/pkg/data/model/volume.go b/internal/pkg/v2/datamodel/volume.go similarity index 99% rename from internal/pkg/data/model/volume.go rename to internal/pkg/v2/datamodel/volume.go index 2eac4d87..bae87c5f 100644 --- a/internal/pkg/data/model/volume.go +++ b/internal/pkg/v2/datamodel/volume.go @@ -1,4 +1,4 @@ -package model +package datamodel import ( "bytes" diff --git a/internal/pkg/data/mock_session/actions.go b/internal/pkg/v2/mock_registry/actions.go similarity index 83% rename from internal/pkg/data/mock_session/actions.go rename to internal/pkg/v2/mock_registry/actions.go index 6308a7ce..8a601955 100644 --- a/internal/pkg/data/mock_session/actions.go +++ b/internal/pkg/v2/mock_registry/actions.go @@ -1,13 +1,13 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/data/session/actions.go +// Source: internal/pkg/data/session/session_registry.go -// Package mock_session is a generated GoMock package. -package mock_session +// Package mock_registry is a generated GoMock package. +package mock_registry import ( - model "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - gomock "github.com/golang/mock/gomock" - reflect "reflect" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/golang/mock/gomock" + "reflect" ) // MockActions is a mock of Actions interface @@ -34,7 +34,7 @@ func (m *MockActions) EXPECT() *MockActionsMockRecorder { } // CreateSessionVolume mocks base method -func (m *MockActions) CreateSessionVolume(session model.Session) error { +func (m *MockActions) CreateSessionVolume(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSessionVolume", session) ret0, _ := ret[0].(error) @@ -48,7 +48,7 @@ func (mr *MockActionsMockRecorder) CreateSessionVolume(session interface{}) *gom } // DeleteSession mocks base method -func (m *MockActions) DeleteSession(session model.Session) error { +func (m *MockActions) DeleteSession(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteSession", session) ret0, _ := ret[0].(error) @@ -62,7 +62,7 @@ func (mr *MockActionsMockRecorder) DeleteSession(session interface{}) *gomock.Ca } // DataIn mocks base method -func (m *MockActions) DataIn(session model.Session) error { +func (m *MockActions) DataIn(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DataIn", session) ret0, _ := ret[0].(error) @@ -76,7 +76,7 @@ func (mr *MockActionsMockRecorder) DataIn(session interface{}) *gomock.Call { } // AttachVolumes mocks base method -func (m *MockActions) AttachVolumes(session model.Session) error { +func (m *MockActions) AttachVolumes(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AttachVolumes", session) ret0, _ := ret[0].(error) @@ -90,7 +90,7 @@ func (mr *MockActionsMockRecorder) AttachVolumes(session interface{}) *gomock.Ca } // DetachVolumes mocks base method -func (m *MockActions) DetachVolumes(session model.Session) error { +func (m *MockActions) DetachVolumes(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DetachVolumes", session) ret0, _ := ret[0].(error) @@ -104,7 +104,7 @@ func (mr *MockActionsMockRecorder) DetachVolumes(session interface{}) *gomock.Ca } // DataOut mocks base method -func (m *MockActions) DataOut(session model.Session) error { +func (m *MockActions) DataOut(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DataOut", session) ret0, _ := ret[0].(error) diff --git a/internal/pkg/v2/mock_registry/allocation.go b/internal/pkg/v2/mock_registry/allocation.go new file mode 100644 index 00000000..6be1b0de --- /dev/null +++ b/internal/pkg/v2/mock_registry/allocation.go @@ -0,0 +1,94 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/allocation.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockAllocationRegistry is a mock of AllocationRegistry interface +type MockAllocationRegistry struct { + ctrl *gomock.Controller + recorder *MockAllocationRegistryMockRecorder +} + +// MockAllocationRegistryMockRecorder is the mock recorder for MockAllocationRegistry +type MockAllocationRegistryMockRecorder struct { + mock *MockAllocationRegistry +} + +// NewMockAllocationRegistry creates a new mock instance +func NewMockAllocationRegistry(ctrl *gomock.Controller) *MockAllocationRegistry { + mock := &MockAllocationRegistry{ctrl: ctrl} + mock.recorder = &MockAllocationRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAllocationRegistry) EXPECT() *MockAllocationRegistryMockRecorder { + return m.recorder +} + +// GetBricksByPool mocks base method +func (m *MockAllocationRegistry) GetBricksByPool() ([]datamodel.PoolInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBricksByPool") + ret0, _ := ret[0].([]datamodel.PoolInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBricksByPool indicates an expected call of GetBricksByPool +func (mr *MockAllocationRegistryMockRecorder) GetBricksByPool() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBricksByPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetBricksByPool)) +} + +// GetAllocationMutex mocks base method +func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllocationMutex") + ret0, _ := ret[0].(store.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllocationMutex indicates an expected call of GetAllocationMutex +func (mr *MockAllocationRegistryMockRecorder) GetAllocationMutex() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllocationMutex)) +} + +// CreateAllocations mocks base method +func (m *MockAllocationRegistry) CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateAllocations", sessionName, allocations) + ret0, _ := ret[0].([]datamodel.BrickAllocation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateAllocations indicates an expected call of CreateAllocations +func (mr *MockAllocationRegistryMockRecorder) CreateAllocations(sessionName, allocations interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateAllocations", reflect.TypeOf((*MockAllocationRegistry)(nil).CreateAllocations), sessionName, allocations) +} + +// DeleteAllocations mocks base method +func (m *MockAllocationRegistry) DeleteAllocations(allocations []datamodel.BrickAllocation) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAllocations", allocations) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllocations indicates an expected call of DeleteAllocations +func (mr *MockAllocationRegistryMockRecorder) DeleteAllocations(allocations interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllocations", reflect.TypeOf((*MockAllocationRegistry)(nil).DeleteAllocations), allocations) +} diff --git a/internal/pkg/v2/mock_registry/brick.go b/internal/pkg/v2/mock_registry/brick.go new file mode 100644 index 00000000..46c6bdd8 --- /dev/null +++ b/internal/pkg/v2/mock_registry/brick.go @@ -0,0 +1,93 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/brick.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + context "context" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockBrickRegistry is a mock of BrickRegistry interface +type MockBrickRegistry struct { + ctrl *gomock.Controller + recorder *MockBrickRegistryMockRecorder +} + +// MockBrickRegistryMockRecorder is the mock recorder for MockBrickRegistry +type MockBrickRegistryMockRecorder struct { + mock *MockBrickRegistry +} + +// NewMockBrickRegistry creates a new mock instance +func NewMockBrickRegistry(ctrl *gomock.Controller) *MockBrickRegistry { + mock := &MockBrickRegistry{ctrl: ctrl} + mock.recorder = &MockBrickRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockBrickRegistry) EXPECT() *MockBrickRegistryMockRecorder { + return m.recorder +} + +// UpdateBrickHost mocks base method +func (m *MockBrickRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBrickHost", brickHostInfo) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateBrickHost indicates an expected call of UpdateBrickHost +func (mr *MockBrickRegistryMockRecorder) UpdateBrickHost(brickHostInfo interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickRegistry)(nil).UpdateBrickHost), brickHostInfo) +} + +// GetSessionActions mocks base method +func (m *MockBrickRegistry) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSessionActions indicates an expected call of GetSessionActions +func (mr *MockBrickRegistryMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockBrickRegistry)(nil).GetSessionActions), ctxt, brickHostName) +} + +// KeepAliveHost mocks base method +func (m *MockBrickRegistry) KeepAliveHost(brickHostName datamodel.BrickHostName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeepAliveHost", brickHostName) + ret0, _ := ret[0].(error) + return ret0 +} + +// KeepAliveHost indicates an expected call of KeepAliveHost +func (mr *MockBrickRegistryMockRecorder) KeepAliveHost(brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickRegistry)(nil).KeepAliveHost), brickHostName) +} + +// IsBrickHostAlive mocks base method +func (m *MockBrickRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsBrickHostAlive", brickHostName) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsBrickHostAlive indicates an expected call of IsBrickHostAlive +func (mr *MockBrickRegistryMockRecorder) IsBrickHostAlive(brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsBrickHostAlive", reflect.TypeOf((*MockBrickRegistry)(nil).IsBrickHostAlive), brickHostName) +} diff --git a/internal/pkg/v2/mock_registry/pool.go b/internal/pkg/v2/mock_registry/pool.go new file mode 100644 index 00000000..035f7a59 --- /dev/null +++ b/internal/pkg/v2/mock_registry/pool.go @@ -0,0 +1,64 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/pool.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockPoolRegistry is a mock of PoolRegistry interface +type MockPoolRegistry struct { + ctrl *gomock.Controller + recorder *MockPoolRegistryMockRecorder +} + +// MockPoolRegistryMockRecorder is the mock recorder for MockPoolRegistry +type MockPoolRegistryMockRecorder struct { + mock *MockPoolRegistry +} + +// NewMockPoolRegistry creates a new mock instance +func NewMockPoolRegistry(ctrl *gomock.Controller) *MockPoolRegistry { + mock := &MockPoolRegistry{ctrl: ctrl} + mock.recorder = &MockPoolRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockPoolRegistry) EXPECT() *MockPoolRegistryMockRecorder { + return m.recorder +} + +// GetPools mocks base method +func (m *MockPoolRegistry) GetPools() ([]datamodel.Pool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPools") + ret0, _ := ret[0].([]datamodel.Pool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPools indicates an expected call of GetPools +func (mr *MockPoolRegistryMockRecorder) GetPools() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPools", reflect.TypeOf((*MockPoolRegistry)(nil).GetPools)) +} + +// EnsurePoolCreated mocks base method +func (m *MockPoolRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGB) + ret0, _ := ret[0].(datamodel.Pool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnsurePoolCreated indicates an expected call of EnsurePoolCreated +func (mr *MockPoolRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGB interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockPoolRegistry)(nil).EnsurePoolCreated), poolName, granularityGB) +} diff --git a/internal/pkg/v2/mock_registry/session.go b/internal/pkg/v2/mock_registry/session.go new file mode 100644 index 00000000..a2ec1baa --- /dev/null +++ b/internal/pkg/v2/mock_registry/session.go @@ -0,0 +1,124 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/session.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockSessionRegistry is a mock of SessionRegistry interface +type MockSessionRegistry struct { + ctrl *gomock.Controller + recorder *MockSessionRegistryMockRecorder +} + +// MockSessionRegistryMockRecorder is the mock recorder for MockSessionRegistry +type MockSessionRegistryMockRecorder struct { + mock *MockSessionRegistry +} + +// NewMockSessionRegistry creates a new mock instance +func NewMockSessionRegistry(ctrl *gomock.Controller) *MockSessionRegistry { + mock := &MockSessionRegistry{ctrl: ctrl} + mock.recorder = &MockSessionRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSessionRegistry) EXPECT() *MockSessionRegistryMockRecorder { + return m.recorder +} + +// CreateSession mocks base method +func (m *MockSessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSession", session) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSession indicates an expected call of CreateSession +func (mr *MockSessionRegistryMockRecorder) CreateSession(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockSessionRegistry)(nil).CreateSession), session) +} + +// GetSession mocks base method +func (m *MockSessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSession", sessionName) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSession indicates an expected call of GetSession +func (mr *MockSessionRegistryMockRecorder) GetSession(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockSessionRegistry)(nil).GetSession), sessionName) +} + +// GetAllSessions mocks base method +func (m *MockSessionRegistry) GetAllSessions() ([]datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllSessions") + ret0, _ := ret[0].([]datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllSessions indicates an expected call of GetAllSessions +func (mr *MockSessionRegistryMockRecorder) GetAllSessions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockSessionRegistry)(nil).GetAllSessions)) +} + +// UpdateSession mocks base method +func (m *MockSessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateSession", session) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateSession indicates an expected call of UpdateSession +func (mr *MockSessionRegistryMockRecorder) UpdateSession(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSession", reflect.TypeOf((*MockSessionRegistry)(nil).UpdateSession), session) +} + +// DeleteSession mocks base method +func (m *MockSessionRegistry) DeleteSession(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSession", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSession indicates an expected call of DeleteSession +func (mr *MockSessionRegistryMockRecorder) DeleteSession(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSessionRegistry)(nil).DeleteSession), session) +} + +// GetSessionMutex mocks base method +func (m *MockSessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSessionMutex", sessionName) + ret0, _ := ret[0].(store.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSessionMutex indicates an expected call of GetSessionMutex +func (mr *MockSessionRegistryMockRecorder) GetSessionMutex(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionMutex", reflect.TypeOf((*MockSessionRegistry)(nil).GetSessionMutex), sessionName) +} diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go new file mode 100644 index 00000000..e6e3fdda --- /dev/null +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -0,0 +1,109 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/session_actions.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + context "context" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockSessionActions is a mock of SessionActions interface +type MockSessionActions struct { + ctrl *gomock.Controller + recorder *MockSessionActionsMockRecorder +} + +// MockSessionActionsMockRecorder is the mock recorder for MockSessionActions +type MockSessionActionsMockRecorder struct { + mock *MockSessionActions +} + +// NewMockSessionActions creates a new mock instance +func NewMockSessionActions(ctrl *gomock.Controller) *MockSessionActions { + mock := &MockSessionActions{ctrl: ctrl} + mock.recorder = &MockSessionActionsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSessionActions) EXPECT() *MockSessionActionsMockRecorder { + return m.recorder +} + +// CreateSessionVolume mocks base method +func (m *MockSessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSessionVolume", ctxt, sessionName) + ret0, _ := ret[0].(<-chan datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateSessionVolume indicates an expected call of CreateSessionVolume +func (mr *MockSessionActionsMockRecorder) CreateSessionVolume(ctxt, sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockSessionActions)(nil).CreateSessionVolume), ctxt, sessionName) +} + +// SendSessionAction mocks base method +func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendSessionAction", ctxt, actionType, session) + ret0, _ := ret[0].(<-chan datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SendSessionAction indicates an expected call of SendSessionAction +func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSessionAction", reflect.TypeOf((*MockSessionActions)(nil).SendSessionAction), ctxt, actionType, session) +} + +// GetCreateSessionVolumeRequests mocks base method +func (m *MockSessionActions) GetCreateSessionVolumeRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCreateSessionVolumeRequests", ctxt, brickHostName) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCreateSessionVolumeRequests indicates an expected call of GetCreateSessionVolumeRequests +func (mr *MockSessionActionsMockRecorder) GetCreateSessionVolumeRequests(ctxt, brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCreateSessionVolumeRequests", reflect.TypeOf((*MockSessionActions)(nil).GetCreateSessionVolumeRequests), ctxt, brickHostName) +} + +// GetSessionActions mocks base method +func (m *MockSessionActions) GetSessionActions(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSessionActions", ctxt, sessionName) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSessionActions indicates an expected call of GetSessionActions +func (mr *MockSessionActionsMockRecorder) GetSessionActions(ctxt, sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActions), ctxt, sessionName) +} + +// CompleteSessionAction mocks base method +func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteSessionAction", action, err) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteSessionAction indicates an expected call of CompleteSessionAction +func (mr *MockSessionActionsMockRecorder) CompleteSessionAction(action, err interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteSessionAction", reflect.TypeOf((*MockSessionActions)(nil).CompleteSessionAction), action, err) +} diff --git a/internal/pkg/v2/mock_workflow/session.go b/internal/pkg/v2/mock_workflow/session.go new file mode 100644 index 00000000..104fb214 --- /dev/null +++ b/internal/pkg/v2/mock_workflow/session.go @@ -0,0 +1,163 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/workflow/session.go + +// Package mock_workflow is a generated GoMock package. +package mock_workflow + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockSession is a mock of Session interface +type MockSession struct { + ctrl *gomock.Controller + recorder *MockSessionMockRecorder +} + +// MockSessionMockRecorder is the mock recorder for MockSession +type MockSessionMockRecorder struct { + mock *MockSession +} + +// NewMockSession creates a new mock instance +func NewMockSession(ctrl *gomock.Controller) *MockSession { + mock := &MockSession{ctrl: ctrl} + mock.recorder = &MockSessionMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSession) EXPECT() *MockSessionMockRecorder { + return m.recorder +} + +// CreateSessionVolume mocks base method +func (m *MockSession) CreateSessionVolume(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateSessionVolume", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateSessionVolume indicates an expected call of CreateSessionVolume +func (mr *MockSessionMockRecorder) CreateSessionVolume(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockSession)(nil).CreateSessionVolume), session) +} + +// DeleteSession mocks base method +func (m *MockSession) DeleteSession(sessionName datamodel.SessionName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteSession", sessionName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteSession indicates an expected call of DeleteSession +func (mr *MockSessionMockRecorder) DeleteSession(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSession)(nil).DeleteSession), sessionName) +} + +// DataIn mocks base method +func (m *MockSession) DataIn(sessionName datamodel.SessionName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataIn", sessionName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataIn indicates an expected call of DataIn +func (mr *MockSessionMockRecorder) DataIn(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataIn", reflect.TypeOf((*MockSession)(nil).DataIn), sessionName) +} + +// AttachVolumes mocks base method +func (m *MockSession) AttachVolumes(sessionName datamodel.SessionName, attachHosts []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AttachVolumes", sessionName, attachHosts) + ret0, _ := ret[0].(error) + return ret0 +} + +// AttachVolumes indicates an expected call of AttachVolumes +func (mr *MockSessionMockRecorder) AttachVolumes(sessionName, attachHosts interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockSession)(nil).AttachVolumes), sessionName, attachHosts) +} + +// DetachVolumes mocks base method +func (m *MockSession) DetachVolumes(sessionName datamodel.SessionName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DetachVolumes", sessionName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DetachVolumes indicates an expected call of DetachVolumes +func (mr *MockSessionMockRecorder) DetachVolumes(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachVolumes", reflect.TypeOf((*MockSession)(nil).DetachVolumes), sessionName) +} + +// DataOut mocks base method +func (m *MockSession) DataOut(sessionName datamodel.SessionName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataOut", sessionName) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataOut indicates an expected call of DataOut +func (mr *MockSessionMockRecorder) DataOut(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataOut", reflect.TypeOf((*MockSession)(nil).DataOut), sessionName) +} + +// GetPools mocks base method +func (m *MockSession) GetPools() ([]datamodel.PoolInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPools") + ret0, _ := ret[0].([]datamodel.PoolInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPools indicates an expected call of GetPools +func (mr *MockSessionMockRecorder) GetPools() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPools", reflect.TypeOf((*MockSession)(nil).GetPools)) +} + +// GetSession mocks base method +func (m *MockSession) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSession", sessionName) + ret0, _ := ret[0].(datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSession indicates an expected call of GetSession +func (mr *MockSessionMockRecorder) GetSession(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockSession)(nil).GetSession), sessionName) +} + +// GetAllSessions mocks base method +func (m *MockSession) GetAllSessions() ([]datamodel.Session, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllSessions") + ret0, _ := ret[0].([]datamodel.Session) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllSessions indicates an expected call of GetAllSessions +func (mr *MockSessionMockRecorder) GetAllSessions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockSession)(nil).GetAllSessions)) +} diff --git a/internal/pkg/v2/registry/allocation.go b/internal/pkg/v2/registry/allocation.go new file mode 100644 index 00000000..62507ee8 --- /dev/null +++ b/internal/pkg/v2/registry/allocation.go @@ -0,0 +1,24 @@ +package registry + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" +) + +type AllocationRegistry interface { + // Get brick availability by pool + GetBricksByPool() ([]datamodel.PoolInfo, error) + + // Caller should acquire this mutex before calling GetAllPools then CreateAllocations + GetAllocationMutex() (store.Mutex, error) + + // Allocations written (by the client), while holding above mutex + // + // Error if any bricks already have an allocation + CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) + + // Allocations deleted by server when bricks no being used + // + // Does not error if given allocation has already been dropped + DeleteAllocations(allocations []datamodel.BrickAllocation) error +} diff --git a/internal/pkg/v2/registry/brick.go b/internal/pkg/v2/registry/brick.go new file mode 100644 index 00000000..d2bb41f3 --- /dev/null +++ b/internal/pkg/v2/registry/brick.go @@ -0,0 +1,34 @@ +package registry + +import ( + "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +) + +type BrickRegistry interface { + // BrickHost updates bricks on startup + // This will error if we remove a brick that has an allocation + // for a Session that isn't in an error state + UpdateBrickHost(brickHostInfo datamodel.BrickHost) error + + // Gets all new actions for the given Session + // This confirms that dacd is ready to service requests for this session + // and updates the session with an appropriate SessionActionPrefix + // The channel tracks all actions sent to that sub keys of the above prefix + // This is also called when dacd is restarted and you want to resume sending + // any actions that are not marked as complete + // New tasks are only sent if the keepalive key is active + GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) + + // While the process is still running this notifies others the host is up + // + // When a host is dead non of its bricks will get new volumes assigned, + // and no bricks will get cleaned up until the next service start. + // Error will be returned if the host info has not yet been written. + KeepAliveHost(brickHostName datamodel.BrickHostName) error + + // Check if given brick host is alive + // + // Error if brick host doesn't exist + IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) +} diff --git a/internal/pkg/v2/registry/pool.go b/internal/pkg/v2/registry/pool.go new file mode 100644 index 00000000..b2d9f9fe --- /dev/null +++ b/internal/pkg/v2/registry/pool.go @@ -0,0 +1,14 @@ +package registry + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +) + +type PoolRegistry interface { + // Get all registered pools + GetPools() ([]datamodel.Pool, error) + + // Creates the pool if it doesn't exist + // error if the granularity doesn't match and existing pool + EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) +} diff --git a/internal/pkg/v2/registry/session.go b/internal/pkg/v2/registry/session.go new file mode 100644 index 00000000..c582cf86 --- /dev/null +++ b/internal/pkg/v2/registry/session.go @@ -0,0 +1,40 @@ +package registry + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" +) + +type SessionRegistry interface { + // Update provided session + // + // Error is session already exists + CreateSession(session datamodel.Session) (datamodel.Session, error) + + // Get requested session + // + // Error if session does not exist + GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) + + // Get all sessions + GetAllSessions() ([]datamodel.Session, error) + + // Update provided session + // + // Error if current revision does not match (i.e. caller has a stale copy of Session) + // Error if session does not exist + UpdateSession(session datamodel.Session) (datamodel.Session, error) + + // This is called before confirming the Session delete request, + // after all bricks have been de-allocated + // + // Error if session has any allocations + // Error if session doesn't match current revision + // No error if session has already been deleted + DeleteSession(session datamodel.Session) error + + // This mutex should be held before doing any operations on given session + // + // No error if the session doesn't exist, as this is also used when creating a session + GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) +} diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go new file mode 100644 index 00000000..15488796 --- /dev/null +++ b/internal/pkg/v2/registry/session_actions.go @@ -0,0 +1,44 @@ +package registry + +import ( + "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +) + +type SessionActions interface { + // Client requests session volume is created + // + // Error if session does not have bricks allocated + // Error if session volume has already been created + // Error if primary brick host is not alive or not enabled + // Error is context is cancelled or timed-out + CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) + + // Updates session, then requests action + // + // Error if current revision of session doesn't match + // Error if context is cancelled or timed-out + SendSessionAction( + ctxt context.Context, actionType datamodel.SessionActionType, + session datamodel.Session) (<-chan datamodel.Session, error) + + // Get session volume create requests, + // where given hostname is the primary brick host + // Called very time brick host is started + // + // Error is context is cancelled or timed-out + GetCreateSessionVolumeRequests( + ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) + + // Gets all new actions for the given Session + // + // Error if context is cancelled or timed-out + GetSessionActions( + ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) + + // Server reports given action is complete + // Includes callbacks for Create Session Volume + // + // Error if action has already completed or doesn't exist + CompleteSessionAction(action datamodel.SessionAction, err error) error +} diff --git a/internal/pkg/data/session_impl/actions.go b/internal/pkg/v2/registry_impl/actions.go similarity index 65% rename from internal/pkg/data/session_impl/actions.go rename to internal/pkg/v2/registry_impl/actions.go index 7fbd559f..dfaf05ae 100644 --- a/internal/pkg/data/session_impl/actions.go +++ b/internal/pkg/v2/registry_impl/actions.go @@ -1,19 +1,19 @@ -package session_impl +package registry_impl import ( "context" "encoding/json" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/model" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/session" - "github.com/RSE-Cambridge/data-acc/internal/pkg/data/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/google/uuid" "log" "time" ) // TODO: this is the client side, need server side too -func NewSessionActions(keystore store.Keystore) session.Actions { +func NewSessionActions(keystore store.Keystore) registry.SessionActions { return &sessionActions{keystore: keystore, defaultTimeout: time.Minute * 20} } @@ -22,6 +22,30 @@ type sessionActions struct { defaultTimeout time.Duration } +func (s *sessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionActions) SendSessionAction( + ctxt context.Context, actionType datamodel.SessionActionType, + session datamodel.Session) (<-chan datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionActions) GetCreateSessionVolumeRequests( + ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { + panic("implement me") +} + +func (s *sessionActions) GetSessionActions( + ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { + panic("implement me") +} + +func (s *sessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { + panic("implement me") +} + func toJson(message interface{}) string { b, error := json.Marshal(message) if error != nil { @@ -30,7 +54,7 @@ func toJson(message interface{}) string { return string(b) } -func (s *sessionActions) sendAction(session model.Session, action string) error { +func (s *sessionActions) sendAction(session datamodel.Session, action string) error { // TODO: update session? // TODO: check primary session host is alive before sending event @@ -82,27 +106,3 @@ func (s *sessionActions) sendAction(session model.Session, action string) error } return nil } - -func (*sessionActions) CreateSessionVolume(session model.Session) error { - panic("implement me") -} - -func (*sessionActions) DeleteSession(session model.Session) error { - panic("implement me") -} - -func (*sessionActions) DataIn(session model.Session) error { - panic("implement me") -} - -func (*sessionActions) AttachVolumes(session model.Session) error { - panic("implement me") -} - -func (*sessionActions) DetachVolumes(session model.Session) error { - panic("implement me") -} - -func (*sessionActions) DataOut(session model.Session) error { - panic("implement me") -} diff --git a/internal/pkg/data/store/interface.go b/internal/pkg/v2/store/keystore.go similarity index 100% rename from internal/pkg/data/store/interface.go rename to internal/pkg/v2/store/keystore.go diff --git a/internal/pkg/v2/workflow/session.go b/internal/pkg/v2/workflow/session.go new file mode 100644 index 00000000..a84ad8ae --- /dev/null +++ b/internal/pkg/v2/workflow/session.go @@ -0,0 +1,37 @@ +package workflow + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + +// Each volume has an associated primary brick +// that is responsible for responding to any actions +// All the calls block until they are complete, or an error occurs +type Session interface { + // Allocates storage and + CreateSessionVolume(session datamodel.Session) error + + // Deletes the requested volume and session allocation + DeleteSession(sessionName datamodel.SessionName) error + + // Update the session and trigger requested data copy in + DataIn(sessionName datamodel.SessionName) error + + // Update session hosts and attach volumes as needed + AttachVolumes(sessionName datamodel.SessionName, attachHosts []string) error + + // Attempt to detach volumes + DetachVolumes(sessionName datamodel.SessionName) error + + // Update the session and trigger requested data copy out + DataOut(sessionName datamodel.SessionName) error + + // Get brick availability by pool + GetPools() ([]datamodel.PoolInfo, error) + + // Get requested session + // + // Error if session does not exist + GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) + + // Get all sessions + GetAllSessions() ([]datamodel.Session, error) +} From a74b837f410f568e36c5bdc52adfa988dbb90772 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 17:23:18 +0100 Subject: [PATCH 019/191] Add mock store --- build/rebuild_mocks.sh | 6 + internal/pkg/v2/mock_store/keystore.go | 229 +++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 internal/pkg/v2/mock_store/keystore.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 6095c67d..e2d185c4 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -38,3 +38,9 @@ for i in $items; do mockgen -source=internal/pkg/v2/workflow/${i}.go \ >internal/pkg/v2/mock_workflow/${i}.go done + +items="keystore" +for i in $items; do + mockgen -source=internal/pkg/v2/store/${i}.go \ + >internal/pkg/v2/mock_store/${i}.go +done diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go new file mode 100644 index 00000000..70acd435 --- /dev/null +++ b/internal/pkg/v2/mock_store/keystore.go @@ -0,0 +1,229 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/store/keystore.go + +// Package mock_store is a generated GoMock package. +package mock_store + +import ( + context "context" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockKeystore is a mock of Keystore interface +type MockKeystore struct { + ctrl *gomock.Controller + recorder *MockKeystoreMockRecorder +} + +// MockKeystoreMockRecorder is the mock recorder for MockKeystore +type MockKeystoreMockRecorder struct { + mock *MockKeystore +} + +// NewMockKeystore creates a new mock instance +func NewMockKeystore(ctrl *gomock.Controller) *MockKeystore { + mock := &MockKeystore{ctrl: ctrl} + mock.recorder = &MockKeystoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockKeystore) EXPECT() *MockKeystoreMockRecorder { + return m.recorder +} + +// Close mocks base method +func (m *MockKeystore) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockKeystoreMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockKeystore)(nil).Close)) +} + +// CleanPrefix mocks base method +func (m *MockKeystore) CleanPrefix(prefix string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CleanPrefix", prefix) + ret0, _ := ret[0].(error) + return ret0 +} + +// CleanPrefix indicates an expected call of CleanPrefix +func (mr *MockKeystoreMockRecorder) CleanPrefix(prefix interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanPrefix", reflect.TypeOf((*MockKeystore)(nil).CleanPrefix), prefix) +} + +// Add mocks base method +func (m *MockKeystore) Add(keyValue store.KeyValue) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", keyValue) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add +func (mr *MockKeystoreMockRecorder) Add(keyValue interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockKeystore)(nil).Add), keyValue) +} + +// Update mocks base method +func (m *MockKeystore) Update(keyValue store.KeyValueVersion) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", keyValue) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockKeystoreMockRecorder) Update(keyValue interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockKeystore)(nil).Update), keyValue) +} + +// DeleteAll mocks base method +func (m *MockKeystore) DeleteAll(keyValues []store.KeyValueVersion) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAll", keyValues) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAll indicates an expected call of DeleteAll +func (mr *MockKeystoreMockRecorder) DeleteAll(keyValues interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAll", reflect.TypeOf((*MockKeystore)(nil).DeleteAll), keyValues) +} + +// GetAll mocks base method +func (m *MockKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAll", prefix) + ret0, _ := ret[0].([]store.KeyValueVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAll indicates an expected call of GetAll +func (mr *MockKeystoreMockRecorder) GetAll(prefix interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockKeystore)(nil).GetAll), prefix) +} + +// Get mocks base method +func (m *MockKeystore) Get(key string) (store.KeyValueVersion, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", key) + ret0, _ := ret[0].(store.KeyValueVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get +func (mr *MockKeystoreMockRecorder) Get(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKeystore)(nil).Get), key) +} + +// Watch mocks base method +func (m *MockKeystore) Watch(ctxt context.Context, key string, withPrefix bool) store.KeyValueUpdateChan { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch", ctxt, key, withPrefix) + ret0, _ := ret[0].(store.KeyValueUpdateChan) + return ret0 +} + +// Watch indicates an expected call of Watch +func (mr *MockKeystoreMockRecorder) Watch(ctxt, key, withPrefix interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockKeystore)(nil).Watch), ctxt, key, withPrefix) +} + +// KeepAliveKey mocks base method +func (m *MockKeystore) KeepAliveKey(key string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeepAliveKey", key) + ret0, _ := ret[0].(error) + return ret0 +} + +// KeepAliveKey indicates an expected call of KeepAliveKey +func (mr *MockKeystoreMockRecorder) KeepAliveKey(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveKey", reflect.TypeOf((*MockKeystore)(nil).KeepAliveKey), key) +} + +// NewMutex mocks base method +func (m *MockKeystore) NewMutex(lockKey string) (store.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewMutex", lockKey) + ret0, _ := ret[0].(store.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewMutex indicates an expected call of NewMutex +func (mr *MockKeystoreMockRecorder) NewMutex(lockKey interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewMutex", reflect.TypeOf((*MockKeystore)(nil).NewMutex), lockKey) +} + +// MockMutex is a mock of Mutex interface +type MockMutex struct { + ctrl *gomock.Controller + recorder *MockMutexMockRecorder +} + +// MockMutexMockRecorder is the mock recorder for MockMutex +type MockMutexMockRecorder struct { + mock *MockMutex +} + +// NewMockMutex creates a new mock instance +func NewMockMutex(ctrl *gomock.Controller) *MockMutex { + mock := &MockMutex{ctrl: ctrl} + mock.recorder = &MockMutexMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockMutex) EXPECT() *MockMutexMockRecorder { + return m.recorder +} + +// Lock mocks base method +func (m *MockMutex) Lock(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lock", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Lock indicates an expected call of Lock +func (mr *MockMutexMockRecorder) Lock(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockMutex)(nil).Lock), ctx) +} + +// Unlock mocks base method +func (m *MockMutex) Unlock(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Unlock", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Unlock indicates an expected call of Unlock +func (mr *MockMutexMockRecorder) Unlock(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unlock", reflect.TypeOf((*MockMutex)(nil).Unlock), ctx) +} From 623f699b25ed9e0eba8f86c8e245884ae93b029a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 17:31:40 +0100 Subject: [PATCH 020/191] Fix up bits broken by refactor ... but just so tests pass for now. --- .../v2/dacctl/actions_impl/actions_test.go | 45 ++++++++++--------- .../pkg/v2/dacctl/actions_impl/job_test.go | 5 ++- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index 6e98ab01..8e3f5b21 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -53,8 +54,8 @@ func (c *mockCliContext) Int(name string) int { func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} registry.EXPECT().CreateSession(datamodel.Session{ @@ -81,13 +82,13 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { func TestDacctlActions_DeleteBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession("token").Return(fakeSession, nil) + registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DeleteSession(fakeSession).Return(fakeError) + session.EXPECT().DeleteSession(fakeSession.Name).Return(fakeError) actions := NewDacctlActions(registry, session, nil) err := actions.DeleteBuffer(&mockCliContext{}) @@ -98,13 +99,13 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { func TestDacctlActions_DataIn(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession("token").Return(fakeSession, nil) + registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DataIn(fakeSession).Return(fakeError) + session.EXPECT().DataIn(fakeSession.Name).Return(fakeError) actions := NewDacctlActions(registry, session, nil) err := actions.DataIn(&mockCliContext{}) @@ -115,13 +116,13 @@ func TestDacctlActions_DataIn(t *testing.T) { func TestDacctlActions_DataOut(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession("token").Return(fakeSession, nil) + registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DataOut(fakeSession).Return(fakeError) + session.EXPECT().DataOut(fakeSession.Name).Return(fakeError) actions := NewDacctlActions(registry, session, nil) err := actions.DataOut(&mockCliContext{}) @@ -132,13 +133,13 @@ func TestDacctlActions_DataOut(t *testing.T) { func TestDacctlActions_PreRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession("token").Return(fakeSession, nil) + registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(fakeSession).Return(fakeError) + session.EXPECT().AttachVolumes(fakeSession.Name, nil).Return(fakeError) actions := NewDacctlActions(registry, session, nil) err := actions.PreRun(&mockCliContext{}) @@ -149,13 +150,13 @@ func TestDacctlActions_PreRun(t *testing.T) { func TestDacctlActions_PostRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession("token").Return(fakeSession, nil) + registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DetachVolumes(fakeSession).Return(fakeError) + session.EXPECT().DetachVolumes(fakeSession.Name).Return(fakeError) actions := NewDacctlActions(registry, session, nil) err := actions.PostRun(&mockCliContext{}) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 41756346..a91d0975 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -4,6 +4,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -12,8 +13,8 @@ import ( func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockRegistry(mockCtrl) - session := mock_registry.NewMockActions(mockCtrl) + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) lines := []string{ From 52038f217a4b18daa8bcd4a2260cf965cc8b434f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 22:07:36 +0100 Subject: [PATCH 021/191] Make the size parsing more robust --- .../dacctl/actions_impl/parsers/capacity.go | 8 ++-- .../actions_impl/parsers/capacity_test.go | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go index 0351a85c..1b0dab7b 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go @@ -3,6 +3,7 @@ package parsers import ( "errors" "fmt" + "math" "strconv" "strings" ) @@ -25,12 +26,13 @@ func ParseSize(raw string) (int, error) { } for suffix, multiplier := range sizeSuffixMultiplier { if strings.HasSuffix(raw, suffix) { - rawInt := strings.TrimSuffix(raw, suffix) - intVal, err := strconv.Atoi(rawInt) + rawInt := strings.TrimSpace(strings.TrimSuffix(raw, suffix)) + floatVal, err := strconv.ParseFloat(rawInt, 64) if err != nil { return 0, err } - return intVal * multiplier, nil + floatBytes := floatVal * float64(multiplier) + return int(math.Ceil(floatBytes)), nil } } return 0, fmt.Errorf("unable to parse size: %s", raw) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go new file mode 100644 index 00000000..d01f3cc6 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go @@ -0,0 +1,40 @@ +package parsers + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestParseSize(t *testing.T) { + size, err := ParseSize("10GiB") + assert.Nil(t, err) + assert.Equal(t, 10737418240, size) + + size, err = ParseSize("10GB") + assert.Nil(t, err) + assert.Equal(t, 10000000000, size) + + size, err = ParseSize("10 GB") + assert.Nil(t, err) + assert.Equal(t, 10000000000, size) + + size, err = ParseSize("10B") + assert.Equal(t,"unable to parse size: 10B", err.Error()) + + size, err = ParseSize("10.1234567MB") + assert.Nil(t, err) + assert.Equal(t, 10123457, size) + + size, err = ParseSize("1TiB") + assert.Nil(t, err) + assert.Equal(t, 1099511627776, size) + + size, err = ParseSize("1TB") + assert.Nil(t, err) + assert.Equal(t, 1000000000000, size) + + size, err = ParseSize("1MiB") + assert.Nil(t, err) + assert.Equal(t, 1048576, size) + +} \ No newline at end of file From eaeb13bc5261ceda88c4215870471249036fb180 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 22:16:35 +0100 Subject: [PATCH 022/191] Make ParseCapacityBytes more robust --- .../dacctl/actions_impl/parsers/capacity.go | 4 ++-- .../actions_impl/parsers/capacity_test.go | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go index 1b0dab7b..c4f24ee0 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go @@ -43,8 +43,8 @@ func ParseCapacityBytes(raw string) (string, int, error) { if len(parts) != 2 { return "", 0, errors.New("must format capacity correctly and include pool") } - pool := parts[0] - rawCapacity := parts[1] + pool := strings.TrimSpace(parts[0]) + rawCapacity := strings.TrimSpace(parts[1]) sizeBytes, err := ParseSize(rawCapacity) if err != nil { return "", 0, err diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go index d01f3cc6..a6443d42 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go @@ -36,5 +36,26 @@ func TestParseSize(t *testing.T) { size, err = ParseSize("1MiB") assert.Nil(t, err) assert.Equal(t, 1048576, size) +} +func TestParseCapacityBytes(t *testing.T) { + pool, size, err := ParseCapacityBytes("foo:1MB") + assert.Nil(t, err) + assert.Equal(t, "foo", pool) + assert.Equal(t, 1000000, size) + + pool, size, err = ParseCapacityBytes("foo : 1 MB") + assert.Equal(t, "foo", pool) + assert.Equal(t, 1000000, size) + assert.Nil(t, err) + + pool, size, err = ParseCapacityBytes("foo1MB") + assert.Equal(t, "must format capacity correctly and include pool", err.Error()) + assert.Equal(t, "", pool) + assert.Equal(t, 0, size) + + pool, size, err = ParseCapacityBytes("foo:1B") + assert.Equal(t, "unable to parse size: 1B", err.Error()) + assert.Equal(t, "", pool) + assert.Equal(t, 0, size) } \ No newline at end of file From 202b181524cb7061de20fa73fe9b34e7e0065a26 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 22:41:30 +0100 Subject: [PATCH 023/191] Better error handling in parseJobRequest --- .../pkg/v2/dacctl/actions_impl/parsers/job.go | 38 ++++++------------- .../dacctl/actions_impl/parsers/job_test.go | 29 ++++++++++++++ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go index ada535e4..0cca3033 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go @@ -1,7 +1,6 @@ package parsers import ( - "encoding/json" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" @@ -15,23 +14,11 @@ type jobSummary struct { Attachments []datamodel.VolumeName DataIn []datamodel.DataCopyRequest DataOut []datamodel.DataCopyRequest + // TODO: support create and destroy persistent? //createPersistent *cmdCreatePersistent //destroyPersistent *cmdDestroyPersistent } -func (s jobSummary) String() string { - return toJson(s) -} - -func toJson(message interface{}) string { - b, error := json.Marshal(message) - if error != nil { - log.Fatal(error) - } - return string(b) -} - -// Parse a given job file func ParseJobFile(disk fileio.Disk, filename string) (jobSummary, error) { lines, err := disk.Lines(filename) if err != nil { @@ -91,7 +78,7 @@ var stringToAccessMode = map[string]datamodel.AccessMode{ "striped,private": datamodel.PrivateAndStriped, } -func AccessModeFromString(raw string) datamodel.AccessMode { +func accessModeFromString(raw string) datamodel.AccessMode { return stringToAccessMode[strings.ToLower(raw)] } @@ -109,7 +96,7 @@ type cmdCreatePersistent struct { GenericCmd bool } -func BufferTypeFromString(raw string) datamodel.BufferType { +func bufferTypeFromString(raw string) datamodel.BufferType { return stringToBufferType[strings.ToLower(raw)] } @@ -188,14 +175,14 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { size, err := ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) - continue + return nil, err } command = cmdCreatePersistent{ Name: argKeyPair["name"], CapacityBytes: size, GenericCmd: isGeneric, - AccessMode: AccessModeFromString(argKeyPair["access_mode"]), - BufferType: BufferTypeFromString(argKeyPair["type"]), + AccessMode: accessModeFromString(argKeyPair["access_mode"]), + BufferType: bufferTypeFromString(argKeyPair["type"]), } case "destroy_persistent": command = cmdDestroyPersistent(argKeyPair["name"]) @@ -205,21 +192,21 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { size, err := ParseSize(argKeyPair["capacity"]) if err != nil { log.Println(err) - continue + return nil, err } command = cmdPerJobBuffer{ CapacityBytes: size, GenericCmd: isGeneric, - AccessMode: AccessModeFromString(argKeyPair["access_mode"]), - BufferType: BufferTypeFromString(argKeyPair["type"]), + AccessMode: accessModeFromString(argKeyPair["access_mode"]), + BufferType: bufferTypeFromString(argKeyPair["type"]), } case "swap": if len(args) != 1 { - log.Println("Unable to parse swap command:", line) + return nil, fmt.Errorf("unable to parse swap command: %s", line) } if size, err := ParseSize(args[0]); err != nil { log.Println(err) - continue + return nil, err } else { command = cmdAttachPerJobSwap{SizeBytes: size} } @@ -236,8 +223,7 @@ func parseJobRequest(lines []string) ([]jobCommand, error) { SourceType: sourceTypeFromString(argKeyPair["type"]), } default: - log.Println("unrecognised command:", cmd, "with argument length", len(args)) - continue + return nil, fmt.Errorf("unrecognised command: %s with arguments: %s", cmd, args) } commands = append(commands, command) } diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index a2f3eec8..0bd27094 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -42,6 +42,8 @@ func TestGetJobSummary(t *testing.T) { `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, `#DW stage_in source=/global/cscratch1/filename2 destination=$DW_JOB_STRIPED/filename2 type=file`, `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, + `skipping other lines that we`, + `don't understand`, } result, err := getJobSummary(lines) @@ -59,3 +61,30 @@ func TestGetJobSummary(t *testing.T) { assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) assert.Equal(t, 4000000, result.Swap.SizeBytes) } + +func TestGetJobSummary_Errors(t *testing.T) { + lines := []string{`#DW bad_command asdf=asdf`} + result, err := getJobSummary(lines) + assert.Equal(t, "unrecognised command: bad_command with arguments: [asdf=asdf]", err.Error()) + assert.Nil(t, result.PerJobBuffer) + + lines = []string{`#DW swap 1B asdf`} + result, err = getJobSummary(lines) + assert.Equal(t, "unable to parse swap command: #DW swap 1B asdf", err.Error()) + assert.Nil(t, result.PerJobBuffer) + + lines = []string{`#DW swap 1B`} + result, err = getJobSummary(lines) + assert.Equal(t, "unable to parse size: 1B", err.Error()) + assert.Nil(t, result.PerJobBuffer) + + lines = []string{`#DW jobdw capacity=4B access_mode=striped,private type=scratch`} + result, err = getJobSummary(lines) + assert.Equal(t, "unable to parse size: 4B", err.Error()) + assert.Nil(t, result.PerJobBuffer) + + lines = []string{`#BB create_persistent name=myBBname capacity=100B access_mode=striped type=scratch`} + result, err = getJobSummary(lines) + assert.Equal(t, "unable to parse size: 100B", err.Error()) + assert.Nil(t, result.PerJobBuffer) +} From 0b52aafee8f0e53639c50a0c99a7b664dee41451 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 14 Aug 2019 22:56:01 +0100 Subject: [PATCH 024/191] Test ParseJobFile --- .../actions_impl/parsers/capacity_test.go | 4 +++ .../pkg/v2/dacctl/actions_impl/parsers/job.go | 6 ++-- .../dacctl/actions_impl/parsers/job_test.go | 36 +++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go index a6443d42..6c500104 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go @@ -36,6 +36,10 @@ func TestParseSize(t *testing.T) { size, err = ParseSize("1MiB") assert.Nil(t, err) assert.Equal(t, 1048576, size) + + size, err = ParseSize("AMiB") + assert.Equal(t, "strconv.ParseFloat: parsing \"A\": invalid syntax", err.Error()) + assert.Equal(t, 0, size) } func TestParseCapacityBytes(t *testing.T) { diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go index 0cca3033..05ca7c29 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go @@ -40,13 +40,13 @@ func getJobSummary(lines []string) (jobSummary, error) { if summary.PerJobBuffer == nil { summary.PerJobBuffer = &c } else { - return summary, fmt.Errorf("only one per job buffer allowed") + return jobSummary{}, fmt.Errorf("only one per job buffer allowed") } case cmdAttachPersistent: summary.Attachments = append(summary.Attachments, datamodel.VolumeName(c)) case cmdAttachPerJobSwap: if summary.Swap != nil { - return summary, fmt.Errorf("only one swap request allowed") + return jobSummary{}, fmt.Errorf("only one swap request allowed") } summary.Swap = &c case cmdStageOutData: @@ -61,8 +61,6 @@ func getJobSummary(lines []string) (jobSummary, error) { Source: c.Source, Destination: c.Destination, }) - default: - // do nothing } } return summary, nil diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index 0bd27094..e8828e82 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -1,12 +1,35 @@ package parsers import ( + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "log" "testing" ) +func TestParseJobFile(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockFileIO := mocks.NewMockDisk(mockCtrl) + mockFileIO.EXPECT().Lines("testfile").Return(nil, errors.New("asdf")) + + lines, err := ParseJobFile(mockFileIO, "testfile") + assert.Equal(t, jobSummary{}, lines) + assert.Equal(t, "asdf", err.Error()) + + mockFileIO = mocks.NewMockDisk(mockCtrl) + mockFileIO.EXPECT().Lines("testfile").Return([]string{`#DW swap asdf`}, nil) + + lines, err = ParseJobFile(mockFileIO, "testfile") + assert.Equal(t, jobSummary{}, lines) + assert.Equal(t, "unable to parse size: asdf", err.Error()) + +} + func TestParseJobRequest(t *testing.T) { jobRequest := []string{ `#BB create_persistent name=myBBname capacity=100GB access_mode=striped type=scratch`, @@ -87,4 +110,17 @@ func TestGetJobSummary_Errors(t *testing.T) { result, err = getJobSummary(lines) assert.Equal(t, "unable to parse size: 100B", err.Error()) assert.Nil(t, result.PerJobBuffer) + + lines = []string{`#DW swap 1MB`, `#DW swap 2MB`} + result, err = getJobSummary(lines) + assert.Equal(t, "only one swap request allowed", err.Error()) + assert.Nil(t, result.PerJobBuffer) + + lines = []string{ + `#DW jobdw capacity=4MiB access_mode=private type=scratch`, + `#DW jobdw capacity=5MiB access_mode=striped type=scratch`, + } + result, err = getJobSummary(lines) + assert.Equal(t, "only one per job buffer allowed", err.Error()) + assert.Nil(t, result.PerJobBuffer) } From b293753a5ff3c946f6bd868361a719a348c637c3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 10:23:52 +0100 Subject: [PATCH 025/191] Rework v2/dacctl --- .../pkg/v2/dacctl/actions_impl/actions.go | 52 +++++--- .../v2/dacctl/actions_impl/actions_test.go | 125 +++++++----------- .../pkg/v2/dacctl/actions_impl/job_test.go | 2 +- .../pkg/v2/dacctl/actions_impl/persistent.go | 2 +- .../v2/dacctl/actions_impl/persistent_test.go | 38 ++++++ internal/pkg/v2/dacctl/interface.go | 1 + internal/pkg/v2/mock_workflow/session.go | 16 +-- internal/pkg/v2/workflow/session.go | 6 +- 8 files changed, 133 insertions(+), 109 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/persistent_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index f89a63ab..a66075b0 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -1,6 +1,7 @@ package actions_impl import ( + "errors" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" @@ -8,6 +9,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" + "regexp" "strings" ) @@ -25,7 +27,7 @@ type dacctlActions struct { disk fileio.Disk } -func checkRequiredStrings(c dacctl.CliContext, flags ...string) { +func checkRequiredStrings(c dacctl.CliContext, flags ...string) error { var errs []string for _, flag := range flags { if str := c.String(flag); str == "" { @@ -33,59 +35,67 @@ func checkRequiredStrings(c dacctl.CliContext, flags ...string) { } } if len(errs) > 0 { - log.Fatalf("Please provide these required parameters: %s", strings.Join(errs, ", ")) + errStr := fmt.Sprintf("Please provide these required parameters: %s", strings.Join(errs, ", ")) + log.Println(errStr) + return errors.New(errStr) } + return nil } -func (d *dacctlActions) getSession(c dacctl.CliContext) (datamodel.Session, error) { - // TODO - not sure we need this now...? - checkRequiredStrings(c, "token") - token := c.String("token") - sessionName := datamodel.SessionName(token) - s, err := d.registry.GetSession(sessionName) +func (d *dacctlActions) getSessionName(c dacctl.CliContext) (datamodel.SessionName, error) { + err := checkRequiredStrings(c, "token") if err != nil { - return s, fmt.Errorf("unable to find session for token %s", token) + return "", err + } + + token := c.String("token") + re := regexp.MustCompile("[a-zA-Z0-9]*") + if !re.Match([]byte(`seafood fool`)) { + return "", fmt.Errorf("badly formatted session name: %s", token) } - return s, nil + + return datamodel.SessionName(token), nil } func (d *dacctlActions) DeleteBuffer(c dacctl.CliContext) error { - s, err := d.getSession(c) + sessionName, err := d.getSessionName(c) if err != nil { return err } - return d.session.DeleteSession(s.Name) + hurry := c.Bool("hurry") + return d.session.DeleteSession(sessionName, hurry) } func (d *dacctlActions) DataIn(c dacctl.CliContext) error { - s, err := d.getSession(c) + sessionName, err := d.getSessionName(c) if err != nil { return err } - return d.session.DataIn(s.Name) + return d.session.DataIn(sessionName) } func (d *dacctlActions) PreRun(c dacctl.CliContext) error { - s, err := d.getSession(c) + sessionName, err := d.getSessionName(c) if err != nil { return err } - // TODO - fix attach hosts - return d.session.AttachVolumes(s.Name, nil) + + // TODO - fix hosts + return d.session.AttachVolumes(sessionName, nil, nil) } func (d *dacctlActions) PostRun(c dacctl.CliContext) error { - s, err := d.getSession(c) + sessionName, err := d.getSessionName(c) if err != nil { return err } - return d.session.DetachVolumes(s.Name) + return d.session.DetachVolumes(sessionName) } func (d *dacctlActions) DataOut(c dacctl.CliContext) error { - s, err := d.getSession(c) + sessionName, err := d.getSessionName(c) if err != nil { return err } - return d.session.DataOut(s.Name) + return d.session.DataOut(sessionName) } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index 8e3f5b21..afaaaabe 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -12,71 +12,40 @@ import ( ) type mockCliContext struct { - capacity int + strings map[string]string + integers map[string]int + booleans map[string]bool } func (c *mockCliContext) String(name string) string { - switch name { - case "capacity": - return fmt.Sprintf("pool1:%dGiB", c.capacity) - case "token": - return "token" - case "caller": - return "caller" - case "user": - return "user" - case "access": - return "access" - case "type": - return "type" - case "job": - return "jobfile" - case "nodehostnamefile": - return "nodehostnamefile1" - case "pathfile": - return "pathfile1" - default: - return "foobar1" - } + return c.strings[name] } func (c *mockCliContext) Int(name string) int { - switch name { - case "user": - return 1001 - case "group": - return 1002 - default: - return 42 + len(name) - } + return c.integers[name] } -func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) - session := mock_workflow.NewMockSession(mockCtrl) - - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().CreateSession(datamodel.Session{ - Name: "token", - Owner: 1001, - Group: 1002, - CreatedAt: 123, - VolumeRequest: datamodel.VolumeRequest{ - MultiJob: true, - Caller: "caller", - PoolName: "pool1", - TotalCapacityBytes: 2147483648, - }, - }).Return(fakeSession, nil) - session.EXPECT().CreateSessionVolume(fakeSession) - fakeTime = 123 - - actions := NewDacctlActions(registry, session, nil) - err := actions.CreatePersistentBuffer(&mockCliContext{capacity: 2}) +func (c *mockCliContext) Bool(name string) bool { + return c.booleans[name] +} - assert.Nil(t, err) +func getMockCliContext(capacity int) *mockCliContext { + ctxt := mockCliContext{} + ctxt.strings = map[string]string{ + "capacity": fmt.Sprintf("pool1:%dGiB", capacity), + "token": "token", + "caller": "caller", + "access": "asdf", + "type": "type", + "job": "jobfile", + "nodehostnamefile": "nodehostnamefile1", + "pathfile": "pathfile1", + } + ctxt.integers = map[string]int{ + "user": 1001, + "group": 1002, + } + return &ctxt } func TestDacctlActions_DeleteBuffer(t *testing.T) { @@ -85,13 +54,14 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DeleteSession(fakeSession.Name).Return(fakeError) + session.EXPECT().DeleteSession(datamodel.SessionName("bar"), true).Return(fakeError) actions := NewDacctlActions(registry, session, nil) - err := actions.DeleteBuffer(&mockCliContext{}) + err := actions.DeleteBuffer(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + booleans: map[string]bool{"hurry": true}, + }) assert.Equal(t, fakeError, err) } @@ -102,15 +72,18 @@ func TestDacctlActions_DataIn(t *testing.T) { registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DataIn(fakeSession.Name).Return(fakeError) + session.EXPECT().DataIn(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(registry, session, nil) - err := actions.DataIn(&mockCliContext{}) + err := actions.DataIn(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + }) assert.Equal(t, fakeError, err) + + err = actions.DataIn(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) } func TestDacctlActions_DataOut(t *testing.T) { @@ -119,13 +92,13 @@ func TestDacctlActions_DataOut(t *testing.T) { registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DataOut(fakeSession.Name).Return(fakeError) + session.EXPECT().DataOut(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(registry, session, nil) - err := actions.DataOut(&mockCliContext{}) + err := actions.DataOut(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + }) assert.Equal(t, fakeError, err) } @@ -136,13 +109,13 @@ func TestDacctlActions_PreRun(t *testing.T) { registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(fakeSession.Name, nil).Return(fakeError) + session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), nil, nil).Return(fakeError) actions := NewDacctlActions(registry, session, nil) - err := actions.PreRun(&mockCliContext{}) + err := actions.PreRun(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + }) assert.Equal(t, fakeError, err) } @@ -153,13 +126,13 @@ func TestDacctlActions_PostRun(t *testing.T) { registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().GetSession(datamodel.SessionName("token")).Return(fakeSession, nil) fakeError := errors.New("fake") - session.EXPECT().DetachVolumes(fakeSession.Name).Return(fakeError) + session.EXPECT().DetachVolumes(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(registry, session, nil) - err := actions.PostRun(&mockCliContext{}) + err := actions.PostRun(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + }) assert.Equal(t, fakeError, err) } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index a91d0975..1f8ffe16 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -71,7 +71,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { fakeTime = 123 actions := NewDacctlActions(registry, session, disk) - err := actions.CreatePerJobBuffer(&mockCliContext{capacity: 2}) + err := actions.CreatePerJobBuffer(getMockCliContext(2)) assert.Nil(t, err) } diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go index 91405105..168d474d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -40,7 +40,7 @@ func getNow() uint { } func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { - checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") + checkRequiredStrings(c, "token", "caller", "capacity", "access", "type") pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) if err != nil { return err diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go new file mode 100644 index 00000000..54ee9696 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go @@ -0,0 +1,38 @@ +package actions_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) + + fakeSession := datamodel.Session{Name: "foo"} + registry.EXPECT().CreateSession(datamodel.Session{ + Name: "token", + Owner: 1001, + Group: 1002, + CreatedAt: 123, + VolumeRequest: datamodel.VolumeRequest{ + MultiJob: true, + Caller: "caller", + PoolName: "pool1", + TotalCapacityBytes: 2147483648, + }, + }).Return(fakeSession, nil) + session.EXPECT().CreateSessionVolume(fakeSession) + fakeTime = 123 + + actions := NewDacctlActions(registry, session, nil) + err := actions.CreatePersistentBuffer(getMockCliContext(2)) + + assert.Nil(t, err) +} diff --git a/internal/pkg/v2/dacctl/interface.go b/internal/pkg/v2/dacctl/interface.go index 246ec2af..9c760ff0 100644 --- a/internal/pkg/v2/dacctl/interface.go +++ b/internal/pkg/v2/dacctl/interface.go @@ -3,6 +3,7 @@ package dacctl type CliContext interface { String(name string) string Int(name string) int + Bool(name string) bool } type DacctlActions interface { diff --git a/internal/pkg/v2/mock_workflow/session.go b/internal/pkg/v2/mock_workflow/session.go index 104fb214..a3b48021 100644 --- a/internal/pkg/v2/mock_workflow/session.go +++ b/internal/pkg/v2/mock_workflow/session.go @@ -48,17 +48,17 @@ func (mr *MockSessionMockRecorder) CreateSessionVolume(session interface{}) *gom } // DeleteSession mocks base method -func (m *MockSession) DeleteSession(sessionName datamodel.SessionName) error { +func (m *MockSession) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSession", sessionName) + ret := m.ctrl.Call(m, "DeleteSession", sessionName, hurry) ret0, _ := ret[0].(error) return ret0 } // DeleteSession indicates an expected call of DeleteSession -func (mr *MockSessionMockRecorder) DeleteSession(sessionName interface{}) *gomock.Call { +func (mr *MockSessionMockRecorder) DeleteSession(sessionName, hurry interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSession)(nil).DeleteSession), sessionName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSession)(nil).DeleteSession), sessionName, hurry) } // DataIn mocks base method @@ -76,17 +76,17 @@ func (mr *MockSessionMockRecorder) DataIn(sessionName interface{}) *gomock.Call } // AttachVolumes mocks base method -func (m *MockSession) AttachVolumes(sessionName datamodel.SessionName, attachHosts []string) error { +func (m *MockSession) AttachVolumes(sessionName datamodel.SessionName, computeNodes, loginNodes []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AttachVolumes", sessionName, attachHosts) + ret := m.ctrl.Call(m, "AttachVolumes", sessionName, computeNodes, loginNodes) ret0, _ := ret[0].(error) return ret0 } // AttachVolumes indicates an expected call of AttachVolumes -func (mr *MockSessionMockRecorder) AttachVolumes(sessionName, attachHosts interface{}) *gomock.Call { +func (mr *MockSessionMockRecorder) AttachVolumes(sessionName, computeNodes, loginNodes interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockSession)(nil).AttachVolumes), sessionName, attachHosts) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockSession)(nil).AttachVolumes), sessionName, computeNodes, loginNodes) } // DetachVolumes mocks base method diff --git a/internal/pkg/v2/workflow/session.go b/internal/pkg/v2/workflow/session.go index a84ad8ae..7db40959 100644 --- a/internal/pkg/v2/workflow/session.go +++ b/internal/pkg/v2/workflow/session.go @@ -10,13 +10,15 @@ type Session interface { CreateSessionVolume(session datamodel.Session) error // Deletes the requested volume and session allocation - DeleteSession(sessionName datamodel.SessionName) error + // If hurry, there is no stage-out attempted + // Unmount is always attempted before deleting the buffer + DeleteSession(sessionName datamodel.SessionName, hurry bool) error // Update the session and trigger requested data copy in DataIn(sessionName datamodel.SessionName) error // Update session hosts and attach volumes as needed - AttachVolumes(sessionName datamodel.SessionName, attachHosts []string) error + AttachVolumes(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error // Attempt to detach volumes DetachVolumes(sessionName datamodel.SessionName) error From d6e7e3177f3a18903c7ee182bf34fb3efee13d4f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 12:13:39 +0100 Subject: [PATCH 026/191] Improve coverage and robustness of dacctl actions --- .../pkg/v2/dacctl/actions_impl/actions.go | 29 +++- .../v2/dacctl/actions_impl/actions_test.go | 134 ++++++++++++++++-- .../dacctl/actions_impl/parsers/hostnames.go | 30 ++++ .../actions_impl/parsers/hostnames_test.go | 54 +++++++ 4 files changed, 229 insertions(+), 18 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go create mode 100644 internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index a66075b0..c00f6a73 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -5,11 +5,11 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" - "regexp" "strings" ) @@ -49,8 +49,7 @@ func (d *dacctlActions) getSessionName(c dacctl.CliContext) (datamodel.SessionNa } token := c.String("token") - re := regexp.MustCompile("[a-zA-Z0-9]*") - if !re.Match([]byte(`seafood fool`)) { + if !parsers.IsValidName(token) { return "", fmt.Errorf("badly formatted session name: %s", token) } @@ -79,9 +78,29 @@ func (d *dacctlActions) PreRun(c dacctl.CliContext) error { if err != nil { return err } + err = checkRequiredStrings(c, "nodehostnamefile") + if err != nil { + return err + } + + computeHosts, err := parsers.GetHostnamesFromFile(d.disk, c.String("nodehostnamefile")) + if err != nil { + return err + } + if len(computeHosts) < 1 { + return errors.New("unable to mount to zero compute hosts") + } + + loginNodeFilename := c.String("jobexecutionnodefile") + var loginNodeHosts []string + if loginNodeFilename != "" { + loginNodeHosts, err = parsers.GetHostnamesFromFile(d.disk, loginNodeFilename) + if err != nil { + return err + } + } - // TODO - fix hosts - return d.session.AttachVolumes(sessionName, nil, nil) + return d.session.AttachVolumes(sessionName, computeHosts, loginNodeHosts) } func (d *dacctlActions) PostRun(c dacctl.CliContext) error { diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index afaaaabe..576e6cbe 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -3,8 +3,8 @@ package actions_impl import ( "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -51,31 +51,32 @@ func getMockCliContext(capacity int) *mockCliContext { func TestDacctlActions_DeleteBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) fakeError := errors.New("fake") session.EXPECT().DeleteSession(datamodel.SessionName("bar"), true).Return(fakeError) - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(nil, session, nil) err := actions.DeleteBuffer(&mockCliContext{ strings: map[string]string{"token": "bar"}, booleans: map[string]bool{"hurry": true}, }) assert.Equal(t, fakeError, err) + + err = actions.DeleteBuffer(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) } func TestDacctlActions_DataIn(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) fakeError := errors.New("fake") session.EXPECT().DataIn(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(nil, session, nil) err := actions.DataIn(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -84,55 +85,162 @@ func TestDacctlActions_DataIn(t *testing.T) { err = actions.DataIn(&mockCliContext{}) assert.Equal(t, "Please provide these required parameters: token", err.Error()) + + err = actions.DataIn(&mockCliContext{ + strings: map[string]string{"token": "bad token"}, + }) + assert.Equal(t, "badly formatted session name: bad token", err.Error()) } func TestDacctlActions_DataOut(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) fakeError := errors.New("fake") session.EXPECT().DataOut(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(nil, session, nil) err := actions.DataOut(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) assert.Equal(t, fakeError, err) + + err = actions.DataOut(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) } func TestDacctlActions_PreRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + computeHosts := []string{"host1", "host2"} + loginHosts := []string{"login"} + disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) + disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), nil, nil).Return(fakeError) + session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(nil, session, disk) err := actions.PreRun(&mockCliContext{ - strings: map[string]string{"token": "bar"}, + strings: map[string]string{ + "token": "bar", + "nodehostnamefile": "computehostfile", + "jobexecutionnodefile": "loginhostfile", + }, }) assert.Equal(t, fakeError, err) + + err = actions.PreRun(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) + + err = actions.PreRun(&mockCliContext{strings: map[string]string{"token":"bar"}}) + assert.Equal(t, "Please provide these required parameters: nodehostnamefile", err.Error()) +} + +func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + + computeHosts := []string{"host1", "host2"} + disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) + fakeError := errors.New("fake") + session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, nil).Return(fakeError) + + actions := NewDacctlActions(nil, session, disk) + err := actions.PreRun(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "nodehostnamefile": "computehostfile", + }, + }) + + assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_PreRun_BadHosts(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + disk := mocks.NewMockDisk(mockCtrl) + + computeHosts := []string{"host1", "host/2"} + disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) + + actions := NewDacctlActions(nil, nil, disk) + err := actions.PreRun(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "nodehostnamefile": "computehostfile", + }, + }) + + assert.Equal(t, "invalid hostname in: [host/2]", err.Error()) +} + +func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + + computeHosts := []string{"host1", "host2"} + loginHosts := []string{"login/asdf"} + disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) + disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) + fakeError := errors.New("fake") + session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) + + actions := NewDacctlActions(nil, session, disk) + err := actions.PreRun(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "nodehostnamefile": "computehostfile", + "jobexecutionnodefile": "loginhostfile", + }, + }) + + assert.Equal(t, "invalid hostname in: [login/asdf]", err.Error()) +} + +func TestDacctlActions_PreRun_NoHosts(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + disk := mocks.NewMockDisk(mockCtrl) + + disk.EXPECT().Lines("computehostfile").Return(nil, nil) + + actions := NewDacctlActions(nil, nil, disk) + err := actions.PreRun(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "nodehostnamefile": "computehostfile", + }, + }) + + assert.Equal(t, "unable to mount to zero compute hosts", err.Error()) } func TestDacctlActions_PostRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) fakeError := errors.New("fake") session.EXPECT().DetachVolumes(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(nil, session, nil) err := actions.PostRun(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) assert.Equal(t, fakeError, err) + + err = actions.PostRun(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) } diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go new file mode 100644 index 00000000..3b08e7cd --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go @@ -0,0 +1,30 @@ +package parsers + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "regexp" +) + +var nameRegex = regexp.MustCompile("^[a-zA-Z0-9.]*$") + +func IsValidName(name string) bool { + return nameRegex.Match([]byte(name)) +} + +func GetHostnamesFromFile(disk fileio.Disk, filename string) ([]string, error) { + computeHosts, err := disk.Lines(filename) + if err != nil { + return nil, err + } + var invalidHosts []string + for _, computeHost := range computeHosts { + if !IsValidName(computeHost) { + invalidHosts = append(invalidHosts, computeHost) + } + } + if len(invalidHosts) > 0 { + return nil, fmt.Errorf("invalid hostname in: %s", invalidHosts) + } + return computeHosts, nil +} \ No newline at end of file diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go new file mode 100644 index 00000000..e1a608bf --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go @@ -0,0 +1,54 @@ +package parsers + +import ( + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetHostnamesFromFile(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + disk := mocks.NewMockDisk(mockCtrl) + + fakeHosts := []string{"test1", "test2"} + disk.EXPECT().Lines("file").Return(fakeHosts, nil) + + hosts, err := GetHostnamesFromFile(disk, "file") + assert.Nil(t, err) + assert.Equal(t, fakeHosts, hosts) +} + +func TestGetHostnamesFromFile_Empty(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + disk := mocks.NewMockDisk(mockCtrl) + + disk.EXPECT().Lines("file").Return(nil, nil) + + hosts, err := GetHostnamesFromFile(disk, "file") + assert.Nil(t, err) + var fakeHosts []string + assert.Equal(t, fakeHosts, hosts) + + fakeErr := errors.New("bob") + disk.EXPECT().Lines("file").Return(nil, fakeErr) + hosts, err = GetHostnamesFromFile(disk, "file") + assert.Equal(t, fakeHosts, hosts) + assert.Equal(t, "bob", err.Error()) +} + +func TestGetHostnamesFromFile_ErrorOnBadHostname(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + disk := mocks.NewMockDisk(mockCtrl) + + fakeHosts := []string{"Test", "test", "test1", "test2.com", "bad hostname", "foo/bar"} + disk.EXPECT().Lines("file").Return(fakeHosts, nil) + + hosts, err := GetHostnamesFromFile(disk, "file") + assert.Nil(t, hosts) + assert.Equal(t, "invalid hostname in: [bad hostname foo/bar]", err.Error()) +} From 1e8d5eeb68940cb014862bf5f3e4be2afa36df7a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 12:22:20 +0100 Subject: [PATCH 027/191] Add coverage for validate job --- internal/pkg/v2/dacctl/actions_impl/job_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 1f8ffe16..ad328d7d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -10,6 +10,23 @@ import ( "testing" ) +func TestDacctlActions_ValidateJob(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + session := mock_workflow.NewMockSession(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + + lines := []string{ + `#DW bad cmd`, + } + disk.EXPECT().Lines("jobfile").Return(lines, nil) + actions := NewDacctlActions(registry, session, disk) + err := actions.ValidateJob(getMockCliContext(2)) + + assert.Equal(t, "unrecognised command: bad with arguments: [cmd]", err.Error()) +} + func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() From a39a4494ff1d26b15fcdd4def497caf1911e2beb Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 12:30:58 +0100 Subject: [PATCH 028/191] Fix up formatting --- .../v2/dacctl/actions_impl/actions_test.go | 22 +++++++++---------- internal/pkg/v2/dacctl/actions_impl/job.go | 8 +++++-- .../pkg/v2/dacctl/actions_impl/job_test.go | 15 ++++++++----- .../actions_impl/parsers/capacity_test.go | 4 ++-- .../dacctl/actions_impl/parsers/hostnames.go | 2 +- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index 576e6cbe..6b0dc693 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -12,7 +12,7 @@ import ( ) type mockCliContext struct { - strings map[string]string + strings map[string]string integers map[string]int booleans map[string]bool } @@ -58,7 +58,7 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { actions := NewDacctlActions(nil, session, nil) err := actions.DeleteBuffer(&mockCliContext{ - strings: map[string]string{"token": "bar"}, + strings: map[string]string{"token": "bar"}, booleans: map[string]bool{"hurry": true}, }) @@ -127,8 +127,8 @@ func TestDacctlActions_PreRun(t *testing.T) { actions := NewDacctlActions(nil, session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ - "token": "bar", - "nodehostnamefile": "computehostfile", + "token": "bar", + "nodehostnamefile": "computehostfile", "jobexecutionnodefile": "loginhostfile", }, }) @@ -138,7 +138,7 @@ func TestDacctlActions_PreRun(t *testing.T) { err = actions.PreRun(&mockCliContext{}) assert.Equal(t, "Please provide these required parameters: token", err.Error()) - err = actions.PreRun(&mockCliContext{strings: map[string]string{"token":"bar"}}) + err = actions.PreRun(&mockCliContext{strings: map[string]string{"token": "bar"}}) assert.Equal(t, "Please provide these required parameters: nodehostnamefile", err.Error()) } @@ -156,7 +156,7 @@ func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { actions := NewDacctlActions(nil, session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ - "token": "bar", + "token": "bar", "nodehostnamefile": "computehostfile", }, }) @@ -175,7 +175,7 @@ func TestDacctlActions_PreRun_BadHosts(t *testing.T) { actions := NewDacctlActions(nil, nil, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ - "token": "bar", + "token": "bar", "nodehostnamefile": "computehostfile", }, }) @@ -193,14 +193,12 @@ func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { loginHosts := []string{"login/asdf"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) - fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) actions := NewDacctlActions(nil, session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ - "token": "bar", - "nodehostnamefile": "computehostfile", + "token": "bar", + "nodehostnamefile": "computehostfile", "jobexecutionnodefile": "loginhostfile", }, }) @@ -218,7 +216,7 @@ func TestDacctlActions_PreRun_NoHosts(t *testing.T) { actions := NewDacctlActions(nil, nil, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ - "token": "bar", + "token": "bar", "nodehostnamefile": "computehostfile", }, }) diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index 33d2e8a0..38912abe 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -9,7 +9,11 @@ import ( ) func (d *dacctlActions) ValidateJob(c dacctl.CliContext) error { - checkRequiredStrings(c, "job") + err := checkRequiredStrings(c, "job") + if err != nil { + return err + } + jobFile := c.String("job") summary, err := parsers.ParseJobFile(d.disk, jobFile) if err != nil { @@ -51,7 +55,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { bufferType := datamodel.Scratch if summary.PerJobBuffer != nil { access = summary.PerJobBuffer.AccessMode - if summary.PerJobBuffer.BufferType == datamodel.Cache { + if summary.PerJobBuffer.BufferType != datamodel.Scratch { return fmt.Errorf("cache is not supported") } } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index ad328d7d..40c0c88c 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -10,21 +10,26 @@ import ( "testing" ) -func TestDacctlActions_ValidateJob(t *testing.T) { +func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) - lines := []string{ - `#DW bad cmd`, - } + lines := []string{`#DW bad cmd`} disk.EXPECT().Lines("jobfile").Return(lines, nil) actions := NewDacctlActions(registry, session, disk) - err := actions.ValidateJob(getMockCliContext(2)) + err := actions.ValidateJob(&mockCliContext{ + strings: map[string]string{ + "job": "jobfile", + }, + }) assert.Equal(t, "unrecognised command: bad with arguments: [cmd]", err.Error()) + + err = actions.ValidateJob(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: job", err.Error()) } func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go index 6c500104..b130621f 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go @@ -19,7 +19,7 @@ func TestParseSize(t *testing.T) { assert.Equal(t, 10000000000, size) size, err = ParseSize("10B") - assert.Equal(t,"unable to parse size: 10B", err.Error()) + assert.Equal(t, "unable to parse size: 10B", err.Error()) size, err = ParseSize("10.1234567MB") assert.Nil(t, err) @@ -62,4 +62,4 @@ func TestParseCapacityBytes(t *testing.T) { assert.Equal(t, "unable to parse size: 1B", err.Error()) assert.Equal(t, "", pool) assert.Equal(t, 0, size) -} \ No newline at end of file +} diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go index 3b08e7cd..23a6033d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go @@ -27,4 +27,4 @@ func GetHostnamesFromFile(disk fileio.Disk, filename string) ([]string, error) { return nil, fmt.Errorf("invalid hostname in: %s", invalidHosts) } return computeHosts, nil -} \ No newline at end of file +} From 9afca39985286f9cc478fbb7e0a0452f4cae635b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 12:38:10 +0100 Subject: [PATCH 029/191] Remove registry from dacctl actions --- internal/pkg/v2/dacctl/actions_impl/actions.go | 13 +++++-------- .../pkg/v2/dacctl/actions_impl/actions_test.go | 18 +++++++++--------- internal/pkg/v2/dacctl/actions_impl/job.go | 4 ---- .../pkg/v2/dacctl/actions_impl/job_test.go | 15 +++++---------- .../pkg/v2/dacctl/actions_impl/persistent.go | 4 ---- .../v2/dacctl/actions_impl/persistent_test.go | 10 +++------- internal/pkg/v2/dacctl/actions_impl/show.go | 12 ++++++------ 7 files changed, 28 insertions(+), 48 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index c00f6a73..c6056272 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -7,24 +7,21 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" "strings" ) -func NewDacctlActions(registry registry.SessionRegistry, actions workflow.Session, disk fileio.Disk) dacctl.DacctlActions { +func NewDacctlActions(actions workflow.Session, disk fileio.Disk) dacctl.DacctlActions { return &dacctlActions{ - registry: registry, - session: actions, - disk: disk, + session: actions, + disk: disk, } } type dacctlActions struct { - registry registry.SessionRegistry - session workflow.Session - disk fileio.Disk + session workflow.Session + disk fileio.Disk } func checkRequiredStrings(c dacctl.CliContext, flags ...string) error { diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index 6b0dc693..f2c7ad4a 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -56,7 +56,7 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().DeleteSession(datamodel.SessionName("bar"), true).Return(fakeError) - actions := NewDacctlActions(nil, session, nil) + actions := NewDacctlActions(session, nil) err := actions.DeleteBuffer(&mockCliContext{ strings: map[string]string{"token": "bar"}, booleans: map[string]bool{"hurry": true}, @@ -76,7 +76,7 @@ func TestDacctlActions_DataIn(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().DataIn(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(nil, session, nil) + actions := NewDacctlActions(session, nil) err := actions.DataIn(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -100,7 +100,7 @@ func TestDacctlActions_DataOut(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().DataOut(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(nil, session, nil) + actions := NewDacctlActions(session, nil) err := actions.DataOut(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -124,7 +124,7 @@ func TestDacctlActions_PreRun(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) - actions := NewDacctlActions(nil, session, disk) + actions := NewDacctlActions(session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -153,7 +153,7 @@ func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, nil).Return(fakeError) - actions := NewDacctlActions(nil, session, disk) + actions := NewDacctlActions(session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -172,7 +172,7 @@ func TestDacctlActions_PreRun_BadHosts(t *testing.T) { computeHosts := []string{"host1", "host/2"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) - actions := NewDacctlActions(nil, nil, disk) + actions := NewDacctlActions(nil, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -194,7 +194,7 @@ func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) - actions := NewDacctlActions(nil, session, disk) + actions := NewDacctlActions(session, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -213,7 +213,7 @@ func TestDacctlActions_PreRun_NoHosts(t *testing.T) { disk.EXPECT().Lines("computehostfile").Return(nil, nil) - actions := NewDacctlActions(nil, nil, disk) + actions := NewDacctlActions(nil, disk) err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -232,7 +232,7 @@ func TestDacctlActions_PostRun(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().DetachVolumes(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(nil, session, nil) + actions := NewDacctlActions(session, nil) err := actions.PostRun(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index 38912abe..aa0c5c01 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -85,10 +85,6 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { } session.Paths = getPaths(session) - session, err = d.registry.CreateSession(session) - if err != nil { - return err - } return d.session.CreateSessionVolume(session) } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 40c0c88c..b649c97b 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -3,7 +3,6 @@ package actions_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -13,13 +12,12 @@ import ( func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) lines := []string{`#DW bad cmd`} disk.EXPECT().Lines("jobfile").Return(lines, nil) - actions := NewDacctlActions(registry, session, disk) + actions := NewDacctlActions(session, disk) err := actions.ValidateJob(&mockCliContext{ strings: map[string]string{ "job": "jobfile", @@ -35,7 +33,6 @@ func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) @@ -49,8 +46,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, } disk.EXPECT().Lines("jobfile").Return(lines, nil) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().CreateSession(datamodel.Session{ + session.EXPECT().CreateSessionVolume(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, @@ -88,11 +84,10 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { "DW_PERSISTENT_STRIPED_myBBname1": "/dac/token_persistent_myBBname1", "DW_PERSISTENT_STRIPED_myBBname2": "/dac/token_persistent_myBBname2", }, - }).Return(fakeSession, nil) - session.EXPECT().CreateSessionVolume(fakeSession) - fakeTime = 123 + }).Return(nil) - actions := NewDacctlActions(registry, session, disk) + fakeTime = 123 + actions := NewDacctlActions(session, disk) err := actions.CreatePerJobBuffer(getMockCliContext(2)) assert.Nil(t, err) diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go index 168d474d..e5a15216 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -60,9 +60,5 @@ func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { Group: uint(c.Int("group")), CreatedAt: getNow(), } - session, err = d.registry.CreateSession(session) - if err != nil { - return err - } return d.session.CreateSessionVolume(session) } diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go index 54ee9696..990c0e71 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go @@ -2,7 +2,6 @@ package actions_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -12,11 +11,9 @@ import ( func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionRegistry(mockCtrl) session := mock_workflow.NewMockSession(mockCtrl) - fakeSession := datamodel.Session{Name: "foo"} - registry.EXPECT().CreateSession(datamodel.Session{ + session.EXPECT().CreateSessionVolume(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, @@ -27,11 +24,10 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { PoolName: "pool1", TotalCapacityBytes: 2147483648, }, - }).Return(fakeSession, nil) - session.EXPECT().CreateSessionVolume(fakeSession) + }).Return(nil) fakeTime = 123 - actions := NewDacctlActions(registry, session, nil) + actions := NewDacctlActions(session, nil) err := actions.CreatePersistentBuffer(getMockCliContext(2)) assert.Nil(t, err) diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index fa529beb..4864885f 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -4,26 +4,26 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" ) -func (*dacctlActions) RealSize(c dacctl.CliContext) error { +func (d *dacctlActions) RealSize(c dacctl.CliContext) error { panic("implement me") } -func (*dacctlActions) Paths(c dacctl.CliContext) error { +func (d *dacctlActions) Paths(c dacctl.CliContext) error { panic("implement me") } -func (*dacctlActions) ShowInstances() error { +func (d *dacctlActions) ShowInstances() error { panic("implement me") } -func (*dacctlActions) ShowSessions() error { +func (d *dacctlActions) ShowSessions() error { panic("implement me") } -func (*dacctlActions) ListPools() error { +func (d *dacctlActions) ListPools() error { panic("implement me") } -func (*dacctlActions) ShowConfigurations() error { +func (d *dacctlActions) ShowConfigurations() error { panic("implement me") } From cefc1d0fbc94720b6580d4e52e0583ab78e5e380 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 12:50:36 +0100 Subject: [PATCH 030/191] Implement RealSize --- internal/pkg/v2/dacctl/actions_impl/show.go | 15 +++++++++-- .../pkg/v2/dacctl/actions_impl/show_test.go | 27 +++++++++++++++++++ internal/pkg/v2/dacctl/interface.go | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/show_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index 4864885f..05b7fc7c 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -1,11 +1,22 @@ package actions_impl import ( + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" ) -func (d *dacctlActions) RealSize(c dacctl.CliContext) error { - panic("implement me") +func (d *dacctlActions) RealSize(c dacctl.CliContext) (string, error) { + sessionName, err := d.getSessionName(c) + if err != nil { + return "", err + } + session, err := d.session.GetSession(sessionName) + if err != nil { + return "", err + } + return fmt.Sprintf( + `{"token":"%s", "capacity":%d, "units":"bytes"}`, + session.Name, session.ActualSizeBytes), nil } func (d *dacctlActions) Paths(c dacctl.CliContext) error { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go new file mode 100644 index 00000000..3c251809 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -0,0 +1,27 @@ +package actions_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDacctlActions_RealSize(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{ + Name: datamodel.SessionName("bar"), + ActualSizeBytes: 123, + }, nil) + + actions := NewDacctlActions(session, nil) + output, err := actions.RealSize(&mockCliContext{ + strings: map[string]string{"token": "bar"}, + }) + + assert.Nil(t, err) + assert.Equal(t, `{"token":"bar", "capacity":123, "units":"bytes"}`, output) +} diff --git a/internal/pkg/v2/dacctl/interface.go b/internal/pkg/v2/dacctl/interface.go index 9c760ff0..804ed98f 100644 --- a/internal/pkg/v2/dacctl/interface.go +++ b/internal/pkg/v2/dacctl/interface.go @@ -15,7 +15,7 @@ type DacctlActions interface { ListPools() error ShowConfigurations() error ValidateJob(c CliContext) error - RealSize(c CliContext) error + RealSize(c CliContext) (string, error) DataIn(c CliContext) error Paths(c CliContext) error PreRun(c CliContext) error From 9f754d0f3fa4f096a717f1198700d90fe4060ece Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 13:08:44 +0100 Subject: [PATCH 031/191] Add datactl actions Paths --- internal/pkg/v2/dacctl/actions_impl/show.go | 27 +++++++++-- .../pkg/v2/dacctl/actions_impl/show_test.go | 45 +++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index 05b7fc7c..e694b8bd 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -3,14 +3,19 @@ package actions_impl import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" ) -func (d *dacctlActions) RealSize(c dacctl.CliContext) (string, error) { +func (d *dacctlActions) getSession(c dacctl.CliContext) (datamodel.Session, error) { sessionName, err := d.getSessionName(c) if err != nil { - return "", err + return datamodel.Session{}, err } - session, err := d.session.GetSession(sessionName) + return d.session.GetSession(sessionName) +} + +func (d *dacctlActions) RealSize(c dacctl.CliContext) (string, error) { + session, err := d.getSession(c) if err != nil { return "", err } @@ -20,7 +25,21 @@ func (d *dacctlActions) RealSize(c dacctl.CliContext) (string, error) { } func (d *dacctlActions) Paths(c dacctl.CliContext) error { - panic("implement me") + err := checkRequiredStrings(c, "token", "pathfile") + if err != nil { + return err + } + + session, err := d.getSession(c) + if err != nil { + return err + } + + var paths []string + for key, value := range session.Paths { + paths = append(paths, fmt.Sprintf("%s=%s", key, value)) + } + return d.disk.Write(c.String("pathfile"), paths) } func (d *dacctlActions) ShowInstances() error { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index 3c251809..b961ef9d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -1,6 +1,8 @@ package actions_impl import ( + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" @@ -24,4 +26,47 @@ func TestDacctlActions_RealSize(t *testing.T) { assert.Nil(t, err) assert.Equal(t, `{"token":"bar", "capacity":123, "units":"bytes"}`, output) + + _, err = actions.RealSize(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token", err.Error()) +} + +func TestDacctlActions_Paths(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + disk := mocks.NewMockDisk(mockCtrl) + + session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{ + Name: datamodel.SessionName("bar"), + Paths: map[string]string{ + "foo1": "bar1", + "foo2": "bar2", + }, + }, nil) + disk.EXPECT().Write("paths", []string{"foo1=bar1", "foo2=bar2"}) + + actions := NewDacctlActions(session, disk) + err := actions.Paths(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "pathfile": "paths", + }, + }) + + assert.Nil(t, err) + + err = actions.Paths(&mockCliContext{}) + assert.Equal(t, "Please provide these required parameters: token, pathfile", err.Error()) + + fakeError := errors.New("fake") + session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{}, fakeError) + err = actions.Paths(&mockCliContext{ + strings: map[string]string{ + "token": "bar", + "pathfile": "paths", + }, + }) + assert.Equal(t, fakeError, err) + } From 262c24c110c7d78f1bef114843c6cc57b07d1f53 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 13:13:21 +0100 Subject: [PATCH 032/191] Tweak interface to return strings --- internal/pkg/v2/dacctl/actions_impl/show.go | 8 ++++---- internal/pkg/v2/dacctl/actions_impl/show_test.go | 3 +++ internal/pkg/v2/dacctl/interface.go | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index e694b8bd..4f9b7cab 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -42,18 +42,18 @@ func (d *dacctlActions) Paths(c dacctl.CliContext) error { return d.disk.Write(c.String("pathfile"), paths) } -func (d *dacctlActions) ShowInstances() error { +func (d *dacctlActions) ShowInstances() (string, error) { panic("implement me") } -func (d *dacctlActions) ShowSessions() error { +func (d *dacctlActions) ShowSessions() (string, error) { panic("implement me") } -func (d *dacctlActions) ListPools() error { +func (d *dacctlActions) ListPools() (string, error) { panic("implement me") } -func (d *dacctlActions) ShowConfigurations() error { +func (d *dacctlActions) ShowConfigurations() (string, error) { panic("implement me") } diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index b961ef9d..c64485d1 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -68,5 +68,8 @@ func TestDacctlActions_Paths(t *testing.T) { }, }) assert.Equal(t, fakeError, err) +} + +func TestDacctlActions_ShowInstances(t *testing.T) { } diff --git a/internal/pkg/v2/dacctl/interface.go b/internal/pkg/v2/dacctl/interface.go index 804ed98f..3ab1ad5e 100644 --- a/internal/pkg/v2/dacctl/interface.go +++ b/internal/pkg/v2/dacctl/interface.go @@ -10,10 +10,10 @@ type DacctlActions interface { CreatePersistentBuffer(c CliContext) error DeleteBuffer(c CliContext) error CreatePerJobBuffer(c CliContext) error - ShowInstances() error - ShowSessions() error - ListPools() error - ShowConfigurations() error + ShowInstances() (string, error) + ShowSessions() (string, error) + ListPools() (string, error) + ShowConfigurations() (string, error) ValidateJob(c CliContext) error RealSize(c CliContext) (string, error) DataIn(c CliContext) error From 272b16ae3fcbc2b26edaeb8a0c4a64ac037092b0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 13:31:12 +0100 Subject: [PATCH 033/191] Implement show instances --- go.mod | 1 + go.sum | 3 ++ .../pkg/v2/dacctl/actions_impl/instances.go | 32 +++++++++++++++++++ internal/pkg/v2/dacctl/actions_impl/show.go | 15 ++++++++- .../pkg/v2/dacctl/actions_impl/show_test.go | 21 ++++++++++-- 5 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/instances.go diff --git a/go.mod b/go.mod index 7020a68e..14fa337c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.9.4 // indirect github.com/jonboulle/clockwork v0.1.0 // indirect github.com/prometheus/client_golang v1.0.0 // indirect + github.com/prometheus/common v0.4.1 github.com/soheilhy/cmux v0.1.4 // indirect github.com/stretchr/testify v1.3.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect diff --git a/go.sum b/go.sum index ad2328fa..bf384d51 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -162,6 +164,7 @@ google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dT google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/pkg/v2/dacctl/actions_impl/instances.go b/internal/pkg/v2/dacctl/actions_impl/instances.go new file mode 100644 index 00000000..6a28c490 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/instances.go @@ -0,0 +1,32 @@ +package actions_impl + +import ( + "encoding/json" + "github.com/prometheus/common/log" +) + +type instanceCapacity struct { + Bytes uint `json:"bytes"` + Nodes uint `json:"nodes"` +} + +type instanceLinks struct { + Session string `json:"session"` +} + +type instance struct { + Id string `json:"id"` + Capacity instanceCapacity `json:"capacity"` + Links instanceLinks `json:"links"` +} + +type instances []instance + +func instancesToString(list []instance) string { + message := map[string]instances{"instances": list} + output, err := json.Marshal(message) + if err != nil { + log.Fatal(err.Error()) + } + return string(output) +} diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index 4f9b7cab..b133bb72 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -43,7 +43,20 @@ func (d *dacctlActions) Paths(c dacctl.CliContext) error { } func (d *dacctlActions) ShowInstances() (string, error) { - panic("implement me") + allSessions, err := d.session.GetAllSessions() + if err != nil { + return "", err + } + + var instances []instance + for _, session := range allSessions { + instances = append(instances, instance{ + Id: string(session.Name), + Capacity: instanceCapacity{Bytes: uint(session.ActualSizeBytes)}, + Links: instanceLinks{string(session.Name)}, + }) + } + return instancesToString(instances), nil } func (d *dacctlActions) ShowSessions() (string, error) { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index c64485d1..0a144fbc 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -41,10 +41,9 @@ func TestDacctlActions_Paths(t *testing.T) { Name: datamodel.SessionName("bar"), Paths: map[string]string{ "foo1": "bar1", - "foo2": "bar2", }, }, nil) - disk.EXPECT().Write("paths", []string{"foo1=bar1", "foo2=bar2"}) + disk.EXPECT().Write("paths", []string{"foo1=bar1"}) actions := NewDacctlActions(session, disk) err := actions.Paths(&mockCliContext{ @@ -71,5 +70,23 @@ func TestDacctlActions_Paths(t *testing.T) { } func TestDacctlActions_ShowInstances(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + session.EXPECT().GetAllSessions().Return([]datamodel.Session{ + { + Name: datamodel.SessionName("foo"), + ActualSizeBytes: 123, + }, + { + Name: datamodel.SessionName("bar"), + ActualSizeBytes: 456, + }, + }, nil) + actions := NewDacctlActions(session, nil) + output, err := actions.ShowInstances() + assert.Nil(t, err) + expected := `{"instances":[{"id":"foo","capacity":{"bytes":123,"nodes":0},"links":{"session":"foo"}},{"id":"bar","capacity":{"bytes":456,"nodes":0},"links":{"session":"bar"}}]}` + assert.Equal(t, expected, output) } From 8cffbfeedf510c57a822b6a2ec6a303c7b54f5e9 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 14:03:44 +0100 Subject: [PATCH 034/191] Improve coverage for showinstances --- internal/pkg/v2/dacctl/actions_impl/show_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index 0a144fbc..af169ca7 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -89,4 +89,10 @@ func TestDacctlActions_ShowInstances(t *testing.T) { assert.Nil(t, err) expected := `{"instances":[{"id":"foo","capacity":{"bytes":123,"nodes":0},"links":{"session":"foo"}},{"id":"bar","capacity":{"bytes":456,"nodes":0},"links":{"session":"bar"}}]}` assert.Equal(t, expected, output) + + fakeErr := errors.New("fake") + session.EXPECT().GetAllSessions().Return(nil, fakeErr) + output, err = actions.ShowInstances() + assert.Equal(t, "", output) + assert.Equal(t, fakeErr, err) } From 053d494b423afb3c02056f7383df7645b8d08395 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 14:23:58 +0100 Subject: [PATCH 035/191] Implement show Sessions --- .../pkg/v2/dacctl/actions_impl/sessions.go | 24 ++++++++++ internal/pkg/v2/dacctl/actions_impl/show.go | 18 +++++++- .../pkg/v2/dacctl/actions_impl/show_test.go | 46 ++++++++++++++++++- 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/sessions.go diff --git a/internal/pkg/v2/dacctl/actions_impl/sessions.go b/internal/pkg/v2/dacctl/actions_impl/sessions.go new file mode 100644 index 00000000..d3649161 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/sessions.go @@ -0,0 +1,24 @@ +package actions_impl + +import ( + "encoding/json" + "log" +) + +type session struct { + Id string `json:"id"` + Created uint `json:"created"` + Owner uint `json:"owner"` + Token string `json:"token"` +} + +type sessions []session + +func sessonsToString(list []session) string { + message := map[string]sessions{"sessions": list} + output, err := json.Marshal(message) + if err != nil { + log.Fatal(err.Error()) + } + return string(output) +} diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index b133bb72..a1d83f2e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -48,7 +48,7 @@ func (d *dacctlActions) ShowInstances() (string, error) { return "", err } - var instances []instance + instances := []instance{} for _, session := range allSessions { instances = append(instances, instance{ Id: string(session.Name), @@ -60,7 +60,21 @@ func (d *dacctlActions) ShowInstances() (string, error) { } func (d *dacctlActions) ShowSessions() (string, error) { - panic("implement me") + allSessions, err := d.session.GetAllSessions() + if err != nil { + return "", err + } + + sessions := sessions{} + for _, s := range allSessions { + sessions = append(sessions, session{ + Id: string(s.Name), + Created: s.CreatedAt, + Owner: s.Owner, + Token: string(s.Name), + }) + } + return sessonsToString(sessions), nil } func (d *dacctlActions) ListPools() (string, error) { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index af169ca7..53b29056 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -83,9 +83,10 @@ func TestDacctlActions_ShowInstances(t *testing.T) { ActualSizeBytes: 456, }, }, nil) - actions := NewDacctlActions(session, nil) + output, err := actions.ShowInstances() + assert.Nil(t, err) expected := `{"instances":[{"id":"foo","capacity":{"bytes":123,"nodes":0},"links":{"session":"foo"}},{"id":"bar","capacity":{"bytes":456,"nodes":0},"links":{"session":"bar"}}]}` assert.Equal(t, expected, output) @@ -95,4 +96,47 @@ func TestDacctlActions_ShowInstances(t *testing.T) { output, err = actions.ShowInstances() assert.Equal(t, "", output) assert.Equal(t, fakeErr, err) + + session.EXPECT().GetAllSessions().Return(nil, nil) + output, err = actions.ShowInstances() + assert.Nil(t, err) + expected = `{"instances":[]}` + assert.Equal(t, expected, output) +} + +func TestDacctlActions_ShowSessions(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + session.EXPECT().GetAllSessions().Return([]datamodel.Session{ + { + Name: datamodel.SessionName("foo"), + Owner: 42, + CreatedAt: 1234, + }, + { + Name: datamodel.SessionName("bar"), + Owner: 43, + CreatedAt: 5678, + }, + }, nil) + actions := NewDacctlActions(session, nil) + + output, err := actions.ShowSessions() + + assert.Nil(t, err) + expected := `{"sessions":[{"id":"foo","created":1234,"owner":42,"token":"foo"},{"id":"bar","created":5678,"owner":43,"token":"bar"}]}` + assert.Equal(t, expected, output) + + fakeErr := errors.New("fake") + session.EXPECT().GetAllSessions().Return(nil, fakeErr) + output, err = actions.ShowSessions() + assert.Equal(t, "", output) + assert.Equal(t, fakeErr, err) + + session.EXPECT().GetAllSessions().Return(nil, nil) + output, err = actions.ShowSessions() + assert.Nil(t, err) + expected = `{"sessions":[]}` + assert.Equal(t, expected, output) } From c0aff4a15f7a4b8f978016a92815694d49549241 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 14:34:12 +0100 Subject: [PATCH 036/191] Implemenet show configurations at least the stub we need to make Slurm happy --- .../v2/dacctl/actions_impl/configurations.go | 17 +++++++++++++++++ internal/pkg/v2/dacctl/actions_impl/pools.go | 1 + internal/pkg/v2/dacctl/actions_impl/show.go | 3 ++- .../pkg/v2/dacctl/actions_impl/show_test.go | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 internal/pkg/v2/dacctl/actions_impl/configurations.go create mode 100644 internal/pkg/v2/dacctl/actions_impl/pools.go diff --git a/internal/pkg/v2/dacctl/actions_impl/configurations.go b/internal/pkg/v2/dacctl/actions_impl/configurations.go new file mode 100644 index 00000000..c9a054c8 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/configurations.go @@ -0,0 +1,17 @@ +package actions_impl + +import ( + "encoding/json" + "log" +) + +type configurations []string + +func configurationToString(list configurations) string { + message := map[string]configurations{"configurations": list} + output, err := json.Marshal(message) + if err != nil { + log.Fatal(err.Error()) + } + return string(output) +} diff --git a/internal/pkg/v2/dacctl/actions_impl/pools.go b/internal/pkg/v2/dacctl/actions_impl/pools.go new file mode 100644 index 00000000..b3688304 --- /dev/null +++ b/internal/pkg/v2/dacctl/actions_impl/pools.go @@ -0,0 +1 @@ +package actions_impl diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index a1d83f2e..1ff2c163 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -82,5 +82,6 @@ func (d *dacctlActions) ListPools() (string, error) { } func (d *dacctlActions) ShowConfigurations() (string, error) { - panic("implement me") + // NOTE: Slurm doesn't read any of the output, so we don't send anything + return configurationToString(configurations{}), nil } diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index 53b29056..d34a70fa 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -140,3 +140,17 @@ func TestDacctlActions_ShowSessions(t *testing.T) { expected = `{"sessions":[]}` assert.Equal(t, expected, output) } + +func TestDacctlActions_ListPools(t *testing.T) { + actions := NewDacctlActions(nil, nil) + output, err := actions.ListPools() + assert.Nil(t, err) + assert.Equal(t, `{"configurations":[]}`, output) +} + +func TestDacctlActions_ShowConfigurations(t *testing.T) { + actions := NewDacctlActions(nil, nil) + output, err := actions.ShowConfigurations() + assert.Nil(t, err) + assert.Equal(t, `{"configurations":[]}`, output) +} From af1137f633dfd485be0cc9866fae378e5a189b49 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 14:50:55 +0100 Subject: [PATCH 037/191] Implement dacctl list pools --- .idea/dictionaries/john.xml | 1 + internal/pkg/v2/dacctl/actions_impl/pools.go | 24 ++++++++++++ internal/pkg/v2/dacctl/actions_impl/show.go | 21 ++++++++++- .../pkg/v2/dacctl/actions_impl/show_test.go | 37 ++++++++++++++++++- internal/pkg/v2/datamodel/pool.go | 2 +- 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/.idea/dictionaries/john.xml b/.idea/dictionaries/john.xml index 8e65c41e..1aeb6e0d 100644 --- a/.idea/dictionaries/john.xml +++ b/.idea/dictionaries/john.xml @@ -5,6 +5,7 @@ dacctl dacd nodehostnamefile + slurm \ No newline at end of file diff --git a/internal/pkg/v2/dacctl/actions_impl/pools.go b/internal/pkg/v2/dacctl/actions_impl/pools.go index b3688304..1d5cac81 100644 --- a/internal/pkg/v2/dacctl/actions_impl/pools.go +++ b/internal/pkg/v2/dacctl/actions_impl/pools.go @@ -1 +1,25 @@ package actions_impl + +import ( + "encoding/json" + "log" +) + +type pool struct { + Id string `json:"id"` + Units string `json:"units"` + Granularity uint `json:"granularity"` + Quantity uint `json:"quantity"` + Free uint `json:"free"` +} + +type pools []pool + +func getPoolsAsString(list pools) string { + message := map[string]pools{"pools": list} + output, err := json.Marshal(message) + if err != nil { + log.Fatal(err.Error()) + } + return string(output) +} diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index 1ff2c163..48eaed6c 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -48,7 +48,7 @@ func (d *dacctlActions) ShowInstances() (string, error) { return "", err } - instances := []instance{} + instances := instances{} for _, session := range allSessions { instances = append(instances, instance{ Id: string(session.Name), @@ -78,7 +78,24 @@ func (d *dacctlActions) ShowSessions() (string, error) { } func (d *dacctlActions) ListPools() (string, error) { - panic("implement me") + allPools, err := d.session.GetPools() + if err != nil { + return "", err + } + + pools := pools{} + for _, regPool := range allPools { + free := len(regPool.AvailableBricks) + quantity := free + len(regPool.AllocatedBricks) + pools = append(pools, pool{ + Id: string(regPool.Pool.Name), + Units: "bytes", + Granularity: regPool.Pool.GranularityBytes, + Quantity: uint(quantity), + Free: uint(free), + }) + } + return getPoolsAsString(pools), nil } func (d *dacctlActions) ShowConfigurations() (string, error) { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index d34a70fa..209e75bb 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -142,10 +142,43 @@ func TestDacctlActions_ShowSessions(t *testing.T) { } func TestDacctlActions_ListPools(t *testing.T) { - actions := NewDacctlActions(nil, nil) + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + session := mock_workflow.NewMockSession(mockCtrl) + session.EXPECT().GetPools().Return([]datamodel.PoolInfo{ + { + Pool: datamodel.Pool{ + Name: "default", + GranularityBytes: 1024, + }, + AllocatedBricks: []datamodel.BrickAllocation{ + { + Brick: datamodel.Brick{Device: "sda"}, + }, + }, + AvailableBricks: []datamodel.Brick{ + {Device: "sdb"}, + {Device: "sdc"}, + }, + }, + }, nil) + actions := NewDacctlActions(session, nil) + output, err := actions.ListPools() assert.Nil(t, err) - assert.Equal(t, `{"configurations":[]}`, output) + expexted := `{"pools":[{"id":"default","units":"bytes","granularity":1024,"quantity":3,"free":2}]}` + assert.Equal(t, expexted, output) + + session.EXPECT().GetPools().Return(nil, nil) + output, err = actions.ListPools() + assert.Nil(t, err) + assert.Equal(t, `{"pools":[]}`, output) + + fakeErr := errors.New("fake") + session.EXPECT().GetPools().Return(nil, fakeErr) + output, err = actions.ListPools() + assert.Equal(t, fakeErr, err) + assert.Equal(t, "", output) } func TestDacctlActions_ShowConfigurations(t *testing.T) { diff --git a/internal/pkg/v2/datamodel/pool.go b/internal/pkg/v2/datamodel/pool.go index 53c18e1e..2d226cb3 100644 --- a/internal/pkg/v2/datamodel/pool.go +++ b/internal/pkg/v2/datamodel/pool.go @@ -8,7 +8,7 @@ type Pool struct { // This is the allocation unit for the pool // It is the minimum size of any registered brick - GranularityGB uint + GranularityBytes uint } type PoolInfo struct { From bfdbbb96619c9c6c9bfb3efa8852ed9be670f468 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 15:11:29 +0100 Subject: [PATCH 038/191] Add config object in brick manager --- .../brick_manager_impl/brick_manager_test.go | 24 ++++++++++++++++++ .../dacd/brick_manager_impl/brick_manger.go | 21 +++++----------- .../pkg/v2/dacd/brick_manager_impl/config.go | 25 +++++++++++++++++++ 3 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go create mode 100644 internal/pkg/v2/dacd/brick_manager_impl/config.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go new file mode 100644 index 00000000..f82696d1 --- /dev/null +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -0,0 +1,24 @@ +package brick_manager_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestBrickManager_Hostname(t *testing.T) { + brickManager := NewBrickManager(nil) + assert.Equal(t, getHostname(), brickManager.Hostname()) +} + +func TestBrickManager_Startup(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + brickRegistry := mock_registry.NewMockBrickRegistry(mockCtrl) + brickManager := NewBrickManager(brickRegistry) + + err := brickManager.Startup(false) + + assert.Nil(t, err) +} \ No newline at end of file diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index 483738c6..f71bd830 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -2,29 +2,20 @@ package brick_manager_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" - "log" - "os" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" ) -func NewBrickManager(store store.Keystore) brick_manager.BrickManager { - return &brickManager{getHostname()} -} - -func getHostname() string { - hostname, err := os.Hostname() - if err != nil { - log.Fatal(err) - } - return hostname +func NewBrickManager(brickRegistry registry.BrickRegistry) brick_manager.BrickManager { + return &brickManager{config: getConfig(), brickRegistry: brickRegistry} } type brickManager struct { - hostname string + config brickManagerConfiguration + brickRegistry registry.BrickRegistry } func (bm *brickManager) Hostname() string { - return bm.hostname + return bm.config.hostname } func (bm *brickManager) Startup(drainSessions bool) error { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/config.go b/internal/pkg/v2/dacd/brick_manager_impl/config.go new file mode 100644 index 00000000..d234ffe9 --- /dev/null +++ b/internal/pkg/v2/dacd/brick_manager_impl/config.go @@ -0,0 +1,25 @@ +package brick_manager_impl + +import ( + "log" + "os" +) + +type brickManagerConfiguration struct { + hostname string +} + +func getConfig() brickManagerConfiguration { + config := brickManagerConfiguration{ + getHostname(), + } + return config +} + +func getHostname() string { + hostname, err := os.Hostname() + if err != nil { + log.Fatal(err) + } + return hostname +} \ No newline at end of file From c8dffc52a0932ae07254f62738f8b552d2c409d3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 15:16:06 +0100 Subject: [PATCH 039/191] Move config --- .../v2/dacd/brick_manager_impl/brick_manager_test.go | 7 ++++--- .../pkg/v2/dacd/brick_manager_impl/brick_manger.go | 7 ++++--- .../v2/dacd/{brick_manager_impl => config}/config.go | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) rename internal/pkg/v2/dacd/{brick_manager_impl => config}/config.go (51%) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index f82696d1..ce0e9f8a 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -1,6 +1,7 @@ package brick_manager_impl import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -8,8 +9,8 @@ import ( ) func TestBrickManager_Hostname(t *testing.T) { - brickManager := NewBrickManager(nil) - assert.Equal(t, getHostname(), brickManager.Hostname()) + brickManager := brickManager{config:config.BrickManagerConfig{Hostname:"host"}} + assert.Equal(t, "host", brickManager.Hostname()) } func TestBrickManager_Startup(t *testing.T) { @@ -21,4 +22,4 @@ func TestBrickManager_Startup(t *testing.T) { err := brickManager.Startup(false) assert.Nil(t, err) -} \ No newline at end of file +} diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index f71bd830..68e6f469 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -2,20 +2,21 @@ package brick_manager_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" ) func NewBrickManager(brickRegistry registry.BrickRegistry) brick_manager.BrickManager { - return &brickManager{config: getConfig(), brickRegistry: brickRegistry} + return &brickManager{config: config.GetBrickManagerConfig(), brickRegistry: brickRegistry} } type brickManager struct { - config brickManagerConfiguration + config config.BrickManagerConfig brickRegistry registry.BrickRegistry } func (bm *brickManager) Hostname() string { - return bm.config.hostname + return bm.config.Hostname } func (bm *brickManager) Startup(drainSessions bool) error { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/config.go b/internal/pkg/v2/dacd/config/config.go similarity index 51% rename from internal/pkg/v2/dacd/brick_manager_impl/config.go rename to internal/pkg/v2/dacd/config/config.go index d234ffe9..70743a9e 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -1,16 +1,16 @@ -package brick_manager_impl +package config import ( "log" "os" ) -type brickManagerConfiguration struct { - hostname string +type BrickManagerConfig struct { + Hostname string } -func getConfig() brickManagerConfiguration { - config := brickManagerConfiguration{ +func GetBrickManagerConfig() BrickManagerConfig { + config := BrickManagerConfig{ getHostname(), } return config @@ -22,4 +22,4 @@ func getHostname() string { log.Fatal(err) } return hostname -} \ No newline at end of file +} From d287a3bfa656c9bc2daeb2d2552e137793eb1373 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 16:07:25 +0100 Subject: [PATCH 040/191] Start reporting bricks --- go.sum | 1 + .../brick_manager_impl/brick_manager_test.go | 1 + .../dacd/brick_manager_impl/brick_manger.go | 8 +- .../v2/dacd/brick_manager_impl/host_bricks.go | 35 +++++++++ internal/pkg/v2/dacd/config/config.go | 78 +++++++++++++++++-- internal/pkg/v2/datamodel/brick_host.go | 2 +- 6 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go diff --git a/go.sum b/go.sum index bf384d51..f86bd382 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index ce0e9f8a..5faa4c8d 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -18,6 +18,7 @@ func TestBrickManager_Startup(t *testing.T) { defer mockCtrl.Finish() brickRegistry := mock_registry.NewMockBrickRegistry(mockCtrl) brickManager := NewBrickManager(brickRegistry) + brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) err := brickManager.Startup(false) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index 68e6f469..da3ec28d 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -7,7 +7,9 @@ import ( ) func NewBrickManager(brickRegistry registry.BrickRegistry) brick_manager.BrickManager { - return &brickManager{config: config.GetBrickManagerConfig(), brickRegistry: brickRegistry} + return &brickManager{ + config: config.GetBrickManagerConfig(config.DefaultEnv), + brickRegistry: brickRegistry} } type brickManager struct { @@ -16,11 +18,11 @@ type brickManager struct { } func (bm *brickManager) Hostname() string { - return bm.config.Hostname + return string(bm.config.Hostname) } func (bm *brickManager) Startup(drainSessions bool) error { - panic("implement me") + return bm.brickRegistry.UpdateBrickHost(getBrickHost(bm.config)) // * update current brick status // ** error out if removing bricks with existing assignments? // * start listening for create sessions (new primary bricks) and session actions diff --git a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go new file mode 100644 index 00000000..572e19b1 --- /dev/null +++ b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go @@ -0,0 +1,35 @@ +package brick_manager_impl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +) + +func getDevices(brickManagerConfig config.BrickManagerConfig) []string { + // TODO: should check these devices exist + var bricks []string + for i := 0; i < int(brickManagerConfig.DeviceCount); i++ { + device := fmt.Sprintf(brickManagerConfig.DeviceAddressPattern, i) + bricks = append(bricks, device) + } + return bricks +} + +func getBrickHost(brickManagerConfig config.BrickManagerConfig) datamodel.BrickHost { + var bricks []datamodel.Brick + for _, device := range getDevices(brickManagerConfig) { + bricks = append(bricks, datamodel.Brick{ + Device: device, + BrickHostName: brickManagerConfig.Hostname, + PoolName: brickManagerConfig.PoolName, + CapacityGiB: brickManagerConfig.DeviceCapacityGiB, + }) + } + + return datamodel.BrickHost{ + Name: brickManagerConfig.Hostname, + Bricks: bricks, + Enabled: brickManagerConfig.HostEnabled, + } +} diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 70743a9e..2635d621 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -1,25 +1,91 @@ package config import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" - "os" + "strconv" ) type BrickManagerConfig struct { - Hostname string + Hostname datamodel.BrickHostName + PoolName datamodel.PoolName + DeviceCapacityGiB uint + DeviceCount uint + DeviceAddressPattern string + HostEnabled bool } -func GetBrickManagerConfig() BrickManagerConfig { +type ReadEnvironemnt interface { + LookupEnv(key string) (string, bool) + Hostname() (string, error) +} + +// TODO: need additional validation here +func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { config := BrickManagerConfig{ - getHostname(), + datamodel.BrickHostName(getHostname(env)), + datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), + getUint(env, "DAC_BRICK_COUNT", 12), + getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), + getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), + getBool(env, "DAC_HOST_ENABLED", true), } + log.Println("Got brick manager config:", config) return config } -func getHostname() string { - hostname, err := os.Hostname() +func getHostname(env ReadEnvironemnt) string { + hostname, err := env.Hostname() if err != nil { log.Fatal(err) } return hostname } + +func getUint(env ReadEnvironemnt, key string, defaultVal uint) uint { + val, ok := env.LookupEnv(key) + if !ok { + return defaultVal + } + intVal, err := strconv.ParseUint(val, 10, 32) + if err != nil { + log.Printf("error parsing %s", key) + return defaultVal + } + return uint(intVal) +} + +func getString(env ReadEnvironemnt, key string, defaultVal string) string { + val, ok := env.LookupEnv(key) + if !ok { + return defaultVal + } + return val +} + +func getBool(env ReadEnvironemnt, key string, defaultVal bool) bool { + val, ok := env.LookupEnv(key) + if !ok { + return defaultVal + } + boolVal, err := strconv.ParseBool(val) + if err != nil { + log.Printf("error parsing %s", key) + return defaultVal + } + return boolVal +} + +type systemEnv struct {} + +func (systemEnv) LookupEnv(key string) (string, bool) { + // TODO return os.LookupEnv(key) + return "", false +} + +func (systemEnv) Hostname() (string, error) { + // TODO return os.Hostname() + return "hostname", nil +} + +var DefaultEnv ReadEnvironemnt = systemEnv{} \ No newline at end of file diff --git a/internal/pkg/v2/datamodel/brick_host.go b/internal/pkg/v2/datamodel/brick_host.go index 5b0ca333..a9e5f8f9 100644 --- a/internal/pkg/v2/datamodel/brick_host.go +++ b/internal/pkg/v2/datamodel/brick_host.go @@ -33,5 +33,5 @@ type Brick struct { PoolName PoolName // Size of the brick, defines the pool granularity - CapacityGB uint + CapacityGiB uint } From c9aeb3957cc3c28afd29394231aeef0b81d21827 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 16:39:25 +0100 Subject: [PATCH 041/191] Added concept of session action handler --- build/rebuild_mocks.sh | 2 +- internal/pkg/registry/volume.go | 2 +- .../brick_manager_impl/brick_manager_test.go | 12 ++++- .../dacd/brick_manager_impl/brick_manger.go | 40 +++++++++++----- .../v2/dacd/brick_manager_impl/host_bricks.go | 4 +- .../session_action_handler.go | 16 +++++++ internal/pkg/v2/dacd/config/config.go | 4 +- internal/pkg/v2/datamodel/volume.go | 2 +- internal/pkg/v2/mock_registry/brick.go | 8 ++-- .../mock_workflow/session_action_handler.go | 46 +++++++++++++++++++ internal/pkg/v2/registry/brick.go | 2 +- .../pkg/v2/workflow/session_action_handler.go | 7 +++ 12 files changed, 120 insertions(+), 25 deletions(-) create mode 100644 internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go create mode 100644 internal/pkg/v2/mock_workflow/session_action_handler.go create mode 100644 internal/pkg/v2/workflow/session_action_handler.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index e2d185c4..61522103 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -33,7 +33,7 @@ for i in $items; do >internal/pkg/v2/mock_registry/${i}.go done -items="session" +items="session session_action_handler" for i in $items; do mockgen -source=internal/pkg/v2/workflow/${i}.go \ >internal/pkg/v2/mock_workflow/${i}.go diff --git a/internal/pkg/registry/volume.go b/internal/pkg/registry/volume.go index c2c33753..03855174 100644 --- a/internal/pkg/registry/volume.go +++ b/internal/pkg/registry/volume.go @@ -271,7 +271,7 @@ const ( ) type Attachment struct { - // Hostname, Job and Volume name uniquely identify an attachment + // BrickHostName, Job and Volume name uniquely identify an attachment Hostname string // Associated jobName diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index 5faa4c8d..a0884ec9 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -1,15 +1,18 @@ package brick_manager_impl import ( + "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" ) func TestBrickManager_Hostname(t *testing.T) { - brickManager := brickManager{config:config.BrickManagerConfig{Hostname:"host"}} + brickManager := brickManager{config:config.BrickManagerConfig{BrickHostName: "host"}} assert.Equal(t, "host", brickManager.Hostname()) } @@ -17,8 +20,13 @@ func TestBrickManager_Startup(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() brickRegistry := mock_registry.NewMockBrickRegistry(mockCtrl) - brickManager := NewBrickManager(brickRegistry) + handler := mock_workflow.NewMockSessionActionHandler(mockCtrl) + brickManager := NewBrickManager(brickRegistry, handler) + + // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) + brickRegistry.EXPECT().GetSessionActions(context.TODO(), gomock.Any()) + brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName("hostname")) err := brickManager.Startup(false) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index da3ec28d..add91972 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -1,35 +1,53 @@ package brick_manager_impl import ( + "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "log" ) -func NewBrickManager(brickRegistry registry.BrickRegistry) brick_manager.BrickManager { +func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.SessionActionHandler) brick_manager.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), - brickRegistry: brickRegistry} + brickRegistry: brickRegistry, + sessionActionHandler: handler, + } } type brickManager struct { config config.BrickManagerConfig brickRegistry registry.BrickRegistry + sessionActionHandler workflow.SessionActionHandler } func (bm *brickManager) Hostname() string { - return string(bm.config.Hostname) + return string(bm.config.BrickHostName) } func (bm *brickManager) Startup(drainSessions bool) error { - return bm.brickRegistry.UpdateBrickHost(getBrickHost(bm.config)) - // * update current brick status - // ** error out if removing bricks with existing assignments? - // * start listening for create sessions (new primary bricks) and session actions - // * report we are listening with keep-alive - // * check all brick assignments - // ** ensure all brick hosts are up, warn if there are issues - // ** if primary brick, refresh Ansible run (i.e. recover from host reboot) + err := bm.brickRegistry.UpdateBrickHost(getBrickHost(bm.config)) + if err != nil { + return err + } + + // If we are are enabled, this includes new create session requests + events, err := bm.brickRegistry.GetSessionActions(context.TODO(), bm.config.BrickHostName) + + go func() { + for event := range events { + bm.sessionActionHandler.ProcessSessionAction(event) + } + log.Println("ERROR: stopped waiting for new Session Actions") + }() + + // TODO: try to recover all existing filesystems on restart + // including a check to make sure all related brick hosts are alive + + // Tell everyone we are listening + return bm.brickRegistry.KeepAliveHost(context.TODO(), bm.config.BrickHostName) } func (bm *brickManager) Shutdown() error { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go index 572e19b1..f3d8e8f3 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go @@ -21,14 +21,14 @@ func getBrickHost(brickManagerConfig config.BrickManagerConfig) datamodel.BrickH for _, device := range getDevices(brickManagerConfig) { bricks = append(bricks, datamodel.Brick{ Device: device, - BrickHostName: brickManagerConfig.Hostname, + BrickHostName: brickManagerConfig.BrickHostName, PoolName: brickManagerConfig.PoolName, CapacityGiB: brickManagerConfig.DeviceCapacityGiB, }) } return datamodel.BrickHost{ - Name: brickManagerConfig.Hostname, + Name: brickManagerConfig.BrickHostName, Bricks: bricks, Enabled: brickManagerConfig.HostEnabled, } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go new file mode 100644 index 00000000..57638798 --- /dev/null +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -0,0 +1,16 @@ +package brick_manager_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" +) + +func NewSessionActionHandler() workflow.SessionActionHandler { + return &sessionActionHandler{} +} + +type sessionActionHandler struct {} + +func (*sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { + panic("implement me") +} diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 2635d621..7ec40d4e 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -7,7 +7,7 @@ import ( ) type BrickManagerConfig struct { - Hostname datamodel.BrickHostName + BrickHostName datamodel.BrickHostName PoolName datamodel.PoolName DeviceCapacityGiB uint DeviceCount uint @@ -84,7 +84,7 @@ func (systemEnv) LookupEnv(key string) (string, bool) { } func (systemEnv) Hostname() (string, error) { - // TODO return os.Hostname() + // TODO return os.BrickHostName() return "hostname", nil } diff --git a/internal/pkg/v2/datamodel/volume.go b/internal/pkg/v2/datamodel/volume.go index bae87c5f..a4b80bd2 100644 --- a/internal/pkg/v2/datamodel/volume.go +++ b/internal/pkg/v2/datamodel/volume.go @@ -70,7 +70,7 @@ type Volume struct { } type Attachment struct { - // Hostname, Job and Volume name uniquely identify an attachment + // BrickHostName, Job and Volume name uniquely identify an attachment Hostname string // Associated jobName diff --git a/internal/pkg/v2/mock_registry/brick.go b/internal/pkg/v2/mock_registry/brick.go index 46c6bdd8..164b94c7 100644 --- a/internal/pkg/v2/mock_registry/brick.go +++ b/internal/pkg/v2/mock_registry/brick.go @@ -64,17 +64,17 @@ func (mr *MockBrickRegistryMockRecorder) GetSessionActions(ctxt, brickHostName i } // KeepAliveHost mocks base method -func (m *MockBrickRegistry) KeepAliveHost(brickHostName datamodel.BrickHostName) error { +func (m *MockBrickRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeepAliveHost", brickHostName) + ret := m.ctrl.Call(m, "KeepAliveHost", ctxt, brickHostName) ret0, _ := ret[0].(error) return ret0 } // KeepAliveHost indicates an expected call of KeepAliveHost -func (mr *MockBrickRegistryMockRecorder) KeepAliveHost(brickHostName interface{}) *gomock.Call { +func (mr *MockBrickRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickRegistry)(nil).KeepAliveHost), brickHostName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickRegistry)(nil).KeepAliveHost), ctxt, brickHostName) } // IsBrickHostAlive mocks base method diff --git a/internal/pkg/v2/mock_workflow/session_action_handler.go b/internal/pkg/v2/mock_workflow/session_action_handler.go new file mode 100644 index 00000000..ef1f51fc --- /dev/null +++ b/internal/pkg/v2/mock_workflow/session_action_handler.go @@ -0,0 +1,46 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/workflow/session_action_handler.go + +// Package mock_workflow is a generated GoMock package. +package mock_workflow + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockSessionActionHandler is a mock of SessionActionHandler interface +type MockSessionActionHandler struct { + ctrl *gomock.Controller + recorder *MockSessionActionHandlerMockRecorder +} + +// MockSessionActionHandlerMockRecorder is the mock recorder for MockSessionActionHandler +type MockSessionActionHandlerMockRecorder struct { + mock *MockSessionActionHandler +} + +// NewMockSessionActionHandler creates a new mock instance +func NewMockSessionActionHandler(ctrl *gomock.Controller) *MockSessionActionHandler { + mock := &MockSessionActionHandler{ctrl: ctrl} + mock.recorder = &MockSessionActionHandlerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSessionActionHandler) EXPECT() *MockSessionActionHandlerMockRecorder { + return m.recorder +} + +// ProcessSessionAction mocks base method +func (m *MockSessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ProcessSessionAction", action) +} + +// ProcessSessionAction indicates an expected call of ProcessSessionAction +func (mr *MockSessionActionHandlerMockRecorder) ProcessSessionAction(action interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSessionAction", reflect.TypeOf((*MockSessionActionHandler)(nil).ProcessSessionAction), action) +} diff --git a/internal/pkg/v2/registry/brick.go b/internal/pkg/v2/registry/brick.go index d2bb41f3..cd8a569b 100644 --- a/internal/pkg/v2/registry/brick.go +++ b/internal/pkg/v2/registry/brick.go @@ -25,7 +25,7 @@ type BrickRegistry interface { // When a host is dead non of its bricks will get new volumes assigned, // and no bricks will get cleaned up until the next service start. // Error will be returned if the host info has not yet been written. - KeepAliveHost(brickHostName datamodel.BrickHostName) error + KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error // Check if given brick host is alive // diff --git a/internal/pkg/v2/workflow/session_action_handler.go b/internal/pkg/v2/workflow/session_action_handler.go new file mode 100644 index 00000000..7592a152 --- /dev/null +++ b/internal/pkg/v2/workflow/session_action_handler.go @@ -0,0 +1,7 @@ +package workflow + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + +type SessionActionHandler interface { + ProcessSessionAction(action datamodel.SessionAction) +} From 798bd360e223796d2c33cda9942e5b05e69f2602 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 16:56:32 +0100 Subject: [PATCH 042/191] Get session action handler marking actions as complete --- .../brick_manager_impl/brick_manager_test.go | 2 +- .../dacd/brick_manager_impl/brick_manger.go | 8 ++++---- .../session_action_hander_test.go | 20 +++++++++++++++++++ .../session_action_handler.go | 20 ++++++++++++++----- internal/pkg/v2/dacd/config/config.go | 4 ++-- 5 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index a0884ec9..a62daca4 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -12,7 +12,7 @@ import ( ) func TestBrickManager_Hostname(t *testing.T) { - brickManager := brickManager{config:config.BrickManagerConfig{BrickHostName: "host"}} + brickManager := brickManager{config: config.BrickManagerConfig{BrickHostName: "host"}} assert.Equal(t, "host", brickManager.Hostname()) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go index add91972..d6f623f8 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go @@ -11,15 +11,15 @@ import ( func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.SessionActionHandler) brick_manager.BrickManager { return &brickManager{ - config: config.GetBrickManagerConfig(config.DefaultEnv), - brickRegistry: brickRegistry, + config: config.GetBrickManagerConfig(config.DefaultEnv), + brickRegistry: brickRegistry, sessionActionHandler: handler, } } type brickManager struct { - config config.BrickManagerConfig - brickRegistry registry.BrickRegistry + config config.BrickManagerConfig + brickRegistry registry.BrickRegistry sessionActionHandler workflow.SessionActionHandler } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go new file mode 100644 index 00000000..89fc0d3b --- /dev/null +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go @@ -0,0 +1,20 @@ +package brick_manager_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/golang/mock/gomock" + "testing" +) + +func TestSessionActionHandler_ProcessSessionAction(t *testing.T) { + action := datamodel.SessionAction{} + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + registry := mock_registry.NewMockSessionActions(mockCtrl) + registry.EXPECT().CompleteSessionAction(action, nil) + handler := NewSessionActionHandler(registry) + + // TODO... + handler.ProcessSessionAction(action) +} diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 57638798..ede51a6a 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -2,15 +2,25 @@ package brick_manager_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "log" ) -func NewSessionActionHandler() workflow.SessionActionHandler { - return &sessionActionHandler{} +func NewSessionActionHandler(actions registry.SessionActions) workflow.SessionActionHandler { + return &sessionActionHandler{actions: actions} } -type sessionActionHandler struct {} +type sessionActionHandler struct { + actions registry.SessionActions +} -func (*sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { - panic("implement me") +func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { + log.Println("Started to process:", action) + err := s.actions.CompleteSessionAction(action, nil) + if err != nil { + log.Println("Failed to complete Action:", err) + return + } + log.Println("Stopped processing action:", action) } diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 7ec40d4e..e52acf55 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -76,7 +76,7 @@ func getBool(env ReadEnvironemnt, key string, defaultVal bool) bool { return boolVal } -type systemEnv struct {} +type systemEnv struct{} func (systemEnv) LookupEnv(key string) (string, bool) { // TODO return os.LookupEnv(key) @@ -88,4 +88,4 @@ func (systemEnv) Hostname() (string, error) { return "hostname", nil } -var DefaultEnv ReadEnvironemnt = systemEnv{} \ No newline at end of file +var DefaultEnv ReadEnvironemnt = systemEnv{} From b5ebf0958f3bb409922509d10d38347e8f7e3ab1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 16:57:50 +0100 Subject: [PATCH 043/191] Correct some filename spellings --- .../dacd/brick_manager_impl/{brick_manger.go => brick_manager.go} | 0 ...ssion_action_hander_test.go => session_action_handler_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename internal/pkg/v2/dacd/brick_manager_impl/{brick_manger.go => brick_manager.go} (100%) rename internal/pkg/v2/dacd/brick_manager_impl/{session_action_hander_test.go => session_action_handler_test.go} (100%) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go similarity index 100% rename from internal/pkg/v2/dacd/brick_manager_impl/brick_manger.go rename to internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go similarity index 100% rename from internal/pkg/v2/dacd/brick_manager_impl/session_action_hander_test.go rename to internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go From fb3b2c4d8a7d50afa0b6dbf6881cc8bce556e31d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 17:00:58 +0100 Subject: [PATCH 044/191] Make dacd and dacctl interfaces consistent --- internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go | 4 ++-- internal/pkg/v2/dacd/{brick_manager => }/interface.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename internal/pkg/v2/dacd/{brick_manager => }/interface.go (95%) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index d6f623f8..eee2b163 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -2,14 +2,14 @@ package brick_manager_impl import ( "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" ) -func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.SessionActionHandler) brick_manager.BrickManager { +func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.SessionActionHandler) dacd.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), brickRegistry: brickRegistry, diff --git a/internal/pkg/v2/dacd/brick_manager/interface.go b/internal/pkg/v2/dacd/interface.go similarity index 95% rename from internal/pkg/v2/dacd/brick_manager/interface.go rename to internal/pkg/v2/dacd/interface.go index 72085e85..0f63b70b 100644 --- a/internal/pkg/v2/dacd/brick_manager/interface.go +++ b/internal/pkg/v2/dacd/interface.go @@ -1,4 +1,4 @@ -package brick_manager +package dacd type BrickManager interface { // Get the current hostname key that is being kept alive From 93e64e26cf4951119bf5e1606f1eeb2dc3b13d34 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 20:19:31 +0100 Subject: [PATCH 045/191] Add placeholder for session workflow impl --- .../pkg/v2/dacctl/workflow_impl/session.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 internal/pkg/v2/dacctl/workflow_impl/session.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go new file mode 100644 index 00000000..50f2fd54 --- /dev/null +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -0,0 +1,57 @@ +package workflow_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" +) + +func NewSessionWorkflow() workflow.Session { + return sessionWorkflow{} +} + +type sessionWorkflow struct { + registry registry.SessionRegistry + actions registry.SessionActions +} + +func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { + // TODO needs to get the allocation mutex, create the session, then create the allocations + // failing if the pool isn't known, or doesn't have enough space + panic("implement me") +} + +func (s sessionWorkflow) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { + // TODO get the session mutex, then update the session, then send the event + // note the actions registry will error out if the host is not up + // release the mutex and wait for the server to complete its work or we timeout + panic("implement me") +} + +func (s sessionWorkflow) DataIn(sessionName datamodel.SessionName) error { + panic("implement me") +} + +func (s sessionWorkflow) AttachVolumes(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error { + panic("implement me") +} + +func (s sessionWorkflow) DetachVolumes(sessionName datamodel.SessionName) error { + panic("implement me") +} + +func (s sessionWorkflow) DataOut(sessionName datamodel.SessionName) error { + panic("implement me") +} + +func (s sessionWorkflow) GetPools() ([]datamodel.PoolInfo, error) { + panic("implement me") +} + +func (s sessionWorkflow) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { + panic("implement me") +} + +func (s sessionWorkflow) GetAllSessions() ([]datamodel.Session, error) { + panic("implement me") +} From dce33df3b6a41ebc819fb64698a38bb9b1a6101e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 21:26:45 +0100 Subject: [PATCH 046/191] Sketch out create volume --- .../pkg/v2/dacctl/workflow_impl/session.go | 134 +++++++++++++++++- internal/pkg/v2/datamodel/session.go | 5 +- internal/pkg/v2/registry/allocation.go | 5 + internal/pkg/v2/registry/pool.go | 2 +- internal/pkg/v2/registry/session_actions.go | 2 +- 5 files changed, 142 insertions(+), 6 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 50f2fd54..70f96ed1 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -1,9 +1,14 @@ package workflow_impl import ( + "context" + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "math" + "math/rand" + "time" ) func NewSessionWorkflow() workflow.Session { @@ -11,14 +16,139 @@ func NewSessionWorkflow() workflow.Session { } type sessionWorkflow struct { - registry registry.SessionRegistry + session registry.SessionRegistry actions registry.SessionActions + allocations registry.AllocationRegistry + pool registry.PoolRegistry } func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { // TODO needs to get the allocation mutex, create the session, then create the allocations // failing if the pool isn't known, or doesn't have enough space - panic("implement me") + err := s.validateSession(session) + if err != nil { + return err + } + + // Get session lock, drop on function exit + sessionMutex, err := s.session.GetSessionMutex(session.Name) + if err != nil { + return fmt.Errorf("unable to get session mutex: %s due to: %s", session.Name, err) + } + err = sessionMutex.Lock(context.TODO()) + if err != nil { + return fmt.Errorf("unable to lock session mutex: %s due to: %s", session.Name, err) + } + + // Allocate bricks, and choose brick host server + session, err = s.doSessionAllocation(session) + if err != nil { + sessionMutex.Unlock(context.TODO()) + return err + } + + // Create filesystem on the brick host server + // TODO: add timeout + eventChan, err := s.actions.CreateSessionVolume(context.TODO(), session.Name) + if err != nil { + return err + } + + // Drop mutex so the server can take it + err = sessionMutex.Unlock(context.TODO()) + if err != nil { + // TODO: cancel above action? + return err + } + + // Wait for the server to create the filesystem + sessionAction := <-eventChan + return sessionAction.Error +} + +func (s sessionWorkflow) validateSession(session datamodel.Session) error { + _, err := s.pool.GetPool(session.VolumeRequest.PoolName) + if err != nil { + return fmt.Errorf("invalid session, unable to find pool %s", session.VolumeRequest.PoolName) + } + // TODO: check for multi-job restrictions, etc? + return nil +} + + +func (s sessionWorkflow) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { + allocationMutex, err := s.allocations.GetAllocationMutex() + if err != nil { + return session, err + } + + err = allocationMutex.Lock(context.TODO()) + if err != nil { + return session, err + } + defer allocationMutex.Unlock(context.TODO()) + + // Write allocations first + actualSizeBytes, allocations, err := s.getBricks(session.VolumeRequest.PoolName, session.VolumeRequest.TotalCapacityBytes) + if err != nil { + return session, fmt.Errorf("can't allocate for session: %s due to %s", session.Name, err) + } + session.ActualSizeBytes = actualSizeBytes + s.allocations.CreateAllocations(session.Name, allocations) + + // Create initial version of session + session, err = s.session.CreateSession(session) + if err != nil { + // TODO: remove allocations + return session, err + } + return session, err +} + +func (s sessionWorkflow) getBricks(poolName datamodel.PoolName, bytes int) (int, []datamodel.Brick, error) { + pool, err := s.allocations.GetPoolInfo(poolName) + if err != nil { + return 0, nil, err + } + + bricksRequired := int(math.Ceil(float64(bytes) / float64(pool.Pool.GranularityBytes))) + actualSize := bricksRequired * int(pool.Pool.GranularityBytes) + + bricks := getBricks(bricksRequired, pool) + if len(bricks) != bricksRequired { + return 0, nil, fmt.Errorf( + "unable to get number of requested bricks (%d) for given pool (%s)", + bricksRequired, pool.Pool.Name) + } + return actualSize, bricks, nil +} + +func getBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Brick { + // pick some of the available bricks + s := rand.NewSource(time.Now().Unix()) + r := rand.New(s) // initialize local pseudorandom generator + + var chosenBricks []datamodel.Brick + randomWalk := r.Perm(len(poolInfo.AvailableBricks)) + for _, i := range randomWalk { + candidateBrick := poolInfo.AvailableBricks[i] + + // TODO: should not the random walk mean this isn't needed! + goodCandidate := true + for _, brick := range chosenBricks { + if brick == candidateBrick { + goodCandidate = false + break + } + } + if goodCandidate { + chosenBricks = append(chosenBricks, candidateBrick) + } + if len(chosenBricks) >= bricksRequired { + break + } + } + return chosenBricks } func (s sessionWorkflow) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index deb9512e..520125d2 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -61,7 +61,7 @@ type Session struct { // If not nil, the session has an unresolved error // and can't be mounted by any new sessions - // it can be deleted + // but it can be deleted Error error } @@ -69,6 +69,7 @@ type SessionAction struct { Uuid string Session Session Action SessionActionType + Error error } // TODO: turn into enum @@ -78,7 +79,7 @@ type VolumeRequest struct { MultiJob bool Caller string TotalCapacityBytes int - PoolName string + PoolName PoolName Access AccessMode Type BufferType SwapBytes int diff --git a/internal/pkg/v2/registry/allocation.go b/internal/pkg/v2/registry/allocation.go index 62507ee8..3493aef2 100644 --- a/internal/pkg/v2/registry/allocation.go +++ b/internal/pkg/v2/registry/allocation.go @@ -9,6 +9,11 @@ type AllocationRegistry interface { // Get brick availability by pool GetBricksByPool() ([]datamodel.PoolInfo, error) + // Get brick availability for one pool + // bricks are only available if corresponding host currently alive + GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) + + // Caller should acquire this mutex before calling GetAllPools then CreateAllocations GetAllocationMutex() (store.Mutex, error) diff --git a/internal/pkg/v2/registry/pool.go b/internal/pkg/v2/registry/pool.go index b2d9f9fe..0be78242 100644 --- a/internal/pkg/v2/registry/pool.go +++ b/internal/pkg/v2/registry/pool.go @@ -6,7 +6,7 @@ import ( type PoolRegistry interface { // Get all registered pools - GetPools() ([]datamodel.Pool, error) + GetPool(name datamodel.PoolName) (datamodel.Pool, error) // Creates the pool if it doesn't exist // error if the granularity doesn't match and existing pool diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go index 15488796..b420bbb9 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/v2/registry/session_actions.go @@ -12,7 +12,7 @@ type SessionActions interface { // Error if session volume has already been created // Error if primary brick host is not alive or not enabled // Error is context is cancelled or timed-out - CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) + CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) // Updates session, then requests action // From f02764c19e7fc1c54fd90021ff75354b80653198 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Aug 2019 22:01:22 +0100 Subject: [PATCH 047/191] Sketch out delete volume --- internal/pkg/v2/dacctl/actions_impl/job.go | 22 ++-- .../pkg/v2/dacctl/actions_impl/job_test.go | 10 +- .../pkg/v2/dacctl/actions_impl/parsers/job.go | 4 +- .../dacctl/actions_impl/parsers/job_test.go | 4 +- .../pkg/v2/dacctl/actions_impl/persistent.go | 7 +- .../pkg/v2/dacctl/workflow_impl/session.go | 80 ++++++++++--- internal/pkg/v2/datamodel/datamodel_test.go | 2 +- internal/pkg/v2/datamodel/session.go | 35 +++--- internal/pkg/v2/mock_registry/allocation.go | 15 +++ internal/pkg/v2/mock_registry/pool.go | 14 +-- .../pkg/v2/mock_registry/session_actions.go | 8 +- internal/pkg/v2/registry/allocation.go | 1 - internal/pkg/v2/registry/session_actions.go | 2 +- internal/pkg/v2/registry_impl/actions.go | 108 ------------------ 14 files changed, 136 insertions(+), 176 deletions(-) delete mode 100644 internal/pkg/v2/registry_impl/actions.go diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index aa0c5c01..aa1856f1 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -59,7 +59,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { return fmt.Errorf("cache is not supported") } } - var multiJobVolumes []datamodel.VolumeName + var multiJobVolumes []datamodel.SessionName for _, attachment := range summary.Attachments { multiJobVolumes = append(multiJobVolumes, attachment) } @@ -68,20 +68,20 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { MultiJob: false, Caller: c.String("caller"), TotalCapacityBytes: capacityBytes, - PoolName: pool, + PoolName: datamodel.PoolName(pool), Access: access, Type: bufferType, SwapBytes: swapBytes, } session := datamodel.Session{ - Name: datamodel.SessionName(c.String("token")), - Owner: uint(c.Int("user")), - Group: uint(c.Int("group")), - CreatedAt: getNow(), - VolumeRequest: request, - MultiJobVolumes: multiJobVolumes, - StageInRequests: summary.DataIn, - StageOutRequests: summary.DataOut, + Name: datamodel.SessionName(c.String("token")), + Owner: uint(c.Int("user")), + Group: uint(c.Int("group")), + CreatedAt: getNow(), + VolumeRequest: request, + MultiJobAttachments: multiJobVolumes, + StageInRequests: summary.DataIn, + StageOutRequests: summary.DataOut, } session.Paths = getPaths(session) @@ -98,7 +98,7 @@ func getPaths(session datamodel.Session) map[string]string { paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", session.Name) } } - for _, multiJobVolume := range session.MultiJobVolumes { + for _, multiJobVolume := range session.MultiJobAttachments { paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf( "/dac/%s_persistent_%s", session.Name, multiJobVolume) } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index b649c97b..4cea4f7d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -47,11 +47,11 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { } disk.EXPECT().Lines("jobfile").Return(lines, nil) session.EXPECT().CreateSessionVolume(datamodel.Session{ - Name: "token", - Owner: 1001, - Group: 1002, - CreatedAt: 123, - MultiJobVolumes: []datamodel.VolumeName{"myBBname1", "myBBname2"}, + Name: "token", + Owner: 1001, + Group: 1002, + CreatedAt: 123, + MultiJobAttachments: []datamodel.SessionName{"myBBname1", "myBBname2"}, StageInRequests: []datamodel.DataCopyRequest{ { SourceType: datamodel.File, diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go index 05ca7c29..797552a2 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go @@ -11,7 +11,7 @@ import ( type jobSummary struct { PerJobBuffer *cmdPerJobBuffer Swap *cmdAttachPerJobSwap - Attachments []datamodel.VolumeName + Attachments []datamodel.SessionName DataIn []datamodel.DataCopyRequest DataOut []datamodel.DataCopyRequest // TODO: support create and destroy persistent? @@ -43,7 +43,7 @@ func getJobSummary(lines []string) (jobSummary, error) { return jobSummary{}, fmt.Errorf("only one per job buffer allowed") } case cmdAttachPersistent: - summary.Attachments = append(summary.Attachments, datamodel.VolumeName(c)) + summary.Attachments = append(summary.Attachments, datamodel.SessionName(c)) case cmdAttachPerJobSwap: if summary.Swap != nil { return jobSummary{}, fmt.Errorf("only one swap request allowed") diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index e8828e82..d823eb42 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -78,8 +78,8 @@ func TestGetJobSummary(t *testing.T) { assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut[0].Source) assert.Equal(t, 2, len(result.Attachments)) - assert.Equal(t, datamodel.VolumeName("myBBname1"), result.Attachments[0]) - assert.Equal(t, datamodel.VolumeName("myBBname2"), result.Attachments[1]) + assert.Equal(t, datamodel.SessionName("myBBname1"), result.Attachments[0]) + assert.Equal(t, datamodel.SessionName("myBBname2"), result.Attachments[1]) assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) assert.Equal(t, 4000000, result.Swap.SizeBytes) diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go index e5a15216..859321cd 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -40,7 +40,10 @@ func getNow() uint { } func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { - checkRequiredStrings(c, "token", "caller", "capacity", "access", "type") + err := checkRequiredStrings(c, "token", "caller", "capacity", "access", "type") + if err != nil { + return err + } pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) if err != nil { return err @@ -49,7 +52,7 @@ func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { MultiJob: true, Caller: c.String("caller"), TotalCapacityBytes: capacityBytes, - PoolName: pool, + PoolName: datamodel.PoolName(pool), Access: accessModeFromString(c.String("access")), Type: bufferTypeFromString(c.String("type")), } diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 70f96ed1..f028e376 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -6,6 +6,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "log" "math" "math/rand" "time" @@ -16,10 +17,10 @@ func NewSessionWorkflow() workflow.Session { } type sessionWorkflow struct { - session registry.SessionRegistry - actions registry.SessionActions + session registry.SessionRegistry + actions registry.SessionActions allocations registry.AllocationRegistry - pool registry.PoolRegistry + pool registry.PoolRegistry } func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { @@ -30,7 +31,7 @@ func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { return err } - // Get session lock, drop on function exit + // Get session lock sessionMutex, err := s.session.GetSessionMutex(session.Name) if err != nil { return fmt.Errorf("unable to get session mutex: %s due to: %s", session.Name, err) @@ -75,7 +76,6 @@ func (s sessionWorkflow) validateSession(session datamodel.Session) error { return nil } - func (s sessionWorkflow) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { allocationMutex, err := s.allocations.GetAllocationMutex() if err != nil { @@ -88,15 +88,24 @@ func (s sessionWorkflow) doSessionAllocation(session datamodel.Session) (datamod } defer allocationMutex.Unlock(context.TODO()) - // Write allocations first - actualSizeBytes, allocations, err := s.getBricks(session.VolumeRequest.PoolName, session.VolumeRequest.TotalCapacityBytes) - if err != nil { - return session, fmt.Errorf("can't allocate for session: %s due to %s", session.Name, err) + if session.VolumeRequest.TotalCapacityBytes > 0 { + // Write allocations first + actualSizeBytes, chosenBricks, err := s.getBricks(session.VolumeRequest.PoolName, session.VolumeRequest.TotalCapacityBytes) + if err != nil { + return session, fmt.Errorf("can't allocate for session: %s due to %s", session.Name, err) + } + session.ActualSizeBytes = actualSizeBytes + + allocations, err := s.allocations.CreateAllocations(session.Name, chosenBricks) + if err != nil { + return session, err + } + + session.Allocations = allocations + session.PrimaryBrickHost = allocations[0].Brick.BrickHostName } - session.ActualSizeBytes = actualSizeBytes - s.allocations.CreateAllocations(session.Name, allocations) - // Create initial version of session + // Store initial version of session session, err = s.session.CreateSession(session) if err != nil { // TODO: remove allocations @@ -152,10 +161,49 @@ func getBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Bric } func (s sessionWorkflow) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { - // TODO get the session mutex, then update the session, then send the event - // note the actions registry will error out if the host is not up - // release the mutex and wait for the server to complete its work or we timeout - panic("implement me") + // Get session lock + sessionMutex, err := s.session.GetSessionMutex(sessionName) + if err != nil { + return fmt.Errorf("unable to get session mutex: %s due to: %s", sessionName, err) + } + err = sessionMutex.Lock(context.TODO()) + if err != nil { + return fmt.Errorf("unable to lock session mutex: %s due to: %s", sessionName, err) + } + + session, err := s.session.GetSession(sessionName) + if err != nil { + log.Println("Unable to find session:", sessionName) + sessionMutex.Unlock(context.TODO()) + return nil + } + + // Record we want this deleted, in case host is not alive + // can be deleted when it is next stated + session.Status.DeleteRequested = true + session, err = s.session.UpdateSession(session) + if err != nil { + sessionMutex.Unlock(context.TODO()) + return err + } + // TODO: send the hurry, i.e. request data copy out first + + // This will error out if the host is not currently up + sessionAction, err := s.actions.SendSessionAction(context.TODO(), datamodel.SessionActionType("delete"), session) + if err != nil { + return err + } + + // Drop mutex to allow server to lock the session + err = sessionMutex.Unlock(context.TODO()) + if err != nil { + // TODO: cancel above waiting around? + return err + } + + // wait for server to complete, or timeout + result := <-sessionAction + return result.Error } func (s sessionWorkflow) DataIn(sessionName datamodel.SessionName) error { diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go index 872af464..df5334ff 100644 --- a/internal/pkg/v2/datamodel/datamodel_test.go +++ b/internal/pkg/v2/datamodel/datamodel_test.go @@ -30,7 +30,7 @@ func Test_Session(t *testing.T) { assert.Nil(t, err) // TODO: not very human readable for Type and Action - expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"DeleteRequested":false,"StageInRequests":null,"StageOutRequests":null,"AttachHosts":null,"JobVolume":"","MultiJobVolumes":null,"Paths":null,"Allocations":null,"SessionActionPrefix":"","ActualSizeBytes":0,"Error":null}` + expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"DeleteRequested":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","AttachHosts":null}` assert.Equal(t, expected, string(sessionAsString)) var sessionFromString Session diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 520125d2..58e7e621 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -23,8 +23,8 @@ type Session struct { // Details of what was requested VolumeRequest VolumeRequest - // Records if we have started trying to delete - DeleteRequested bool + // Flags about current state of the buffer + Status SessionStatus // Request certain files to be staged in // Not currently allowed for multi job volumes @@ -34,35 +34,38 @@ type Session struct { // Not currently allowed for multi job volumes StageOutRequests []DataCopyRequest - // The hosts that want to mount the storage - // Note: to allow for copy in/out the brick hosts are assumed to have an attachment - AttachHosts []string - - // If non-zero capacity requested, a volume is created for this job - // It may be exposed to the attach hosts in a variety of ways, as defined by the volume - JobVolume VolumeName - // There maybe be attachments to multiple shared volumes - MultiJobVolumes []VolumeName + MultiJobAttachments []SessionName // Environment variables for each volume associated with the job Paths map[string]string + // Resources used by session once pool granularity is taken into account + ActualSizeBytes int + // List of the bricks allocated to implement the JobVolume // One is the primary brick that should be watching for all actions Allocations []BrickAllocation - // If not empty, says where to send actions too - // If empty the session has not yet been acknowledged by the dacd process - SessionActionPrefix string + // Where session requests should be sent + PrimaryBrickHost BrickHostName - // Resources used by session once pool granularity is taken into account - ActualSizeBytes int + // The hosts that want to mount the storage + // Note: to allow for copy in/out the brick hosts are assumed to have an attachment + AttachHosts []string +} +type SessionStatus struct { // If not nil, the session has an unresolved error // and can't be mounted by any new sessions // but it can be deleted Error error + + // CreateVolume has succeeded, so other actions can now happen + FileSystemCreated bool + + // Records if we have started trying to delete + DeleteRequested bool } type SessionAction struct { diff --git a/internal/pkg/v2/mock_registry/allocation.go b/internal/pkg/v2/mock_registry/allocation.go index 6be1b0de..1f75f7e6 100644 --- a/internal/pkg/v2/mock_registry/allocation.go +++ b/internal/pkg/v2/mock_registry/allocation.go @@ -49,6 +49,21 @@ func (mr *MockAllocationRegistryMockRecorder) GetBricksByPool() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBricksByPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetBricksByPool)) } +// GetPoolInfo mocks base method +func (m *MockAllocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPoolInfo", poolName) + ret0, _ := ret[0].(datamodel.PoolInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPoolInfo indicates an expected call of GetPoolInfo +func (mr *MockAllocationRegistryMockRecorder) GetPoolInfo(poolName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPoolInfo", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPoolInfo), poolName) +} + // GetAllocationMutex mocks base method func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/mock_registry/pool.go b/internal/pkg/v2/mock_registry/pool.go index 035f7a59..bc39faf2 100644 --- a/internal/pkg/v2/mock_registry/pool.go +++ b/internal/pkg/v2/mock_registry/pool.go @@ -33,19 +33,19 @@ func (m *MockPoolRegistry) EXPECT() *MockPoolRegistryMockRecorder { return m.recorder } -// GetPools mocks base method -func (m *MockPoolRegistry) GetPools() ([]datamodel.Pool, error) { +// GetPool mocks base method +func (m *MockPoolRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPools") - ret0, _ := ret[0].([]datamodel.Pool) + ret := m.ctrl.Call(m, "GetPool", name) + ret0, _ := ret[0].(datamodel.Pool) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetPools indicates an expected call of GetPools -func (mr *MockPoolRegistryMockRecorder) GetPools() *gomock.Call { +// GetPool indicates an expected call of GetPool +func (mr *MockPoolRegistryMockRecorder) GetPool(name interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPools", reflect.TypeOf((*MockPoolRegistry)(nil).GetPools)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockPoolRegistry)(nil).GetPool), name) } // EnsurePoolCreated mocks base method diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go index e6e3fdda..c1034490 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -35,10 +35,10 @@ func (m *MockSessionActions) EXPECT() *MockSessionActionsMockRecorder { } // CreateSessionVolume mocks base method -func (m *MockSessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) { +func (m *MockSessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSessionVolume", ctxt, sessionName) - ret0, _ := ret[0].(<-chan datamodel.Session) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -50,10 +50,10 @@ func (mr *MockSessionActionsMockRecorder) CreateSessionVolume(ctxt, sessionName } // SendSessionAction mocks base method -func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.Session, error) { +func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SendSessionAction", ctxt, actionType, session) - ret0, _ := ret[0].(<-chan datamodel.Session) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/internal/pkg/v2/registry/allocation.go b/internal/pkg/v2/registry/allocation.go index 3493aef2..538a33e6 100644 --- a/internal/pkg/v2/registry/allocation.go +++ b/internal/pkg/v2/registry/allocation.go @@ -13,7 +13,6 @@ type AllocationRegistry interface { // bricks are only available if corresponding host currently alive GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) - // Caller should acquire this mutex before calling GetAllPools then CreateAllocations GetAllocationMutex() (store.Mutex, error) diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go index b420bbb9..51bdb7e5 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/v2/registry/session_actions.go @@ -20,7 +20,7 @@ type SessionActions interface { // Error if context is cancelled or timed-out SendSessionAction( ctxt context.Context, actionType datamodel.SessionActionType, - session datamodel.Session) (<-chan datamodel.Session, error) + session datamodel.Session) (<-chan datamodel.SessionAction, error) // Get session volume create requests, // where given hostname is the primary brick host diff --git a/internal/pkg/v2/registry_impl/actions.go b/internal/pkg/v2/registry_impl/actions.go deleted file mode 100644 index dfaf05ae..00000000 --- a/internal/pkg/v2/registry_impl/actions.go +++ /dev/null @@ -1,108 +0,0 @@ -package registry_impl - -import ( - "context" - "encoding/json" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" - "github.com/google/uuid" - "log" - "time" -) - -// TODO: this is the client side, need server side too -func NewSessionActions(keystore store.Keystore) registry.SessionActions { - return &sessionActions{keystore: keystore, defaultTimeout: time.Minute * 20} -} - -type sessionActions struct { - keystore store.Keystore - defaultTimeout time.Duration -} - -func (s *sessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.Session, error) { - panic("implement me") -} - -func (s *sessionActions) SendSessionAction( - ctxt context.Context, actionType datamodel.SessionActionType, - session datamodel.Session) (<-chan datamodel.Session, error) { - panic("implement me") -} - -func (s *sessionActions) GetCreateSessionVolumeRequests( - ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { - panic("implement me") -} - -func (s *sessionActions) GetSessionActions( - ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { - panic("implement me") -} - -func (s *sessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { - panic("implement me") -} - -func toJson(message interface{}) string { - b, error := json.Marshal(message) - if error != nil { - log.Fatal(error) - } - return string(b) -} - -func (s *sessionActions) sendAction(session datamodel.Session, action string) error { - // TODO: update session? - - // TODO: check primary session host is alive before sending event - // TODO: If we timeout, cancel the event only if the host is now not alive - - actionId := uuid.New().String() - sessionPrefix := fmt.Sprintf("/Session/Actions/%s", session.Name) - actionPrefix := fmt.Sprintf("%s/%s", sessionPrefix, actionId) - - mutex, err := s.keystore.NewMutex(sessionPrefix) - if err != nil { - return fmt.Errorf("unable to start action due to: %s", err.Error()) - } - mctxt, cancel := context.WithTimeout(context.Background(), s.defaultTimeout) - defer cancel() - err = mutex.Lock(mctxt) - if err != nil { - return fmt.Errorf("unable to start action due to: %s", err.Error()) - } - defer mutex.Unlock(context.Background()) - - ctxt, cancel := context.WithTimeout(context.Background(), s.defaultTimeout) - defer cancel() - responses := s.keystore.Watch(ctxt, fmt.Sprintf("%s/%s", actionPrefix, "output"), false) - - err = s.keystore.Add(store.KeyValue{ - Key: fmt.Sprintf("%s/%s", actionPrefix, "input"), - // TODO: need format request object - Value: action, - }) - if err != nil { - return err - } - - // TODO: how do we detect a timeout here? - for response := range responses { - if response.Err != nil { - return response.Err - } - - if response.IsCreate { - // TODO: need formal response object? - if response.New.Value != "" { - return fmt.Errorf("error while sending action") - } - // TODO: who deletes the action key, if anyone? - } - return nil - } - return nil -} From 515c88e2f6a31b7ac92e5246163e5699f5b96617 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 09:12:36 +0100 Subject: [PATCH 048/191] Rename workflow to facade --- build/rebuild_mocks.sh | 4 +- cmd/dacctl/main_test.go | 8 +-- .../pkg/v2/dacctl/actions_impl/actions.go | 14 ++-- .../v2/dacctl/actions_impl/actions_test.go | 26 ++++---- internal/pkg/v2/dacctl/actions_impl/job.go | 2 +- .../pkg/v2/dacctl/actions_impl/job_test.go | 8 +-- .../pkg/v2/dacctl/actions_impl/persistent.go | 2 +- .../v2/dacctl/actions_impl/persistent_test.go | 6 +- .../pkg/v2/dacctl/actions_impl/show_test.go | 12 ++-- .../pkg/v2/dacctl/workflow_impl/session.go | 32 ++++----- .../v2/dacctl/workflow_impl/session_test.go | 11 ++++ .../dacd/brick_manager_impl/brick_manager.go | 6 +- .../brick_manager_impl/brick_manager_test.go | 4 +- .../session_action_handler.go | 4 +- .../pkg/v2/{workflow => facade}/session.go | 12 ++-- .../session_action_handler.go | 2 +- .../{mock_workflow => mock_facade}/session.go | 66 +++++++++---------- .../session_action_handler.go | 6 +- internal/pkg/v2/mock_registry/actions.go | 40 +++++------ 19 files changed, 138 insertions(+), 127 deletions(-) create mode 100644 internal/pkg/v2/dacctl/workflow_impl/session_test.go rename internal/pkg/v2/{workflow => facade}/session.go (75%) rename internal/pkg/v2/{workflow => facade}/session_action_handler.go (90%) rename internal/pkg/v2/{mock_workflow => mock_facade}/session.go (60%) rename internal/pkg/v2/{mock_workflow => mock_facade}/session_action_handler.go (91%) diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 61522103..96bdd041 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -35,8 +35,8 @@ done items="session session_action_handler" for i in $items; do - mockgen -source=internal/pkg/v2/workflow/${i}.go \ - >internal/pkg/v2/mock_workflow/${i}.go + mockgen -source=internal/pkg/v2/facade/${i}.go \ + >internal/pkg/v2/mock_facade/${i}.go done items="keystore" diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 1edf6837..e5ae4d96 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -125,7 +125,7 @@ func TestFlow(t *testing.T) { assert.Equal(t, "RealSize", err.Error()) err = runCli([]string{"--function", "data_in", "--token", "a", "--job", "b"}) - assert.Equal(t, "DataIn", err.Error()) + assert.Equal(t, "CopyDataIn", err.Error()) err = runCli([]string{"--function", "paths", "--token", "a", "--job", "b", "--pathfile", "c"}) assert.Equal(t, "Paths", err.Error()) @@ -137,7 +137,7 @@ func TestFlow(t *testing.T) { assert.Equal(t, "PostRun", err.Error()) err = runCli([]string{"--function", "data_out", "--token", "a", "--job", "b"}) - assert.Equal(t, "DataOut", err.Error()) + assert.Equal(t, "CopyDataOut", err.Error()) } type stubKeystore struct{} @@ -209,7 +209,7 @@ func (*stubDacctlActions) RealSize(c actions.CliContext) error { return errors.New("RealSize") } func (*stubDacctlActions) DataIn(c actions.CliContext) error { - return errors.New("DataIn") + return errors.New("CopyDataIn") } func (*stubDacctlActions) Paths(c actions.CliContext) error { return errors.New("Paths") @@ -221,5 +221,5 @@ func (*stubDacctlActions) PostRun(c actions.CliContext) error { return errors.New("PostRun") } func (*stubDacctlActions) DataOut(c actions.CliContext) error { - return errors.New("DataOut") + return errors.New("CopyDataOut") } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index c6056272..baf901d9 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -7,12 +7,12 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "log" "strings" ) -func NewDacctlActions(actions workflow.Session, disk fileio.Disk) dacctl.DacctlActions { +func NewDacctlActions(actions facade.Session, disk fileio.Disk) dacctl.DacctlActions { return &dacctlActions{ session: actions, disk: disk, @@ -20,7 +20,7 @@ func NewDacctlActions(actions workflow.Session, disk fileio.Disk) dacctl.DacctlA } type dacctlActions struct { - session workflow.Session + session facade.Session disk fileio.Disk } @@ -67,7 +67,7 @@ func (d *dacctlActions) DataIn(c dacctl.CliContext) error { if err != nil { return err } - return d.session.DataIn(sessionName) + return d.session.CopyDataIn(sessionName) } func (d *dacctlActions) PreRun(c dacctl.CliContext) error { @@ -97,7 +97,7 @@ func (d *dacctlActions) PreRun(c dacctl.CliContext) error { } } - return d.session.AttachVolumes(sessionName, computeHosts, loginNodeHosts) + return d.session.Mount(sessionName, computeHosts, loginNodeHosts) } func (d *dacctlActions) PostRun(c dacctl.CliContext) error { @@ -105,7 +105,7 @@ func (d *dacctlActions) PostRun(c dacctl.CliContext) error { if err != nil { return err } - return d.session.DetachVolumes(sessionName) + return d.session.Unmount(sessionName) } func (d *dacctlActions) DataOut(c dacctl.CliContext) error { @@ -113,5 +113,5 @@ func (d *dacctlActions) DataOut(c dacctl.CliContext) error { if err != nil { return err } - return d.session.DataOut(sessionName) + return d.session.CopyDataOut(sessionName) } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index f2c7ad4a..a6f9838e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -51,7 +51,7 @@ func getMockCliContext(capacity int) *mockCliContext { func TestDacctlActions_DeleteBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) fakeError := errors.New("fake") session.EXPECT().DeleteSession(datamodel.SessionName("bar"), true).Return(fakeError) @@ -71,10 +71,10 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { func TestDacctlActions_DataIn(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) fakeError := errors.New("fake") - session.EXPECT().DataIn(datamodel.SessionName("bar")).Return(fakeError) + session.EXPECT().CopyDataIn(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(session, nil) err := actions.DataIn(&mockCliContext{ @@ -95,10 +95,10 @@ func TestDacctlActions_DataIn(t *testing.T) { func TestDacctlActions_DataOut(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) fakeError := errors.New("fake") - session.EXPECT().DataOut(datamodel.SessionName("bar")).Return(fakeError) + session.EXPECT().CopyDataOut(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(session, nil) err := actions.DataOut(&mockCliContext{ @@ -114,7 +114,7 @@ func TestDacctlActions_DataOut(t *testing.T) { func TestDacctlActions_PreRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} @@ -122,7 +122,7 @@ func TestDacctlActions_PreRun(t *testing.T) { disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) + session.EXPECT().Mount(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) actions := NewDacctlActions(session, disk) err := actions.PreRun(&mockCliContext{ @@ -145,13 +145,13 @@ func TestDacctlActions_PreRun(t *testing.T) { func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) fakeError := errors.New("fake") - session.EXPECT().AttachVolumes(datamodel.SessionName("bar"), computeHosts, nil).Return(fakeError) + session.EXPECT().Mount(datamodel.SessionName("bar"), computeHosts, nil).Return(fakeError) actions := NewDacctlActions(session, disk) err := actions.PreRun(&mockCliContext{ @@ -186,7 +186,7 @@ func TestDacctlActions_PreRun_BadHosts(t *testing.T) { func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} @@ -227,10 +227,10 @@ func TestDacctlActions_PreRun_NoHosts(t *testing.T) { func TestDacctlActions_PostRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) fakeError := errors.New("fake") - session.EXPECT().DetachVolumes(datamodel.SessionName("bar")).Return(fakeError) + session.EXPECT().Unmount(datamodel.SessionName("bar")).Return(fakeError) actions := NewDacctlActions(session, nil) err := actions.PostRun(&mockCliContext{ diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index aa1856f1..c9066e70 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -85,7 +85,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { } session.Paths = getPaths(session) - return d.session.CreateSessionVolume(session) + return d.session.CreateSession(session) } func getPaths(session datamodel.Session) map[string]string { diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 4cea4f7d..5d100598 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -3,7 +3,7 @@ package actions_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -12,7 +12,7 @@ import ( func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) lines := []string{`#DW bad cmd`} @@ -33,7 +33,7 @@ func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) lines := []string{ @@ -46,7 +46,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, } disk.EXPECT().Lines("jobfile").Return(lines, nil) - session.EXPECT().CreateSessionVolume(datamodel.Session{ + session.EXPECT().CreateSession(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go index 859321cd..43cee938 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -63,5 +63,5 @@ func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { Group: uint(c.Int("group")), CreatedAt: getNow(), } - return d.session.CreateSessionVolume(session) + return d.session.CreateSession(session) } diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go index 990c0e71..c02a2012 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go @@ -2,7 +2,7 @@ package actions_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -11,9 +11,9 @@ import ( func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) - session.EXPECT().CreateSessionVolume(datamodel.Session{ + session.EXPECT().CreateSession(datamodel.Session{ Name: "token", Owner: 1001, Group: 1002, diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index 209e75bb..daa2de40 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -13,7 +13,7 @@ import ( func TestDacctlActions_RealSize(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{ Name: datamodel.SessionName("bar"), ActualSizeBytes: 123, @@ -34,7 +34,7 @@ func TestDacctlActions_RealSize(t *testing.T) { func TestDacctlActions_Paths(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) disk := mocks.NewMockDisk(mockCtrl) session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{ @@ -72,7 +72,7 @@ func TestDacctlActions_Paths(t *testing.T) { func TestDacctlActions_ShowInstances(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) session.EXPECT().GetAllSessions().Return([]datamodel.Session{ { Name: datamodel.SessionName("foo"), @@ -107,7 +107,7 @@ func TestDacctlActions_ShowInstances(t *testing.T) { func TestDacctlActions_ShowSessions(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) session.EXPECT().GetAllSessions().Return([]datamodel.Session{ { Name: datamodel.SessionName("foo"), @@ -144,7 +144,7 @@ func TestDacctlActions_ShowSessions(t *testing.T) { func TestDacctlActions_ListPools(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - session := mock_workflow.NewMockSession(mockCtrl) + session := mock_facade.NewMockSession(mockCtrl) session.EXPECT().GetPools().Return([]datamodel.PoolInfo{ { Pool: datamodel.Pool{ diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index f028e376..49ac26f3 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -4,26 +4,26 @@ import ( "context" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" "math" "math/rand" "time" ) -func NewSessionWorkflow() workflow.Session { - return sessionWorkflow{} +func NewSessionFacade() facade.Session { + return sessionFacade{} } -type sessionWorkflow struct { +type sessionFacade struct { session registry.SessionRegistry actions registry.SessionActions allocations registry.AllocationRegistry pool registry.PoolRegistry } -func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { +func (s sessionFacade) CreateSession(session datamodel.Session) error { // TODO needs to get the allocation mutex, create the session, then create the allocations // failing if the pool isn't known, or doesn't have enough space err := s.validateSession(session) @@ -67,7 +67,7 @@ func (s sessionWorkflow) CreateSessionVolume(session datamodel.Session) error { return sessionAction.Error } -func (s sessionWorkflow) validateSession(session datamodel.Session) error { +func (s sessionFacade) validateSession(session datamodel.Session) error { _, err := s.pool.GetPool(session.VolumeRequest.PoolName) if err != nil { return fmt.Errorf("invalid session, unable to find pool %s", session.VolumeRequest.PoolName) @@ -76,7 +76,7 @@ func (s sessionWorkflow) validateSession(session datamodel.Session) error { return nil } -func (s sessionWorkflow) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { +func (s sessionFacade) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { allocationMutex, err := s.allocations.GetAllocationMutex() if err != nil { return session, err @@ -114,7 +114,7 @@ func (s sessionWorkflow) doSessionAllocation(session datamodel.Session) (datamod return session, err } -func (s sessionWorkflow) getBricks(poolName datamodel.PoolName, bytes int) (int, []datamodel.Brick, error) { +func (s sessionFacade) getBricks(poolName datamodel.PoolName, bytes int) (int, []datamodel.Brick, error) { pool, err := s.allocations.GetPoolInfo(poolName) if err != nil { return 0, nil, err @@ -160,7 +160,7 @@ func getBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Bric return chosenBricks } -func (s sessionWorkflow) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { +func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { // Get session lock sessionMutex, err := s.session.GetSessionMutex(sessionName) if err != nil { @@ -206,30 +206,30 @@ func (s sessionWorkflow) DeleteSession(sessionName datamodel.SessionName, hurry return result.Error } -func (s sessionWorkflow) DataIn(sessionName datamodel.SessionName) error { +func (s sessionFacade) CopyDataIn(sessionName datamodel.SessionName) error { panic("implement me") } -func (s sessionWorkflow) AttachVolumes(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error { +func (s sessionFacade) Mount(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error { panic("implement me") } -func (s sessionWorkflow) DetachVolumes(sessionName datamodel.SessionName) error { +func (s sessionFacade) Unmount(sessionName datamodel.SessionName) error { panic("implement me") } -func (s sessionWorkflow) DataOut(sessionName datamodel.SessionName) error { +func (s sessionFacade) CopyDataOut(sessionName datamodel.SessionName) error { panic("implement me") } -func (s sessionWorkflow) GetPools() ([]datamodel.PoolInfo, error) { +func (s sessionFacade) GetPools() ([]datamodel.PoolInfo, error) { panic("implement me") } -func (s sessionWorkflow) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { +func (s sessionFacade) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { panic("implement me") } -func (s sessionWorkflow) GetAllSessions() ([]datamodel.Session, error) { +func (s sessionFacade) GetAllSessions() ([]datamodel.Session, error) { panic("implement me") } diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go new file mode 100644 index 00000000..61170d2e --- /dev/null +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -0,0 +1,11 @@ +package workflow_impl + +import "testing" + +func TestSessionFacade_CreateSession(t *testing.T) { + +} + +func TestSessionFacade_DeleteSession(t *testing.T) { + +} diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index eee2b163..65d6924f 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -4,12 +4,12 @@ import ( "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" ) -func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.SessionActionHandler) dacd.BrickManager { +func NewBrickManager(brickRegistry registry.BrickRegistry, handler facade.SessionActionHandler) dacd.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), brickRegistry: brickRegistry, @@ -20,7 +20,7 @@ func NewBrickManager(brickRegistry registry.BrickRegistry, handler workflow.Sess type brickManager struct { config config.BrickManagerConfig brickRegistry registry.BrickRegistry - sessionActionHandler workflow.SessionActionHandler + sessionActionHandler facade.SessionActionHandler } func (bm *brickManager) Hostname() string { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index a62daca4..8a44043c 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -4,8 +4,8 @@ import ( "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_workflow" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -20,7 +20,7 @@ func TestBrickManager_Startup(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() brickRegistry := mock_registry.NewMockBrickRegistry(mockCtrl) - handler := mock_workflow.NewMockSessionActionHandler(mockCtrl) + handler := mock_facade.NewMockSessionActionHandler(mockCtrl) brickManager := NewBrickManager(brickRegistry, handler) // TODO... diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index ede51a6a..77a20795 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -2,12 +2,12 @@ package brick_manager_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/workflow" "log" ) -func NewSessionActionHandler(actions registry.SessionActions) workflow.SessionActionHandler { +func NewSessionActionHandler(actions registry.SessionActions) facade.SessionActionHandler { return &sessionActionHandler{actions: actions} } diff --git a/internal/pkg/v2/workflow/session.go b/internal/pkg/v2/facade/session.go similarity index 75% rename from internal/pkg/v2/workflow/session.go rename to internal/pkg/v2/facade/session.go index 7db40959..af61127f 100644 --- a/internal/pkg/v2/workflow/session.go +++ b/internal/pkg/v2/facade/session.go @@ -1,4 +1,4 @@ -package workflow +package facade import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" @@ -7,7 +7,7 @@ import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" // All the calls block until they are complete, or an error occurs type Session interface { // Allocates storage and - CreateSessionVolume(session datamodel.Session) error + CreateSession(session datamodel.Session) error // Deletes the requested volume and session allocation // If hurry, there is no stage-out attempted @@ -15,16 +15,16 @@ type Session interface { DeleteSession(sessionName datamodel.SessionName, hurry bool) error // Update the session and trigger requested data copy in - DataIn(sessionName datamodel.SessionName) error + CopyDataIn(sessionName datamodel.SessionName) error // Update session hosts and attach volumes as needed - AttachVolumes(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error + Mount(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error // Attempt to detach volumes - DetachVolumes(sessionName datamodel.SessionName) error + Unmount(sessionName datamodel.SessionName) error // Update the session and trigger requested data copy out - DataOut(sessionName datamodel.SessionName) error + CopyDataOut(sessionName datamodel.SessionName) error // Get brick availability by pool GetPools() ([]datamodel.PoolInfo, error) diff --git a/internal/pkg/v2/workflow/session_action_handler.go b/internal/pkg/v2/facade/session_action_handler.go similarity index 90% rename from internal/pkg/v2/workflow/session_action_handler.go rename to internal/pkg/v2/facade/session_action_handler.go index 7592a152..0727d9af 100644 --- a/internal/pkg/v2/workflow/session_action_handler.go +++ b/internal/pkg/v2/facade/session_action_handler.go @@ -1,4 +1,4 @@ -package workflow +package facade import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" diff --git a/internal/pkg/v2/mock_workflow/session.go b/internal/pkg/v2/mock_facade/session.go similarity index 60% rename from internal/pkg/v2/mock_workflow/session.go rename to internal/pkg/v2/mock_facade/session.go index a3b48021..c19048a7 100644 --- a/internal/pkg/v2/mock_workflow/session.go +++ b/internal/pkg/v2/mock_facade/session.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/workflow/session.go +// Source: internal/pkg/v2/facade/session.go -// Package mock_workflow is a generated GoMock package. -package mock_workflow +// Package mock_facade is a generated GoMock package. +package mock_facade import ( datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" @@ -33,18 +33,18 @@ func (m *MockSession) EXPECT() *MockSessionMockRecorder { return m.recorder } -// CreateSessionVolume mocks base method -func (m *MockSession) CreateSessionVolume(session datamodel.Session) error { +// CreateSession mocks base method +func (m *MockSession) CreateSession(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSessionVolume", session) + ret := m.ctrl.Call(m, "CreateSession", session) ret0, _ := ret[0].(error) return ret0 } -// CreateSessionVolume indicates an expected call of CreateSessionVolume -func (mr *MockSessionMockRecorder) CreateSessionVolume(session interface{}) *gomock.Call { +// CreateSession indicates an expected call of CreateSession +func (mr *MockSessionMockRecorder) CreateSession(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockSession)(nil).CreateSessionVolume), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockSession)(nil).CreateSession), session) } // DeleteSession mocks base method @@ -61,60 +61,60 @@ func (mr *MockSessionMockRecorder) DeleteSession(sessionName, hurry interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSession)(nil).DeleteSession), sessionName, hurry) } -// DataIn mocks base method -func (m *MockSession) DataIn(sessionName datamodel.SessionName) error { +// CopyDataIn mocks base method +func (m *MockSession) CopyDataIn(sessionName datamodel.SessionName) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DataIn", sessionName) + ret := m.ctrl.Call(m, "CopyDataIn", sessionName) ret0, _ := ret[0].(error) return ret0 } -// DataIn indicates an expected call of DataIn -func (mr *MockSessionMockRecorder) DataIn(sessionName interface{}) *gomock.Call { +// CopyDataIn indicates an expected call of CopyDataIn +func (mr *MockSessionMockRecorder) CopyDataIn(sessionName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataIn", reflect.TypeOf((*MockSession)(nil).DataIn), sessionName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataIn", reflect.TypeOf((*MockSession)(nil).CopyDataIn), sessionName) } -// AttachVolumes mocks base method -func (m *MockSession) AttachVolumes(sessionName datamodel.SessionName, computeNodes, loginNodes []string) error { +// Mount mocks base method +func (m *MockSession) Mount(sessionName datamodel.SessionName, computeNodes, loginNodes []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AttachVolumes", sessionName, computeNodes, loginNodes) + ret := m.ctrl.Call(m, "Mount", sessionName, computeNodes, loginNodes) ret0, _ := ret[0].(error) return ret0 } -// AttachVolumes indicates an expected call of AttachVolumes -func (mr *MockSessionMockRecorder) AttachVolumes(sessionName, computeNodes, loginNodes interface{}) *gomock.Call { +// Mount indicates an expected call of Mount +func (mr *MockSessionMockRecorder) Mount(sessionName, computeNodes, loginNodes interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockSession)(nil).AttachVolumes), sessionName, computeNodes, loginNodes) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mount", reflect.TypeOf((*MockSession)(nil).Mount), sessionName, computeNodes, loginNodes) } -// DetachVolumes mocks base method -func (m *MockSession) DetachVolumes(sessionName datamodel.SessionName) error { +// Unmount mocks base method +func (m *MockSession) Unmount(sessionName datamodel.SessionName) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DetachVolumes", sessionName) + ret := m.ctrl.Call(m, "Unmount", sessionName) ret0, _ := ret[0].(error) return ret0 } -// DetachVolumes indicates an expected call of DetachVolumes -func (mr *MockSessionMockRecorder) DetachVolumes(sessionName interface{}) *gomock.Call { +// Unmount indicates an expected call of Unmount +func (mr *MockSessionMockRecorder) Unmount(sessionName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachVolumes", reflect.TypeOf((*MockSession)(nil).DetachVolumes), sessionName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unmount", reflect.TypeOf((*MockSession)(nil).Unmount), sessionName) } -// DataOut mocks base method -func (m *MockSession) DataOut(sessionName datamodel.SessionName) error { +// CopyDataOut mocks base method +func (m *MockSession) CopyDataOut(sessionName datamodel.SessionName) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DataOut", sessionName) + ret := m.ctrl.Call(m, "CopyDataOut", sessionName) ret0, _ := ret[0].(error) return ret0 } -// DataOut indicates an expected call of DataOut -func (mr *MockSessionMockRecorder) DataOut(sessionName interface{}) *gomock.Call { +// CopyDataOut indicates an expected call of CopyDataOut +func (mr *MockSessionMockRecorder) CopyDataOut(sessionName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataOut", reflect.TypeOf((*MockSession)(nil).DataOut), sessionName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataOut", reflect.TypeOf((*MockSession)(nil).CopyDataOut), sessionName) } // GetPools mocks base method diff --git a/internal/pkg/v2/mock_workflow/session_action_handler.go b/internal/pkg/v2/mock_facade/session_action_handler.go similarity index 91% rename from internal/pkg/v2/mock_workflow/session_action_handler.go rename to internal/pkg/v2/mock_facade/session_action_handler.go index ef1f51fc..bf6745fa 100644 --- a/internal/pkg/v2/mock_workflow/session_action_handler.go +++ b/internal/pkg/v2/mock_facade/session_action_handler.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/workflow/session_action_handler.go +// Source: internal/pkg/v2/facade/session_action_handler.go -// Package mock_workflow is a generated GoMock package. -package mock_workflow +// Package mock_facade is a generated GoMock package. +package mock_facade import ( datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" diff --git a/internal/pkg/v2/mock_registry/actions.go b/internal/pkg/v2/mock_registry/actions.go index 8a601955..00ef4b37 100644 --- a/internal/pkg/v2/mock_registry/actions.go +++ b/internal/pkg/v2/mock_registry/actions.go @@ -33,18 +33,18 @@ func (m *MockActions) EXPECT() *MockActionsMockRecorder { return m.recorder } -// CreateSessionVolume mocks base method +// CreateSession mocks base method func (m *MockActions) CreateSessionVolume(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSessionVolume", session) + ret := m.ctrl.Call(m, "CreateSession", session) ret0, _ := ret[0].(error) return ret0 } -// CreateSessionVolume indicates an expected call of CreateSessionVolume +// CreateSession indicates an expected call of CreateSession func (mr *MockActionsMockRecorder) CreateSessionVolume(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockActions)(nil).CreateSessionVolume), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockActions)(nil).CreateSessionVolume), session) } // DeleteSession mocks base method @@ -61,58 +61,58 @@ func (mr *MockActionsMockRecorder) DeleteSession(session interface{}) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockActions)(nil).DeleteSession), session) } -// DataIn mocks base method +// CopyDataIn mocks base method func (m *MockActions) DataIn(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DataIn", session) + ret := m.ctrl.Call(m, "CopyDataIn", session) ret0, _ := ret[0].(error) return ret0 } -// DataIn indicates an expected call of DataIn +// CopyDataIn indicates an expected call of CopyDataIn func (mr *MockActionsMockRecorder) DataIn(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataIn", reflect.TypeOf((*MockActions)(nil).DataIn), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataIn", reflect.TypeOf((*MockActions)(nil).DataIn), session) } -// AttachVolumes mocks base method +// Mount mocks base method func (m *MockActions) AttachVolumes(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AttachVolumes", session) + ret := m.ctrl.Call(m, "Mount", session) ret0, _ := ret[0].(error) return ret0 } -// AttachVolumes indicates an expected call of AttachVolumes +// Mount indicates an expected call of Mount func (mr *MockActionsMockRecorder) AttachVolumes(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachVolumes", reflect.TypeOf((*MockActions)(nil).AttachVolumes), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mount", reflect.TypeOf((*MockActions)(nil).AttachVolumes), session) } -// DetachVolumes mocks base method +// Unmount mocks base method func (m *MockActions) DetachVolumes(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DetachVolumes", session) + ret := m.ctrl.Call(m, "Unmount", session) ret0, _ := ret[0].(error) return ret0 } -// DetachVolumes indicates an expected call of DetachVolumes +// Unmount indicates an expected call of Unmount func (mr *MockActionsMockRecorder) DetachVolumes(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DetachVolumes", reflect.TypeOf((*MockActions)(nil).DetachVolumes), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unmount", reflect.TypeOf((*MockActions)(nil).DetachVolumes), session) } -// DataOut mocks base method +// CopyDataOut mocks base method func (m *MockActions) DataOut(session datamodel.Session) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DataOut", session) + ret := m.ctrl.Call(m, "CopyDataOut", session) ret0, _ := ret[0].(error) return ret0 } -// DataOut indicates an expected call of DataOut +// CopyDataOut indicates an expected call of CopyDataOut func (mr *MockActionsMockRecorder) DataOut(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataOut", reflect.TypeOf((*MockActions)(nil).DataOut), session) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataOut", reflect.TypeOf((*MockActions)(nil).DataOut), session) } From f8c36f54983b430ca36c66f20b9baa44c2481e41 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 11:59:49 +0100 Subject: [PATCH 049/191] Add initial unit tests for create session --- .../pkg/v2/dacctl/workflow_impl/session.go | 53 ++++--- .../v2/dacctl/workflow_impl/session_test.go | 144 +++++++++++++++++- 2 files changed, 173 insertions(+), 24 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 49ac26f3..da0cec55 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -43,17 +43,15 @@ func (s sessionFacade) CreateSession(session datamodel.Session) error { // Allocate bricks, and choose brick host server session, err = s.doSessionAllocation(session) - if err != nil { + // if no bricks allocated, no need to call CreateSessionVolume + if err != nil || session.ActualSizeBytes == 0 { sessionMutex.Unlock(context.TODO()) return err } // Create filesystem on the brick host server // TODO: add timeout - eventChan, err := s.actions.CreateSessionVolume(context.TODO(), session.Name) - if err != nil { - return err - } + eventChan, createErr := s.actions.CreateSessionVolume(context.TODO(), session.Name) // Drop mutex so the server can take it err = sessionMutex.Unlock(context.TODO()) @@ -62,6 +60,12 @@ func (s sessionFacade) CreateSession(session datamodel.Session) error { return err } + // Always drop the mutex, but check error before checking the channel + if createErr != nil { + // TODO: session should go into Error State? Maybe its up the above call in this case? + return createErr + } + // Wait for the server to create the filesystem sessionAction := <-eventChan return sessionAction.Error @@ -77,39 +81,44 @@ func (s sessionFacade) validateSession(session datamodel.Session) error { } func (s sessionFacade) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { - allocationMutex, err := s.allocations.GetAllocationMutex() - if err != nil { - return session, err - } + if session.VolumeRequest.TotalCapacityBytes > 0 { + allocationMutex, err := s.allocations.GetAllocationMutex() + if err != nil { + return session, err + } - err = allocationMutex.Lock(context.TODO()) - if err != nil { - return session, err - } - defer allocationMutex.Unlock(context.TODO()) + err = allocationMutex.Lock(context.TODO()) + if err != nil { + return session, err + } + defer allocationMutex.Unlock(context.TODO()) - if session.VolumeRequest.TotalCapacityBytes > 0 { - // Write allocations first + // Write allocations before creating the session actualSizeBytes, chosenBricks, err := s.getBricks(session.VolumeRequest.PoolName, session.VolumeRequest.TotalCapacityBytes) if err != nil { return session, fmt.Errorf("can't allocate for session: %s due to %s", session.Name, err) } - session.ActualSizeBytes = actualSizeBytes allocations, err := s.allocations.CreateAllocations(session.Name, chosenBricks) if err != nil { return session, err } + session.ActualSizeBytes = actualSizeBytes session.Allocations = allocations session.PrimaryBrickHost = allocations[0].Brick.BrickHostName } // Store initial version of session - session, err = s.session.CreateSession(session) + // returned session will have updated revision info + session, err := s.session.CreateSession(session) if err != nil { - // TODO: remove allocations - return session, err + if session.Allocations != nil { + deleteErr := s.allocations.DeleteAllocations(session.Allocations) + if deleteErr != nil { + log.Println("Failed to clean up allocations due to:", deleteErr) + } + } } return session, err } @@ -123,7 +132,7 @@ func (s sessionFacade) getBricks(poolName datamodel.PoolName, bytes int) (int, [ bricksRequired := int(math.Ceil(float64(bytes) / float64(pool.Pool.GranularityBytes))) actualSize := bricksRequired * int(pool.Pool.GranularityBytes) - bricks := getBricks(bricksRequired, pool) + bricks := pickBricks(bricksRequired, pool) if len(bricks) != bricksRequired { return 0, nil, fmt.Errorf( "unable to get number of requested bricks (%d) for given pool (%s)", @@ -132,7 +141,7 @@ func (s sessionFacade) getBricks(poolName datamodel.PoolName, bytes int) (int, [ return actualSize, bricks, nil } -func getBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Brick { +func pickBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Brick { // pick some of the available bricks s := rand.NewSource(time.Now().Unix()) r := rand.New(s) // initialize local pseudorandom generator diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 61170d2e..d9685c93 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -1,11 +1,151 @@ package workflow_impl -import "testing" +import ( + "context" + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) -func TestSessionFacade_CreateSession(t *testing.T) { +func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { + initialSession := datamodel.Session{ + Name:"foo", + VolumeRequest: datamodel.VolumeRequest{ + PoolName: datamodel.PoolName("pool1"), + TotalCapacityBytes: 0, + }, + } + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) + poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) + allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) + facade := sessionFacade{ + session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + } + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + sessionMutex := mock_store.NewMockMutex(mockCtrl) + sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) + sessionMutex.EXPECT().Lock(context.TODO()) + sessionRegistry.EXPECT().CreateSession(initialSession).Return(initialSession, nil) + sessionMutex.EXPECT().Unlock(context.TODO()) + + err := facade.CreateSession(initialSession) + + assert.Nil(t, err) +} + +func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { + initialSession := datamodel.Session{ + Name:"foo", + VolumeRequest: datamodel.VolumeRequest{ + PoolName: datamodel.PoolName("pool1"), + TotalCapacityBytes: 2048, + }, + } + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) + poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) + allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) + facade := sessionFacade{ + session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + } + + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + sessionMutex := mock_store.NewMockMutex(mockCtrl) + sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) + sessionMutex.EXPECT().Lock(context.TODO()) + allocationMutex := mock_store.NewMockMutex(mockCtrl) + allocations.EXPECT().GetAllocationMutex().Return(allocationMutex, nil) + allocationMutex.EXPECT().Lock(context.TODO()) + allocations.EXPECT().GetPoolInfo(initialSession.VolumeRequest.PoolName).Return(datamodel.PoolInfo{ + Pool: datamodel.Pool{ + Name: "pool1", GranularityBytes: 1024, + }, + }, nil) + allocationMutex.EXPECT().Unlock(context.TODO()) + sessionMutex.EXPECT().Unlock(context.TODO()) + + err := facade.CreateSession(initialSession) + + assert.Equal(t, "can't allocate for session: foo due to unable to get number of requested bricks (2) for given pool (pool1)", err.Error()) +} + +func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) { + initialSession := datamodel.Session{ + Name:"foo", + VolumeRequest: datamodel.VolumeRequest{ + PoolName: datamodel.PoolName("pool1"), + TotalCapacityBytes: 1024, + }, + } + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) + poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) + allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) + facade := sessionFacade{ + session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + } + + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + sessionMutex := mock_store.NewMockMutex(mockCtrl) + sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) + sessionMutex.EXPECT().Lock(context.TODO()) + allocationMutex := mock_store.NewMockMutex(mockCtrl) + allocations.EXPECT().GetAllocationMutex().Return(allocationMutex, nil) + allocationMutex.EXPECT().Lock(context.TODO()) + brickList := []datamodel.Brick{{Device:"sda", BrickHostName: datamodel.BrickHostName("host1")}} + allocations.EXPECT().GetPoolInfo(initialSession.VolumeRequest.PoolName).Return(datamodel.PoolInfo{ + Pool: datamodel.Pool{ + Name: "pool1", GranularityBytes: 1024, + }, + AvailableBricks: brickList, + }, nil) + allocationList := []datamodel.BrickAllocation{{Brick:brickList[0]}} + allocations.EXPECT().CreateAllocations(initialSession.Name, brickList).Return(allocationList,nil) + updatedSession := datamodel.Session{ + Name: "foo", + VolumeRequest: datamodel.VolumeRequest{ + PoolName: datamodel.PoolName("pool1"), + TotalCapacityBytes: 1024, + }, + ActualSizeBytes: 1024, + Allocations: allocationList, + PrimaryBrickHost: allocationList[0].Brick.BrickHostName, + } + returnedSession := datamodel.Session{ + Name: "foo", + ActualSizeBytes: 1024, + } + sessionRegistry.EXPECT().CreateSession(updatedSession).Return(returnedSession, nil) + allocationMutex.EXPECT().Unlock(context.TODO()) + fakeErr := errors.New("fake") + actions.EXPECT().CreateSessionVolume(context.TODO(), initialSession.Name).Return(nil, fakeErr) + sessionMutex.EXPECT().Unlock(context.TODO()) + + err := facade.CreateSession(initialSession) + + assert.Equal(t, fakeErr, err) } func TestSessionFacade_DeleteSession(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) + facade := sessionFacade{session:sessionRegistry, actions:actions} + + err := facade.DeleteSession(datamodel.SessionName("foo"), true) + assert.Nil(t, err) } From bd1b93683cff148fddbe419eaaecee5842ea394e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 12:01:02 +0100 Subject: [PATCH 050/191] Fix up formatting --- .../pkg/v2/dacctl/workflow_impl/session.go | 2 - .../v2/dacctl/workflow_impl/session_test.go | 38 +++++++++---------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index da0cec55..ba5f0daa 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -24,8 +24,6 @@ type sessionFacade struct { } func (s sessionFacade) CreateSession(session datamodel.Session) error { - // TODO needs to get the allocation mutex, create the session, then create the allocations - // failing if the pool isn't known, or doesn't have enough space err := s.validateSession(session) if err != nil { return err diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index d9685c93..10e89d8e 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -13,9 +13,9 @@ import ( func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { initialSession := datamodel.Session{ - Name:"foo", + Name: "foo", VolumeRequest: datamodel.VolumeRequest{ - PoolName: datamodel.PoolName("pool1"), + PoolName: datamodel.PoolName("pool1"), TotalCapacityBytes: 0, }, } @@ -26,10 +26,10 @@ func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) @@ -43,9 +43,9 @@ func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { initialSession := datamodel.Session{ - Name:"foo", + Name: "foo", VolumeRequest: datamodel.VolumeRequest{ - PoolName: datamodel.PoolName("pool1"), + PoolName: datamodel.PoolName("pool1"), TotalCapacityBytes: 2048, }, } @@ -56,10 +56,10 @@ func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) @@ -81,9 +81,9 @@ func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) { initialSession := datamodel.Session{ - Name:"foo", + Name: "foo", VolumeRequest: datamodel.VolumeRequest{ - PoolName: datamodel.PoolName("pool1"), + PoolName: datamodel.PoolName("pool1"), TotalCapacityBytes: 1024, }, } @@ -94,37 +94,37 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session:sessionRegistry, actions:actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name:"pool1"}, nil) + poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) allocationMutex := mock_store.NewMockMutex(mockCtrl) allocations.EXPECT().GetAllocationMutex().Return(allocationMutex, nil) allocationMutex.EXPECT().Lock(context.TODO()) - brickList := []datamodel.Brick{{Device:"sda", BrickHostName: datamodel.BrickHostName("host1")}} + brickList := []datamodel.Brick{{Device: "sda", BrickHostName: datamodel.BrickHostName("host1")}} allocations.EXPECT().GetPoolInfo(initialSession.VolumeRequest.PoolName).Return(datamodel.PoolInfo{ Pool: datamodel.Pool{ Name: "pool1", GranularityBytes: 1024, }, AvailableBricks: brickList, }, nil) - allocationList := []datamodel.BrickAllocation{{Brick:brickList[0]}} - allocations.EXPECT().CreateAllocations(initialSession.Name, brickList).Return(allocationList,nil) + allocationList := []datamodel.BrickAllocation{{Brick: brickList[0]}} + allocations.EXPECT().CreateAllocations(initialSession.Name, brickList).Return(allocationList, nil) updatedSession := datamodel.Session{ Name: "foo", VolumeRequest: datamodel.VolumeRequest{ PoolName: datamodel.PoolName("pool1"), TotalCapacityBytes: 1024, }, - ActualSizeBytes: 1024, - Allocations: allocationList, + ActualSizeBytes: 1024, + Allocations: allocationList, PrimaryBrickHost: allocationList[0].Brick.BrickHostName, } returnedSession := datamodel.Session{ - Name: "foo", + Name: "foo", ActualSizeBytes: 1024, } sessionRegistry.EXPECT().CreateSession(updatedSession).Return(returnedSession, nil) @@ -143,7 +143,7 @@ func TestSessionFacade_DeleteSession(t *testing.T) { defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) - facade := sessionFacade{session:sessionRegistry, actions:actions} + facade := sessionFacade{session: sessionRegistry, actions: actions} err := facade.DeleteSession(datamodel.SessionName("foo"), true) From 5f8b522053417b1b3d585ef0305ddca15d119ded Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 13:20:10 +0100 Subject: [PATCH 051/191] Add channel send into the unittest --- internal/pkg/v2/dacctl/workflow_impl/session_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 10e89d8e..623974bc 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -130,8 +130,14 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) sessionRegistry.EXPECT().CreateSession(updatedSession).Return(returnedSession, nil) allocationMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") - actions.EXPECT().CreateSessionVolume(context.TODO(), initialSession.Name).Return(nil, fakeErr) + actionChan := make(chan datamodel.SessionAction) + actions.EXPECT().CreateSessionVolume(context.TODO(), initialSession.Name).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) + go func() { + actionChan <- datamodel.SessionAction{Error: fakeErr} + close(actionChan) + }() + defer close(actionChan) err := facade.CreateSession(initialSession) From fd8c9001acbd7fcd766a0114994b7cd1bdbde3e2 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 14:29:21 +0100 Subject: [PATCH 052/191] Fix delete session test --- .idea/data-acc.iml | 6 +++++ .../pkg/v2/dacctl/workflow_impl/session.go | 7 +++++ .../v2/dacctl/workflow_impl/session_test.go | 27 ++++++++++++++++--- internal/pkg/v2/datamodel/datamodel_test.go | 2 +- internal/pkg/v2/datamodel/session.go | 3 +++ 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.idea/data-acc.iml b/.idea/data-acc.iml index c956989b..7f988569 100644 --- a/.idea/data-acc.iml +++ b/.idea/data-acc.iml @@ -1,8 +1,14 @@ + + + + + + \ No newline at end of file diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index ba5f0daa..db906e61 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -188,6 +188,7 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo // Record we want this deleted, in case host is not alive // can be deleted when it is next stated session.Status.DeleteRequested = true + session.Status.DeleteSkipCopyDataOut = hurry session, err = s.session.UpdateSession(session) if err != nil { sessionMutex.Unlock(context.TODO()) @@ -195,6 +196,12 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo } // TODO: send the hurry, i.e. request data copy out first + if session.PrimaryBrickHost == "" { + // TODO: session has no primary brick host + // so need to do local umount + + } + // This will error out if the host is not currently up sessionAction, err := s.actions.SendSessionAction(context.TODO(), datamodel.SessionActionType("delete"), session) if err != nil { diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 623974bc..adfefee4 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -137,7 +137,6 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) actionChan <- datamodel.SessionAction{Error: fakeErr} close(actionChan) }() - defer close(actionChan) err := facade.CreateSession(initialSession) @@ -145,13 +144,35 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) } func TestSessionFacade_DeleteSession(t *testing.T) { + sessionName := datamodel.SessionName("foo") mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) facade := sessionFacade{session: sessionRegistry, actions: actions} + sessionMutex := mock_store.NewMockMutex(mockCtrl) + sessionRegistry.EXPECT().GetSessionMutex(sessionName).Return(sessionMutex, nil) + sessionMutex.EXPECT().Lock(context.TODO()) + initialSession := datamodel.Session{Name: "foo"} + sessionRegistry.EXPECT().GetSession(sessionName).Return(initialSession, nil) + updatedSession := datamodel.Session{ + Name: "foo", + Status: datamodel.SessionStatus{ + DeleteRequested: true, + DeleteSkipCopyDataOut: true, + }, + } + sessionRegistry.EXPECT().UpdateSession(updatedSession).Return(initialSession, nil) + actionChan := make(chan datamodel.SessionAction) + actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionActionType("delete"), initialSession).Return(actionChan, nil) + sessionMutex.EXPECT().Unlock(context.TODO()) + fakeErr := errors.New("fake") + go func() { + actionChan <- datamodel.SessionAction{Error: fakeErr} + close(actionChan) + }() - err := facade.DeleteSession(datamodel.SessionName("foo"), true) + err := facade.DeleteSession(sessionName, true) - assert.Nil(t, err) + assert.Equal(t, fakeErr, err) } diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go index df5334ff..53bef7d7 100644 --- a/internal/pkg/v2/datamodel/datamodel_test.go +++ b/internal/pkg/v2/datamodel/datamodel_test.go @@ -30,7 +30,7 @@ func Test_Session(t *testing.T) { assert.Nil(t, err) // TODO: not very human readable for Type and Action - expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"DeleteRequested":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","AttachHosts":null}` + expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","AttachHosts":null}` assert.Equal(t, expected, string(sessionAsString)) var sessionFromString Session diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 58e7e621..674e067d 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -66,6 +66,9 @@ type SessionStatus struct { // Records if we have started trying to delete DeleteRequested bool + + // Records if we should skip copy data out on delete + DeleteSkipCopyDataOut bool } type SessionAction struct { From fcb94f3bf9cc5767e645d1d3ed7afe1224c738c9 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 14:38:46 +0100 Subject: [PATCH 053/191] Add pass through methods --- internal/pkg/v2/dacctl/workflow_impl/session.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index db906e61..60a06c85 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -237,13 +237,13 @@ func (s sessionFacade) CopyDataOut(sessionName datamodel.SessionName) error { } func (s sessionFacade) GetPools() ([]datamodel.PoolInfo, error) { - panic("implement me") + return s.allocations.GetBricksByPool() } func (s sessionFacade) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { - panic("implement me") + return s.session.GetSession(sessionName) } func (s sessionFacade) GetAllSessions() ([]datamodel.Session, error) { - panic("implement me") + return s.session.GetAllSessions() } From 9bf04a8293d7548eeba8885279dd675a43f2aa91 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 14:40:02 +0100 Subject: [PATCH 054/191] Add placeholder actions --- internal/pkg/v2/dacctl/workflow_impl/session.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 60a06c85..2c39fb2b 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -221,19 +221,24 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo } func (s sessionFacade) CopyDataIn(sessionName datamodel.SessionName) error { - panic("implement me") + // TODO: complete implementation of all actions + log.Println("FAKE CopyDataIn") + return nil } func (s sessionFacade) Mount(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error { - panic("implement me") + log.Println("FAKE Mount") + return nil } func (s sessionFacade) Unmount(sessionName datamodel.SessionName) error { - panic("implement me") + log.Println("FAKE Unmount") + return nil } func (s sessionFacade) CopyDataOut(sessionName datamodel.SessionName) error { - panic("implement me") + log.Println("FAKE CopyDataOut") + return nil } func (s sessionFacade) GetPools() ([]datamodel.PoolInfo, error) { From 0ee733937e76ea8305ae34fb1340143ffab534be Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 15:00:34 +0100 Subject: [PATCH 055/191] Add session action enum --- .../pkg/v2/dacctl/workflow_impl/session.go | 2 +- .../v2/dacctl/workflow_impl/session_test.go | 2 +- internal/pkg/v2/datamodel/session.go | 10 ---------- internal/pkg/v2/datamodel/session_action.go | 20 +++++++++++++++++++ 4 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 internal/pkg/v2/datamodel/session_action.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 2c39fb2b..33b703c1 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -203,7 +203,7 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo } // This will error out if the host is not currently up - sessionAction, err := s.actions.SendSessionAction(context.TODO(), datamodel.SessionActionType("delete"), session) + sessionAction, err := s.actions.SendSessionAction(context.TODO(), datamodel.SessionDelete, session) if err != nil { return err } diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index adfefee4..744af7fb 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -164,7 +164,7 @@ func TestSessionFacade_DeleteSession(t *testing.T) { } sessionRegistry.EXPECT().UpdateSession(updatedSession).Return(initialSession, nil) actionChan := make(chan datamodel.SessionAction) - actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionActionType("delete"), initialSession).Return(actionChan, nil) + actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionDelete, initialSession).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") go func() { diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 674e067d..bce092b5 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -71,16 +71,6 @@ type SessionStatus struct { DeleteSkipCopyDataOut bool } -type SessionAction struct { - Uuid string - Session Session - Action SessionActionType - Error error -} - -// TODO: turn into enum -type SessionActionType string - type VolumeRequest struct { MultiJob bool Caller string diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/v2/datamodel/session_action.go new file mode 100644 index 00000000..08711655 --- /dev/null +++ b/internal/pkg/v2/datamodel/session_action.go @@ -0,0 +1,20 @@ +package datamodel + +type SessionAction struct { + Uuid string + Session Session + Action SessionActionType + Error error +} + +type SessionActionType int + +const ( + UnknownSessionAction SessionActionType = iota + SessionDelete + SessionCreate + SessionCopyDataIn + SessionMount + SessionUnmount + SessionCopyDataOut +) From 97faed26136b503c9bfaf28396fe35f9022fb2b3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 15:32:16 +0100 Subject: [PATCH 056/191] Fill out session handler switch statement --- .../session_action_handler.go | 37 +++++++++++++++++-- .../session_action_handler_test.go | 37 ++++++++++++++----- internal/pkg/v2/datamodel/datamodel_test.go | 2 +- internal/pkg/v2/datamodel/session_action.go | 8 ++-- 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 77a20795..6884f7ba 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -12,14 +12,45 @@ func NewSessionActionHandler(actions registry.SessionActions) facade.SessionActi } type sessionActionHandler struct { - actions registry.SessionActions + actions registry.SessionActions + skipActions bool + actionCalled datamodel.SessionActionType } func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { - log.Println("Started to process:", action) + log.Printf("Started to process: %+v\n", action) + switch action.ActionType { + case datamodel.SessionDelete: + // TODO... must test this better! + s.actionCalled = datamodel.SessionDelete + if !s.skipActions { + go s.handleDelete(action) + } + case datamodel.SessionCreate: + s.actionCalled = datamodel.SessionCreate + if !s.skipActions { + go s.handleCreate(action) + } + default: + log.Panicf("not yet implemented action for %+v", action) + } +} + +func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { + log.Println("create") + err := s.actions.CompleteSessionAction(action, nil) + if err != nil { + log.Println("Failed to complete ActionType:", err) + return + } + log.Println("Stopped processing action:", action) +} + +func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { + log.Println("delete") err := s.actions.CompleteSessionAction(action, nil) if err != nil { - log.Println("Failed to complete Action:", err) + log.Println("Failed to complete ActionType:", err) return } log.Println("Stopped processing action:", action) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index 89fc0d3b..186902de 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -1,20 +1,39 @@ package brick_manager_impl import ( + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" - "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" "testing" ) -func TestSessionActionHandler_ProcessSessionAction(t *testing.T) { +func TestSessionActionHandler_ProcessSessionAction_Unknown(t *testing.T) { action := datamodel.SessionAction{} - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - registry := mock_registry.NewMockSessionActions(mockCtrl) - registry.EXPECT().CompleteSessionAction(action, nil) - handler := NewSessionActionHandler(registry) + handler := NewSessionActionHandler(nil) + + assert.PanicsWithValue(t, + fmt.Sprintf("not yet implemented action for %+v", action), + func() { handler.ProcessSessionAction(action) }) +} + +func TestSessionActionHandler_ProcessSessionAction_Create(t *testing.T) { + action := datamodel.SessionAction{ + ActionType: datamodel.SessionCreate, + } + handler := sessionActionHandler{skipActions: true} - // TODO... handler.ProcessSessionAction(action) + + assert.Equal(t, datamodel.SessionCreate, handler.actionCalled) +} + +func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { + action := datamodel.SessionAction{ + ActionType: datamodel.SessionDelete, + } + handler := sessionActionHandler{skipActions: true} + + handler.ProcessSessionAction(action) + + assert.Equal(t, datamodel.SessionDelete, handler.actionCalled) } diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go index 53bef7d7..08a646d8 100644 --- a/internal/pkg/v2/datamodel/datamodel_test.go +++ b/internal/pkg/v2/datamodel/datamodel_test.go @@ -29,7 +29,7 @@ func Test_Session(t *testing.T) { sessionAsString, err := json.Marshal(session) assert.Nil(t, err) - // TODO: not very human readable for Type and Action + // TODO: not very human readable for Type and ActionType expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","AttachHosts":null}` assert.Equal(t, expected, string(sessionAsString)) diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/v2/datamodel/session_action.go index 08711655..6d05551b 100644 --- a/internal/pkg/v2/datamodel/session_action.go +++ b/internal/pkg/v2/datamodel/session_action.go @@ -1,10 +1,10 @@ package datamodel type SessionAction struct { - Uuid string - Session Session - Action SessionActionType - Error error + Uuid string + Session Session + ActionType SessionActionType + Error error } type SessionActionType int From 9d603c2f1bb9d21798ac4de10af188a57d75e5eb Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 15:36:12 +0100 Subject: [PATCH 057/191] Add tests for handleCreate and handleDelete --- .../session_action_handler_test.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index 186902de..6b7d4be7 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -3,6 +3,8 @@ package brick_manager_impl import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" ) @@ -37,3 +39,31 @@ func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { assert.Equal(t, datamodel.SessionDelete, handler.actionCalled) } + +func TestSessionActionHandler_handleCreate(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + handler := sessionActionHandler{actions:actions} + action := datamodel.SessionAction{ + ActionType: datamodel.SessionCreate, + } + + actions.EXPECT().CompleteSessionAction(action, nil) + + handler.handleCreate(action) +} + +func TestSessionActionHandler_handleDelete(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + actions := mock_registry.NewMockSessionActions(mockCtrl) + handler := sessionActionHandler{actions:actions} + action := datamodel.SessionAction{ + ActionType: datamodel.SessionDelete, + } + + actions.EXPECT().CompleteSessionAction(action, nil) + + handler.handleDelete(action) +} \ No newline at end of file From cbcb3bf0056762dd84ff969785e5327a39df54a2 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 15:38:18 +0100 Subject: [PATCH 058/191] DRY up handleCreate and handleDelete --- .../brick_manager_impl/session_action_handler.go | 12 ++++++------ .../session_action_handler_test.go | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 6884f7ba..ddc22e2b 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -38,19 +38,19 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { log.Println("create") - err := s.actions.CompleteSessionAction(action, nil) - if err != nil { - log.Println("Failed to complete ActionType:", err) - return - } - log.Println("Stopped processing action:", action) + s.reportComplete(action) } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { log.Println("delete") + s.reportComplete(action) +} + +func (s *sessionActionHandler) reportComplete(action datamodel.SessionAction) { err := s.actions.CompleteSessionAction(action, nil) if err != nil { log.Println("Failed to complete ActionType:", err) + // TODO: put session into error state? return } log.Println("Stopped processing action:", action) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index 6b7d4be7..344f3661 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -44,7 +44,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) - handler := sessionActionHandler{actions:actions} + handler := sessionActionHandler{actions: actions} action := datamodel.SessionAction{ ActionType: datamodel.SessionCreate, } @@ -58,7 +58,7 @@ func TestSessionActionHandler_handleDelete(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) - handler := sessionActionHandler{actions:actions} + handler := sessionActionHandler{actions: actions} action := datamodel.SessionAction{ ActionType: datamodel.SessionDelete, } @@ -66,4 +66,4 @@ func TestSessionActionHandler_handleDelete(t *testing.T) { actions.EXPECT().CompleteSessionAction(action, nil) handler.handleDelete(action) -} \ No newline at end of file +} From f5917b73eebdd59a394ae3706b4fa0d2d29f26d5 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 17:52:34 +0100 Subject: [PATCH 059/191] Rework the filesystem provider interface --- build/rebuild_mocks.sh | 6 + internal/pkg/v2/datamodel/datamodel_test.go | 19 +-- internal/pkg/v2/datamodel/session.go | 44 ++++++- internal/pkg/v2/datamodel/volume.go | 132 -------------------- internal/pkg/v2/filesystem/provider.go | 14 +++ internal/pkg/v2/mock_filesystem/provider.go | 119 ++++++++++++++++++ 6 files changed, 180 insertions(+), 154 deletions(-) delete mode 100644 internal/pkg/v2/datamodel/volume.go create mode 100644 internal/pkg/v2/filesystem/provider.go create mode 100644 internal/pkg/v2/mock_filesystem/provider.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 96bdd041..575eb48e 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -44,3 +44,9 @@ for i in $items; do mockgen -source=internal/pkg/v2/store/${i}.go \ >internal/pkg/v2/mock_store/${i}.go done + +items="provider" +for i in $items; do + mockgen -source=internal/pkg/v2/filesystem/${i}.go \ + >internal/pkg/v2/mock_filesystem/${i}.go +done diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go index 08a646d8..2a917f3e 100644 --- a/internal/pkg/v2/datamodel/datamodel_test.go +++ b/internal/pkg/v2/datamodel/datamodel_test.go @@ -6,23 +6,6 @@ import ( "testing" ) -func Test_Volume(t *testing.T) { - volume := Volume{} - - volumeAsString, err := json.Marshal(volume) - - assert.Nil(t, err) - expected := `{"Name":"","UUID":"","MultiJob":false,"Pool":"","SizeBricks":0,"SizeGB":0,"JobName":"","Owner":0,"Group":0,"CreatedBy":"","CreatedAt":0,"Attachments":null,"AttachGlobalNamespace":false,"AttachPrivateNamespace":false,"AttachAsSwapBytes":0,"AttachPrivateCache":null,"StageInRequests":null,"StageOutRequests":null,"ClientPort":0,"HadBricksAssigned":false}` - assert.Equal(t, expected, string(volumeAsString)) - - var volumeFromString Volume - data := []byte(expected) - err = json.Unmarshal(data, &volumeFromString) - - assert.Nil(t, err) - assert.Equal(t, volume, volumeFromString) -} - func Test_Session(t *testing.T) { session := Session{} @@ -30,7 +13,7 @@ func Test_Session(t *testing.T) { assert.Nil(t, err) // TODO: not very human readable for Type and ActionType - expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","AttachHosts":null}` + expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}` assert.Equal(t, expected, string(sessionAsString)) var sessionFromString Session diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index bce092b5..41b3ac45 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -13,8 +13,10 @@ type Session struct { // this is checked when an update is requested Revision int - // unix uid and gid + // unix uid Owner uint + + // unix group id Group uint // utc unix timestamp when buffer created @@ -50,9 +52,39 @@ type Session struct { // Where session requests should be sent PrimaryBrickHost BrickHostName - // The hosts that want to mount the storage - // Note: to allow for copy in/out the brick hosts are assumed to have an attachment - AttachHosts []string + // Compute hosts for this session + // Note: should be empty for multi-job volumes + RequestedAttachHosts []string + + // Used by filesystem provider to store internal state + // and track if the filesystem had a recent error + FilesystemStatus FilesystemStatus + + // For multi-job volumes these are always other sessions + // for job volumes this is always for just this session + CurrentAttachments map[SessionName]AttachmentSessionStatus +} + +type FilesystemStatus struct { + Error error + InternalName string + InternalData string +} + +type AttachmentSession struct { + SessionName SessionName + Hosts []string +} + +type AttachmentSessionStatus struct { + AttachmentSession AttachmentSession + + GlobalMount bool + PrivateMount bool + SwapBytes int + + DetachRequested bool + Error error } type SessionStatus struct { @@ -64,6 +96,10 @@ type SessionStatus struct { // CreateVolume has succeeded, so other actions can now happen FileSystemCreated bool + // Assuming one data in / data out cycle per job + CopyDataInComplete bool + CopyDataOutComplete bool + // Records if we have started trying to delete DeleteRequested bool diff --git a/internal/pkg/v2/datamodel/volume.go b/internal/pkg/v2/datamodel/volume.go deleted file mode 100644 index a4b80bd2..00000000 --- a/internal/pkg/v2/datamodel/volume.go +++ /dev/null @@ -1,132 +0,0 @@ -package datamodel - -import ( - "bytes" - "encoding/json" -) - -type VolumeName string - -type Volume struct { - // e.g. job1 or Foo - Name VolumeName - - // its 8 characters long, so works nicely with lustre - UUID string - - // True if multiple jobs can attach to this volume - MultiJob bool - - // Requested pool of bricks for volume - Pool string // TODO: PoolName? - // Number of bricks requested, calculated from requested capacity - SizeBricks uint - // Actual size of the volume - SizeGB uint - - // Back reference to what job created this volume - JobName string - // e.g. 1001 - Owner uint - // If empty defaults to User - Group uint - // e.g. SLURM or Manila - CreatedBy string - // The unix (utc) timestamp of when this volume was created - CreatedAt uint - - // TODO: need to fill these in... - // They all related to how the volume is attached - - // All current attachments - Attachments []Attachment - // Attach all attachments to a shared global namespace - // Allowed for any volume type - AttachGlobalNamespace bool - // Have an attachment specific namespace mounted, only for non multi job - AttachPrivateNamespace bool - // If not zero, swap of the requested amount mounted for each attachment - // Not allowed for multi job - AttachAsSwapBytes uint - // Add attachment specific cache for each given filesystem path - // Not allowed for multi job - // Note: assumes the same path is cached for all attachments - AttachPrivateCache []string - - // TODO: maybe data copy should be a slice associated with the job? - // Request certain files to be staged in - // Not currently allowed for multi job volumes - StageInRequests []DataCopyRequest - // Request certain files to be staged in - // Not currently allowed for multi job volumes - StageOutRequests []DataCopyRequest - - // BeeGFS wants each fs to be assigned a unique port number - ClientPort int - - // Track if we have had bricks assigned - // if we request delete, no bricks ever assigned, don't ait for dacd! - HadBricksAssigned bool -} - -type Attachment struct { - // BrickHostName, Job and Volume name uniquely identify an attachment - Hostname string - - // Associated jobName - Job string - - State AttachmentState - - // If any error happened, it is reported here - Error error -} - -type AttachmentState int - -const ( - UnknownAttachmentState AttachmentState = iota - RequestAttach - Attached - RequestDetach - Detached AttachmentState = 400 // all bricks correctly deprovisioned unless host down or gone to ERROR - AttachmentError AttachmentState = 500 -) - -var attachStateStrings = map[AttachmentState]string{ - UnknownAttachmentState: "", - RequestAttach: "RequestAttach", - Attached: "Attached", - RequestDetach: "RequestDetach", - Detached: "Detached", - AttachmentError: "AttachmentError", -} -var stringToAttachmentState = map[string]AttachmentState{ - "": UnknownAttachmentState, - "RequestAttach": RequestAttach, - "Attached": Attached, - "RequestDetach": RequestDetach, - "Detached": Detached, - "AttachmentError": AttachmentError, -} - -func (attachmentState AttachmentState) String() string { - return attachStateStrings[attachmentState] -} - -func (attachmentState AttachmentState) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString(`"`) - buffer.WriteString(attachStateStrings[attachmentState]) - buffer.WriteString(`"`) - return buffer.Bytes(), nil -} - -func (attachmentState *AttachmentState) UnmarshalJSON(b []byte) error { - var str string - err := json.Unmarshal(b, &str) - if err != nil { - return err - } - *attachmentState = stringToAttachmentState[str] - return nil -} diff --git a/internal/pkg/v2/filesystem/provider.go b/internal/pkg/v2/filesystem/provider.go new file mode 100644 index 00000000..93d6e6f9 --- /dev/null +++ b/internal/pkg/v2/filesystem/provider.go @@ -0,0 +1,14 @@ +package filesystem + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + +type Provider interface { + Create(session datamodel.Session) (datamodel.FilesystemStatus, error) + Delete(session datamodel.Session) error + + DataCopyIn(session datamodel.Session) error + DataCopyOut(session datamodel.Session) error + + Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus + Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus +} diff --git a/internal/pkg/v2/mock_filesystem/provider.go b/internal/pkg/v2/mock_filesystem/provider.go new file mode 100644 index 00000000..3e0da507 --- /dev/null +++ b/internal/pkg/v2/mock_filesystem/provider.go @@ -0,0 +1,119 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/filesystem/provider.go + +// Package mock_filesystem is a generated GoMock package. +package mock_filesystem + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockProvider is a mock of Provider interface +type MockProvider struct { + ctrl *gomock.Controller + recorder *MockProviderMockRecorder +} + +// MockProviderMockRecorder is the mock recorder for MockProvider +type MockProviderMockRecorder struct { + mock *MockProvider +} + +// NewMockProvider creates a new mock instance +func NewMockProvider(ctrl *gomock.Controller) *MockProvider { + mock := &MockProvider{ctrl: ctrl} + mock.recorder = &MockProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockProvider) EXPECT() *MockProviderMockRecorder { + return m.recorder +} + +// Create mocks base method +func (m *MockProvider) Create(session datamodel.Session) (datamodel.FilesystemStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", session) + ret0, _ := ret[0].(datamodel.FilesystemStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create +func (mr *MockProviderMockRecorder) Create(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockProvider)(nil).Create), session) +} + +// Delete mocks base method +func (m *MockProvider) Delete(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockProviderMockRecorder) Delete(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockProvider)(nil).Delete), session) +} + +// DataCopyIn mocks base method +func (m *MockProvider) DataCopyIn(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataCopyIn", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataCopyIn indicates an expected call of DataCopyIn +func (mr *MockProviderMockRecorder) DataCopyIn(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataCopyIn", reflect.TypeOf((*MockProvider)(nil).DataCopyIn), session) +} + +// DataCopyOut mocks base method +func (m *MockProvider) DataCopyOut(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DataCopyOut", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// DataCopyOut indicates an expected call of DataCopyOut +func (mr *MockProviderMockRecorder) DataCopyOut(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DataCopyOut", reflect.TypeOf((*MockProvider)(nil).DataCopyOut), session) +} + +// Mount mocks base method +func (m *MockProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Mount", session, attachments) + ret0, _ := ret[0].(datamodel.AttachmentSessionStatus) + return ret0 +} + +// Mount indicates an expected call of Mount +func (mr *MockProviderMockRecorder) Mount(session, attachments interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mount", reflect.TypeOf((*MockProvider)(nil).Mount), session, attachments) +} + +// Unmount mocks base method +func (m *MockProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Unmount", session, attachments) + ret0, _ := ret[0].(datamodel.AttachmentSessionStatus) + return ret0 +} + +// Unmount indicates an expected call of Unmount +func (mr *MockProviderMockRecorder) Unmount(session, attachments interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unmount", reflect.TypeOf((*MockProvider)(nil).Unmount), session, attachments) +} From 7493b72d92f5bb9f6b1f67f89582461813a381f8 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 16 Aug 2019 21:04:45 +0100 Subject: [PATCH 060/191] Add tests for handleCreate --- .../session_action_handler.go | 61 +++++++++++++++---- .../session_action_handler_test.go | 28 ++++++++- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index ddc22e2b..bf188bb4 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -1,8 +1,10 @@ package brick_manager_impl import ( + "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "log" ) @@ -12,7 +14,9 @@ func NewSessionActionHandler(actions registry.SessionActions) facade.SessionActi } type sessionActionHandler struct { + registry registry.SessionRegistry actions registry.SessionActions + fsProvider filesystem.Provider skipActions bool actionCalled datamodel.SessionActionType } @@ -37,21 +41,52 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi } func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { - log.Println("create") - s.reportComplete(action) -} + sessionName := action.Session.Name + sessionMutex, err := s.registry.GetSessionMutex(sessionName) + if err != nil { + log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) + s.actions.CompleteSessionAction(action, err) + return + } + err = sessionMutex.Lock(context.TODO()) + if err != nil { + log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) + s.actions.CompleteSessionAction(action, err) + return + } + defer func() { + if err := sessionMutex.Unlock(context.TODO()); err != nil { + log.Println("failed to drop mutex for:", sessionName) + } + }() + log.Printf("starting create for %+v\n", sessionName) -func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { - log.Println("delete") - s.reportComplete(action) -} + // Get latest session now we have the mutex + session, err := s.registry.GetSession(sessionName) + + fsStatus, err := s.fsProvider.Create(action.Session) + session.FilesystemStatus = fsStatus + session.Status.FileSystemCreated = err == nil + session.Status.Error = err -func (s *sessionActionHandler) reportComplete(action datamodel.SessionAction) { - err := s.actions.CompleteSessionAction(action, nil) + session, err = s.registry.UpdateSession(session) if err != nil { - log.Println("Failed to complete ActionType:", err) - // TODO: put session into error state? - return + log.Printf("Failed to update session: %+v", session) + } else { + action.Session = session + } + + if err := s.actions.CompleteSessionAction(action, action.Session.Status.Error); err != nil { + log.Printf("Failed to complete action: %+v", action) } - log.Println("Stopped processing action:", action) + if action.Session.Status.Error != nil { + log.Println("error during create for", sessionName, err) + } else { + log.Printf("completed create for %+v\n", sessionName) + } +} + +func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { + log.Println("delete") + s.actions.CompleteSessionAction(action, nil) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index 344f3661..c7bf20cd 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -1,9 +1,12 @@ package brick_manager_impl import ( + "context" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_filesystem" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -43,13 +46,32 @@ func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { func TestSessionActionHandler_handleCreate(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + registry := mock_registry.NewMockSessionRegistry(mockCtrl) actions := mock_registry.NewMockSessionActions(mockCtrl) - handler := sessionActionHandler{actions: actions} + fsProvider := mock_filesystem.NewMockProvider(mockCtrl) + handler := sessionActionHandler{ + registry: registry, actions: actions, fsProvider: fsProvider, + } action := datamodel.SessionAction{ ActionType: datamodel.SessionCreate, + Session: datamodel.Session{Name: "test"}, } - - actions.EXPECT().CompleteSessionAction(action, nil) + sessionMutex := mock_store.NewMockMutex(mockCtrl) + registry.EXPECT().GetSessionMutex(action.Session.Name).Return(sessionMutex, nil) + sessionMutex.EXPECT().Lock(context.TODO()) + sessionMutex.EXPECT().Unlock(context.TODO()) + registry.EXPECT().GetSession(action.Session.Name).Return(action.Session, nil) + fsProvider.EXPECT().Create(action.Session) + updatedSession := datamodel.Session{ + Name: action.Session.Name, + Status: datamodel.SessionStatus{FileSystemCreated: true}, + } + registry.EXPECT().UpdateSession(updatedSession).Return(updatedSession, nil) + updatedAction := datamodel.SessionAction{ + ActionType: datamodel.SessionCreate, + Session: updatedSession, + } + actions.EXPECT().CompleteSessionAction(updatedAction, nil) handler.handleCreate(action) } From 335906d5bfb2bde445d8bc4567bbe18607191127 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 10:01:25 +0100 Subject: [PATCH 061/191] Add filesystem impl boilerplate --- build/rebuild_mocks.sh | 2 +- internal/pkg/v2/filesystem/ansible.go | 25 +++++++ internal/pkg/v2/filesystem_impl/provider.go | 38 +++++++++++ internal/pkg/v2/mock_filesystem/ansible.go | 76 +++++++++++++++++++++ 4 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 internal/pkg/v2/filesystem/ansible.go create mode 100644 internal/pkg/v2/filesystem_impl/provider.go create mode 100644 internal/pkg/v2/mock_filesystem/ansible.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 575eb48e..07e1f8c3 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -45,7 +45,7 @@ for i in $items; do >internal/pkg/v2/mock_store/${i}.go done -items="provider" +items="provider ansible" for i in $items; do mockgen -source=internal/pkg/v2/filesystem/${i}.go \ >internal/pkg/v2/mock_filesystem/${i}.go diff --git a/internal/pkg/v2/filesystem/ansible.go b/internal/pkg/v2/filesystem/ansible.go new file mode 100644 index 00000000..df1253dd --- /dev/null +++ b/internal/pkg/v2/filesystem/ansible.go @@ -0,0 +1,25 @@ +package filesystem + +import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + +type AnsibleEnv struct { + directory string +} + +type Ansible interface { + CreateEnvironment(session datamodel.Session, createAllPlaybooks bool, groupAllVars map[string]string) (AnsibleEnv, error) + DestroyEnvironment(env AnsibleEnv) error + RunPlaybook(env AnsibleEnv, playbook AnsiblePlaybook, extraVars map[string]string) +} + +type AnsiblePlaybook int + +const ( + Create AnsiblePlaybook = iota + Delete + DataCopyIn + DataCopyOut + // Mount and unmount require extraVars with the hosts to be mounted + Mount + Unmount +) \ No newline at end of file diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go new file mode 100644 index 00000000..d7c9d4d2 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -0,0 +1,38 @@ +package filesystem_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" +) + +func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { + return &fileSystemProvider{ansible:ansible} +} + +type fileSystemProvider struct { + ansible filesystem.Ansible +} + +func (f *fileSystemProvider) Create(session datamodel.Session) (datamodel.FilesystemStatus, error) { + panic("implement me") +} + +func (f *fileSystemProvider) Delete(session datamodel.Session) error { + panic("implement me") +} + +func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { + panic("implement me") +} + +func (f *fileSystemProvider) DataCopyOut(session datamodel.Session) error { + panic("implement me") +} + +func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { + panic("implement me") +} + +func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { + panic("implement me") +} diff --git a/internal/pkg/v2/mock_filesystem/ansible.go b/internal/pkg/v2/mock_filesystem/ansible.go new file mode 100644 index 00000000..52a0a151 --- /dev/null +++ b/internal/pkg/v2/mock_filesystem/ansible.go @@ -0,0 +1,76 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/filesystem/ansible.go + +// Package mock_filesystem is a generated GoMock package. +package mock_filesystem + +import ( + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + filesystem "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockAnsible is a mock of Ansible interface +type MockAnsible struct { + ctrl *gomock.Controller + recorder *MockAnsibleMockRecorder +} + +// MockAnsibleMockRecorder is the mock recorder for MockAnsible +type MockAnsibleMockRecorder struct { + mock *MockAnsible +} + +// NewMockAnsible creates a new mock instance +func NewMockAnsible(ctrl *gomock.Controller) *MockAnsible { + mock := &MockAnsible{ctrl: ctrl} + mock.recorder = &MockAnsibleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAnsible) EXPECT() *MockAnsibleMockRecorder { + return m.recorder +} + +// CreateEnvironment mocks base method +func (m *MockAnsible) CreateEnvironment(session datamodel.Session, createAllPlaybooks bool, groupAllVars map[string]string) (filesystem.AnsibleEnv, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateEnvironment", session, createAllPlaybooks, groupAllVars) + ret0, _ := ret[0].(filesystem.AnsibleEnv) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateEnvironment indicates an expected call of CreateEnvironment +func (mr *MockAnsibleMockRecorder) CreateEnvironment(session, createAllPlaybooks, groupAllVars interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEnvironment", reflect.TypeOf((*MockAnsible)(nil).CreateEnvironment), session, createAllPlaybooks, groupAllVars) +} + +// DestroyEnvironment mocks base method +func (m *MockAnsible) DestroyEnvironment(env filesystem.AnsibleEnv) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DestroyEnvironment", env) + ret0, _ := ret[0].(error) + return ret0 +} + +// DestroyEnvironment indicates an expected call of DestroyEnvironment +func (mr *MockAnsibleMockRecorder) DestroyEnvironment(env interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DestroyEnvironment", reflect.TypeOf((*MockAnsible)(nil).DestroyEnvironment), env) +} + +// RunPlaybook mocks base method +func (m *MockAnsible) RunPlaybook(env filesystem.AnsibleEnv, playbook filesystem.AnsiblePlaybook, extraVars map[string]string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RunPlaybook", env, playbook, extraVars) +} + +// RunPlaybook indicates an expected call of RunPlaybook +func (mr *MockAnsibleMockRecorder) RunPlaybook(env, playbook, extraVars interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunPlaybook", reflect.TypeOf((*MockAnsible)(nil).RunPlaybook), env, playbook, extraVars) +} From 8507a4c195aa772383bcee8a1a8ecdad52a2cf0c Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 11:02:01 +0100 Subject: [PATCH 062/191] Remove pool registry, and tidy up registry interfaces --- build/rebuild_mocks.sh | 2 +- .../pkg/v2/dacctl/workflow_impl/session.go | 11 ++-- .../v2/dacctl/workflow_impl/session_test.go | 17 ++--- .../session_action_handler.go | 6 +- .../session_action_handler_test.go | 14 ++-- internal/pkg/v2/datamodel/session_action.go | 2 +- internal/pkg/v2/filesystem/ansible.go | 2 +- internal/pkg/v2/filesystem_impl/provider.go | 2 +- internal/pkg/v2/mock_registry/allocation.go | 27 ++++++-- internal/pkg/v2/mock_registry/brick.go | 15 ----- internal/pkg/v2/mock_registry/pool.go | 64 ------------------- internal/pkg/v2/mock_registry/session.go | 30 ++++----- .../pkg/v2/mock_registry/session_actions.go | 30 --------- internal/pkg/v2/registry/allocation.go | 10 ++- internal/pkg/v2/registry/brick.go | 4 +- internal/pkg/v2/registry/pool.go | 14 ---- internal/pkg/v2/registry/session.go | 10 +-- internal/pkg/v2/registry/session_actions.go | 16 ----- 18 files changed, 81 insertions(+), 195 deletions(-) delete mode 100644 internal/pkg/v2/mock_registry/pool.go delete mode 100644 internal/pkg/v2/registry/pool.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 07e1f8c3..9aec9d89 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -27,7 +27,7 @@ done mockgen -source=internal/pkg/pfsprovider/interface.go \ -package mocks >internal/pkg/mocks/pfsprovider_mock.go -items="allocation brick pool session session_actions" +items="allocation brick session session_actions" for i in $items; do mockgen -source=internal/pkg/v2/registry/${i}.go \ >internal/pkg/v2/mock_registry/${i}.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 33b703c1..8a7f46d3 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -20,7 +20,6 @@ type sessionFacade struct { session registry.SessionRegistry actions registry.SessionActions allocations registry.AllocationRegistry - pool registry.PoolRegistry } func (s sessionFacade) CreateSession(session datamodel.Session) error { @@ -40,7 +39,7 @@ func (s sessionFacade) CreateSession(session datamodel.Session) error { } // Allocate bricks, and choose brick host server - session, err = s.doSessionAllocation(session) + session, err = s.doAllocationAndWriteSession(session) // if no bricks allocated, no need to call CreateSessionVolume if err != nil || session.ActualSizeBytes == 0 { sessionMutex.Unlock(context.TODO()) @@ -49,7 +48,7 @@ func (s sessionFacade) CreateSession(session datamodel.Session) error { // Create filesystem on the brick host server // TODO: add timeout - eventChan, createErr := s.actions.CreateSessionVolume(context.TODO(), session.Name) + eventChan, createErr := s.actions.SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, session) // Drop mutex so the server can take it err = sessionMutex.Unlock(context.TODO()) @@ -70,7 +69,7 @@ func (s sessionFacade) CreateSession(session datamodel.Session) error { } func (s sessionFacade) validateSession(session datamodel.Session) error { - _, err := s.pool.GetPool(session.VolumeRequest.PoolName) + _, err := s.allocations.GetPool(session.VolumeRequest.PoolName) if err != nil { return fmt.Errorf("invalid session, unable to find pool %s", session.VolumeRequest.PoolName) } @@ -78,7 +77,7 @@ func (s sessionFacade) validateSession(session datamodel.Session) error { return nil } -func (s sessionFacade) doSessionAllocation(session datamodel.Session) (datamodel.Session, error) { +func (s sessionFacade) doAllocationAndWriteSession(session datamodel.Session) (datamodel.Session, error) { if session.VolumeRequest.TotalCapacityBytes > 0 { allocationMutex, err := s.allocations.GetAllocationMutex() if err != nil { @@ -242,7 +241,7 @@ func (s sessionFacade) CopyDataOut(sessionName datamodel.SessionName) error { } func (s sessionFacade) GetPools() ([]datamodel.PoolInfo, error) { - return s.allocations.GetBricksByPool() + return s.allocations.GetAllPoolInfos() } func (s sessionFacade) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 744af7fb..bfffe53e 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -23,13 +23,12 @@ func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) - poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) + allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) @@ -53,13 +52,12 @@ func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) - poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) + allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) @@ -91,13 +89,12 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) - poolRegistry := mock_registry.NewMockPoolRegistry(mockCtrl) allocations := mock_registry.NewMockAllocationRegistry(mockCtrl) facade := sessionFacade{ - session: sessionRegistry, actions: actions, pool: poolRegistry, allocations: allocations, + session: sessionRegistry, actions: actions, allocations: allocations, } - poolRegistry.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) + allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) @@ -131,7 +128,7 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) allocationMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") actionChan := make(chan datamodel.SessionAction) - actions.EXPECT().CreateSessionVolume(context.TODO(), initialSession.Name).Return(actionChan, nil) + actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, returnedSession).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) go func() { actionChan <- datamodel.SessionAction{Error: fakeErr} diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index bf188bb4..d4f6ed34 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -30,8 +30,8 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi if !s.skipActions { go s.handleDelete(action) } - case datamodel.SessionCreate: - s.actionCalled = datamodel.SessionCreate + case datamodel.SessionCreateFilesystem: + s.actionCalled = datamodel.SessionCreateFilesystem if !s.skipActions { go s.handleCreate(action) } @@ -88,5 +88,7 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { log.Println("delete") + // TODO: clearly need mutex here, etc + s.registry.DeleteSession(action.Session) s.actions.CompleteSessionAction(action, nil) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index c7bf20cd..fc9c2362 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -23,13 +23,13 @@ func TestSessionActionHandler_ProcessSessionAction_Unknown(t *testing.T) { func TestSessionActionHandler_ProcessSessionAction_Create(t *testing.T) { action := datamodel.SessionAction{ - ActionType: datamodel.SessionCreate, + ActionType: datamodel.SessionCreateFilesystem, } handler := sessionActionHandler{skipActions: true} handler.ProcessSessionAction(action) - assert.Equal(t, datamodel.SessionCreate, handler.actionCalled) + assert.Equal(t, datamodel.SessionCreateFilesystem, handler.actionCalled) } func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { @@ -53,7 +53,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { registry: registry, actions: actions, fsProvider: fsProvider, } action := datamodel.SessionAction{ - ActionType: datamodel.SessionCreate, + ActionType: datamodel.SessionCreateFilesystem, Session: datamodel.Session{Name: "test"}, } sessionMutex := mock_store.NewMockMutex(mockCtrl) @@ -68,7 +68,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { } registry.EXPECT().UpdateSession(updatedSession).Return(updatedSession, nil) updatedAction := datamodel.SessionAction{ - ActionType: datamodel.SessionCreate, + ActionType: datamodel.SessionCreateFilesystem, Session: updatedSession, } actions.EXPECT().CompleteSessionAction(updatedAction, nil) @@ -80,11 +80,13 @@ func TestSessionActionHandler_handleDelete(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) - handler := sessionActionHandler{actions: actions} + registry := mock_registry.NewMockSessionRegistry(mockCtrl) + handler := sessionActionHandler{actions: actions, registry: registry} action := datamodel.SessionAction{ ActionType: datamodel.SessionDelete, } - + // TODO: need to pass session better? who deletes allocations? + registry.EXPECT().DeleteSession(action.Session) actions.EXPECT().CompleteSessionAction(action, nil) handler.handleDelete(action) diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/v2/datamodel/session_action.go index 6d05551b..e638ccae 100644 --- a/internal/pkg/v2/datamodel/session_action.go +++ b/internal/pkg/v2/datamodel/session_action.go @@ -11,8 +11,8 @@ type SessionActionType int const ( UnknownSessionAction SessionActionType = iota + SessionCreateFilesystem SessionDelete - SessionCreate SessionCopyDataIn SessionMount SessionUnmount diff --git a/internal/pkg/v2/filesystem/ansible.go b/internal/pkg/v2/filesystem/ansible.go index df1253dd..682d3455 100644 --- a/internal/pkg/v2/filesystem/ansible.go +++ b/internal/pkg/v2/filesystem/ansible.go @@ -22,4 +22,4 @@ const ( // Mount and unmount require extraVars with the hosts to be mounted Mount Unmount -) \ No newline at end of file +) diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index d7c9d4d2..8409a625 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -6,7 +6,7 @@ import ( ) func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { - return &fileSystemProvider{ansible:ansible} + return &fileSystemProvider{ansible: ansible} } type fileSystemProvider struct { diff --git a/internal/pkg/v2/mock_registry/allocation.go b/internal/pkg/v2/mock_registry/allocation.go index 1f75f7e6..4132a03b 100644 --- a/internal/pkg/v2/mock_registry/allocation.go +++ b/internal/pkg/v2/mock_registry/allocation.go @@ -34,19 +34,34 @@ func (m *MockAllocationRegistry) EXPECT() *MockAllocationRegistryMockRecorder { return m.recorder } -// GetBricksByPool mocks base method -func (m *MockAllocationRegistry) GetBricksByPool() ([]datamodel.PoolInfo, error) { +// GetPool mocks base method +func (m *MockAllocationRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBricksByPool") + ret := m.ctrl.Call(m, "GetPool", name) + ret0, _ := ret[0].(datamodel.Pool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPool indicates an expected call of GetPool +func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPool), name) +} + +// GetAllPoolInfos mocks base method +func (m *MockAllocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllPoolInfos") ret0, _ := ret[0].([]datamodel.PoolInfo) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetBricksByPool indicates an expected call of GetBricksByPool -func (mr *MockAllocationRegistryMockRecorder) GetBricksByPool() *gomock.Call { +// GetAllPoolInfos indicates an expected call of GetAllPoolInfos +func (mr *MockAllocationRegistryMockRecorder) GetAllPoolInfos() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBricksByPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetBricksByPool)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPoolInfos", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllPoolInfos)) } // GetPoolInfo mocks base method diff --git a/internal/pkg/v2/mock_registry/brick.go b/internal/pkg/v2/mock_registry/brick.go index 164b94c7..1102749d 100644 --- a/internal/pkg/v2/mock_registry/brick.go +++ b/internal/pkg/v2/mock_registry/brick.go @@ -76,18 +76,3 @@ func (mr *MockBrickRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName inter mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickRegistry)(nil).KeepAliveHost), ctxt, brickHostName) } - -// IsBrickHostAlive mocks base method -func (m *MockBrickRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsBrickHostAlive", brickHostName) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsBrickHostAlive indicates an expected call of IsBrickHostAlive -func (mr *MockBrickRegistryMockRecorder) IsBrickHostAlive(brickHostName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsBrickHostAlive", reflect.TypeOf((*MockBrickRegistry)(nil).IsBrickHostAlive), brickHostName) -} diff --git a/internal/pkg/v2/mock_registry/pool.go b/internal/pkg/v2/mock_registry/pool.go deleted file mode 100644 index bc39faf2..00000000 --- a/internal/pkg/v2/mock_registry/pool.go +++ /dev/null @@ -1,64 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/pool.go - -// Package mock_registry is a generated GoMock package. -package mock_registry - -import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockPoolRegistry is a mock of PoolRegistry interface -type MockPoolRegistry struct { - ctrl *gomock.Controller - recorder *MockPoolRegistryMockRecorder -} - -// MockPoolRegistryMockRecorder is the mock recorder for MockPoolRegistry -type MockPoolRegistryMockRecorder struct { - mock *MockPoolRegistry -} - -// NewMockPoolRegistry creates a new mock instance -func NewMockPoolRegistry(ctrl *gomock.Controller) *MockPoolRegistry { - mock := &MockPoolRegistry{ctrl: ctrl} - mock.recorder = &MockPoolRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockPoolRegistry) EXPECT() *MockPoolRegistryMockRecorder { - return m.recorder -} - -// GetPool mocks base method -func (m *MockPoolRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPool", name) - ret0, _ := ret[0].(datamodel.Pool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPool indicates an expected call of GetPool -func (mr *MockPoolRegistryMockRecorder) GetPool(name interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockPoolRegistry)(nil).GetPool), name) -} - -// EnsurePoolCreated mocks base method -func (m *MockPoolRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGB) - ret0, _ := ret[0].(datamodel.Pool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EnsurePoolCreated indicates an expected call of EnsurePoolCreated -func (mr *MockPoolRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGB interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockPoolRegistry)(nil).EnsurePoolCreated), poolName, granularityGB) -} diff --git a/internal/pkg/v2/mock_registry/session.go b/internal/pkg/v2/mock_registry/session.go index a2ec1baa..ad60e97d 100644 --- a/internal/pkg/v2/mock_registry/session.go +++ b/internal/pkg/v2/mock_registry/session.go @@ -34,6 +34,21 @@ func (m *MockSessionRegistry) EXPECT() *MockSessionRegistryMockRecorder { return m.recorder } +// GetSessionMutex mocks base method +func (m *MockSessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSessionMutex", sessionName) + ret0, _ := ret[0].(store.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSessionMutex indicates an expected call of GetSessionMutex +func (mr *MockSessionRegistryMockRecorder) GetSessionMutex(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionMutex", reflect.TypeOf((*MockSessionRegistry)(nil).GetSessionMutex), sessionName) +} + // CreateSession mocks base method func (m *MockSessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { m.ctrl.T.Helper() @@ -107,18 +122,3 @@ func (mr *MockSessionRegistryMockRecorder) DeleteSession(session interface{}) *g mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockSessionRegistry)(nil).DeleteSession), session) } - -// GetSessionMutex mocks base method -func (m *MockSessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionMutex", sessionName) - ret0, _ := ret[0].(store.Mutex) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSessionMutex indicates an expected call of GetSessionMutex -func (mr *MockSessionRegistryMockRecorder) GetSessionMutex(sessionName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionMutex", reflect.TypeOf((*MockSessionRegistry)(nil).GetSessionMutex), sessionName) -} diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go index c1034490..61f4afbb 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -34,21 +34,6 @@ func (m *MockSessionActions) EXPECT() *MockSessionActionsMockRecorder { return m.recorder } -// CreateSessionVolume mocks base method -func (m *MockSessionActions) CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSessionVolume", ctxt, sessionName) - ret0, _ := ret[0].(<-chan datamodel.SessionAction) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateSessionVolume indicates an expected call of CreateSessionVolume -func (mr *MockSessionActionsMockRecorder) CreateSessionVolume(ctxt, sessionName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSessionVolume", reflect.TypeOf((*MockSessionActions)(nil).CreateSessionVolume), ctxt, sessionName) -} - // SendSessionAction mocks base method func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() @@ -64,21 +49,6 @@ func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, se return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSessionAction", reflect.TypeOf((*MockSessionActions)(nil).SendSessionAction), ctxt, actionType, session) } -// GetCreateSessionVolumeRequests mocks base method -func (m *MockSessionActions) GetCreateSessionVolumeRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCreateSessionVolumeRequests", ctxt, brickHostName) - ret0, _ := ret[0].(<-chan datamodel.SessionAction) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetCreateSessionVolumeRequests indicates an expected call of GetCreateSessionVolumeRequests -func (mr *MockSessionActionsMockRecorder) GetCreateSessionVolumeRequests(ctxt, brickHostName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCreateSessionVolumeRequests", reflect.TypeOf((*MockSessionActions)(nil).GetCreateSessionVolumeRequests), ctxt, brickHostName) -} - // GetSessionActions mocks base method func (m *MockSessionActions) GetSessionActions(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/registry/allocation.go b/internal/pkg/v2/registry/allocation.go index 538a33e6..72afa3cd 100644 --- a/internal/pkg/v2/registry/allocation.go +++ b/internal/pkg/v2/registry/allocation.go @@ -6,8 +6,16 @@ import ( ) type AllocationRegistry interface { + // Get all registered pools + GetPool(name datamodel.PoolName) (datamodel.Pool, error) + + // Creates the pool if it doesn't exist + // error if the granularity doesn't match and existing pool + // TODO package method called by brick registry? + //EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) + // Get brick availability by pool - GetBricksByPool() ([]datamodel.PoolInfo, error) + GetAllPoolInfos() ([]datamodel.PoolInfo, error) // Get brick availability for one pool // bricks are only available if corresponding host currently alive diff --git a/internal/pkg/v2/registry/brick.go b/internal/pkg/v2/registry/brick.go index cd8a569b..5adcda3f 100644 --- a/internal/pkg/v2/registry/brick.go +++ b/internal/pkg/v2/registry/brick.go @@ -9,6 +9,7 @@ type BrickRegistry interface { // BrickHost updates bricks on startup // This will error if we remove a brick that has an allocation // for a Session that isn't in an error state + // This includes ensuring the pool exists and is consistent with the given brick host info UpdateBrickHost(brickHostInfo datamodel.BrickHost) error // Gets all new actions for the given Session @@ -30,5 +31,6 @@ type BrickRegistry interface { // Check if given brick host is alive // // Error if brick host doesn't exist - IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) + // TODO: private method? + //IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) } diff --git a/internal/pkg/v2/registry/pool.go b/internal/pkg/v2/registry/pool.go deleted file mode 100644 index 0be78242..00000000 --- a/internal/pkg/v2/registry/pool.go +++ /dev/null @@ -1,14 +0,0 @@ -package registry - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" -) - -type PoolRegistry interface { - // Get all registered pools - GetPool(name datamodel.PoolName) (datamodel.Pool, error) - - // Creates the pool if it doesn't exist - // error if the granularity doesn't match and existing pool - EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) -} diff --git a/internal/pkg/v2/registry/session.go b/internal/pkg/v2/registry/session.go index c582cf86..1a4bfbdd 100644 --- a/internal/pkg/v2/registry/session.go +++ b/internal/pkg/v2/registry/session.go @@ -6,6 +6,11 @@ import ( ) type SessionRegistry interface { + // This mutex should be held before doing any operations on given session + // + // No error if the session doesn't exist, as this is also used when creating a session + GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) + // Update provided session // // Error is session already exists @@ -32,9 +37,4 @@ type SessionRegistry interface { // Error if session doesn't match current revision // No error if session has already been deleted DeleteSession(session datamodel.Session) error - - // This mutex should be held before doing any operations on given session - // - // No error if the session doesn't exist, as this is also used when creating a session - GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) } diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go index 51bdb7e5..6c24f9df 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/v2/registry/session_actions.go @@ -6,14 +6,6 @@ import ( ) type SessionActions interface { - // Client requests session volume is created - // - // Error if session does not have bricks allocated - // Error if session volume has already been created - // Error if primary brick host is not alive or not enabled - // Error is context is cancelled or timed-out - CreateSessionVolume(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) - // Updates session, then requests action // // Error if current revision of session doesn't match @@ -22,14 +14,6 @@ type SessionActions interface { ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) - // Get session volume create requests, - // where given hostname is the primary brick host - // Called very time brick host is started - // - // Error is context is cancelled or timed-out - GetCreateSessionVolumeRequests( - ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) - // Gets all new actions for the given Session // // Error if context is cancelled or timed-out From 31e156372b1257a0d1cfc9f19d234866da71e190 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 11:43:09 +0100 Subject: [PATCH 063/191] Tidy up new Keystore interface --- build/rebuild_mocks.sh | 2 +- .../dacd/brick_manager_impl/brick_manager.go | 4 +- .../brick_manager_impl/brick_manager_test.go | 2 +- internal/pkg/v2/mock_registry/actions.go | 118 ------------------ internal/pkg/v2/mock_registry/brick.go | 78 ------------ .../{allocation.go => brick_allocation.go} | 2 +- internal/pkg/v2/mock_registry/brick_host.go | 78 ++++++++++++ internal/pkg/v2/mock_store/keystore.go | 74 +++++------ .../{allocation.go => brick_allocation.go} | 0 .../v2/registry/{brick.go => brick_host.go} | 2 +- internal/pkg/v2/registry_impl/session.go | 39 ++++++ internal/pkg/v2/store/keystore.go | 44 ++----- 12 files changed, 173 insertions(+), 270 deletions(-) delete mode 100644 internal/pkg/v2/mock_registry/actions.go delete mode 100644 internal/pkg/v2/mock_registry/brick.go rename internal/pkg/v2/mock_registry/{allocation.go => brick_allocation.go} (98%) create mode 100644 internal/pkg/v2/mock_registry/brick_host.go rename internal/pkg/v2/registry/{allocation.go => brick_allocation.go} (100%) rename internal/pkg/v2/registry/{brick.go => brick_host.go} (97%) create mode 100644 internal/pkg/v2/registry_impl/session.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 9aec9d89..a2d929a5 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -27,7 +27,7 @@ done mockgen -source=internal/pkg/pfsprovider/interface.go \ -package mocks >internal/pkg/mocks/pfsprovider_mock.go -items="allocation brick session session_actions" +items="brick_allocation brick_host session session_actions" for i in $items; do mockgen -source=internal/pkg/v2/registry/${i}.go \ >internal/pkg/v2/mock_registry/${i}.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index 65d6924f..f0850a67 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -9,7 +9,7 @@ import ( "log" ) -func NewBrickManager(brickRegistry registry.BrickRegistry, handler facade.SessionActionHandler) dacd.BrickManager { +func NewBrickManager(brickRegistry registry.BrickHostRegistry, handler facade.SessionActionHandler) dacd.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), brickRegistry: brickRegistry, @@ -19,7 +19,7 @@ func NewBrickManager(brickRegistry registry.BrickRegistry, handler facade.Sessio type brickManager struct { config config.BrickManagerConfig - brickRegistry registry.BrickRegistry + brickRegistry registry.BrickHostRegistry sessionActionHandler facade.SessionActionHandler } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index 8a44043c..d6f337b2 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -19,7 +19,7 @@ func TestBrickManager_Hostname(t *testing.T) { func TestBrickManager_Startup(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - brickRegistry := mock_registry.NewMockBrickRegistry(mockCtrl) + brickRegistry := mock_registry.NewMockBrickHostRegistry(mockCtrl) handler := mock_facade.NewMockSessionActionHandler(mockCtrl) brickManager := NewBrickManager(brickRegistry, handler) diff --git a/internal/pkg/v2/mock_registry/actions.go b/internal/pkg/v2/mock_registry/actions.go deleted file mode 100644 index 00ef4b37..00000000 --- a/internal/pkg/v2/mock_registry/actions.go +++ /dev/null @@ -1,118 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/data/session/session_registry.go - -// Package mock_registry is a generated GoMock package. -package mock_registry - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/golang/mock/gomock" - "reflect" -) - -// MockActions is a mock of Actions interface -type MockActions struct { - ctrl *gomock.Controller - recorder *MockActionsMockRecorder -} - -// MockActionsMockRecorder is the mock recorder for MockActions -type MockActionsMockRecorder struct { - mock *MockActions -} - -// NewMockActions creates a new mock instance -func NewMockActions(ctrl *gomock.Controller) *MockActions { - mock := &MockActions{ctrl: ctrl} - mock.recorder = &MockActionsMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockActions) EXPECT() *MockActionsMockRecorder { - return m.recorder -} - -// CreateSession mocks base method -func (m *MockActions) CreateSessionVolume(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateSession", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateSession indicates an expected call of CreateSession -func (mr *MockActionsMockRecorder) CreateSessionVolume(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockActions)(nil).CreateSessionVolume), session) -} - -// DeleteSession mocks base method -func (m *MockActions) DeleteSession(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteSession", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteSession indicates an expected call of DeleteSession -func (mr *MockActionsMockRecorder) DeleteSession(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSession", reflect.TypeOf((*MockActions)(nil).DeleteSession), session) -} - -// CopyDataIn mocks base method -func (m *MockActions) DataIn(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CopyDataIn", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// CopyDataIn indicates an expected call of CopyDataIn -func (mr *MockActionsMockRecorder) DataIn(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataIn", reflect.TypeOf((*MockActions)(nil).DataIn), session) -} - -// Mount mocks base method -func (m *MockActions) AttachVolumes(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Mount", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// Mount indicates an expected call of Mount -func (mr *MockActionsMockRecorder) AttachVolumes(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mount", reflect.TypeOf((*MockActions)(nil).AttachVolumes), session) -} - -// Unmount mocks base method -func (m *MockActions) DetachVolumes(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Unmount", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// Unmount indicates an expected call of Unmount -func (mr *MockActionsMockRecorder) DetachVolumes(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unmount", reflect.TypeOf((*MockActions)(nil).DetachVolumes), session) -} - -// CopyDataOut mocks base method -func (m *MockActions) DataOut(session datamodel.Session) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CopyDataOut", session) - ret0, _ := ret[0].(error) - return ret0 -} - -// CopyDataOut indicates an expected call of CopyDataOut -func (mr *MockActionsMockRecorder) DataOut(session interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataOut", reflect.TypeOf((*MockActions)(nil).DataOut), session) -} diff --git a/internal/pkg/v2/mock_registry/brick.go b/internal/pkg/v2/mock_registry/brick.go deleted file mode 100644 index 1102749d..00000000 --- a/internal/pkg/v2/mock_registry/brick.go +++ /dev/null @@ -1,78 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/brick.go - -// Package mock_registry is a generated GoMock package. -package mock_registry - -import ( - context "context" - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockBrickRegistry is a mock of BrickRegistry interface -type MockBrickRegistry struct { - ctrl *gomock.Controller - recorder *MockBrickRegistryMockRecorder -} - -// MockBrickRegistryMockRecorder is the mock recorder for MockBrickRegistry -type MockBrickRegistryMockRecorder struct { - mock *MockBrickRegistry -} - -// NewMockBrickRegistry creates a new mock instance -func NewMockBrickRegistry(ctrl *gomock.Controller) *MockBrickRegistry { - mock := &MockBrickRegistry{ctrl: ctrl} - mock.recorder = &MockBrickRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockBrickRegistry) EXPECT() *MockBrickRegistryMockRecorder { - return m.recorder -} - -// UpdateBrickHost mocks base method -func (m *MockBrickRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateBrickHost", brickHostInfo) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateBrickHost indicates an expected call of UpdateBrickHost -func (mr *MockBrickRegistryMockRecorder) UpdateBrickHost(brickHostInfo interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickRegistry)(nil).UpdateBrickHost), brickHostInfo) -} - -// GetSessionActions mocks base method -func (m *MockBrickRegistry) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) - ret0, _ := ret[0].(<-chan datamodel.SessionAction) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSessionActions indicates an expected call of GetSessionActions -func (mr *MockBrickRegistryMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockBrickRegistry)(nil).GetSessionActions), ctxt, brickHostName) -} - -// KeepAliveHost mocks base method -func (m *MockBrickRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeepAliveHost", ctxt, brickHostName) - ret0, _ := ret[0].(error) - return ret0 -} - -// KeepAliveHost indicates an expected call of KeepAliveHost -func (mr *MockBrickRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickRegistry)(nil).KeepAliveHost), ctxt, brickHostName) -} diff --git a/internal/pkg/v2/mock_registry/allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go similarity index 98% rename from internal/pkg/v2/mock_registry/allocation.go rename to internal/pkg/v2/mock_registry/brick_allocation.go index 4132a03b..99b1b345 100644 --- a/internal/pkg/v2/mock_registry/allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/allocation.go +// Source: internal/pkg/v2/registry/brick_allocation.go // Package mock_registry is a generated GoMock package. package mock_registry diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/v2/mock_registry/brick_host.go new file mode 100644 index 00000000..0aea2107 --- /dev/null +++ b/internal/pkg/v2/mock_registry/brick_host.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/v2/registry/brick_host.go + +// Package mock_registry is a generated GoMock package. +package mock_registry + +import ( + context "context" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockBrickHostRegistry is a mock of BrickHostRegistry interface +type MockBrickHostRegistry struct { + ctrl *gomock.Controller + recorder *MockBrickHostRegistryMockRecorder +} + +// MockBrickHostRegistryMockRecorder is the mock recorder for MockBrickHostRegistry +type MockBrickHostRegistryMockRecorder struct { + mock *MockBrickHostRegistry +} + +// NewMockBrickHostRegistry creates a new mock instance +func NewMockBrickHostRegistry(ctrl *gomock.Controller) *MockBrickHostRegistry { + mock := &MockBrickHostRegistry{ctrl: ctrl} + mock.recorder = &MockBrickHostRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockBrickHostRegistry) EXPECT() *MockBrickHostRegistryMockRecorder { + return m.recorder +} + +// UpdateBrickHost mocks base method +func (m *MockBrickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateBrickHost", brickHostInfo) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateBrickHost indicates an expected call of UpdateBrickHost +func (mr *MockBrickHostRegistryMockRecorder) UpdateBrickHost(brickHostInfo interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).UpdateBrickHost), brickHostInfo) +} + +// GetSessionActions mocks base method +func (m *MockBrickHostRegistry) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) + ret0, _ := ret[0].(<-chan datamodel.SessionAction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSessionActions indicates an expected call of GetSessionActions +func (mr *MockBrickHostRegistryMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockBrickHostRegistry)(nil).GetSessionActions), ctxt, brickHostName) +} + +// KeepAliveHost mocks base method +func (m *MockBrickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KeepAliveHost", ctxt, brickHostName) + ret0, _ := ret[0].(error) + return ret0 +} + +// KeepAliveHost indicates an expected call of KeepAliveHost +func (mr *MockBrickHostRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).KeepAliveHost), ctxt, brickHostName) +} diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index 70acd435..6a731962 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -48,75 +48,77 @@ func (mr *MockKeystoreMockRecorder) Close() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockKeystore)(nil).Close)) } -// CleanPrefix mocks base method -func (m *MockKeystore) CleanPrefix(prefix string) error { +// Create mocks base method +func (m *MockKeystore) Create(key, value string) (store.KeyValueVersion, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CleanPrefix", prefix) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "Create", key, value) + ret0, _ := ret[0].(store.KeyValueVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// CleanPrefix indicates an expected call of CleanPrefix -func (mr *MockKeystoreMockRecorder) CleanPrefix(prefix interface{}) *gomock.Call { +// Create indicates an expected call of Create +func (mr *MockKeystoreMockRecorder) Create(key, value interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanPrefix", reflect.TypeOf((*MockKeystore)(nil).CleanPrefix), prefix) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockKeystore)(nil).Create), key, value) } -// Add mocks base method -func (m *MockKeystore) Add(keyValue store.KeyValue) error { +// Update mocks base method +func (m *MockKeystore) Update(key, value string, modRevision int) (store.KeyValueVersion, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Add", keyValue) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "Update", key, value, modRevision) + ret0, _ := ret[0].(store.KeyValueVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// Add indicates an expected call of Add -func (mr *MockKeystoreMockRecorder) Add(keyValue interface{}) *gomock.Call { +// Update indicates an expected call of Update +func (mr *MockKeystoreMockRecorder) Update(key, value, modRevision interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockKeystore)(nil).Add), keyValue) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockKeystore)(nil).Update), key, value, modRevision) } -// Update mocks base method -func (m *MockKeystore) Update(keyValue store.KeyValueVersion) error { +// Delete mocks base method +func (m *MockKeystore) Delete(key string, modRevision int) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", keyValue) + ret := m.ctrl.Call(m, "Delete", key, modRevision) ret0, _ := ret[0].(error) return ret0 } -// Update indicates an expected call of Update -func (mr *MockKeystoreMockRecorder) Update(keyValue interface{}) *gomock.Call { +// Delete indicates an expected call of Delete +func (mr *MockKeystoreMockRecorder) Delete(key, modRevision interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockKeystore)(nil).Update), keyValue) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockKeystore)(nil).Delete), key, modRevision) } -// DeleteAll mocks base method -func (m *MockKeystore) DeleteAll(keyValues []store.KeyValueVersion) error { +// DeleteAllKeysWithPrefix mocks base method +func (m *MockKeystore) DeleteAllKeysWithPrefix(keyPrefix string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAll", keyValues) + ret := m.ctrl.Call(m, "DeleteAllKeysWithPrefix", keyPrefix) ret0, _ := ret[0].(error) return ret0 } -// DeleteAll indicates an expected call of DeleteAll -func (mr *MockKeystoreMockRecorder) DeleteAll(keyValues interface{}) *gomock.Call { +// DeleteAllKeysWithPrefix indicates an expected call of DeleteAllKeysWithPrefix +func (mr *MockKeystoreMockRecorder) DeleteAllKeysWithPrefix(keyPrefix interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAll", reflect.TypeOf((*MockKeystore)(nil).DeleteAll), keyValues) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllKeysWithPrefix", reflect.TypeOf((*MockKeystore)(nil).DeleteAllKeysWithPrefix), keyPrefix) } // GetAll mocks base method -func (m *MockKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error) { +func (m *MockKeystore) GetAll(keyPrefix string) ([]store.KeyValueVersion, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAll", prefix) + ret := m.ctrl.Call(m, "GetAll", keyPrefix) ret0, _ := ret[0].([]store.KeyValueVersion) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAll indicates an expected call of GetAll -func (mr *MockKeystoreMockRecorder) GetAll(prefix interface{}) *gomock.Call { +func (mr *MockKeystoreMockRecorder) GetAll(keyPrefix interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockKeystore)(nil).GetAll), prefix) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockKeystore)(nil).GetAll), keyPrefix) } // Get mocks base method @@ -149,17 +151,17 @@ func (mr *MockKeystoreMockRecorder) Watch(ctxt, key, withPrefix interface{}) *go } // KeepAliveKey mocks base method -func (m *MockKeystore) KeepAliveKey(key string) error { +func (m *MockKeystore) KeepAliveKey(ctxt context.Context, key string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeepAliveKey", key) + ret := m.ctrl.Call(m, "KeepAliveKey", ctxt, key) ret0, _ := ret[0].(error) return ret0 } // KeepAliveKey indicates an expected call of KeepAliveKey -func (mr *MockKeystoreMockRecorder) KeepAliveKey(key interface{}) *gomock.Call { +func (mr *MockKeystoreMockRecorder) KeepAliveKey(ctxt, key interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveKey", reflect.TypeOf((*MockKeystore)(nil).KeepAliveKey), key) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveKey", reflect.TypeOf((*MockKeystore)(nil).KeepAliveKey), ctxt, key) } // NewMutex mocks base method diff --git a/internal/pkg/v2/registry/allocation.go b/internal/pkg/v2/registry/brick_allocation.go similarity index 100% rename from internal/pkg/v2/registry/allocation.go rename to internal/pkg/v2/registry/brick_allocation.go diff --git a/internal/pkg/v2/registry/brick.go b/internal/pkg/v2/registry/brick_host.go similarity index 97% rename from internal/pkg/v2/registry/brick.go rename to internal/pkg/v2/registry/brick_host.go index 5adcda3f..d13fdec3 100644 --- a/internal/pkg/v2/registry/brick.go +++ b/internal/pkg/v2/registry/brick_host.go @@ -5,7 +5,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" ) -type BrickRegistry interface { +type BrickHostRegistry interface { // BrickHost updates bricks on startup // This will error if we remove a brick that has an allocation // for a Session that isn't in an error state diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go new file mode 100644 index 00000000..4a342f82 --- /dev/null +++ b/internal/pkg/v2/registry_impl/session.go @@ -0,0 +1,39 @@ +package registry_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" +) + +func GetNewSessionRegistry(store store.Keystore) registry.SessionRegistry { + return &sessionRegistry{store} +} + +type sessionRegistry struct { + store store.Keystore +} + +func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { + panic("implement me") +} + +func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { + panic("implement me") +} + +func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { + panic("implement me") +} diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index 39f5660b..c26658b1 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -2,8 +2,6 @@ package store import ( "context" - "encoding/json" - "log" ) type Keystore interface { @@ -11,37 +9,35 @@ type Keystore interface { // ... such as a connection to etcd. Close() error - // Removes any key starting with the given prefix. - // An error is returned if nothing was deleted, - // which some users may choose to safely ignore. - CleanPrefix(prefix string) error - // Atomically add all the key value pairs // // If an error occurs no keyvalues are written. // Error is returned if any key already exists. - Add(keyValue KeyValue) error + Create(key string, value string) (KeyValueVersion, error) - // Update the specifed key values, atomically + // Update the specified key values, atomically // // If ModRevision is 0, it is ignored. // Otherwise if the revisions of any key doesn't // match the current revision of that key, the update fails. // When update fails an error is returned and no keyValues are updated - Update(keyValue KeyValueVersion) error + Update(key string, value string, modRevision int) (KeyValueVersion, error) - // Delete the specifed key values, atomically + // Delete the specified key values, atomically // // Similar to update, checks ModRevision matches current key, // ignores ModRevision if not zero. // If any keys are not currently present, the request fails. // Deletes no keys if an error is returned - DeleteAll(keyValues []KeyValueVersion) error + Delete(key string, modRevision int) error + + // Removes all keys with given prefix + DeleteAllKeysWithPrefix(keyPrefix string) error // Get all key values for a given prefix. - GetAll(prefix string) ([]KeyValueVersion, error) + GetAll(keyPrefix string) ([]KeyValueVersion, error) - // Get all keys for a given prefix. + // Get given key Get(key string) (KeyValueVersion, error) // Get a channel containing all KeyValueUpdate events @@ -52,7 +48,8 @@ type Keystore interface { // Add a key, and remove it when calling process dies // Error is returned if the key already exists - KeepAliveKey(key string) error + // can be cancelled via the context + KeepAliveKey(ctxt context.Context, key string) error // Get a new mutex associated with the specified key NewMutex(lockKey string) (Mutex, error) @@ -60,11 +57,6 @@ type Keystore interface { type KeyValueUpdateChan <-chan KeyValueUpdate -type KeyValue struct { - Key string - Value string // TODO: should this be []byte? Or have a json parsed version? -} - type KeyValueVersion struct { Key string Value string @@ -81,18 +73,6 @@ type KeyValueUpdate struct { Err error } -func (kvv KeyValueVersion) String() string { - return toJson(kvv) -} - -func toJson(message interface{}) string { - b, error := json.Marshal(message) - if error != nil { - log.Fatal(error) - } - return string(b) -} - type Mutex interface { Lock(ctx context.Context) error Unlock(ctx context.Context) error From 1f54aaf3a7b669f0a4dcde3ab70d113dbccf2bca Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 13:28:58 +0100 Subject: [PATCH 064/191] Implement Get Session Mutex --- internal/pkg/v2/registry_impl/session.go | 5 ++-- internal/pkg/v2/registry_impl/session_test.go | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 internal/pkg/v2/registry_impl/session_test.go diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 4a342f82..8dffa3b2 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -1,12 +1,13 @@ package registry_impl import ( + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" ) -func GetNewSessionRegistry(store store.Keystore) registry.SessionRegistry { +func NewSessionRegistry(store store.Keystore) registry.SessionRegistry { return &sessionRegistry{store} } @@ -15,7 +16,7 @@ type sessionRegistry struct { } func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { - panic("implement me") + return s.store.NewMutex(fmt.Sprintf("/session_lock/%s", sessionName)) } func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go new file mode 100644 index 00000000..c380f353 --- /dev/null +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -0,0 +1,23 @@ +package registry_impl + +import ( + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSessionRegistry_GetSessionMutex(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + store := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(store) + fakeErr := errors.New("fake") + store.EXPECT().NewMutex("/session_lock/foo").Return(nil, fakeErr) + + mutex, err := registry.GetSessionMutex("foo") + + assert.Nil(t, mutex) + assert.Equal(t, fakeErr, err) +} From 0457c8b790017c017345b4f700647bcd9ad5f842 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 14:39:41 +0100 Subject: [PATCH 065/191] Validate sessionName --- internal/pkg/v2/registry_impl/session.go | 15 ++++++++++++++- internal/pkg/v2/registry_impl/session_test.go | 7 +++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 8dffa3b2..5131559c 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -2,6 +2,7 @@ package registry_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" @@ -16,7 +17,19 @@ type sessionRegistry struct { } func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { - return s.store.NewMutex(fmt.Sprintf("/session_lock/%s", sessionName)) + sessionKey, err := getSessionKey(sessionName) + if err != nil { + return nil, err + } + lockKey := fmt.Sprintf("/lock%s", sessionKey) + return s.store.NewMutex(lockKey) +} + +func getSessionKey(sessionName datamodel.SessionName) (string, error) { + if !parsers.IsValidName(string(sessionName)) { + return "", fmt.Errorf("invalid session name %s", sessionName) + } + return fmt.Sprintf("/session/%s", sessionName), nil } func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index c380f353..1345af31 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -14,10 +14,13 @@ func TestSessionRegistry_GetSessionMutex(t *testing.T) { store := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(store) fakeErr := errors.New("fake") - store.EXPECT().NewMutex("/session_lock/foo").Return(nil, fakeErr) + store.EXPECT().NewMutex("/lock/session/foo").Return(nil, fakeErr) mutex, err := registry.GetSessionMutex("foo") - assert.Nil(t, mutex) assert.Equal(t, fakeErr, err) + + mutex, err = registry.GetSessionMutex("foo/bar") + assert.Nil(t, mutex) + assert.Equal(t, "invalid session name foo/bar", err.Error()) } From 6e95573ccde9c7c3ae8244271cb31ce2c6d97aa7 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 14:52:10 +0100 Subject: [PATCH 066/191] Implement Create Session --- internal/pkg/v2/datamodel/session.go | 2 +- internal/pkg/v2/registry_impl/session.go | 20 +++++++++++++++- internal/pkg/v2/registry_impl/session_test.go | 23 ++++++++++++++++--- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 41b3ac45..54d562f0 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -11,7 +11,7 @@ type Session struct { // Currently stored revision // this is checked when an update is requested - Revision int + Revision int64 // unix uid Owner uint diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 5131559c..0a8702d8 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -1,6 +1,7 @@ package registry_impl import ( + "encoding/json" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" @@ -33,7 +34,24 @@ func getSessionKey(sessionName datamodel.SessionName) (string, error) { } func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { - panic("implement me") + sessionKey, err := getSessionKey(session.Name) + if err != nil { + return session, err + } + + sessionAsString, err := json.Marshal(session) + if err != nil { + return session, fmt.Errorf("unable to convert session to json due to: %s", err) + } + + keyValueVersion, err := s.store.Create(sessionKey, string(sessionAsString)) + if err != nil { + return session, fmt.Errorf("unable to create session due to: %s", err) + } + + // Return the last modification revision + session.Revision = keyValueVersion.ModRevision + return session, nil } func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 1345af31..db785346 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -2,7 +2,9 @@ package registry_impl import ( "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -11,10 +13,10 @@ import ( func TestSessionRegistry_GetSessionMutex(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - store := mock_store.NewMockKeystore(mockCtrl) - registry := NewSessionRegistry(store) + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) fakeErr := errors.New("fake") - store.EXPECT().NewMutex("/lock/session/foo").Return(nil, fakeErr) + keystore.EXPECT().NewMutex("/lock/session/foo").Return(nil, fakeErr) mutex, err := registry.GetSessionMutex("foo") assert.Nil(t, mutex) @@ -24,3 +26,18 @@ func TestSessionRegistry_GetSessionMutex(t *testing.T) { assert.Nil(t, mutex) assert.Equal(t, "invalid session name foo/bar", err.Error()) } + +func TestSessionRegistry_CreateSession(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) + session := datamodel.Session{Name:"foo"} + expectedValue := `{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}` + keystore.EXPECT().Create("/session/foo", expectedValue).Return(store.KeyValueVersion{ModRevision:42}, nil) + + session, err := registry.CreateSession(session) + + assert.Nil(t, err) + assert.Equal(t, int64(42), session.Revision) +} \ No newline at end of file From 16ee71b6a6a1bbdfefe34e4c75e55c0b8a58a472 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 15:11:12 +0100 Subject: [PATCH 067/191] Add some validation --- internal/pkg/v2/registry_impl/session.go | 17 ++++++++++++ internal/pkg/v2/registry_impl/session_test.go | 27 ++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 0a8702d8..b1e81874 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -39,6 +39,23 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se return session, err } + // TODO: more validation? + if session.ActualSizeBytes > 0 { + if len(session.Allocations) == 0 { + return session, fmt.Errorf("session must have allocations before being created") + } + if session.PrimaryBrickHost == "" { + return session, fmt.Errorf("session must have a primary brick host set") + } + } else { + if len(session.Allocations) != 0 { + return session, fmt.Errorf("allocations out of sync with ActualSizeBytes") + } + if session.PrimaryBrickHost != "" { + return session, fmt.Errorf("PrimaryBrickHost should be empty if no bricks assigned") + } + } + sessionAsString, err := json.Marshal(session) if err != nil { return session, fmt.Errorf("unable to convert session to json due to: %s", err) diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index db785346..f106c62f 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -32,12 +32,33 @@ func TestSessionRegistry_CreateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - session := datamodel.Session{Name:"foo"} expectedValue := `{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}` keystore.EXPECT().Create("/session/foo", expectedValue).Return(store.KeyValueVersion{ModRevision:42}, nil) - session, err := registry.CreateSession(session) - + session, err := registry.CreateSession(datamodel.Session{Name:"foo"}) assert.Nil(t, err) assert.Equal(t, int64(42), session.Revision) + + _, err = registry.CreateSession(datamodel.Session{Name:"foo", ActualSizeBytes:1024}) + assert.NotNil(t, err) + assert.Equal(t, "session must have allocations before being created", err.Error()) + + _, err = registry.CreateSession(datamodel.Session{ + Name:"foo", + ActualSizeBytes:1024, + Allocations:[]datamodel.BrickAllocation{{}}, + }) + assert.NotNil(t, err) + assert.Equal(t, "session must have a primary brick host set", err.Error()) + + _, err = registry.CreateSession(datamodel.Session{ + Name:"foo", + Allocations:[]datamodel.BrickAllocation{{}}, + }) + assert.NotNil(t, err) + assert.Equal(t, "allocations out of sync with ActualSizeBytes", err.Error()) + + _, err = registry.CreateSession(datamodel.Session{Name:"foo", PrimaryBrickHost:"foo"}) + assert.NotNil(t, err) + assert.Equal(t, "PrimaryBrickHost should be empty if no bricks assigned", err.Error()) } \ No newline at end of file From 0ea2c4fa14631eefe69ff88ba8a0f7c866bc0046 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 15:12:27 +0100 Subject: [PATCH 068/191] Check name is validated on create --- internal/pkg/v2/registry_impl/session_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index f106c62f..5d6fd2d8 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -39,6 +39,10 @@ func TestSessionRegistry_CreateSession(t *testing.T) { assert.Nil(t, err) assert.Equal(t, int64(42), session.Revision) + session, err = registry.CreateSession(datamodel.Session{Name:"foo/bar"}) + assert.NotNil(t, err) + assert.Equal(t, "invalid session name foo/bar", err.Error()) + _, err = registry.CreateSession(datamodel.Session{Name:"foo", ActualSizeBytes:1024}) assert.NotNil(t, err) assert.Equal(t, "session must have allocations before being created", err.Error()) From 4aa60107b8e42b7678e7a6153bc74ba2e769512e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:05:30 +0100 Subject: [PATCH 069/191] Test GetSession --- internal/pkg/v2/mock_store/keystore.go | 4 +- internal/pkg/v2/registry_impl/session.go | 21 ++++++- internal/pkg/v2/registry_impl/session_test.go | 56 +++++++++++++++---- internal/pkg/v2/store/keystore.go | 6 +- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index 6a731962..f9b7fa61 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -49,7 +49,7 @@ func (mr *MockKeystoreMockRecorder) Close() *gomock.Call { } // Create mocks base method -func (m *MockKeystore) Create(key, value string) (store.KeyValueVersion, error) { +func (m *MockKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", key, value) ret0, _ := ret[0].(store.KeyValueVersion) @@ -64,7 +64,7 @@ func (mr *MockKeystoreMockRecorder) Create(key, value interface{}) *gomock.Call } // Update mocks base method -func (m *MockKeystore) Update(key, value string, modRevision int) (store.KeyValueVersion, error) { +func (m *MockKeystore) Update(key string, value []byte, modRevision int) (store.KeyValueVersion, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Update", key, value, modRevision) ret0, _ := ret[0].(store.KeyValueVersion) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index b1e81874..ca9a2ead 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -61,7 +61,7 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se return session, fmt.Errorf("unable to convert session to json due to: %s", err) } - keyValueVersion, err := s.store.Create(sessionKey, string(sessionAsString)) + keyValueVersion, err := s.store.Create(sessionKey, sessionAsString) if err != nil { return session, fmt.Errorf("unable to create session due to: %s", err) } @@ -72,7 +72,24 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se } func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { - panic("implement me") + sessionKey, err := getSessionKey(sessionName) + if err != nil { + return datamodel.Session{}, err + } + + keyValueVersion, err := s.store.Get(sessionKey) + if err != nil { + return datamodel.Session{}, fmt.Errorf("unable to get session due to: %s", err) + } + + session := datamodel.Session{} + err = json.Unmarshal(keyValueVersion.Value, &session) + if err != nil { + return datamodel.Session{}, fmt.Errorf("unable parse session from store due to: %s", err) + } + + session.Revision = keyValueVersion.ModRevision + return session, nil } func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 5d6fd2d8..1588570e 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -10,6 +10,8 @@ import ( "testing" ) +var emptySessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) + func TestSessionRegistry_GetSessionMutex(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -32,37 +34,67 @@ func TestSessionRegistry_CreateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - expectedValue := `{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}` - keystore.EXPECT().Create("/session/foo", expectedValue).Return(store.KeyValueVersion{ModRevision:42}, nil) + keystore.EXPECT().Create("/session/foo", emptySessionString).Return(store.KeyValueVersion{ModRevision: 42}, nil) - session, err := registry.CreateSession(datamodel.Session{Name:"foo"}) + session, err := registry.CreateSession(datamodel.Session{Name: "foo"}) assert.Nil(t, err) assert.Equal(t, int64(42), session.Revision) - session, err = registry.CreateSession(datamodel.Session{Name:"foo/bar"}) + session, err = registry.CreateSession(datamodel.Session{Name: "foo/bar"}) assert.NotNil(t, err) assert.Equal(t, "invalid session name foo/bar", err.Error()) - _, err = registry.CreateSession(datamodel.Session{Name:"foo", ActualSizeBytes:1024}) + _, err = registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024}) assert.NotNil(t, err) assert.Equal(t, "session must have allocations before being created", err.Error()) _, err = registry.CreateSession(datamodel.Session{ - Name:"foo", - ActualSizeBytes:1024, - Allocations:[]datamodel.BrickAllocation{{}}, + Name: "foo", + ActualSizeBytes: 1024, + Allocations: []datamodel.BrickAllocation{{}}, }) assert.NotNil(t, err) assert.Equal(t, "session must have a primary brick host set", err.Error()) _, err = registry.CreateSession(datamodel.Session{ - Name:"foo", - Allocations:[]datamodel.BrickAllocation{{}}, + Name: "foo", + Allocations: []datamodel.BrickAllocation{{}}, }) assert.NotNil(t, err) assert.Equal(t, "allocations out of sync with ActualSizeBytes", err.Error()) - _, err = registry.CreateSession(datamodel.Session{Name:"foo", PrimaryBrickHost:"foo"}) + _, err = registry.CreateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "foo"}) assert.NotNil(t, err) assert.Equal(t, "PrimaryBrickHost should be empty if no bricks assigned", err.Error()) -} \ No newline at end of file +} + +func TestSessionRegistry_GetSession(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) + keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{ + ModRevision: 42, + Value: emptySessionString, + }, nil) + + session, err := registry.GetSession("foo") + + assert.Nil(t, err) + assert.Equal(t, datamodel.Session{Name: "foo", Revision: 42}, session) + + session, err = registry.GetSession("foo/bar") + assert.NotNil(t, err) + assert.Equal(t, "invalid session name foo/bar", err.Error()) + + fakeErr := errors.New("fake") + keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{}, fakeErr) + session, err = registry.GetSession("foo") + assert.NotNil(t, err) + assert.Equal(t, "unable to get session due to: fake", err.Error()) + + keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{}, nil) + session, err = registry.GetSession("foo") + assert.NotNil(t, err) + assert.Equal(t, "unable parse session from store due to: unexpected end of JSON input", err.Error()) +} diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index c26658b1..ef144189 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -13,7 +13,7 @@ type Keystore interface { // // If an error occurs no keyvalues are written. // Error is returned if any key already exists. - Create(key string, value string) (KeyValueVersion, error) + Create(key string, value []byte) (KeyValueVersion, error) // Update the specified key values, atomically // @@ -21,7 +21,7 @@ type Keystore interface { // Otherwise if the revisions of any key doesn't // match the current revision of that key, the update fails. // When update fails an error is returned and no keyValues are updated - Update(key string, value string, modRevision int) (KeyValueVersion, error) + Update(key string, value []byte, modRevision int) (KeyValueVersion, error) // Delete the specified key values, atomically // @@ -59,7 +59,7 @@ type KeyValueUpdateChan <-chan KeyValueUpdate type KeyValueVersion struct { Key string - Value string + Value []byte CreateRevision int64 ModRevision int64 } From c1e3e07a892f6a50a7d5662cc48092bb494748b9 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:18:36 +0100 Subject: [PATCH 070/191] Implement GetAllSessions --- internal/pkg/v2/registry_impl/session.go | 23 +++++++++++++++++-- internal/pkg/v2/registry_impl/session_test.go | 22 ++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index ca9a2ead..4a16a178 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -7,6 +7,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "log" ) func NewSessionRegistry(store store.Keystore) registry.SessionRegistry { @@ -26,11 +27,13 @@ func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (st return s.store.NewMutex(lockKey) } +const sessionPrefix = "/session/" + func getSessionKey(sessionName datamodel.SessionName) (string, error) { if !parsers.IsValidName(string(sessionName)) { return "", fmt.Errorf("invalid session name %s", sessionName) } - return fmt.Sprintf("/session/%s", sessionName), nil + return fmt.Sprintf("%s%s", sessionPrefix, sessionName), nil } func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { @@ -93,7 +96,23 @@ func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamod } func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { - panic("implement me") + results, err := s.store.GetAll(sessionPrefix) + if err != nil { + return nil, fmt.Errorf("unable to get all sessions due to: %s", err.Error()) + } + + var sessions []datamodel.Session + for _, keyValueVersion := range results { + session := datamodel.Session{} + err = json.Unmarshal(keyValueVersion.Value, &session) + if err != nil { + log.Panicf("unable parse session from store due to: %s", err) + } + session.Revision = keyValueVersion.ModRevision + sessions = append(sessions, session) + } + + return sessions, nil } func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 1588570e..dc47eb04 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -98,3 +98,25 @@ func TestSessionRegistry_GetSession(t *testing.T) { assert.NotNil(t, err) assert.Equal(t, "unable parse session from store due to: unexpected end of JSON input", err.Error()) } + +func TestSessionRegistry_GetAllSessions(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) + keystore.EXPECT().GetAll("/session/").Return([]store.KeyValueVersion{{ + ModRevision: 42, + Value: emptySessionString, + }}, nil) + + sessions, err := registry.GetAllSessions() + assert.Nil(t, err) + assert.Equal(t, []datamodel.Session{{Name: "foo", Revision: 42}}, sessions) + + fakeErr := errors.New("fake") + keystore.EXPECT().GetAll("/session/").Return(nil, fakeErr) + sessions, err = registry.GetAllSessions() + assert.Nil(t, sessions) + assert.NotNil(t, err) + assert.Equal(t, "unable to get all sessions due to: fake", err.Error()) +} \ No newline at end of file From dfed551ccaa3af885be751e3d38101d33792742f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:22:34 +0100 Subject: [PATCH 071/191] Improve coverage of getAllSessions --- internal/pkg/v2/registry_impl/session_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index dc47eb04..5f760ed5 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -119,4 +119,14 @@ func TestSessionRegistry_GetAllSessions(t *testing.T) { assert.Nil(t, sessions) assert.NotNil(t, err) assert.Equal(t, "unable to get all sessions due to: fake", err.Error()) -} \ No newline at end of file + + keystore.EXPECT().GetAll("/session/").Return(nil, nil) + sessions, err = registry.GetAllSessions() + assert.Nil(t, err) + assert.Nil(t, sessions) + + keystore.EXPECT().GetAll("/session/").Return([]store.KeyValueVersion{{}}, nil) + assert.PanicsWithValue(t, + "unable parse session from store due to: unexpected end of JSON input", + func() { registry.GetAllSessions() }) +} From cc38c89af1aed0f6b13b3ebb60ae3e7d814458ca Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:26:13 +0100 Subject: [PATCH 072/191] Ensure hostname can't be empty --- internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go | 2 +- internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go index 23a6033d..86a4ac30 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go @@ -6,7 +6,7 @@ import ( "regexp" ) -var nameRegex = regexp.MustCompile("^[a-zA-Z0-9.]*$") +var nameRegex = regexp.MustCompile("^[a-zA-Z0-9.]+$") func IsValidName(name string) bool { return nameRegex.Match([]byte(name)) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go index e1a608bf..da163da1 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go @@ -45,10 +45,10 @@ func TestGetHostnamesFromFile_ErrorOnBadHostname(t *testing.T) { defer mockCtrl.Finish() disk := mocks.NewMockDisk(mockCtrl) - fakeHosts := []string{"Test", "test", "test1", "test2.com", "bad hostname", "foo/bar"} + fakeHosts := []string{"Test", "test", "test1", "test2.com", "bad hostname", "foo/bar", ""} disk.EXPECT().Lines("file").Return(fakeHosts, nil) hosts, err := GetHostnamesFromFile(disk, "file") assert.Nil(t, hosts) - assert.Equal(t, "invalid hostname in: [bad hostname foo/bar]", err.Error()) + assert.Equal(t, "invalid hostname in: [bad hostname foo/bar ]", err.Error()) } From 559ed9ed29e509227266e27735eace5e70ea2f97 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:43:41 +0100 Subject: [PATCH 073/191] Add DeleteSession and UpdateSession --- internal/pkg/v2/mock_store/keystore.go | 4 +-- internal/pkg/v2/registry_impl/session.go | 29 +++++++++++++++++-- internal/pkg/v2/registry_impl/session_test.go | 23 +++++++++++++++ internal/pkg/v2/store/keystore.go | 4 +-- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index f9b7fa61..f11453fe 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -64,7 +64,7 @@ func (mr *MockKeystoreMockRecorder) Create(key, value interface{}) *gomock.Call } // Update mocks base method -func (m *MockKeystore) Update(key string, value []byte, modRevision int) (store.KeyValueVersion, error) { +func (m *MockKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Update", key, value, modRevision) ret0, _ := ret[0].(store.KeyValueVersion) @@ -79,7 +79,7 @@ func (mr *MockKeystoreMockRecorder) Update(key, value, modRevision interface{}) } // Delete mocks base method -func (m *MockKeystore) Delete(key string, modRevision int) error { +func (m *MockKeystore) Delete(key string, modRevision int64) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Delete", key, modRevision) ret0, _ := ret[0].(error) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 4a16a178..0d23b646 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -116,9 +116,34 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { } func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { - panic("implement me") + sessionKey, err := getSessionKey(session.Name) + if err != nil { + log.Panicf("invalid session name") + } + + sessionAsStr, err := json.Marshal(session) + if err != nil { + log.Panicf("unable to convert session to json due to: %s", err.Error()) + } + + keyValueVersion, err := s.store.Update(sessionKey, sessionAsStr, session.Revision) + if err != nil { + return session, fmt.Errorf("unable to update session due to: %s", err.Error()) + } + + newSession := datamodel.Session{} + err = json.Unmarshal(keyValueVersion.Value, &newSession) + if err != nil { + log.Panicf("unable parse session from store due to: %s", err) + } + newSession.Revision = keyValueVersion.ModRevision + return newSession, nil } func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { - panic("implement me") + sessionKey, err := getSessionKey(session.Name) + if err != nil { + log.Panicf("invalid session name") + } + return s.store.Delete(sessionKey, session.Revision) } diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 5f760ed5..96759820 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -130,3 +130,26 @@ func TestSessionRegistry_GetAllSessions(t *testing.T) { "unable parse session from store due to: unexpected end of JSON input", func() { registry.GetAllSessions() }) } + +func TestSessionRegistry_UpdateSession(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) + keystore.EXPECT().Update("/session/foo", emptySessionString, int64(0)).Return(store.KeyValueVersion{ + ModRevision: 42, + Value: emptySessionString, + }, nil) + + registry.UpdateSession(datamodel.Session{Name: "foo", Revision: 0}) +} + +func TestSessionRegistry_DeleteSession(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + keystore := mock_store.NewMockKeystore(mockCtrl) + registry := NewSessionRegistry(keystore) + keystore.EXPECT().Delete("/session/foo", int64(40)) + + registry.DeleteSession(datamodel.Session{Name: "foo", Revision: 40}) +} diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index ef144189..78de06d5 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -21,7 +21,7 @@ type Keystore interface { // Otherwise if the revisions of any key doesn't // match the current revision of that key, the update fails. // When update fails an error is returned and no keyValues are updated - Update(key string, value []byte, modRevision int) (KeyValueVersion, error) + Update(key string, value []byte, modRevision int64) (KeyValueVersion, error) // Delete the specified key values, atomically // @@ -29,7 +29,7 @@ type Keystore interface { // ignores ModRevision if not zero. // If any keys are not currently present, the request fails. // Deletes no keys if an error is returned - Delete(key string, modRevision int) error + Delete(key string, modRevision int64) error // Removes all keys with given prefix DeleteAllKeysWithPrefix(keyPrefix string) error From 17c298ee42011707f320fe05dad09ecfac58a787 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:48:15 +0100 Subject: [PATCH 074/191] Add proper assertions for test update and delete session --- internal/pkg/v2/registry_impl/session_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 96759820..9844dea3 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -137,11 +137,14 @@ func TestSessionRegistry_UpdateSession(t *testing.T) { keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) keystore.EXPECT().Update("/session/foo", emptySessionString, int64(0)).Return(store.KeyValueVersion{ - ModRevision: 42, + ModRevision: 44, Value: emptySessionString, }, nil) - registry.UpdateSession(datamodel.Session{Name: "foo", Revision: 0}) + session, err := registry.UpdateSession(datamodel.Session{Name: "foo", Revision: 0}) + + assert.Nil(t, err) + assert.Equal(t, datamodel.Session{Name: "foo", Revision: 44}, session) } func TestSessionRegistry_DeleteSession(t *testing.T) { @@ -149,7 +152,10 @@ func TestSessionRegistry_DeleteSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - keystore.EXPECT().Delete("/session/foo", int64(40)) + fakeErr := errors.New("fake") + keystore.EXPECT().Delete("/session/foo", int64(40)).Return(fakeErr) + + err := registry.DeleteSession(datamodel.Session{Name: "foo", Revision: 40}) - registry.DeleteSession(datamodel.Session{Name: "foo", Revision: 40}) + assert.Equal(t, fakeErr, err) } From fc0b836efec0f182762079b6ae8398ce5a891f41 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:54:18 +0100 Subject: [PATCH 075/191] Invalid session name should panic --- internal/pkg/v2/registry_impl/session.go | 31 +++++-------------- internal/pkg/v2/registry_impl/session_test.go | 19 ++++++------ 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 0d23b646..9939dcb9 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -19,28 +19,22 @@ type sessionRegistry struct { } func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { - sessionKey, err := getSessionKey(sessionName) - if err != nil { - return nil, err - } + sessionKey := getSessionKey(sessionName) lockKey := fmt.Sprintf("/lock%s", sessionKey) return s.store.NewMutex(lockKey) } const sessionPrefix = "/session/" -func getSessionKey(sessionName datamodel.SessionName) (string, error) { +func getSessionKey(sessionName datamodel.SessionName) string { if !parsers.IsValidName(string(sessionName)) { - return "", fmt.Errorf("invalid session name %s", sessionName) + log.Panicf("invalid session name: '%s'", sessionName) } - return fmt.Sprintf("%s%s", sessionPrefix, sessionName), nil + return fmt.Sprintf("%s%s", sessionPrefix, sessionName) } func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { - sessionKey, err := getSessionKey(session.Name) - if err != nil { - return session, err - } + sessionKey := getSessionKey(session.Name) // TODO: more validation? if session.ActualSizeBytes > 0 { @@ -75,10 +69,7 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se } func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { - sessionKey, err := getSessionKey(sessionName) - if err != nil { - return datamodel.Session{}, err - } + sessionKey := getSessionKey(sessionName) keyValueVersion, err := s.store.Get(sessionKey) if err != nil { @@ -116,10 +107,7 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { } func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { - sessionKey, err := getSessionKey(session.Name) - if err != nil { - log.Panicf("invalid session name") - } + sessionKey := getSessionKey(session.Name) sessionAsStr, err := json.Marshal(session) if err != nil { @@ -141,9 +129,6 @@ func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Se } func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { - sessionKey, err := getSessionKey(session.Name) - if err != nil { - log.Panicf("invalid session name") - } + sessionKey := getSessionKey(session.Name) return s.store.Delete(sessionKey, session.Revision) } diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 9844dea3..d4bd390e 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -24,9 +24,9 @@ func TestSessionRegistry_GetSessionMutex(t *testing.T) { assert.Nil(t, mutex) assert.Equal(t, fakeErr, err) - mutex, err = registry.GetSessionMutex("foo/bar") - assert.Nil(t, mutex) - assert.Equal(t, "invalid session name foo/bar", err.Error()) + assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { + registry.GetSessionMutex("foo/bar") + }) } func TestSessionRegistry_CreateSession(t *testing.T) { @@ -40,10 +40,9 @@ func TestSessionRegistry_CreateSession(t *testing.T) { assert.Nil(t, err) assert.Equal(t, int64(42), session.Revision) - session, err = registry.CreateSession(datamodel.Session{Name: "foo/bar"}) - assert.NotNil(t, err) - assert.Equal(t, "invalid session name foo/bar", err.Error()) - + assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { + registry.CreateSession(datamodel.Session{Name: "foo/bar"}) + }) _, err = registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024}) assert.NotNil(t, err) assert.Equal(t, "session must have allocations before being created", err.Error()) @@ -83,9 +82,9 @@ func TestSessionRegistry_GetSession(t *testing.T) { assert.Nil(t, err) assert.Equal(t, datamodel.Session{Name: "foo", Revision: 42}, session) - session, err = registry.GetSession("foo/bar") - assert.NotNil(t, err) - assert.Equal(t, "invalid session name foo/bar", err.Error()) + assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { + registry.GetSession("foo/bar") + }) fakeErr := errors.New("fake") keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{}, fakeErr) From 4021dcad68d5732f29a7176689e55141aa306a99 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 17:58:22 +0100 Subject: [PATCH 076/191] Refactor updateSession --- internal/pkg/v2/registry_impl/session.go | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 9939dcb9..c411af4b 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -109,21 +109,12 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { sessionKey := getSessionKey(session.Name) - sessionAsStr, err := json.Marshal(session) - if err != nil { - log.Panicf("unable to convert session to json due to: %s", err.Error()) - } - - keyValueVersion, err := s.store.Update(sessionKey, sessionAsStr, session.Revision) + keyValueVersion, err := s.store.Update(sessionKey, sessionToRaw(session), session.Revision) if err != nil { return session, fmt.Errorf("unable to update session due to: %s", err.Error()) } - newSession := datamodel.Session{} - err = json.Unmarshal(keyValueVersion.Value, &newSession) - if err != nil { - log.Panicf("unable parse session from store due to: %s", err) - } + newSession := sessionFromRaw(keyValueVersion.Value) newSession.Revision = keyValueVersion.ModRevision return newSession, nil } @@ -132,3 +123,20 @@ func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { sessionKey := getSessionKey(session.Name) return s.store.Delete(sessionKey, session.Revision) } + +func sessionToRaw(session datamodel.Session) []byte { + rawSession, err := json.Marshal(session) + if err != nil { + log.Panicf("unable to convert session to json due to: %s", err.Error()) + } + return rawSession +} + +func sessionFromRaw(raw []byte) datamodel.Session { + session := datamodel.Session{} + err := json.Unmarshal(raw, &session) + if err != nil { + log.Panicf("unable parse session from store due to: %s", err) + } + return session +} \ No newline at end of file From 00aad67f88445e286206f9c1e78335b2fc59b2bc Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 18:03:28 +0100 Subject: [PATCH 077/191] Simplify with sessionFromRaw and toRaw --- internal/pkg/v2/registry_impl/session.go | 33 ++++--------------- internal/pkg/v2/registry_impl/session_test.go | 8 +++-- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index c411af4b..3c14de99 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -53,12 +53,7 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se } } - sessionAsString, err := json.Marshal(session) - if err != nil { - return session, fmt.Errorf("unable to convert session to json due to: %s", err) - } - - keyValueVersion, err := s.store.Create(sessionKey, sessionAsString) + keyValueVersion, err := s.store.Create(sessionKey, sessionToRaw(session)) if err != nil { return session, fmt.Errorf("unable to create session due to: %s", err) } @@ -69,19 +64,12 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se } func (s *sessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { - sessionKey := getSessionKey(sessionName) - - keyValueVersion, err := s.store.Get(sessionKey) + keyValueVersion, err := s.store.Get(getSessionKey(sessionName)) if err != nil { return datamodel.Session{}, fmt.Errorf("unable to get session due to: %s", err) } - session := datamodel.Session{} - err = json.Unmarshal(keyValueVersion.Value, &session) - if err != nil { - return datamodel.Session{}, fmt.Errorf("unable parse session from store due to: %s", err) - } - + session := sessionFromRaw(keyValueVersion.Value) session.Revision = keyValueVersion.ModRevision return session, nil } @@ -94,11 +82,7 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { var sessions []datamodel.Session for _, keyValueVersion := range results { - session := datamodel.Session{} - err = json.Unmarshal(keyValueVersion.Value, &session) - if err != nil { - log.Panicf("unable parse session from store due to: %s", err) - } + session := sessionFromRaw(keyValueVersion.Value) session.Revision = keyValueVersion.ModRevision sessions = append(sessions, session) } @@ -107,9 +91,7 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { } func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { - sessionKey := getSessionKey(session.Name) - - keyValueVersion, err := s.store.Update(sessionKey, sessionToRaw(session), session.Revision) + keyValueVersion, err := s.store.Update(getSessionKey(session.Name), sessionToRaw(session), session.Revision) if err != nil { return session, fmt.Errorf("unable to update session due to: %s", err.Error()) } @@ -120,8 +102,7 @@ func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Se } func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { - sessionKey := getSessionKey(session.Name) - return s.store.Delete(sessionKey, session.Revision) + return s.store.Delete(getSessionKey(session.Name), session.Revision) } func sessionToRaw(session datamodel.Session) []byte { @@ -139,4 +120,4 @@ func sessionFromRaw(raw []byte) datamodel.Session { log.Panicf("unable parse session from store due to: %s", err) } return session -} \ No newline at end of file +} diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index d4bd390e..a9aced7c 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -93,9 +93,11 @@ func TestSessionRegistry_GetSession(t *testing.T) { assert.Equal(t, "unable to get session due to: fake", err.Error()) keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{}, nil) - session, err = registry.GetSession("foo") - assert.NotNil(t, err) - assert.Equal(t, "unable parse session from store due to: unexpected end of JSON input", err.Error()) + assert.PanicsWithValue(t, + "unable parse session from store due to: unexpected end of JSON input", + func() { + registry.GetSession("foo") + }) } func TestSessionRegistry_GetAllSessions(t *testing.T) { From f2c5c5596cf5e2be8cc259c63aa9a73b3fa08dea Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 22:26:55 +0100 Subject: [PATCH 078/191] Convert invalid create session to painc --- internal/pkg/v2/registry_impl/session.go | 8 ++-- internal/pkg/v2/registry_impl/session_test.go | 37 +++++++++---------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 3c14de99..319851d5 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -39,17 +39,17 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se // TODO: more validation? if session.ActualSizeBytes > 0 { if len(session.Allocations) == 0 { - return session, fmt.Errorf("session must have allocations before being created") + log.Panicf("session must have allocations before being created: %s", session.Name) } if session.PrimaryBrickHost == "" { - return session, fmt.Errorf("session must have a primary brick host set") + log.Panicf("session must have a primary brick host set: %s", session.Name) } } else { if len(session.Allocations) != 0 { - return session, fmt.Errorf("allocations out of sync with ActualSizeBytes") + log.Panicf("allocations out of sync with ActualSizeBytes: %s", session.Name) } if session.PrimaryBrickHost != "" { - return session, fmt.Errorf("PrimaryBrickHost should be empty if no bricks assigned") + log.Panicf("PrimaryBrickHost should be empty if no bricks assigned: %s", session.Name) } } diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index a9aced7c..14fd5624 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -43,28 +43,25 @@ func TestSessionRegistry_CreateSession(t *testing.T) { assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { registry.CreateSession(datamodel.Session{Name: "foo/bar"}) }) - _, err = registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024}) - assert.NotNil(t, err) - assert.Equal(t, "session must have allocations before being created", err.Error()) - - _, err = registry.CreateSession(datamodel.Session{ - Name: "foo", - ActualSizeBytes: 1024, - Allocations: []datamodel.BrickAllocation{{}}, + assert.PanicsWithValue(t, "session must have allocations before being created: foo", func() { + registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024}) }) - assert.NotNil(t, err) - assert.Equal(t, "session must have a primary brick host set", err.Error()) - - _, err = registry.CreateSession(datamodel.Session{ - Name: "foo", - Allocations: []datamodel.BrickAllocation{{}}, + assert.PanicsWithValue(t, "session must have a primary brick host set: foo", func() { + registry.CreateSession(datamodel.Session{ + Name: "foo", + ActualSizeBytes: 1024, + Allocations: []datamodel.BrickAllocation{{}}, + }) + }) + assert.PanicsWithValue(t, "allocations out of sync with ActualSizeBytes: foo", func() { + registry.CreateSession(datamodel.Session{ + Name: "foo", + Allocations: []datamodel.BrickAllocation{{}}, + }) + }) + assert.PanicsWithValue(t, "PrimaryBrickHost should be empty if no bricks assigned: foo", func() { + registry.CreateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "foo"}) }) - assert.NotNil(t, err) - assert.Equal(t, "allocations out of sync with ActualSizeBytes", err.Error()) - - _, err = registry.CreateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "foo"}) - assert.NotNil(t, err) - assert.Equal(t, "PrimaryBrickHost should be empty if no bricks assigned", err.Error()) } func TestSessionRegistry_GetSession(t *testing.T) { From c5b114a612e71d81a4a1d7f06c8706b99ed9d994 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 22:54:39 +0100 Subject: [PATCH 079/191] Ensure sessions always have a primary brick host Even when they have no bricks allocated --- .../pkg/v2/dacctl/workflow_impl/session.go | 13 ++++++ .../v2/dacctl/workflow_impl/session_test.go | 3 ++ internal/pkg/v2/registry_impl/session.go | 9 ++-- .../pkg/v2/registry_impl/session_actions.go | 35 +++++++++++++++ .../v2/registry_impl/session_actions_test.go | 1 + internal/pkg/v2/registry_impl/session_test.go | 43 ++++++++----------- 6 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 internal/pkg/v2/registry_impl/session_actions.go create mode 100644 internal/pkg/v2/registry_impl/session_actions_test.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 8a7f46d3..4e5ecd62 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -104,6 +104,19 @@ func (s sessionFacade) doAllocationAndWriteSession(session datamodel.Session) (d session.ActualSizeBytes = actualSizeBytes session.Allocations = allocations session.PrimaryBrickHost = allocations[0].Brick.BrickHostName + } else { + // Pick a random alive host to be the PrimaryBrickHost anyway + pools, err := s.allocations.GetAllPoolInfos() + if err != nil { + return session, err + } + if len(pools) == 0 { + return session, fmt.Errorf("unable to find any pools") + } + // TODO: need to pick the default pool, but right now only one + poolInfo := pools[0] + bricks := pickBricks(1, poolInfo) + session.PrimaryBrickHost = bricks[0].BrickHostName } // Store initial version of session diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index bfffe53e..d58a2c66 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -32,6 +32,9 @@ func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) sessionMutex.EXPECT().Lock(context.TODO()) + brickList := []datamodel.Brick{{Device: "sda", BrickHostName: datamodel.BrickHostName("host1")}} + allocations.EXPECT().GetAllPoolInfos().Return([]datamodel.PoolInfo{{AvailableBricks: brickList}}, nil) + initialSession.PrimaryBrickHost = "host1" sessionRegistry.EXPECT().CreateSession(initialSession).Return(initialSession, nil) sessionMutex.EXPECT().Unlock(context.TODO()) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 319851d5..a2a33e6f 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -37,20 +37,17 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se sessionKey := getSessionKey(session.Name) // TODO: more validation? + if session.PrimaryBrickHost == "" { + log.Panicf("PrimaryBrickHost must be set before creating session: %s", session.Name) + } if session.ActualSizeBytes > 0 { if len(session.Allocations) == 0 { log.Panicf("session must have allocations before being created: %s", session.Name) } - if session.PrimaryBrickHost == "" { - log.Panicf("session must have a primary brick host set: %s", session.Name) - } } else { if len(session.Allocations) != 0 { log.Panicf("allocations out of sync with ActualSizeBytes: %s", session.Name) } - if session.PrimaryBrickHost != "" { - log.Panicf("PrimaryBrickHost should be empty if no bricks assigned: %s", session.Name) - } } keyValueVersion, err := s.store.Create(sessionKey, sessionToRaw(session)) diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go new file mode 100644 index 00000000..ac628f03 --- /dev/null +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -0,0 +1,35 @@ +package registry_impl + +import ( + "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" +) + +func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { + return &sessionActions{store} +} + +type sessionActions struct { + store store.Keystore +} + +func (s *sessionActions) SendSessionAction( + ctxt context.Context, actionType datamodel.SessionActionType, + session datamodel.Session) (<-chan datamodel.SessionAction, error) { + + if session.PrimaryBrickHost == "" { + panic("TODO: implement actions without assigned primary brick host") + } + panic("implement me") +} + +func (s *sessionActions) GetSessionActions( + ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { + panic("implement me") +} + +func (s *sessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { + panic("implement me") +} diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/v2/registry_impl/session_actions_test.go new file mode 100644 index 00000000..4864aad6 --- /dev/null +++ b/internal/pkg/v2/registry_impl/session_actions_test.go @@ -0,0 +1 @@ +package registry_impl diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 14fd5624..84450dec 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -10,7 +10,8 @@ import ( "testing" ) -var emptySessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) +var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) +var exampleSession = datamodel.Session{Name: "foo", PrimaryBrickHost: "host1"} func TestSessionRegistry_GetSessionMutex(t *testing.T) { mockCtrl := gomock.NewController(t) @@ -34,33 +35,27 @@ func TestSessionRegistry_CreateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - keystore.EXPECT().Create("/session/foo", emptySessionString).Return(store.KeyValueVersion{ModRevision: 42}, nil) + keystore.EXPECT().Create("/session/foo", exampleSessionString).Return(store.KeyValueVersion{ModRevision: 42}, nil) - session, err := registry.CreateSession(datamodel.Session{Name: "foo"}) + session, err := registry.CreateSession(exampleSession) assert.Nil(t, err) assert.Equal(t, int64(42), session.Revision) assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { - registry.CreateSession(datamodel.Session{Name: "foo/bar"}) + registry.CreateSession(datamodel.Session{Name: "foo/bar", PrimaryBrickHost: "host1"}) }) assert.PanicsWithValue(t, "session must have allocations before being created: foo", func() { - registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024}) - }) - assert.PanicsWithValue(t, "session must have a primary brick host set: foo", func() { - registry.CreateSession(datamodel.Session{ - Name: "foo", - ActualSizeBytes: 1024, - Allocations: []datamodel.BrickAllocation{{}}, - }) + registry.CreateSession(datamodel.Session{Name: "foo", ActualSizeBytes: 1024, PrimaryBrickHost: "host1"}) }) assert.PanicsWithValue(t, "allocations out of sync with ActualSizeBytes: foo", func() { registry.CreateSession(datamodel.Session{ - Name: "foo", - Allocations: []datamodel.BrickAllocation{{}}, + Name: "foo", + Allocations: []datamodel.BrickAllocation{{}}, + PrimaryBrickHost: "host1", }) }) - assert.PanicsWithValue(t, "PrimaryBrickHost should be empty if no bricks assigned: foo", func() { - registry.CreateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "foo"}) + assert.PanicsWithValue(t, "PrimaryBrickHost must be set before creating session: foo", func() { + registry.CreateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: ""}) }) } @@ -71,13 +66,13 @@ func TestSessionRegistry_GetSession(t *testing.T) { registry := NewSessionRegistry(keystore) keystore.EXPECT().Get("/session/foo").Return(store.KeyValueVersion{ ModRevision: 42, - Value: emptySessionString, + Value: exampleSessionString, }, nil) session, err := registry.GetSession("foo") assert.Nil(t, err) - assert.Equal(t, datamodel.Session{Name: "foo", Revision: 42}, session) + assert.Equal(t, datamodel.Session{Name: "foo", Revision: 42, PrimaryBrickHost: "host1"}, session) assert.PanicsWithValue(t, "invalid session name: 'foo/bar'", func() { registry.GetSession("foo/bar") @@ -104,12 +99,12 @@ func TestSessionRegistry_GetAllSessions(t *testing.T) { registry := NewSessionRegistry(keystore) keystore.EXPECT().GetAll("/session/").Return([]store.KeyValueVersion{{ ModRevision: 42, - Value: emptySessionString, + Value: exampleSessionString, }}, nil) sessions, err := registry.GetAllSessions() assert.Nil(t, err) - assert.Equal(t, []datamodel.Session{{Name: "foo", Revision: 42}}, sessions) + assert.Equal(t, []datamodel.Session{{Name: "foo", Revision: 42, PrimaryBrickHost: "host1"}}, sessions) fakeErr := errors.New("fake") keystore.EXPECT().GetAll("/session/").Return(nil, fakeErr) @@ -134,15 +129,15 @@ func TestSessionRegistry_UpdateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - keystore.EXPECT().Update("/session/foo", emptySessionString, int64(0)).Return(store.KeyValueVersion{ + keystore.EXPECT().Update("/session/foo", exampleSessionString, int64(0)).Return(store.KeyValueVersion{ ModRevision: 44, - Value: emptySessionString, + Value: exampleSessionString, }, nil) - session, err := registry.UpdateSession(datamodel.Session{Name: "foo", Revision: 0}) + session, err := registry.UpdateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "host1", Revision: 0}) assert.Nil(t, err) - assert.Equal(t, datamodel.Session{Name: "foo", Revision: 44}, session) + assert.Equal(t, datamodel.Session{Name: "foo", PrimaryBrickHost: "host1", Revision: 44}, session) } func TestSessionRegistry_DeleteSession(t *testing.T) { From f68f89f04700892224b5d178d25c7eb75ade809a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 23:28:37 +0100 Subject: [PATCH 080/191] Untested session action send --- internal/pkg/v2/mock_registry/brick_host.go | 15 +++ internal/pkg/v2/registry/brick_host.go | 3 +- .../pkg/v2/registry_impl/session_actions.go | 107 +++++++++++++++++- 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/v2/mock_registry/brick_host.go index 0aea2107..c1e07885 100644 --- a/internal/pkg/v2/mock_registry/brick_host.go +++ b/internal/pkg/v2/mock_registry/brick_host.go @@ -76,3 +76,18 @@ func (mr *MockBrickHostRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName i mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).KeepAliveHost), ctxt, brickHostName) } + +// IsBrickHostAlive mocks base method +func (m *MockBrickHostRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsBrickHostAlive", brickHostName) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsBrickHostAlive indicates an expected call of IsBrickHostAlive +func (mr *MockBrickHostRegistryMockRecorder) IsBrickHostAlive(brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsBrickHostAlive", reflect.TypeOf((*MockBrickHostRegistry)(nil).IsBrickHostAlive), brickHostName) +} diff --git a/internal/pkg/v2/registry/brick_host.go b/internal/pkg/v2/registry/brick_host.go index d13fdec3..1ca702a5 100644 --- a/internal/pkg/v2/registry/brick_host.go +++ b/internal/pkg/v2/registry/brick_host.go @@ -31,6 +31,5 @@ type BrickHostRegistry interface { // Check if given brick host is alive // // Error if brick host doesn't exist - // TODO: private method? - //IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) + IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) } diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index ac628f03..2345ff1d 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -2,17 +2,75 @@ package registry_impl import ( "context" + "encoding/json" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/google/uuid" + "log" ) func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { - return &sessionActions{store} + // TODO: create brickHostRegistry + return &sessionActions{store, nil} } type sessionActions struct { - store store.Keystore + store store.Keystore + brickHostRegistry registry.BrickHostRegistry +} + +const sessionActionRequestPrefix = "/session_action/request/" + +func getSessionActionRequestHostPrefix(brickHost datamodel.BrickHostName) string { + if !parsers.IsValidName(string(brickHost)) { + log.Panicf("invalid session PrimaryBrickHost") + } + return fmt.Sprintf("%s%s/", sessionActionRequestPrefix, brickHost) +} + +func getSessionActionRequestKey(action datamodel.SessionAction) string { + hostPrefix := getSessionActionRequestHostPrefix(action.Session.PrimaryBrickHost) + if !parsers.IsValidName(action.Uuid) { + log.Panicf("invalid session action uuid") + } + return fmt.Sprintf("%s%s", hostPrefix, action.Uuid) +} + +const sessionActionResponsePrefix = "/session_action/response/" + +func getSessionActionResponseHostPrefix(brickHost datamodel.BrickHostName) string { + if !parsers.IsValidName(string(brickHost)) { + log.Panicf("invalid session PrimaryBrickHost") + } + return fmt.Sprintf("%s%s/", sessionActionResponsePrefix, brickHost) +} + +func getSessionActionResponseKey(action datamodel.SessionAction) string { + hostPrefix := getSessionActionResponseHostPrefix(action.Session.PrimaryBrickHost) + if !parsers.IsValidName(action.Uuid) { + log.Panicf("invalid session action uuid") + } + return fmt.Sprintf("%s%s", hostPrefix, action.Uuid) +} + +func sessionActionToRaw(session datamodel.SessionAction) []byte { + rawSession, err := json.Marshal(session) + if err != nil { + log.Panicf("unable to convert session action to json due to: %s", err.Error()) + } + return rawSession +} + +func sessionActionFromRaw(raw []byte) datamodel.SessionAction { + session := datamodel.SessionAction{} + err := json.Unmarshal(raw, &session) + if err != nil { + log.Panicf("unable parse session action from store due to: %s", err) + } + return session } func (s *sessionActions) SendSessionAction( @@ -20,9 +78,50 @@ func (s *sessionActions) SendSessionAction( session datamodel.Session) (<-chan datamodel.SessionAction, error) { if session.PrimaryBrickHost == "" { - panic("TODO: implement actions without assigned primary brick host") + panic("sessions must have a primary brick host set") } - panic("implement me") + sessionAction := datamodel.SessionAction{ + Session: session, + ActionType: actionType, + Uuid: uuid.New().String(), + } + + isAlive, err := s.brickHostRegistry.IsBrickHostAlive(session.PrimaryBrickHost) + if err != nil { + return nil, fmt.Errorf("unable to check host status: %s", session.PrimaryBrickHost) + } + if !isAlive { + return nil, fmt.Errorf("can't send as primary brick host not alive: %s", session.PrimaryBrickHost) + } + + responseKey := getSessionActionResponseKey(sessionAction) + callbackKeyUpdates := s.store.Watch(ctxt, responseKey, false) + + requestKey := getSessionActionRequestKey(sessionAction) + if _, err := s.store.Create(requestKey, sessionActionToRaw(sessionAction)); err != nil { + return nil, fmt.Errorf("unable to send session action due to: %s", err) + } + + responseChan := make(chan datamodel.SessionAction) + + go func() { + log.Printf("started waiting for action response %+v\n", sessionAction) + for update := range callbackKeyUpdates { + if !update.IsCreate || update.New.Value == nil { + log.Panicf("only expected to see the action response key being created") + } + + responseSessionAction := sessionActionFromRaw(update.New.Value) + log.Printf("found action response %+v\n", responseSessionAction) + + responseChan <- responseSessionAction + close(responseChan) + + log.Printf("completed waiting for action response %+v\n", sessionAction) + return + } + }() + return responseChan, nil } func (s *sessionActions) GetSessionActions( From af6f423e62ea7aebb004a1824efac68f12c0c33b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 19 Aug 2019 23:38:14 +0100 Subject: [PATCH 081/191] Move getSessionActions between interfaces --- .../v2/dacd/brick_manager_impl/brick_manager.go | 7 +++++-- .../dacd/brick_manager_impl/brick_manager_test.go | 5 +++-- internal/pkg/v2/mock_registry/brick_host.go | 15 --------------- internal/pkg/v2/mock_registry/session_actions.go | 8 ++++---- internal/pkg/v2/registry/brick_host.go | 9 --------- internal/pkg/v2/registry/session_actions.go | 7 ++----- internal/pkg/v2/registry_impl/session_actions.go | 8 ++++---- 7 files changed, 18 insertions(+), 41 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index f0850a67..e9f96871 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -9,10 +9,12 @@ import ( "log" ) -func NewBrickManager(brickRegistry registry.BrickHostRegistry, handler facade.SessionActionHandler) dacd.BrickManager { +func NewBrickManager(brickRegistry registry.BrickHostRegistry, sessionActions registry.SessionActions, + handler facade.SessionActionHandler) dacd.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), brickRegistry: brickRegistry, + sessionActions: sessionActions, sessionActionHandler: handler, } } @@ -20,6 +22,7 @@ func NewBrickManager(brickRegistry registry.BrickHostRegistry, handler facade.Se type brickManager struct { config config.BrickManagerConfig brickRegistry registry.BrickHostRegistry + sessionActions registry.SessionActions sessionActionHandler facade.SessionActionHandler } @@ -34,7 +37,7 @@ func (bm *brickManager) Startup(drainSessions bool) error { } // If we are are enabled, this includes new create session requests - events, err := bm.brickRegistry.GetSessionActions(context.TODO(), bm.config.BrickHostName) + events, err := bm.sessionActions.GetSessionActions(context.TODO(), bm.config.BrickHostName) go func() { for event := range events { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index d6f337b2..b8881364 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -20,12 +20,13 @@ func TestBrickManager_Startup(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() brickRegistry := mock_registry.NewMockBrickHostRegistry(mockCtrl) + sessionActions := mock_registry.NewMockSessionActions(mockCtrl) handler := mock_facade.NewMockSessionActionHandler(mockCtrl) - brickManager := NewBrickManager(brickRegistry, handler) + brickManager := NewBrickManager(brickRegistry, sessionActions, handler) // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) - brickRegistry.EXPECT().GetSessionActions(context.TODO(), gomock.Any()) + sessionActions.EXPECT().GetSessionActions(context.TODO(), gomock.Any()) brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName("hostname")) err := brickManager.Startup(false) diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/v2/mock_registry/brick_host.go index c1e07885..f2cc8750 100644 --- a/internal/pkg/v2/mock_registry/brick_host.go +++ b/internal/pkg/v2/mock_registry/brick_host.go @@ -48,21 +48,6 @@ func (mr *MockBrickHostRegistryMockRecorder) UpdateBrickHost(brickHostInfo inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).UpdateBrickHost), brickHostInfo) } -// GetSessionActions mocks base method -func (m *MockBrickHostRegistry) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) - ret0, _ := ret[0].(<-chan datamodel.SessionAction) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetSessionActions indicates an expected call of GetSessionActions -func (mr *MockBrickHostRegistryMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockBrickHostRegistry)(nil).GetSessionActions), ctxt, brickHostName) -} - // KeepAliveHost mocks base method func (m *MockBrickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go index 61f4afbb..bdab25cd 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -50,18 +50,18 @@ func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, se } // GetSessionActions mocks base method -func (m *MockSessionActions) GetSessionActions(ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { +func (m *MockSessionActions) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionActions", ctxt, sessionName) + ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) ret0, _ := ret[0].(<-chan datamodel.SessionAction) ret1, _ := ret[1].(error) return ret0, ret1 } // GetSessionActions indicates an expected call of GetSessionActions -func (mr *MockSessionActionsMockRecorder) GetSessionActions(ctxt, sessionName interface{}) *gomock.Call { +func (mr *MockSessionActionsMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActions), ctxt, sessionName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActions), ctxt, brickHostName) } // CompleteSessionAction mocks base method diff --git a/internal/pkg/v2/registry/brick_host.go b/internal/pkg/v2/registry/brick_host.go index 1ca702a5..81d37a2c 100644 --- a/internal/pkg/v2/registry/brick_host.go +++ b/internal/pkg/v2/registry/brick_host.go @@ -12,15 +12,6 @@ type BrickHostRegistry interface { // This includes ensuring the pool exists and is consistent with the given brick host info UpdateBrickHost(brickHostInfo datamodel.BrickHost) error - // Gets all new actions for the given Session - // This confirms that dacd is ready to service requests for this session - // and updates the session with an appropriate SessionActionPrefix - // The channel tracks all actions sent to that sub keys of the above prefix - // This is also called when dacd is restarted and you want to resume sending - // any actions that are not marked as complete - // New tasks are only sent if the keepalive key is active - GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) - // While the process is still running this notifies others the host is up // // When a host is dead non of its bricks will get new volumes assigned, diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go index 6c24f9df..574bf679 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/v2/registry/session_actions.go @@ -14,11 +14,8 @@ type SessionActions interface { ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) - // Gets all new actions for the given Session - // - // Error if context is cancelled or timed-out - GetSessionActions( - ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) + // Gets all actions for the given host + GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) // Server reports given action is complete // Includes callbacks for Create Session Volume diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index 2345ff1d..01b364d9 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -12,9 +12,9 @@ import ( "log" ) -func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { +func NewSessionActionsRegistry(store store.Keystore, brickHostRegistry registry.BrickHostRegistry) registry.SessionActions { // TODO: create brickHostRegistry - return &sessionActions{store, nil} + return &sessionActions{store, brickHostRegistry} } type sessionActions struct { @@ -124,8 +124,8 @@ func (s *sessionActions) SendSessionAction( return responseChan, nil } -func (s *sessionActions) GetSessionActions( - ctxt context.Context, sessionName datamodel.SessionName) (<-chan datamodel.SessionAction, error) { +func (s *sessionActions) GetSessionActions(ctxt context.Context, + brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { panic("implement me") } From bacdfefc5cec30c38733db55c5162258a118f659 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 00:02:35 +0100 Subject: [PATCH 082/191] Hack in some ideas for session actions --- .../dacd/brick_manager_impl/brick_manager.go | 2 +- .../brick_manager_impl/brick_manager_test.go | 2 +- .../session_action_handler.go | 12 +++-- .../session_action_handler_test.go | 4 +- .../pkg/v2/mock_registry/session_actions.go | 20 ++++---- internal/pkg/v2/registry/session_actions.go | 4 +- .../pkg/v2/registry_impl/session_actions.go | 48 +++++++++++++++++-- 7 files changed, 68 insertions(+), 24 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index e9f96871..cbf31c24 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -37,7 +37,7 @@ func (bm *brickManager) Startup(drainSessions bool) error { } // If we are are enabled, this includes new create session requests - events, err := bm.sessionActions.GetSessionActions(context.TODO(), bm.config.BrickHostName) + events, err := bm.sessionActions.GetSessionActionRequests(context.TODO(), bm.config.BrickHostName) go func() { for event := range events { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index b8881364..a3e8f940 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -26,7 +26,7 @@ func TestBrickManager_Startup(t *testing.T) { // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) - sessionActions.EXPECT().GetSessionActions(context.TODO(), gomock.Any()) + sessionActions.EXPECT().GetSessionActionRequests(context.TODO(), gomock.Any()) brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName("hostname")) err := brickManager.Startup(false) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index d4f6ed34..b49da16c 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -45,13 +45,15 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { sessionMutex, err := s.registry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) - s.actions.CompleteSessionAction(action, err) + action.Error = err + s.actions.CompleteSessionAction(action) return } err = sessionMutex.Lock(context.TODO()) if err != nil { log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) - s.actions.CompleteSessionAction(action, err) + action.Error = err + s.actions.CompleteSessionAction(action) return } defer func() { @@ -72,11 +74,13 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { session, err = s.registry.UpdateSession(session) if err != nil { log.Printf("Failed to update session: %+v", session) + action.Error = err } else { action.Session = session + action.Error = session.Status.Error } - if err := s.actions.CompleteSessionAction(action, action.Session.Status.Error); err != nil { + if err := s.actions.CompleteSessionAction(action); err != nil { log.Printf("Failed to complete action: %+v", action) } if action.Session.Status.Error != nil { @@ -90,5 +94,5 @@ func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { log.Println("delete") // TODO: clearly need mutex here, etc s.registry.DeleteSession(action.Session) - s.actions.CompleteSessionAction(action, nil) + s.actions.CompleteSessionAction(action) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index fc9c2362..d8e064a3 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -71,7 +71,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { ActionType: datamodel.SessionCreateFilesystem, Session: updatedSession, } - actions.EXPECT().CompleteSessionAction(updatedAction, nil) + actions.EXPECT().CompleteSessionAction(updatedAction) handler.handleCreate(action) } @@ -87,7 +87,7 @@ func TestSessionActionHandler_handleDelete(t *testing.T) { } // TODO: need to pass session better? who deletes allocations? registry.EXPECT().DeleteSession(action.Session) - actions.EXPECT().CompleteSessionAction(action, nil) + actions.EXPECT().CompleteSessionAction(action) handler.handleDelete(action) } diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go index bdab25cd..122dfb04 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -49,31 +49,31 @@ func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, se return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSessionAction", reflect.TypeOf((*MockSessionActions)(nil).SendSessionAction), ctxt, actionType, session) } -// GetSessionActions mocks base method -func (m *MockSessionActions) GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { +// GetSessionActionRequests mocks base method +func (m *MockSessionActions) GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionActions", ctxt, brickHostName) + ret := m.ctrl.Call(m, "GetSessionActionRequests", ctxt, brickHostName) ret0, _ := ret[0].(<-chan datamodel.SessionAction) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetSessionActions indicates an expected call of GetSessionActions -func (mr *MockSessionActionsMockRecorder) GetSessionActions(ctxt, brickHostName interface{}) *gomock.Call { +// GetSessionActionRequests indicates an expected call of GetSessionActionRequests +func (mr *MockSessionActionsMockRecorder) GetSessionActionRequests(ctxt, brickHostName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActions", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActions), ctxt, brickHostName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActionRequests", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActionRequests), ctxt, brickHostName) } // CompleteSessionAction mocks base method -func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { +func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CompleteSessionAction", action, err) + ret := m.ctrl.Call(m, "CompleteSessionAction", action) ret0, _ := ret[0].(error) return ret0 } // CompleteSessionAction indicates an expected call of CompleteSessionAction -func (mr *MockSessionActionsMockRecorder) CompleteSessionAction(action, err interface{}) *gomock.Call { +func (mr *MockSessionActionsMockRecorder) CompleteSessionAction(action interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteSessionAction", reflect.TypeOf((*MockSessionActions)(nil).CompleteSessionAction), action, err) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteSessionAction", reflect.TypeOf((*MockSessionActions)(nil).CompleteSessionAction), action) } diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/v2/registry/session_actions.go index 574bf679..efc80aab 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/v2/registry/session_actions.go @@ -15,11 +15,11 @@ type SessionActions interface { session datamodel.Session) (<-chan datamodel.SessionAction, error) // Gets all actions for the given host - GetSessionActions(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) + GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) // Server reports given action is complete // Includes callbacks for Create Session Volume // // Error if action has already completed or doesn't exist - CompleteSessionAction(action datamodel.SessionAction, err error) error + CompleteSessionAction(action datamodel.SessionAction) error } diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index 01b364d9..bbdd3da4 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -124,11 +124,51 @@ func (s *sessionActions) SendSessionAction( return responseChan, nil } -func (s *sessionActions) GetSessionActions(ctxt context.Context, +func (s *sessionActions) GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { - panic("implement me") + requestHostPrefix := getSessionActionRequestHostPrefix(brickHostName) + + // TODO: how do we check for any pending actions that exist before we start watching? + // or do we only care about pending deletes, and we let them just timeout? + requestUpdates := s.store.Watch(ctxt, requestHostPrefix, true) + + sessionActionChan := make(chan datamodel.SessionAction) + go func() { + log.Printf("Starting watching for SessionActionRequests for %s\n", brickHostName) + for update := range requestUpdates { + if update.IsDelete { + log.Printf("Seen SessionActionRequest deleted for %s\n", brickHostName) + continue + } + if !update.IsCreate || update.New.Value == nil { + log.Panicf("don't expect to see updates of session action request key") + } + log.Printf("Seen SessionActionRequest created for %s\n", brickHostName) + + sessionAction := sessionActionFromRaw(update.New.Value) + sessionActionChan <- sessionAction + } + log.Printf("Stopped watching for SessionActionRequests for %s\n", brickHostName) + close(sessionActionChan) + }() + return sessionActionChan, nil } -func (s *sessionActions) CompleteSessionAction(action datamodel.SessionAction, err error) error { - panic("implement me") +func (s *sessionActions) CompleteSessionAction(sessionAction datamodel.SessionAction) error { + // TODO: when you delete a session, you should delete all completion records? + + // Tell caller we are done by writing this key + responseKey := getSessionActionResponseKey(sessionAction) + _, err := s.store.Create(responseKey, sessionActionToRaw(sessionAction)) + if err != nil { + return fmt.Errorf("unable to create response message due to: %s", err) + } + + // Delete the request, not it is processed + requestKey := getSessionActionRequestKey(sessionAction) + err = s.store.Delete(requestKey, 0) + if err != nil { + return fmt.Errorf("unable to delete stale request message due to: %s", err) + } + return nil } From dcb8a61e6c7cb4fce97783ee5cce7d322dea3d53 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 00:04:07 +0100 Subject: [PATCH 083/191] Restore ensurePoolCreated method --- internal/pkg/v2/mock_registry/brick_allocation.go | 15 +++++++++++++++ internal/pkg/v2/registry/brick_allocation.go | 3 +-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go index 99b1b345..ece91960 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -49,6 +49,21 @@ func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPool), name) } +// EnsurePoolCreated mocks base method +func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGB) + ret0, _ := ret[0].(datamodel.Pool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EnsurePoolCreated indicates an expected call of EnsurePoolCreated +func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGB interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityGB) +} + // GetAllPoolInfos mocks base method func (m *MockAllocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/v2/registry/brick_allocation.go index 72afa3cd..937703cd 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/v2/registry/brick_allocation.go @@ -11,8 +11,7 @@ type AllocationRegistry interface { // Creates the pool if it doesn't exist // error if the granularity doesn't match and existing pool - // TODO package method called by brick registry? - //EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) + EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) // Get brick availability by pool GetAllPoolInfos() ([]datamodel.PoolInfo, error) From 0ce6499c381a3c677bd50e0b6672e95ba966a469 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 09:23:41 +0100 Subject: [PATCH 084/191] Add draft for update brick host --- .../pkg/v2/mock_registry/brick_allocation.go | 8 +- internal/pkg/v2/registry/brick_allocation.go | 2 +- internal/pkg/v2/registry_impl/brick_host.go | 76 +++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 internal/pkg/v2/registry_impl/brick_host.go diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go index ece91960..3f979eaa 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -50,18 +50,18 @@ func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock. } // EnsurePoolCreated mocks base method -func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) { +func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGiB uint) (datamodel.Pool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGB) + ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGiB) ret0, _ := ret[0].(datamodel.Pool) ret1, _ := ret[1].(error) return ret0, ret1 } // EnsurePoolCreated indicates an expected call of EnsurePoolCreated -func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGB interface{}) *gomock.Call { +func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGiB interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityGB) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityGiB) } // GetAllPoolInfos mocks base method diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/v2/registry/brick_allocation.go index 937703cd..75d46cc7 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/v2/registry/brick_allocation.go @@ -11,7 +11,7 @@ type AllocationRegistry interface { // Creates the pool if it doesn't exist // error if the granularity doesn't match and existing pool - EnsurePoolCreated(poolName datamodel.PoolName, granularityGB int) (datamodel.Pool, error) + EnsurePoolCreated(poolName datamodel.PoolName, granularityGiB uint) (datamodel.Pool, error) // Get brick availability by pool GetAllPoolInfos() ([]datamodel.PoolInfo, error) diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go new file mode 100644 index 00000000..bace10c7 --- /dev/null +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -0,0 +1,76 @@ +package registry_impl + +import ( + "context" + "encoding/json" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "log" +) + +func NewBrickHostRegistry(store store.Keystore) registry.BrickHostRegistry { + // TODO: create AllocationRegistry + return &brickHostRegistry{store, nil} +} + +type brickHostRegistry struct { + store store.Keystore + allocation registry.AllocationRegistry +} + +func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { + + // find out granularity for each reported pool + if len(brickHostInfo.Bricks) == 0 { + log.Panicf("brick host must have some bricks: %s", brickHostInfo.Name) + } + poolGranularityGiBMap := make(map[datamodel.PoolName]uint) + for _, brick := range brickHostInfo.Bricks { + for !parsers.IsValidName(string(brick.PoolName)) { + log.Panicf("invalid pool name: %+v", brick) + } + poolGranularity, ok := poolGranularityGiBMap[brick.PoolName] + if !ok { + if brick.CapacityGiB <= 0 { + log.Panicf("invalid brick size: %+v", brick) + } + poolGranularityGiBMap[brick.PoolName] = brick.CapacityGiB + } else { + if brick.CapacityGiB != poolGranularity { + log.Panicf("inconsistent brick size: %+v", brick) + } + } + } + + // Check existing pools match what this brick host is reporting + for poolName, granularityGiB := range poolGranularityGiBMap { + _, err := b.allocation.EnsurePoolCreated(poolName, granularityGiB) + if err != nil { + return fmt.Errorf("unable to create pool due to: %s", err) + } + } + + if !parsers.IsValidName(string(brickHostInfo.Name)) { + log.Panicf("invalid brick host name: %s", brickHostInfo.Name) + } + key := fmt.Sprintf("/brickhost/%s", brickHostInfo.Name) + value, err := json.Marshal(brickHostInfo) + if err != nil { + log.Panicf("unable to covert brick host to json: %s", brickHostInfo.Name) + } + + // Always overwrite any pre-existing key + _, err = b.store.Update(key, value, 0) + return err +} + +func (b *brickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { + panic("implement me") +} + +func (b *brickHostRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { + panic("implement me") +} From a2e27f745e1d6748794acc154ffcace56ef3e367 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 09:30:33 +0100 Subject: [PATCH 085/191] Draft KeepAliveHost --- internal/pkg/v2/mock_store/keystore.go | 15 +++++++++++++++ internal/pkg/v2/registry_impl/brick_host.go | 16 +++++++++++++--- internal/pkg/v2/store/keystore.go | 3 +++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index f11453fe..9a0318d3 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -136,6 +136,21 @@ func (mr *MockKeystoreMockRecorder) Get(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKeystore)(nil).Get), key) } +// IsExist mocks base method +func (m *MockKeystore) IsExist(key string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsExist", key) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsExist indicates an expected call of IsExist +func (mr *MockKeystoreMockRecorder) IsExist(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsExist", reflect.TypeOf((*MockKeystore)(nil).IsExist), key) +} + // Watch mocks base method func (m *MockKeystore) Watch(ctxt context.Context, key string, withPrefix bool) store.KeyValueUpdateChan { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index bace10c7..f47534c1 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -21,6 +21,9 @@ type brickHostRegistry struct { allocation registry.AllocationRegistry } +const brickHostPrefix = "/BrickHostStore/" +const keepAlivePrefix = "/BrickHostAlive/" + func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { // find out granularity for each reported pool @@ -56,7 +59,7 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e if !parsers.IsValidName(string(brickHostInfo.Name)) { log.Panicf("invalid brick host name: %s", brickHostInfo.Name) } - key := fmt.Sprintf("/brickhost/%s", brickHostInfo.Name) + key := fmt.Sprintf("%s%s", brickHostPrefix, brickHostInfo.Name) value, err := json.Marshal(brickHostInfo) if err != nil { log.Panicf("unable to covert brick host to json: %s", brickHostInfo.Name) @@ -67,10 +70,17 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e return err } +func getKeepAliveKey(brickHostName datamodel.BrickHostName) string { + if !parsers.IsValidName(string(brickHostName)) { + log.Panicf("invalid brick host name: %s", brickHostName) + } + return fmt.Sprintf("%s%s", keepAlivePrefix, brickHostName) +} + func (b *brickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { - panic("implement me") + return b.store.KeepAliveKey(ctxt, getKeepAliveKey(brickHostName)) } func (b *brickHostRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { - panic("implement me") + return b.store.IsExist(getKeepAliveKey(brickHostName)) } diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index 78de06d5..9093a59c 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -40,6 +40,9 @@ type Keystore interface { // Get given key Get(key string) (KeyValueVersion, error) + // Check if a given key exists + IsExist(key string) (bool, error) + // Get a channel containing all KeyValueUpdate events // // Use the context to control if you watch forever, or if you choose to cancel when a key From b0e686b361e9199ddfde8ce6f958a2f627d99763 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 10:15:59 +0100 Subject: [PATCH 086/191] Add draft ensure pool created --- .../dacctl/actions_impl/parsers/capacity.go | 13 ++- .../dacd/brick_manager_impl/brick_manager.go | 2 + internal/pkg/v2/datamodel/brick_host.go | 1 + .../pkg/v2/mock_registry/brick_allocation.go | 8 +- internal/pkg/v2/registry/brick_allocation.go | 2 +- .../pkg/v2/registry_impl/brick_allocation.go | 86 +++++++++++++++++++ internal/pkg/v2/registry_impl/brick_host.go | 3 +- 7 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 internal/pkg/v2/registry_impl/brick_allocation.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go index c4f24ee0..08b1b01a 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go @@ -3,13 +3,14 @@ package parsers import ( "errors" "fmt" + "log" "math" "strconv" "strings" ) // TODO: missing a few? -var sizeSuffixMultiplier = map[string]int{ +var sizeSuffixMultiplier = map[string]uint{ "TiB": 1099511627776, "TB": 1000000000000, "GiB": 1073741824, @@ -18,6 +19,16 @@ var sizeSuffixMultiplier = map[string]int{ "MB": 1000000, } +// TODO: test me!! +func GetBytes(value uint, unit string) uint { + multiplier, ok := sizeSuffixMultiplier[unit] + if !ok { + log.Panicf("unrecognised unit") + } + return value * multiplier +} + +// TODO: why not uint? func ParseSize(raw string) (int, error) { intVal, err := strconv.Atoi(raw) if err == nil { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index cbf31c24..0f8d00c0 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -31,6 +31,8 @@ func (bm *brickManager) Hostname() string { } func (bm *brickManager) Startup(drainSessions bool) error { + // TODO: should we get the allocation mutex until we are started the keep alive? + err := bm.brickRegistry.UpdateBrickHost(getBrickHost(bm.config)) if err != nil { return err diff --git a/internal/pkg/v2/datamodel/brick_host.go b/internal/pkg/v2/datamodel/brick_host.go index a9e5f8f9..24b81327 100644 --- a/internal/pkg/v2/datamodel/brick_host.go +++ b/internal/pkg/v2/datamodel/brick_host.go @@ -32,6 +32,7 @@ type Brick struct { // It must only contain the characters A-Za-z0-9 PoolName PoolName + // TODO: move this to bytes, and make bytes a special type? // Size of the brick, defines the pool granularity CapacityGiB uint } diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go index 3f979eaa..00698df6 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -50,18 +50,18 @@ func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock. } // EnsurePoolCreated mocks base method -func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityGiB uint) (datamodel.Pool, error) { +func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityBytes uint) (datamodel.Pool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityGiB) + ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityBytes) ret0, _ := ret[0].(datamodel.Pool) ret1, _ := ret[1].(error) return ret0, ret1 } // EnsurePoolCreated indicates an expected call of EnsurePoolCreated -func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granularityGiB interface{}) *gomock.Call { +func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granularityBytes interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityGiB) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityBytes) } // GetAllPoolInfos mocks base method diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/v2/registry/brick_allocation.go index 75d46cc7..997623a3 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/v2/registry/brick_allocation.go @@ -11,7 +11,7 @@ type AllocationRegistry interface { // Creates the pool if it doesn't exist // error if the granularity doesn't match and existing pool - EnsurePoolCreated(poolName datamodel.PoolName, granularityGiB uint) (datamodel.Pool, error) + EnsurePoolCreated(poolName datamodel.PoolName, granularityBytes uint) (datamodel.Pool, error) // Get brick availability by pool GetAllPoolInfos() ([]datamodel.PoolInfo, error) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go new file mode 100644 index 00000000..069a3785 --- /dev/null +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -0,0 +1,86 @@ +package registry_impl + +import ( + "encoding/json" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "log" +) + +func NewAllocationRegistry(store store.Keystore) registry.AllocationRegistry { + // TODO: create brickHostRegistry + return &allocationRegistry{store, nil} +} + +type allocationRegistry struct { + store store.Keystore + brickHostRegistry registry.BrickHostRegistry +} + +const poolPrefix = "/Pool/" + +func getPoolKey(poolName datamodel.PoolName) string { + if !parsers.IsValidName(string(poolName)) { + log.Panicf("invalid session PrimaryBrickHost") + } + return fmt.Sprintf("%s%s", poolPrefix, poolName) +} + +func (a *allocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityBytes uint) (datamodel.Pool, error) { + if granularityBytes <= 0 { + log.Panicf("granularity must be greater than 0") + } + key := getPoolKey(poolName) + poolExists, err := a.store.IsExist(key) + if err != nil { + return datamodel.Pool{}, fmt.Errorf("unable to check if pool exists: %s", err) + } + + if poolExists { + pool, err := a.GetPool(poolName) + if err != nil { + return pool, fmt.Errorf("unable to get pool due to: %s", err) + } + if pool.GranularityBytes != granularityBytes { + return pool, fmt.Errorf("granularity doesn't match existing pool: %d", pool.GranularityBytes) + } + return pool, nil + } + + // TODO: need an admin tool to delete a "bad" pool + // create the pool + pool := datamodel.Pool{Name: poolName, GranularityBytes: granularityBytes} + value, err := json.Marshal(pool) + if err != nil { + log.Panicf("failed to convert pool to json: %s", err) + } + _, err = a.store.Create(key, value) + return pool, err +} + +func (a *allocationRegistry) GetPool(poolName datamodel.PoolName) (datamodel.Pool, error) { + panic("implement me") +} + +func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { + panic("implement me") +} + +func (a *allocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { + panic("implement me") +} + +func (a *allocationRegistry) GetAllocationMutex() (store.Mutex, error) { + panic("implement me") +} + +func (a *allocationRegistry) CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) { + panic("implement me") +} + +func (a *allocationRegistry) DeleteAllocations(allocations []datamodel.BrickAllocation) error { + panic("implement me") +} diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index f47534c1..05e1dce5 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -25,7 +25,6 @@ const brickHostPrefix = "/BrickHostStore/" const keepAlivePrefix = "/BrickHostAlive/" func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { - // find out granularity for each reported pool if len(brickHostInfo.Bricks) == 0 { log.Panicf("brick host must have some bricks: %s", brickHostInfo.Name) @@ -50,7 +49,7 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e // Check existing pools match what this brick host is reporting for poolName, granularityGiB := range poolGranularityGiBMap { - _, err := b.allocation.EnsurePoolCreated(poolName, granularityGiB) + _, err := b.allocation.EnsurePoolCreated(poolName, parsers.GetBytes(granularityGiB, "GiB")) if err != nil { return fmt.Errorf("unable to create pool due to: %s", err) } From f034727baaf1517fffd82e81b27e27b62aca86b7 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 10:21:36 +0100 Subject: [PATCH 087/191] Draft getPool --- internal/pkg/v2/registry_impl/brick_allocation.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 069a3785..1b79f406 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -62,7 +62,18 @@ func (a *allocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, gran } func (a *allocationRegistry) GetPool(poolName datamodel.PoolName) (datamodel.Pool, error) { - panic("implement me") + key := getPoolKey(poolName) + keyValueVersion, err := a.store.Get(key) + pool := datamodel.Pool{} + if err != nil { + return pool, fmt.Errorf("unable to get pool due to: %s", err) + } + + err = json.Unmarshal(keyValueVersion.Value, &pool) + if err != nil { + log.Panicf("unable to parse pool") + } + return pool, nil } func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { From c5ae6fa70e935c52e53a1e669c9c0189923a154d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 10:24:18 +0100 Subject: [PATCH 088/191] Add allocation mutex --- internal/pkg/v2/registry/brick_allocation.go | 6 +++--- internal/pkg/v2/registry_impl/brick_allocation.go | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/v2/registry/brick_allocation.go index 997623a3..3ad925f9 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/v2/registry/brick_allocation.go @@ -6,6 +6,9 @@ import ( ) type AllocationRegistry interface { + // Caller should acquire this mutex before calling GetAllPools then CreateAllocations + GetAllocationMutex() (store.Mutex, error) + // Get all registered pools GetPool(name datamodel.PoolName) (datamodel.Pool, error) @@ -20,9 +23,6 @@ type AllocationRegistry interface { // bricks are only available if corresponding host currently alive GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) - // Caller should acquire this mutex before calling GetAllPools then CreateAllocations - GetAllocationMutex() (store.Mutex, error) - // Allocations written (by the client), while holding above mutex // // Error if any bricks already have an allocation diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 1b79f406..39f848d0 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -21,6 +21,11 @@ type allocationRegistry struct { } const poolPrefix = "/Pool/" +const allocationLockKey = "/LockAllocation/" + +func (a *allocationRegistry) GetAllocationMutex() (store.Mutex, error) { + return a.store.NewMutex(allocationLockKey) +} func getPoolKey(poolName datamodel.PoolName) string { if !parsers.IsValidName(string(poolName)) { @@ -84,10 +89,6 @@ func (a *allocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel panic("implement me") } -func (a *allocationRegistry) GetAllocationMutex() (store.Mutex, error) { - panic("implement me") -} - func (a *allocationRegistry) CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) { panic("implement me") } From 52cf651dd3e9e4d1c191a0dfc4c87ba3ccd6eddb Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 10:53:58 +0100 Subject: [PATCH 089/191] Simplify allocation plan --- .../pkg/v2/dacctl/workflow_impl/session.go | 20 +------ .../v2/dacctl/workflow_impl/session_test.go | 6 +- internal/pkg/v2/datamodel/datamodel_test.go | 25 -------- internal/pkg/v2/datamodel/session.go | 2 +- .../pkg/v2/mock_registry/brick_allocation.go | 59 +++++-------------- internal/pkg/v2/registry/brick_allocation.go | 10 ---- .../pkg/v2/registry_impl/brick_allocation.go | 8 --- internal/pkg/v2/registry_impl/session.go | 4 +- internal/pkg/v2/registry_impl/session_test.go | 4 +- 9 files changed, 25 insertions(+), 113 deletions(-) delete mode 100644 internal/pkg/v2/datamodel/datamodel_test.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 4e5ecd62..f86e9d67 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -96,14 +96,9 @@ func (s sessionFacade) doAllocationAndWriteSession(session datamodel.Session) (d return session, fmt.Errorf("can't allocate for session: %s due to %s", session.Name, err) } - allocations, err := s.allocations.CreateAllocations(session.Name, chosenBricks) - if err != nil { - return session, err - } - session.ActualSizeBytes = actualSizeBytes - session.Allocations = allocations - session.PrimaryBrickHost = allocations[0].Brick.BrickHostName + session.AllocatedBricks = chosenBricks + session.PrimaryBrickHost = chosenBricks[0].BrickHostName } else { // Pick a random alive host to be the PrimaryBrickHost anyway pools, err := s.allocations.GetAllPoolInfos() @@ -121,16 +116,7 @@ func (s sessionFacade) doAllocationAndWriteSession(session datamodel.Session) (d // Store initial version of session // returned session will have updated revision info - session, err := s.session.CreateSession(session) - if err != nil { - if session.Allocations != nil { - deleteErr := s.allocations.DeleteAllocations(session.Allocations) - if deleteErr != nil { - log.Println("Failed to clean up allocations due to:", deleteErr) - } - } - } - return session, err + return s.session.CreateSession(session) } func (s sessionFacade) getBricks(poolName datamodel.PoolName, bytes int) (int, []datamodel.Brick, error) { diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index d58a2c66..6817e44a 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -111,8 +111,6 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) }, AvailableBricks: brickList, }, nil) - allocationList := []datamodel.BrickAllocation{{Brick: brickList[0]}} - allocations.EXPECT().CreateAllocations(initialSession.Name, brickList).Return(allocationList, nil) updatedSession := datamodel.Session{ Name: "foo", VolumeRequest: datamodel.VolumeRequest{ @@ -120,8 +118,8 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) TotalCapacityBytes: 1024, }, ActualSizeBytes: 1024, - Allocations: allocationList, - PrimaryBrickHost: allocationList[0].Brick.BrickHostName, + AllocatedBricks: brickList, + PrimaryBrickHost: brickList[0].BrickHostName, } returnedSession := datamodel.Session{ Name: "foo", diff --git a/internal/pkg/v2/datamodel/datamodel_test.go b/internal/pkg/v2/datamodel/datamodel_test.go deleted file mode 100644 index 2a917f3e..00000000 --- a/internal/pkg/v2/datamodel/datamodel_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package datamodel - -import ( - "encoding/json" - "github.com/stretchr/testify/assert" - "testing" -) - -func Test_Session(t *testing.T) { - session := Session{} - - sessionAsString, err := json.Marshal(session) - - assert.Nil(t, err) - // TODO: not very human readable for Type and ActionType - expected := `{"Name":"","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}` - assert.Equal(t, expected, string(sessionAsString)) - - var sessionFromString Session - data := []byte(expected) - err = json.Unmarshal(data, &sessionFromString) - - assert.Nil(t, err) - assert.Equal(t, session, sessionFromString) -} diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 54d562f0..6138f0e0 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -47,7 +47,7 @@ type Session struct { // List of the bricks allocated to implement the JobVolume // One is the primary brick that should be watching for all actions - Allocations []BrickAllocation + AllocatedBricks []Brick // Where session requests should be sent PrimaryBrickHost BrickHostName diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go index 00698df6..900a1b79 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -34,6 +34,21 @@ func (m *MockAllocationRegistry) EXPECT() *MockAllocationRegistryMockRecorder { return m.recorder } +// GetAllocationMutex mocks base method +func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllocationMutex") + ret0, _ := ret[0].(store.Mutex) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllocationMutex indicates an expected call of GetAllocationMutex +func (mr *MockAllocationRegistryMockRecorder) GetAllocationMutex() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllocationMutex)) +} + // GetPool mocks base method func (m *MockAllocationRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { m.ctrl.T.Helper() @@ -93,47 +108,3 @@ func (mr *MockAllocationRegistryMockRecorder) GetPoolInfo(poolName interface{}) mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPoolInfo", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPoolInfo), poolName) } - -// GetAllocationMutex mocks base method -func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllocationMutex") - ret0, _ := ret[0].(store.Mutex) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllocationMutex indicates an expected call of GetAllocationMutex -func (mr *MockAllocationRegistryMockRecorder) GetAllocationMutex() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllocationMutex)) -} - -// CreateAllocations mocks base method -func (m *MockAllocationRegistry) CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateAllocations", sessionName, allocations) - ret0, _ := ret[0].([]datamodel.BrickAllocation) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateAllocations indicates an expected call of CreateAllocations -func (mr *MockAllocationRegistryMockRecorder) CreateAllocations(sessionName, allocations interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateAllocations", reflect.TypeOf((*MockAllocationRegistry)(nil).CreateAllocations), sessionName, allocations) -} - -// DeleteAllocations mocks base method -func (m *MockAllocationRegistry) DeleteAllocations(allocations []datamodel.BrickAllocation) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAllocations", allocations) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteAllocations indicates an expected call of DeleteAllocations -func (mr *MockAllocationRegistryMockRecorder) DeleteAllocations(allocations interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllocations", reflect.TypeOf((*MockAllocationRegistry)(nil).DeleteAllocations), allocations) -} diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/v2/registry/brick_allocation.go index 3ad925f9..e53485f9 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/v2/registry/brick_allocation.go @@ -22,14 +22,4 @@ type AllocationRegistry interface { // Get brick availability for one pool // bricks are only available if corresponding host currently alive GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) - - // Allocations written (by the client), while holding above mutex - // - // Error if any bricks already have an allocation - CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) - - // Allocations deleted by server when bricks no being used - // - // Does not error if given allocation has already been dropped - DeleteAllocations(allocations []datamodel.BrickAllocation) error } diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 39f848d0..50311754 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -88,11 +88,3 @@ func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { func (a *allocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { panic("implement me") } - -func (a *allocationRegistry) CreateAllocations(sessionName datamodel.SessionName, allocations []datamodel.Brick) ([]datamodel.BrickAllocation, error) { - panic("implement me") -} - -func (a *allocationRegistry) DeleteAllocations(allocations []datamodel.BrickAllocation) error { - panic("implement me") -} diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index a2a33e6f..224a4ee2 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -41,11 +41,11 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se log.Panicf("PrimaryBrickHost must be set before creating session: %s", session.Name) } if session.ActualSizeBytes > 0 { - if len(session.Allocations) == 0 { + if len(session.AllocatedBricks) == 0 { log.Panicf("session must have allocations before being created: %s", session.Name) } } else { - if len(session.Allocations) != 0 { + if len(session.AllocatedBricks) != 0 { log.Panicf("allocations out of sync with ActualSizeBytes: %s", session.Name) } } diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 84450dec..bdcaec0a 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -10,7 +10,7 @@ import ( "testing" ) -var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"Allocations":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) +var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"AllocatedBricks":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) var exampleSession = datamodel.Session{Name: "foo", PrimaryBrickHost: "host1"} func TestSessionRegistry_GetSessionMutex(t *testing.T) { @@ -50,7 +50,7 @@ func TestSessionRegistry_CreateSession(t *testing.T) { assert.PanicsWithValue(t, "allocations out of sync with ActualSizeBytes: foo", func() { registry.CreateSession(datamodel.Session{ Name: "foo", - Allocations: []datamodel.BrickAllocation{{}}, + AllocatedBricks: []datamodel.Brick{{}}, PrimaryBrickHost: "host1", }) }) From fbcf46b68a92a15db77f795ed92761ebfedad579 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 11:13:46 +0100 Subject: [PATCH 090/191] Add concept of getallbrickhosts --- internal/pkg/v2/mock_registry/brick_host.go | 15 +++++++++ internal/pkg/v2/registry/brick_host.go | 3 ++ internal/pkg/v2/registry/session.go | 1 + .../pkg/v2/registry_impl/brick_allocation.go | 33 ++++++++++++++++++- internal/pkg/v2/registry_impl/brick_host.go | 4 +++ internal/pkg/v2/registry_impl/session.go | 1 + 6 files changed, 56 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/v2/mock_registry/brick_host.go index f2cc8750..a42ead05 100644 --- a/internal/pkg/v2/mock_registry/brick_host.go +++ b/internal/pkg/v2/mock_registry/brick_host.go @@ -48,6 +48,21 @@ func (mr *MockBrickHostRegistryMockRecorder) UpdateBrickHost(brickHostInfo inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).UpdateBrickHost), brickHostInfo) } +// GetAllBrickHosts mocks base method +func (m *MockBrickHostRegistry) GetAllBrickHosts() ([]datamodel.BrickHost, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAllBrickHosts") + ret0, _ := ret[0].([]datamodel.BrickHost) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllBrickHosts indicates an expected call of GetAllBrickHosts +func (mr *MockBrickHostRegistryMockRecorder) GetAllBrickHosts() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrickHosts", reflect.TypeOf((*MockBrickHostRegistry)(nil).GetAllBrickHosts)) +} + // KeepAliveHost mocks base method func (m *MockBrickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { m.ctrl.T.Helper() diff --git a/internal/pkg/v2/registry/brick_host.go b/internal/pkg/v2/registry/brick_host.go index 81d37a2c..eb863202 100644 --- a/internal/pkg/v2/registry/brick_host.go +++ b/internal/pkg/v2/registry/brick_host.go @@ -12,6 +12,9 @@ type BrickHostRegistry interface { // This includes ensuring the pool exists and is consistent with the given brick host info UpdateBrickHost(brickHostInfo datamodel.BrickHost) error + // Get all brick hosts + GetAllBrickHosts() ([]datamodel.BrickHost, error) + // While the process is still running this notifies others the host is up // // When a host is dead non of its bricks will get new volumes assigned, diff --git a/internal/pkg/v2/registry/session.go b/internal/pkg/v2/registry/session.go index 1a4bfbdd..4344a685 100644 --- a/internal/pkg/v2/registry/session.go +++ b/internal/pkg/v2/registry/session.go @@ -5,6 +5,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" ) +// TODO: rename to instance? or filesystem? or just in the object model? type SessionRegistry interface { // This mutex should be held before doing any operations on given session // diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 50311754..2b7325fe 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -12,12 +12,13 @@ import ( func NewAllocationRegistry(store store.Keystore) registry.AllocationRegistry { // TODO: create brickHostRegistry - return &allocationRegistry{store, nil} + return &allocationRegistry{store, nil, nil} } type allocationRegistry struct { store store.Keystore brickHostRegistry registry.BrickHostRegistry + sessionRegistry registry.SessionRegistry } const poolPrefix = "/Pool/" @@ -81,7 +82,37 @@ func (a *allocationRegistry) GetPool(poolName datamodel.PoolName) (datamodel.Poo return pool, nil } +func (a *allocationRegistry) getAllPools() (map[datamodel.PoolName]datamodel.Pool, error) { + allKeyValues, err := a.store.GetAll(poolPrefix) + if err != nil { + return nil, fmt.Errorf("unable to get pools due to: %s", err) + } + pools := make(map[datamodel.PoolName]datamodel.Pool) + for _, keyValueVersion := range allKeyValues { + pool := datamodel.Pool{} + err = json.Unmarshal(keyValueVersion.Value, &pool) + if err != nil { + log.Panicf("unable to parse pool") + } + pools[pool.Name] = pool + } + return pools, nil +} + func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { + pools, err := a.getAllPools() + if err != nil { + return nil, fmt.Errorf("unable to get pools due to: %s", err) + } + + sessions, err := a.sessionRegistry.GetAllSessions() + if err != nil { + return nil, fmt.Errorf("unable to get all sessions due to: %s", err) + } + //brickHosts := a.brickHostRegistry.GetAllBrickHosts() + + log.Println(sessions) + log.Println(pools) panic("implement me") } diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index 05e1dce5..82d7576d 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -69,6 +69,10 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e return err } +func (b *brickHostRegistry) GetAllBrickHosts() ([]datamodel.BrickHost, error) { + panic("implement me") +} + func getKeepAliveKey(brickHostName datamodel.BrickHostName) string { if !parsers.IsValidName(string(brickHostName)) { log.Panicf("invalid brick host name: %s", brickHostName) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 224a4ee2..cabba108 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -48,6 +48,7 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se if len(session.AllocatedBricks) != 0 { log.Panicf("allocations out of sync with ActualSizeBytes: %s", session.Name) } + // TODO: ensure not allocated to any other session? } keyValueVersion, err := s.store.Create(sessionKey, sessionToRaw(session)) From 3d3c02a5f09f9da30f8fed4a2b3d4d4c7d2ff8b1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 11:19:31 +0100 Subject: [PATCH 091/191] Add get all brick hosts --- internal/pkg/v2/registry_impl/brick_host.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index 82d7576d..c37ddf74 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -70,7 +70,21 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e } func (b *brickHostRegistry) GetAllBrickHosts() ([]datamodel.BrickHost, error) { - panic("implement me") + allKeyValues, err := b.store.GetAll(brickHostPrefix) + if err != nil { + return nil, fmt.Errorf("unable to get all bricks hosts due to: %s", err) + } + + var allBrickHosts []datamodel.BrickHost + for _, keyValueVersion := range allKeyValues { + brickHost := datamodel.BrickHost{} + err := json.Unmarshal(keyValueVersion.Value, &brickHost) + if err != nil { + log.Panicf("unable to parse brick host due to: %s", err) + } + allBrickHosts = append(allBrickHosts, brickHost) + } + return allBrickHosts, nil } func getKeepAliveKey(brickHostName datamodel.BrickHostName) string { From 930023e83038676391e5e09ecc034cebaafa9f31 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 11:51:26 +0100 Subject: [PATCH 092/191] Add draft of get all poolInfo --- internal/pkg/v2/datamodel/brick_allocation.go | 17 ------- .../pkg/v2/registry_impl/brick_allocation.go | 50 +++++++++++++++++-- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/internal/pkg/v2/datamodel/brick_allocation.go b/internal/pkg/v2/datamodel/brick_allocation.go index e459a3b3..c5f10277 100644 --- a/internal/pkg/v2/datamodel/brick_allocation.go +++ b/internal/pkg/v2/datamodel/brick_allocation.go @@ -8,24 +8,7 @@ type BrickAllocation struct { // Name of the session that owns the brick Session SessionName - // Unique id for this allocation - // matched when delete allocation is called - UUID string - // 0 index allocation is the primary brick, // which is responsible for provisioning the associated volume AllocatedIndex uint - - // One primary prick per volume - // this brick is responsible for watching for - // associated job and volume actions - IsPrimaryBrick bool - - // This is set when server has accepted the allocation - IsAllocationAccepted bool - - // If any allocation sent to deallocate has a host that isn't - // alive, this flag is set rather than have allocations removed. - // A host should check for any allocations - DeallocateRequested bool } diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 2b7325fe..f73d29cc 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -104,16 +104,56 @@ func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { if err != nil { return nil, fmt.Errorf("unable to get pools due to: %s", err) } - sessions, err := a.sessionRegistry.GetAllSessions() if err != nil { return nil, fmt.Errorf("unable to get all sessions due to: %s", err) } - //brickHosts := a.brickHostRegistry.GetAllBrickHosts() + brickHosts, err := a.brickHostRegistry.GetAllBrickHosts() + if err != nil { + return nil, fmt.Errorf("unable to get all briks due to: %s", err) + } - log.Println(sessions) - log.Println(pools) - panic("implement me") + var allPoolInfos []datamodel.PoolInfo + + for _, pool := range pools { + poolInfo := datamodel.PoolInfo{Pool: pool} + + allocatedDevicesByBrickHost := make(map[datamodel.BrickHostName][]string) + for _, session := range sessions { + for i, brick := range session.AllocatedBricks { + if brick.PoolName == pool.Name { + poolInfo.AllocatedBricks = append(poolInfo.AllocatedBricks, datamodel.BrickAllocation{ + Session: session.Name, + Brick: brick, + AllocatedIndex: uint(i), + }) + allocatedDevicesByBrickHost[brick.BrickHostName] = append(allocatedDevicesByBrickHost[brick.BrickHostName], brick.Device) + } + } + } + + for _, brickHost := range brickHosts { + for _, brick := range brickHost.Bricks { + log.Println(brick) + // Check if in allocated list + allocated := false + for _, allocatedDevice := range allocatedDevicesByBrickHost[brick.BrickHostName] { + if allocatedDevice == brick.Device { + if allocated { + log.Panicf("detected duplicated brick allocation: %+v", brick) + } + allocated = true + } + } + if !allocated { + poolInfo.AvailableBricks = append(poolInfo.AvailableBricks, brick) + } + } + } + + allPoolInfos = append(allPoolInfos, poolInfo) + } + return allPoolInfos, nil } func (a *allocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { From 8c4f4ef5864c765697d23fe318de017c099de3aa Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 11:54:53 +0100 Subject: [PATCH 093/191] draft of get pool info --- internal/pkg/v2/registry_impl/brick_allocation.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index f73d29cc..dcae74ee 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -157,5 +157,15 @@ func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { } func (a *allocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { - panic("implement me") + allInfo, err := a.GetAllPoolInfos() + if err != nil { + return datamodel.PoolInfo{}, err + } + + for _, poolInfo := range allInfo { + if poolInfo.Pool.Name == poolName { + return poolInfo, nil + } + } + return datamodel.PoolInfo{}, fmt.Errorf("unable to find pool %s", poolName) } From 4a3f904eb7f0c1c77cdbaae5880cb5a2f9d8ef16 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 13:32:29 +0100 Subject: [PATCH 094/191] Port keystore to new data format and interface --- internal/pkg/v2/mock_store/keystore.go | 7 +- internal/pkg/v2/store/keystore.go | 2 +- internal/pkg/v2/store_impl/keystore.go | 321 +++++++++++++++++++++++++ 3 files changed, 326 insertions(+), 4 deletions(-) create mode 100644 internal/pkg/v2/store_impl/keystore.go diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index 9a0318d3..0b245307 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -93,11 +93,12 @@ func (mr *MockKeystoreMockRecorder) Delete(key, modRevision interface{}) *gomock } // DeleteAllKeysWithPrefix mocks base method -func (m *MockKeystore) DeleteAllKeysWithPrefix(keyPrefix string) error { +func (m *MockKeystore) DeleteAllKeysWithPrefix(keyPrefix string) (int64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteAllKeysWithPrefix", keyPrefix) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 } // DeleteAllKeysWithPrefix indicates an expected call of DeleteAllKeysWithPrefix diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index 9093a59c..faf50790 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -32,7 +32,7 @@ type Keystore interface { Delete(key string, modRevision int64) error // Removes all keys with given prefix - DeleteAllKeysWithPrefix(keyPrefix string) error + DeleteAllKeysWithPrefix(keyPrefix string) (int64, error) // Get all key values for a given prefix. GetAll(keyPrefix string) ([]KeyValueVersion, error) diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go new file mode 100644 index 00000000..329afcda --- /dev/null +++ b/internal/pkg/v2/store_impl/keystore.go @@ -0,0 +1,321 @@ +package store_impl + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3/clientv3util" + "github.com/coreos/etcd/clientv3/concurrency" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + "github.com/coreos/etcd/mvcc/mvccpb" + "github.com/coreos/etcd/pkg/transport" + "log" + "os" + "strings" + "time" +) + +func getTLSConfig() *tls.Config { + certFile := os.Getenv("ETCDCTL_CERT_FILE") + keyFile := os.Getenv("ETCDCTL_KEY_FILE") + caFile := os.Getenv("ETCDCTL_CA_FILE") + + if certFile == "" || keyFile == "" || caFile == "" { + return nil + } + + tlsInfo := transport.TLSInfo{ + CertFile: certFile, + KeyFile: keyFile, + TrustedCAFile: caFile, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + log.Fatal(err) + } + return tlsConfig +} + +func getEndpoints() []string { + endpoints := os.Getenv("ETCDCTL_ENDPOINTS") + if endpoints == "" { + endpoints = os.Getenv("ETCD_ENDPOINTS") + } + if endpoints == "" { + log.Fatalf("Must set ETCDCTL_ENDPOINTS environemnt variable, e.g. export ETCDCTL_ENDPOINTS=127.0.0.1:2379") + } + return strings.Split(endpoints, ",") +} + +func newEtcdClient() *clientv3.Client { + cli, err := clientv3.New(clientv3.Config{ + Endpoints: getEndpoints(), + DialTimeout: 10 * time.Second, + TLS: getTLSConfig(), + }) + if err != nil { + fmt.Println("failed to create client") + log.Fatal(err) + } + return cli +} + +func NewKeystore() store.Keystore { + cli := newEtcdClient() + return &etcKeystore{ + Watcher: cli.Watcher, + KV: cli.KV, + Lease: cli.Lease, + Client: cli, + } +} + +type etcKeystore struct { + Watcher clientv3.Watcher + KV clientv3.KV + Lease clientv3.Lease + Client *clientv3.Client +} + +func (client *etcKeystore) NewMutex(lockKey string) (store.Mutex, error) { + session, err := concurrency.NewSession(client.Client) + if err != nil { + return nil, err + } + key := fmt.Sprintf("/locks/%s", lockKey) + return concurrency.NewMutex(session, key), nil +} + +func handleError(err error) { + if err != nil { + switch err { + case context.Canceled: + log.Fatalf("ctx is canceled by another routine: %v", err) + case context.DeadlineExceeded: + log.Fatalf("ctx is attached with a deadline is exceeded: %v", err) + case rpctypes.ErrEmptyKey: + log.Fatalf("client-side error: %v", err) + default: + log.Fatalf("bad cluster endpoints, which are not etcd servers: %v", err) + } + } +} + +func (client *etcKeystore) Close() error { + return client.Client.Close() +} + +func (client *etcKeystore) runTransaction(ifOps []clientv3.Cmp, thenOps []clientv3.Op) error { + kvc := clientv3.NewKV(client.Client) + kvc.Txn(context.Background()) + response, err := kvc.Txn(context.Background()).If(ifOps...).Then(thenOps...).Commit() + handleError(err) + + if !response.Succeeded { + log.Println(ifOps) + return fmt.Errorf("transaction failed, as condition not met") + } + return nil +} + +func (client *etcKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { + var ifOps []clientv3.Cmp + var thenOps []clientv3.Op + ifOps = append(ifOps, clientv3util.KeyMissing(key)) + thenOps = append(thenOps, clientv3.OpPut(key, string(value))) + err := client.runTransaction(ifOps, thenOps) + if err != nil { + return store.KeyValueVersion{}, fmt.Errorf("unable to update ke: %s", err) + } + return client.Get(key) +} + +func (client *etcKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { + + var ifOps []clientv3.Cmp + var thenOps []clientv3.Op + + ifOps = append(ifOps, clientv3util.KeyExists(key)) + if modRevision > 0 { + checkModRev := clientv3.Compare(clientv3.ModRevision(key), "=", modRevision) + ifOps = append(ifOps, checkModRev) + } + thenOps = append(thenOps, clientv3.OpPut(key, string(value))) + + err := client.runTransaction(ifOps, thenOps) + if err != nil { + return store.KeyValueVersion{}, fmt.Errorf("unable to update ke: %s", err) + } + return client.Get(key) +} + +func (client *etcKeystore) Delete(key string, modRevision int64) error { + + var ifOps []clientv3.Cmp + var thenOps []clientv3.Op + + ifOps = append(ifOps, clientv3util.KeyExists(key)) + if modRevision > 0 { + checkModRev := clientv3.Compare(clientv3.ModRevision(key), "=", modRevision) + ifOps = append(ifOps, checkModRev) + } + thenOps = append(thenOps, clientv3.OpDelete(key)) + + return client.runTransaction(ifOps, thenOps) +} + +func getKeyValueVersion(rawKeyValue *mvccpb.KeyValue) *store.KeyValueVersion { + if rawKeyValue == nil { + return nil + } + return &store.KeyValueVersion{ + Key: string(rawKeyValue.Key), + Value: rawKeyValue.Value, + ModRevision: rawKeyValue.ModRevision, + CreateRevision: rawKeyValue.CreateRevision, + } +} + +func (client *etcKeystore) IsExist(key string) (bool, error) { + kvc := clientv3.NewKV(client.Client) + response, err := kvc.Get(context.Background(), key) + handleError(err) + return response.Count == 1, nil +} + +func (client *etcKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error) { + kvc := clientv3.NewKV(client.Client) + response, err := kvc.Get(context.Background(), prefix, clientv3.WithPrefix()) + handleError(err) + + if response.Count == 0 { + return []store.KeyValueVersion{}, + fmt.Errorf("unable to find any values for prefix: %s", prefix) + } + var values []store.KeyValueVersion + for _, rawKeyValue := range response.Kvs { + values = append(values, *getKeyValueVersion(rawKeyValue)) + } + return values, nil +} + +func (client *etcKeystore) Get(key string) (store.KeyValueVersion, error) { + kvc := clientv3.NewKV(client.Client) + response, err := kvc.Get(context.Background(), key) + handleError(err) + + value := store.KeyValueVersion{} + + if response.Count == 0 { + return value, fmt.Errorf("unable to find any values for key: %s", key) + } + if response.Count > 1 { + panic(errors.New("should never get more than one value for get")) + } + + return *getKeyValueVersion(response.Kvs[0]), nil +} + +func (client *etcKeystore) KeepAliveKey(ctxt context.Context, key string) error { + kvc := clientv3.NewKV(client.Client) + + getResponse, err := kvc.Get(context.Background(), key) + if getResponse.Count == 1 { + // if another host seems to exist, back off for 10 seconds incase we just did a quick restart + time.Sleep(time.Second * 10) + } + + // TODO what about configure timeout and ttl? + var ttl int64 = 10 + grantResponse, err := client.Client.Grant(ctxt, ttl) + if err != nil { + log.Fatal(err) + } + leaseID := grantResponse.ID + + txnResponse, err := kvc.Txn(ctxt). + If(clientv3util.KeyMissing(key)). + Then(clientv3.OpPut(key, "keep-alive", clientv3.WithLease(leaseID), clientv3.WithPrevKV())). + Commit() + handleError(err) + if !txnResponse.Succeeded { + return fmt.Errorf("unable to create keep-alive key: %s", key) + } + + ch, err := client.Client.KeepAlive(ctxt, leaseID) + if err != nil { + log.Fatal(err) + } + + counter := 9 + go func() { + for range ch { + if counter >= 9 { + counter = 0 + log.Println("Still refreshing key:", key) + } else { + counter++ + } + } + // TODO: should allow context to be cancelled + log.Panicf("Unable to refresh key: %s", key) + }() + return nil +} + +func (client *etcKeystore) DeleteAllKeysWithPrefix(prefix string) (int64, error) { + kvc := clientv3.NewKV(client.Client) + response, err := kvc.Delete(context.Background(), prefix, clientv3.WithPrefix()) + handleError(err) + + log.Printf("Cleaned %d keys with prefix: '%s'.\n", response.Deleted, prefix) + return response.Deleted, nil +} + +func (client *etcKeystore) Watch(ctxt context.Context, key string, withPrefix bool) store.KeyValueUpdateChan { + options := []clientv3.OpOption{clientv3.WithPrevKV()} + if withPrefix { + options = append(options, clientv3.WithPrefix()) + } + rch := client.Watcher.Watch(ctxt, key, options...) + + c := make(chan store.KeyValueUpdate) + + go processWatchEvents(rch, c) + + return c +} + +func processWatchEvents(watchChan clientv3.WatchChan, c chan store.KeyValueUpdate) { + for watchResponse := range watchChan { + // if error, send empty update with an error + err := watchResponse.Err() + if err != nil { + c <- store.KeyValueUpdate{Err: err} + } + + // send all events in this watch response + for _, ev := range watchResponse.Events { + update := store.KeyValueUpdate{ + IsCreate: ev.IsCreate(), + IsModify: ev.IsModify(), + IsDelete: ev.Type == clientv3.EventTypeDelete, + } + if update.IsCreate || update.IsModify { + update.New = getKeyValueVersion(ev.Kv) + } + if update.IsDelete || update.IsModify { + update.Old = getKeyValueVersion(ev.PrevKv) + } + + c <- update + } + } + + // Assuming we get here when the context is cancelled or hits its timeout + // i.e. there are no more events, so we close the channel + close(c) +} From a571d8241ea440e48f62d942dc2bed25b3f5ab89 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 13:57:38 +0100 Subject: [PATCH 095/191] Make dacd use the v2 logic --- cmd/dacd/main.go | 25 +++++++++---------- .../dacd/brick_manager_impl/brick_manager.go | 25 +++++++++++-------- .../brick_manager_impl/brick_manager_test.go | 9 ++++--- internal/pkg/v2/dacd/interface.go | 6 ++--- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/cmd/dacd/main.go b/cmd/dacd/main.go index 3be34ac8..46bdbd0d 100644 --- a/cmd/dacd/main.go +++ b/cmd/dacd/main.go @@ -1,37 +1,36 @@ package main import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/etcdregistry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/lifecycle/brickmanager" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager_impl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store_impl" "log" "os" "os/signal" "syscall" ) -func waitForShutdown() { +func waitForShutdown(manager dacd.BrickManager) { c := make(chan os.Signal) signal.Notify(c, os.Interrupt, syscall.SIGINT) <-c log.Println("I have been asked to shutdown, doing tidy up...") + manager.Shutdown() os.Exit(1) } func main() { log.Println("Starting data-accelerator's brick manager") - keystore := etcdregistry.NewKeystore() - defer keystore.Close() - poolRegistry := keystoreregistry.NewPoolRegistry(keystore) - volumeRegistry := keystoreregistry.NewVolumeRegistry(keystore) + keystore := store_impl.NewKeystore() + defer func() { + log.Println("keystore closed with error: ", keystore.Close()) + }() - manager := brickmanager.NewBrickManager(poolRegistry, volumeRegistry) - if err := manager.Start(); err != nil { - log.Fatal(err) - } + manager := brick_manager_impl.NewBrickManager(keystore) + manager.Startup() log.Println("Brick manager started for:", manager.Hostname()) - waitForShutdown() + waitForShutdown(manager) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index 0f8d00c0..4cfd4801 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -6,16 +6,17 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) -func NewBrickManager(brickRegistry registry.BrickHostRegistry, sessionActions registry.SessionActions, - handler facade.SessionActionHandler) dacd.BrickManager { +func NewBrickManager(keystore store.Keystore) dacd.BrickManager { + // TODO: call concrete options return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), - brickRegistry: brickRegistry, - sessionActions: sessionActions, - sessionActionHandler: handler, + brickRegistry: nil, + sessionActions: nil, + sessionActionHandler: nil, } } @@ -30,12 +31,13 @@ func (bm *brickManager) Hostname() string { return string(bm.config.BrickHostName) } -func (bm *brickManager) Startup(drainSessions bool) error { +func (bm *brickManager) Startup() { // TODO: should we get the allocation mutex until we are started the keep alive? + // TODO: add a drain configuration? err := bm.brickRegistry.UpdateBrickHost(getBrickHost(bm.config)) if err != nil { - return err + log.Panicf("failed to update brick host: %s", err) } // If we are are enabled, this includes new create session requests @@ -52,12 +54,15 @@ func (bm *brickManager) Startup(drainSessions bool) error { // including a check to make sure all related brick hosts are alive // Tell everyone we are listening - return bm.brickRegistry.KeepAliveHost(context.TODO(), bm.config.BrickHostName) + err = bm.brickRegistry.KeepAliveHost(context.TODO(), bm.config.BrickHostName) + if err != nil { + log.Panicf("failed to start keep alive host: %s", err) + } } -func (bm *brickManager) Shutdown() error { +func (bm *brickManager) Shutdown() { // Delete the keepalive key, to stop new actions being sent // Wait for existing actions by trying to get a lock on all // sessions we for which we are the primary brick - panic("implement me") + // TODO... } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index a3e8f940..e3254b16 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -22,14 +22,15 @@ func TestBrickManager_Startup(t *testing.T) { brickRegistry := mock_registry.NewMockBrickHostRegistry(mockCtrl) sessionActions := mock_registry.NewMockSessionActions(mockCtrl) handler := mock_facade.NewMockSessionActionHandler(mockCtrl) - brickManager := NewBrickManager(brickRegistry, sessionActions, handler) + brickManager := brickManager{ + config: config.GetBrickManagerConfig(config.DefaultEnv), + brickRegistry: brickRegistry, sessionActions: sessionActions, sessionActionHandler: handler, + } // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) sessionActions.EXPECT().GetSessionActionRequests(context.TODO(), gomock.Any()) brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName("hostname")) - err := brickManager.Startup(false) - - assert.Nil(t, err) + brickManager.Startup() } diff --git a/internal/pkg/v2/dacd/interface.go b/internal/pkg/v2/dacd/interface.go index 0f63b70b..f252a446 100644 --- a/internal/pkg/v2/dacd/interface.go +++ b/internal/pkg/v2/dacd/interface.go @@ -7,10 +7,8 @@ type BrickManager interface { // Tidy up from previous shutdowns // , then start waiting for session actions // notify dacctl we are listening via keep alive key - // if drainSessions = True, we don't allow new - Startup(drainSessions bool) error - + Startup() // Wait for any events to complete // then do any tidy up required for a graceful shutdown - Shutdown() error + Shutdown() } From db18ea0eb2021321a941ccf20c0bfefb743e4f65 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 15:32:04 +0100 Subject: [PATCH 096/191] Try to move to v2 code --- cmd/dacctl/actions.go | 48 +++++---- cmd/dacctl/main.go | 2 +- cmd/dacctl/main_test.go | 98 ++++++++++++------- .../pkg/v2/dacctl/actions_impl/actions.go | 6 +- .../v2/dacctl/actions_impl/actions_test.go | 16 +-- .../pkg/v2/dacctl/actions_impl/job_test.go | 4 +- .../v2/dacctl/actions_impl/persistent_test.go | 2 +- .../pkg/v2/dacctl/actions_impl/show_test.go | 12 +-- .../pkg/v2/dacctl/workflow_impl/session.go | 10 +- .../pkg/v2/registry_impl/session_actions.go | 4 +- 10 files changed, 120 insertions(+), 82 deletions(-) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index 8f8d87c4..ba55763e 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -1,36 +1,33 @@ package main import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/etcdregistry" + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store_impl" "github.com/urfave/cli" ) -var testKeystore keystoreregistry.Keystore +var testKeystore store.Keystore var testDisk fileio.Disk -var testActions actions.DacctlActions +var testActions dacctl.DacctlActions -func getKeystore() keystoreregistry.Keystore { +func getKeystore() store.Keystore { // TODO must be a better way to test this, proper factory? if testKeystore != nil { return testKeystore } - return etcdregistry.NewKeystore() + return store_impl.NewKeystore() } -func getActions(keystore keystoreregistry.Keystore) actions.DacctlActions { +func getActions(keystore store.Keystore) dacctl.DacctlActions { if testActions != nil { return testActions } - volReg := keystoreregistry.NewVolumeRegistry(keystore) - poolReg := keystoreregistry.NewPoolRegistry(keystore) - disk := testDisk - if testDisk == nil { - disk = fileio.NewDisk() - } - return actions.NewDacctlActions(poolReg, volReg, disk) + disk := fileio.NewDisk() + return actions_impl.NewDacctlActions(keystore, disk) } func createPersistent(c *cli.Context) error { @@ -39,28 +36,37 @@ func createPersistent(c *cli.Context) error { return getActions(keystore).CreatePersistentBuffer(c) } +func printOutput(function func() (string, error)) error { + sessions, err := function() + if err != nil { + return err + } + fmt.Println(sessions) + return nil +} + func showInstances(_ *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).ShowInstances() + return printOutput(getActions(keystore).ShowInstances) } func showSessions(_ *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).ShowSessions() + return printOutput(getActions(keystore).ShowSessions) } func listPools(_ *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).ListPools() + return printOutput(getActions(keystore).ListPools) } func showConfigurations(_ *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).ShowConfigurations() + return printOutput(getActions(keystore).ShowConfigurations) } func teardown(c *cli.Context) error { @@ -84,7 +90,9 @@ func setup(c *cli.Context) error { func realSize(c *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).RealSize(c) + return printOutput(func() (s string, e error) { + return getActions(keystore).RealSize(c) + }) } func dataIn(c *cli.Context) error { diff --git a/cmd/dacctl/main.go b/cmd/dacctl/main.go index 94cd528d..3bfc0e17 100644 --- a/cmd/dacctl/main.go +++ b/cmd/dacctl/main.go @@ -172,7 +172,7 @@ func main() { } f, err := os.OpenFile(logFilename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - log.Fatalf("error opening file: %v", err) + log.Fatalf("please use DACCTL_LOG to configure an alternative, as error opening file: %v ", err) } defer f.Close() diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index e5ae4d96..34693bd8 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -4,9 +4,10 @@ import ( "context" "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/stretchr/testify/assert" + "log" "strings" "testing" ) @@ -125,7 +126,7 @@ func TestFlow(t *testing.T) { assert.Equal(t, "RealSize", err.Error()) err = runCli([]string{"--function", "data_in", "--token", "a", "--job", "b"}) - assert.Equal(t, "CopyDataIn", err.Error()) + assert.Equal(t, "DataIn", err.Error()) err = runCli([]string{"--function", "paths", "--token", "a", "--job", "b", "--pathfile", "c"}) assert.Equal(t, "Paths", err.Error()) @@ -137,89 +138,110 @@ func TestFlow(t *testing.T) { assert.Equal(t, "PostRun", err.Error()) err = runCli([]string{"--function", "data_out", "--token", "a", "--job", "b"}) - assert.Equal(t, "CopyDataOut", err.Error()) + assert.Equal(t, "DataOut", err.Error()) } type stubKeystore struct{} func (*stubKeystore) Close() error { + log.Println("closed") return nil } -func (*stubKeystore) CleanPrefix(prefix string) error { - panic("implement me") -} -func (*stubKeystore) Add(keyValues []keystoreregistry.KeyValue) error { + +func (*stubKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { panic("implement me") } -func (*stubKeystore) Update(keyValues []keystoreregistry.KeyValueVersion) error { + +func (*stubKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { panic("implement me") } -func (*stubKeystore) DeleteAll(keyValues []keystoreregistry.KeyValueVersion) error { + +func (*stubKeystore) Delete(key string, modRevision int64) error { panic("implement me") } -func (*stubKeystore) GetAll(prefix string) ([]keystoreregistry.KeyValueVersion, error) { + +func (*stubKeystore) DeleteAllKeysWithPrefix(keyPrefix string) (int64, error) { panic("implement me") } -func (*stubKeystore) Get(key string) (keystoreregistry.KeyValueVersion, error) { + +func (*stubKeystore) GetAll(keyPrefix string) ([]store.KeyValueVersion, error) { panic("implement me") } -func (*stubKeystore) WatchPrefix(prefix string, onUpdate func(old *keystoreregistry.KeyValueVersion, new *keystoreregistry.KeyValueVersion)) { + +func (*stubKeystore) Get(key string) (store.KeyValueVersion, error) { panic("implement me") } -func (*stubKeystore) WatchKey(ctxt context.Context, key string, onUpdate func(old *keystoreregistry.KeyValueVersion, new *keystoreregistry.KeyValueVersion)) { + +func (*stubKeystore) IsExist(key string) (bool, error) { panic("implement me") } -func (*stubKeystore) KeepAliveKey(key string) error { + +func (*stubKeystore) Watch(ctxt context.Context, key string, withPrefix bool) store.KeyValueUpdateChan { panic("implement me") } -func (*stubKeystore) NewMutex(lockKey string) (keystoreregistry.Mutex, error) { + +func (*stubKeystore) KeepAliveKey(ctxt context.Context, key string) error { panic("implement me") } -func (*stubKeystore) Watch(ctxt context.Context, key string, withPrefix bool) keystoreregistry.KeyValueUpdateChan { + +func (*stubKeystore) NewMutex(lockKey string) (store.Mutex, error) { panic("implement me") } type stubDacctlActions struct{} -func (*stubDacctlActions) CreatePersistentBuffer(c actions.CliContext) error { +func (*stubDacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { return fmt.Errorf("CreatePersistentBuffer %s", c.String("token")) } -func (*stubDacctlActions) DeleteBuffer(c actions.CliContext) error { + +func (*stubDacctlActions) DeleteBuffer(c dacctl.CliContext) error { return fmt.Errorf("DeleteBuffer %s", c.String("token")) } -func (*stubDacctlActions) CreatePerJobBuffer(c actions.CliContext) error { + +func (*stubDacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { return errors.New("CreatePerJobBuffer") } -func (*stubDacctlActions) ShowInstances() error { - return errors.New("ShowInstances") + +func (*stubDacctlActions) ShowInstances() (string, error) { + return "", errors.New("ShowInstances") } -func (*stubDacctlActions) ShowSessions() error { - return errors.New("ShowSessions") + +func (*stubDacctlActions) ShowSessions() (string, error) { + return "", errors.New("ShowSessions") } -func (*stubDacctlActions) ListPools() error { - return errors.New("ListPools") + +func (*stubDacctlActions) ListPools() (string, error) { + return "", errors.New("ListPools") } -func (*stubDacctlActions) ShowConfigurations() error { - return errors.New("ShowConfigurations") + +func (*stubDacctlActions) ShowConfigurations() (string, error) { + return "", errors.New("ShowConfigurations") } -func (*stubDacctlActions) ValidateJob(c actions.CliContext) error { + +func (*stubDacctlActions) ValidateJob(c dacctl.CliContext) error { return errors.New("ValidateJob") } -func (*stubDacctlActions) RealSize(c actions.CliContext) error { - return errors.New("RealSize") + +func (*stubDacctlActions) RealSize(c dacctl.CliContext) (string, error) { + return "", errors.New("RealSize") } -func (*stubDacctlActions) DataIn(c actions.CliContext) error { - return errors.New("CopyDataIn") + +func (*stubDacctlActions) DataIn(c dacctl.CliContext) error { + return errors.New("DataIn") } -func (*stubDacctlActions) Paths(c actions.CliContext) error { + +func (*stubDacctlActions) Paths(c dacctl.CliContext) error { return errors.New("Paths") } -func (*stubDacctlActions) PreRun(c actions.CliContext) error { + +func (*stubDacctlActions) PreRun(c dacctl.CliContext) error { return errors.New("PreRun") } -func (*stubDacctlActions) PostRun(c actions.CliContext) error { + +func (*stubDacctlActions) PostRun(c dacctl.CliContext) error { return errors.New("PostRun") } -func (*stubDacctlActions) DataOut(c actions.CliContext) error { - return errors.New("CopyDataOut") + +func (*stubDacctlActions) DataOut(c dacctl.CliContext) error { + return errors.New("DataOut") } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index baf901d9..1c83f49e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -6,15 +6,17 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/workflow_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" "strings" ) -func NewDacctlActions(actions facade.Session, disk fileio.Disk) dacctl.DacctlActions { +func NewDacctlActions(keystore store.Keystore, disk fileio.Disk) dacctl.DacctlActions { return &dacctlActions{ - session: actions, + session: workflow_impl.NewSessionFacade(keystore), disk: disk, } } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index a6f9838e..a58b44c7 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -56,7 +56,7 @@ func TestDacctlActions_DeleteBuffer(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().DeleteSession(datamodel.SessionName("bar"), true).Return(fakeError) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} err := actions.DeleteBuffer(&mockCliContext{ strings: map[string]string{"token": "bar"}, booleans: map[string]bool{"hurry": true}, @@ -76,7 +76,7 @@ func TestDacctlActions_DataIn(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().CopyDataIn(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} err := actions.DataIn(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -100,7 +100,7 @@ func TestDacctlActions_DataOut(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().CopyDataOut(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} err := actions.DataOut(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -124,7 +124,7 @@ func TestDacctlActions_PreRun(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().Mount(datamodel.SessionName("bar"), computeHosts, loginHosts).Return(fakeError) - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -153,7 +153,7 @@ func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().Mount(datamodel.SessionName("bar"), computeHosts, nil).Return(fakeError) - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -172,7 +172,7 @@ func TestDacctlActions_PreRun_BadHosts(t *testing.T) { computeHosts := []string{"host1", "host/2"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) - actions := NewDacctlActions(nil, disk) + actions := dacctlActions{disk: disk} err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -194,7 +194,7 @@ func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) disk.EXPECT().Lines("loginhostfile").Return(loginHosts, nil) - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.PreRun(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -232,7 +232,7 @@ func TestDacctlActions_PostRun(t *testing.T) { fakeError := errors.New("fake") session.EXPECT().Unmount(datamodel.SessionName("bar")).Return(fakeError) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} err := actions.PostRun(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 5d100598..587bf595 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -17,7 +17,7 @@ func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { lines := []string{`#DW bad cmd`} disk.EXPECT().Lines("jobfile").Return(lines, nil) - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.ValidateJob(&mockCliContext{ strings: map[string]string{ "job": "jobfile", @@ -87,7 +87,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { }).Return(nil) fakeTime = 123 - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.CreatePerJobBuffer(getMockCliContext(2)) assert.Nil(t, err) diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go index c02a2012..240fb733 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go @@ -27,7 +27,7 @@ func TestDacctlActions_CreatePersistentBuffer(t *testing.T) { }).Return(nil) fakeTime = 123 - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} err := actions.CreatePersistentBuffer(getMockCliContext(2)) assert.Nil(t, err) diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index daa2de40..f1004cc1 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -19,7 +19,7 @@ func TestDacctlActions_RealSize(t *testing.T) { ActualSizeBytes: 123, }, nil) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} output, err := actions.RealSize(&mockCliContext{ strings: map[string]string{"token": "bar"}, }) @@ -45,7 +45,7 @@ func TestDacctlActions_Paths(t *testing.T) { }, nil) disk.EXPECT().Write("paths", []string{"foo1=bar1"}) - actions := NewDacctlActions(session, disk) + actions := dacctlActions{session: session, disk: disk} err := actions.Paths(&mockCliContext{ strings: map[string]string{ "token": "bar", @@ -83,7 +83,7 @@ func TestDacctlActions_ShowInstances(t *testing.T) { ActualSizeBytes: 456, }, }, nil) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} output, err := actions.ShowInstances() @@ -120,7 +120,7 @@ func TestDacctlActions_ShowSessions(t *testing.T) { CreatedAt: 5678, }, }, nil) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} output, err := actions.ShowSessions() @@ -162,7 +162,7 @@ func TestDacctlActions_ListPools(t *testing.T) { }, }, }, nil) - actions := NewDacctlActions(session, nil) + actions := dacctlActions{session: session} output, err := actions.ListPools() assert.Nil(t, err) @@ -182,7 +182,7 @@ func TestDacctlActions_ListPools(t *testing.T) { } func TestDacctlActions_ShowConfigurations(t *testing.T) { - actions := NewDacctlActions(nil, nil) + actions := dacctlActions{} output, err := actions.ShowConfigurations() assert.Nil(t, err) assert.Equal(t, `{"configurations":[]}`, output) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index f86e9d67..2bae7d5c 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -6,14 +6,20 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" "math" "math/rand" "time" ) -func NewSessionFacade() facade.Session { - return sessionFacade{} +func NewSessionFacade(keystore store.Keystore) facade.Session { + return sessionFacade{ + session: registry_impl.NewSessionRegistry(keystore), + actions: registry_impl.NewSessionActionsRegistry(keystore), + allocations: registry_impl.NewAllocationRegistry(keystore), + } } type sessionFacade struct { diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index bbdd3da4..becff228 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -12,9 +12,9 @@ import ( "log" ) -func NewSessionActionsRegistry(store store.Keystore, brickHostRegistry registry.BrickHostRegistry) registry.SessionActions { +func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { // TODO: create brickHostRegistry - return &sessionActions{store, brickHostRegistry} + return &sessionActions{store, NewBrickHostRegistry(store)} } type sessionActions struct { From 12b0cf265a5b6e5912c65aa33dc5a24e2aecd80a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 16:39:20 +0100 Subject: [PATCH 097/191] Skip requiring the key to exist if no version is given --- internal/pkg/v2/store_impl/keystore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go index 329afcda..8e4e4a56 100644 --- a/internal/pkg/v2/store_impl/keystore.go +++ b/internal/pkg/v2/store_impl/keystore.go @@ -138,8 +138,8 @@ func (client *etcKeystore) Update(key string, value []byte, modRevision int64) ( var ifOps []clientv3.Cmp var thenOps []clientv3.Op - ifOps = append(ifOps, clientv3util.KeyExists(key)) if modRevision > 0 { + ifOps = append(ifOps, clientv3util.KeyExists(key)) checkModRev := clientv3.Compare(clientv3.ModRevision(key), "=", modRevision) ifOps = append(ifOps, checkModRev) } From 70187259b6b8d018d32ad36d7f718bc329a5ca02 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 16:39:43 +0100 Subject: [PATCH 098/191] Fix up concrete chain of constructors --- .../dacd/brick_manager_impl/brick_manager.go | 8 ++--- .../session_action_handler.go | 32 +++++++++++-------- .../session_action_handler_test.go | 8 ++--- internal/pkg/v2/filesystem_impl/provider.go | 7 ++-- .../pkg/v2/registry_impl/brick_allocation.go | 7 ++-- internal/pkg/v2/registry_impl/brick_host.go | 13 ++++---- .../pkg/v2/registry_impl/session_actions.go | 1 - 7 files changed, 41 insertions(+), 35 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index 4cfd4801..fd61c9d2 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -6,17 +6,17 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) func NewBrickManager(keystore store.Keystore) dacd.BrickManager { - // TODO: call concrete options return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), - brickRegistry: nil, - sessionActions: nil, - sessionActionHandler: nil, + brickRegistry: registry_impl.NewBrickHostRegistry(keystore), + sessionActions: registry_impl.NewSessionActionsRegistry(keystore), + sessionActionHandler: NewSessionActionHandler(keystore), } } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index b49da16c..c27294a4 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -5,20 +5,28 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) -func NewSessionActionHandler(actions registry.SessionActions) facade.SessionActionHandler { - return &sessionActionHandler{actions: actions} +func NewSessionActionHandler(keystore store.Keystore) facade.SessionActionHandler { + return &sessionActionHandler{ + registry_impl.NewSessionRegistry(keystore), + registry_impl.NewSessionActionsRegistry(keystore), + // TODO: fix up fsprovider!! + filesystem_impl.NewFileSystemProvider(nil), + false, + } } type sessionActionHandler struct { - registry registry.SessionRegistry - actions registry.SessionActions - fsProvider filesystem.Provider - skipActions bool - actionCalled datamodel.SessionActionType + sessionRegistry registry.SessionRegistry + actions registry.SessionActions + fsProvider filesystem.Provider + skipActions bool } func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { @@ -26,12 +34,10 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi switch action.ActionType { case datamodel.SessionDelete: // TODO... must test this better! - s.actionCalled = datamodel.SessionDelete if !s.skipActions { go s.handleDelete(action) } case datamodel.SessionCreateFilesystem: - s.actionCalled = datamodel.SessionCreateFilesystem if !s.skipActions { go s.handleCreate(action) } @@ -42,7 +48,7 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { sessionName := action.Session.Name - sessionMutex, err := s.registry.GetSessionMutex(sessionName) + sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) action.Error = err @@ -64,14 +70,14 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { log.Printf("starting create for %+v\n", sessionName) // Get latest session now we have the mutex - session, err := s.registry.GetSession(sessionName) + session, err := s.sessionRegistry.GetSession(sessionName) fsStatus, err := s.fsProvider.Create(action.Session) session.FilesystemStatus = fsStatus session.Status.FileSystemCreated = err == nil session.Status.Error = err - session, err = s.registry.UpdateSession(session) + session, err = s.sessionRegistry.UpdateSession(session) if err != nil { log.Printf("Failed to update session: %+v", session) action.Error = err @@ -93,6 +99,6 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { log.Println("delete") // TODO: clearly need mutex here, etc - s.registry.DeleteSession(action.Session) + s.sessionRegistry.DeleteSession(action.Session) s.actions.CompleteSessionAction(action) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index d8e064a3..5242ed51 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -28,8 +28,6 @@ func TestSessionActionHandler_ProcessSessionAction_Create(t *testing.T) { handler := sessionActionHandler{skipActions: true} handler.ProcessSessionAction(action) - - assert.Equal(t, datamodel.SessionCreateFilesystem, handler.actionCalled) } func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { @@ -39,8 +37,6 @@ func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { handler := sessionActionHandler{skipActions: true} handler.ProcessSessionAction(action) - - assert.Equal(t, datamodel.SessionDelete, handler.actionCalled) } func TestSessionActionHandler_handleCreate(t *testing.T) { @@ -50,7 +46,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { actions := mock_registry.NewMockSessionActions(mockCtrl) fsProvider := mock_filesystem.NewMockProvider(mockCtrl) handler := sessionActionHandler{ - registry: registry, actions: actions, fsProvider: fsProvider, + sessionRegistry: registry, actions: actions, fsProvider: fsProvider, } action := datamodel.SessionAction{ ActionType: datamodel.SessionCreateFilesystem, @@ -81,7 +77,7 @@ func TestSessionActionHandler_handleDelete(t *testing.T) { defer mockCtrl.Finish() actions := mock_registry.NewMockSessionActions(mockCtrl) registry := mock_registry.NewMockSessionRegistry(mockCtrl) - handler := sessionActionHandler{actions: actions, registry: registry} + handler := sessionActionHandler{actions: actions, sessionRegistry: registry} action := datamodel.SessionAction{ ActionType: datamodel.SessionDelete, } diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index 8409a625..a31e0c6c 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -3,6 +3,7 @@ package filesystem_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + "log" ) func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { @@ -14,11 +15,13 @@ type fileSystemProvider struct { } func (f *fileSystemProvider) Create(session datamodel.Session) (datamodel.FilesystemStatus, error) { - panic("implement me") + log.Println("FAKE Create") + return datamodel.FilesystemStatus{}, nil } func (f *fileSystemProvider) Delete(session datamodel.Session) error { - panic("implement me") + log.Println("FAKE Delete") + return nil } func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index dcae74ee..12413f74 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -10,9 +10,10 @@ import ( "log" ) -func NewAllocationRegistry(store store.Keystore) registry.AllocationRegistry { - // TODO: create brickHostRegistry - return &allocationRegistry{store, nil, nil} +func NewAllocationRegistry(keystore store.Keystore) registry.AllocationRegistry { + return &allocationRegistry{ + keystore, NewBrickHostRegistry(keystore), NewSessionRegistry(keystore), + } } type allocationRegistry struct { diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index c37ddf74..efb84d83 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -11,14 +11,12 @@ import ( "log" ) -func NewBrickHostRegistry(store store.Keystore) registry.BrickHostRegistry { - // TODO: create AllocationRegistry - return &brickHostRegistry{store, nil} +func NewBrickHostRegistry(keystore store.Keystore) registry.BrickHostRegistry { + return &brickHostRegistry{keystore} } type brickHostRegistry struct { - store store.Keystore - allocation registry.AllocationRegistry + store store.Keystore } const brickHostPrefix = "/BrickHostStore/" @@ -47,9 +45,12 @@ func (b *brickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) e } } + // TODO: odd dependencies here!! + allocationRegistry := NewAllocationRegistry(b.store) + // Check existing pools match what this brick host is reporting for poolName, granularityGiB := range poolGranularityGiBMap { - _, err := b.allocation.EnsurePoolCreated(poolName, parsers.GetBytes(granularityGiB, "GiB")) + _, err := allocationRegistry.EnsurePoolCreated(poolName, parsers.GetBytes(granularityGiB, "GiB")) if err != nil { return fmt.Errorf("unable to create pool due to: %s", err) } diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index becff228..92d5b800 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -13,7 +13,6 @@ import ( ) func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { - // TODO: create brickHostRegistry return &sessionActions{store, NewBrickHostRegistry(store)} } From 752321ec4e514a3fa3d0fa15464d4793c31c1a35 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 17:15:41 +0100 Subject: [PATCH 099/191] Make sure we report the hostname correctly --- .../pkg/v2/dacd/brick_manager_impl/brick_manager_test.go | 4 +++- internal/pkg/v2/dacd/config/config.go | 9 +++++---- internal/pkg/v2/store_impl/keystore.go | 6 +----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index e3254b16..b5dce218 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -8,6 +8,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "os" "testing" ) @@ -30,7 +31,8 @@ func TestBrickManager_Startup(t *testing.T) { // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) sessionActions.EXPECT().GetSessionActionRequests(context.TODO(), gomock.Any()) - brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName("hostname")) + hostname, _ := os.Hostname() + brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName(hostname)) brickManager.Startup() } diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index e52acf55..d265c1db 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -3,6 +3,7 @@ package config import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" + "os" "strconv" ) @@ -79,13 +80,13 @@ func getBool(env ReadEnvironemnt, key string, defaultVal bool) bool { type systemEnv struct{} func (systemEnv) LookupEnv(key string) (string, bool) { - // TODO return os.LookupEnv(key) - return "", false + return os.LookupEnv(key) + //return "", false } func (systemEnv) Hostname() (string, error) { - // TODO return os.BrickHostName() - return "hostname", nil + return os.Hostname() + //return "hostname", nil } var DefaultEnv ReadEnvironemnt = systemEnv{} diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go index 8e4e4a56..daace378 100644 --- a/internal/pkg/v2/store_impl/keystore.go +++ b/internal/pkg/v2/store_impl/keystore.go @@ -191,10 +191,6 @@ func (client *etcKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error response, err := kvc.Get(context.Background(), prefix, clientv3.WithPrefix()) handleError(err) - if response.Count == 0 { - return []store.KeyValueVersion{}, - fmt.Errorf("unable to find any values for prefix: %s", prefix) - } var values []store.KeyValueVersion for _, rawKeyValue := range response.Kvs { values = append(values, *getKeyValueVersion(rawKeyValue)) @@ -242,7 +238,7 @@ func (client *etcKeystore) KeepAliveKey(ctxt context.Context, key string) error Commit() handleError(err) if !txnResponse.Succeeded { - return fmt.Errorf("unable to create keep-alive key: %s", key) + return fmt.Errorf("unable to create keep-alive key %s due to: %+v", key, txnResponse.Responses) } ch, err := client.Client.KeepAlive(ctxt, leaseID) From 55e74ea613820703896d568d6bc2d3c6cecbd39f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 17:30:37 +0100 Subject: [PATCH 100/191] Fix up config parser --- internal/pkg/v2/dacd/config/config.go | 2 +- internal/pkg/v2/dacd/config/config_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 internal/pkg/v2/dacd/config/config_test.go diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index d265c1db..4c8bdb7e 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -26,8 +26,8 @@ func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { config := BrickManagerConfig{ datamodel.BrickHostName(getHostname(env)), datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), - getUint(env, "DAC_BRICK_COUNT", 12), getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), + getUint(env, "DAC_BRICK_COUNT", 12), getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), getBool(env, "DAC_HOST_ENABLED", true), } diff --git a/internal/pkg/v2/dacd/config/config_test.go b/internal/pkg/v2/dacd/config/config_test.go new file mode 100644 index 00000000..ccfbd7c6 --- /dev/null +++ b/internal/pkg/v2/dacd/config/config_test.go @@ -0,0 +1,20 @@ +package config + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestGetBrickManagerConfig(t *testing.T) { + config := GetBrickManagerConfig(DefaultEnv) + + hostname, _ := os.Hostname() + assert.Equal(t, datamodel.BrickHostName(hostname), config.BrickHostName) + assert.Equal(t, uint(12), config.DeviceCount) + assert.Equal(t, datamodel.PoolName("default"), config.PoolName) + assert.Equal(t, true, config.HostEnabled) + assert.Equal(t, "nvme%dn1", config.DeviceAddressPattern) + assert.Equal(t, uint(1400), config.DeviceCapacityGiB) +} From 13530c32ffae82b59bc715689187795b30f93f7e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 17:35:15 +0100 Subject: [PATCH 101/191] Tidy up lock name for session --- internal/pkg/v2/registry_impl/brick_allocation.go | 2 +- internal/pkg/v2/registry_impl/session.go | 3 +-- internal/pkg/v2/registry_impl/session_test.go | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 12413f74..39930e30 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -23,7 +23,7 @@ type allocationRegistry struct { } const poolPrefix = "/Pool/" -const allocationLockKey = "/LockAllocation/" +const allocationLockKey = "LockAllocation" func (a *allocationRegistry) GetAllocationMutex() (store.Mutex, error) { return a.store.NewMutex(allocationLockKey) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index cabba108..30d1680d 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -20,8 +20,7 @@ type sessionRegistry struct { func (s *sessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { sessionKey := getSessionKey(sessionName) - lockKey := fmt.Sprintf("/lock%s", sessionKey) - return s.store.NewMutex(lockKey) + return s.store.NewMutex(sessionKey) } const sessionPrefix = "/session/" diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index bdcaec0a..239519de 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -19,7 +19,7 @@ func TestSessionRegistry_GetSessionMutex(t *testing.T) { keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) fakeErr := errors.New("fake") - keystore.EXPECT().NewMutex("/lock/session/foo").Return(nil, fakeErr) + keystore.EXPECT().NewMutex("/session/foo").Return(nil, fakeErr) mutex, err := registry.GetSessionMutex("foo") assert.Nil(t, mutex) From b63f10c92a5d12fb5850c9d3613f4fc928f23615 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 17:55:01 +0100 Subject: [PATCH 102/191] Allow uuid in keys so session action can be stored --- .../dacctl/actions_impl/parsers/hostnames.go | 3 +- .../pkg/v2/registry_impl/session_actions.go | 3 +- .../v2/registry_impl/session_actions_test.go | 31 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go index 86a4ac30..86e8fa84 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go @@ -6,7 +6,8 @@ import ( "regexp" ) -var nameRegex = regexp.MustCompile("^[a-zA-Z0-9.]+$") +// TODO: allowed "-" so uuid passes validation +var nameRegex = regexp.MustCompile("^[a-zA-Z0-9.-]+$") func IsValidName(name string) bool { return nameRegex.Match([]byte(name)) diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index 92d5b800..3d885c82 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -50,7 +50,7 @@ func getSessionActionResponseHostPrefix(brickHost datamodel.BrickHostName) strin func getSessionActionResponseKey(action datamodel.SessionAction) string { hostPrefix := getSessionActionResponseHostPrefix(action.Session.PrimaryBrickHost) if !parsers.IsValidName(action.Uuid) { - log.Panicf("invalid session action uuid") + log.Panicf("invalid session action uuid %s", action.Uuid) } return fmt.Sprintf("%s%s", hostPrefix, action.Uuid) } @@ -119,6 +119,7 @@ func (s *sessionActions) SendSessionAction( log.Printf("completed waiting for action response %+v\n", sessionAction) return } + // TODO: don't we need to stop the watch above? }() return responseChan, nil } diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/v2/registry_impl/session_actions_test.go index 4864aad6..55570299 100644 --- a/internal/pkg/v2/registry_impl/session_actions_test.go +++ b/internal/pkg/v2/registry_impl/session_actions_test.go @@ -1 +1,32 @@ package registry_impl + +import ( + "context" + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSessionActions_SendSessionAction(t *testing.T) { + // TODO: need way more testing here + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + brickHost := mock_registry.NewMockBrickHostRegistry(mockCtrl) + keystore := mock_store.NewMockKeystore(mockCtrl) + actions := sessionActions{brickHostRegistry: brickHost, store: keystore} + session := datamodel.Session{Name:"foo", PrimaryBrickHost:"host1"} + brickHost.EXPECT().IsBrickHostAlive(session.PrimaryBrickHost).Return(true, nil) + keystore.EXPECT().Watch(context.TODO(), gomock.Any(), false).Return(nil) + fakeErr := errors.New("fake") + keystore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(store.KeyValueVersion{}, fakeErr) + + channel, err := actions.SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, session) + + assert.Nil(t, channel) + assert.Equal(t, "unable to send session action due to: fake", err.Error()) +} From b6187ac979fa8112135e0bdb457817ff0fe5d016 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 17:55:46 +0100 Subject: [PATCH 103/191] Fix format issue --- internal/pkg/v2/registry_impl/session_actions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/v2/registry_impl/session_actions_test.go index 55570299..8239e8b3 100644 --- a/internal/pkg/v2/registry_impl/session_actions_test.go +++ b/internal/pkg/v2/registry_impl/session_actions_test.go @@ -19,7 +19,7 @@ func TestSessionActions_SendSessionAction(t *testing.T) { brickHost := mock_registry.NewMockBrickHostRegistry(mockCtrl) keystore := mock_store.NewMockKeystore(mockCtrl) actions := sessionActions{brickHostRegistry: brickHost, store: keystore} - session := datamodel.Session{Name:"foo", PrimaryBrickHost:"host1"} + session := datamodel.Session{Name: "foo", PrimaryBrickHost: "host1"} brickHost.EXPECT().IsBrickHostAlive(session.PrimaryBrickHost).Return(true, nil) keystore.EXPECT().Watch(context.TODO(), gomock.Any(), false).Return(nil) fakeErr := errors.New("fake") From e3dff582e70b297645e57f634835e01aaba931b4 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 18:13:17 +0100 Subject: [PATCH 104/191] Remove left over debug logging --- internal/pkg/v2/registry_impl/brick_allocation.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 39930e30..9f917a1a 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -135,8 +135,6 @@ func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { for _, brickHost := range brickHosts { for _, brick := range brickHost.Bricks { - log.Println(brick) - // Check if in allocated list allocated := false for _, allocatedDevice := range allocatedDevicesByBrickHost[brick.BrickHostName] { if allocatedDevice == brick.Device { From dc40cc8e8e039cb61baa86eee16141664ccf5d36 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 21:24:23 +0100 Subject: [PATCH 105/191] Correctly report success for create_persistent --- cmd/dacctl/actions.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index ba55763e..ffd493d1 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -33,7 +33,12 @@ func getActions(keystore store.Keystore) dacctl.DacctlActions { func createPersistent(c *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).CreatePersistentBuffer(c) + err := getActions(keystore).CreatePersistentBuffer(c) + if err == nil { + // Slurm is looking for the string "created" to know this worked + fmt.Printf("created %s\n", c.String("token")) + } + return err } func printOutput(function func() (string, error)) error { From c69cce3f25606aa4163a6884c5b8ad61f2f6613d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 22:12:47 +0100 Subject: [PATCH 106/191] Make workflow_impl more DRY --- .../pkg/v2/dacctl/workflow_impl/session.go | 130 ++++++++---------- internal/pkg/v2/filesystem_impl/provider.go | 17 ++- 2 files changed, 66 insertions(+), 81 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 2bae7d5c..087cc0bb 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -28,50 +28,64 @@ type sessionFacade struct { allocations registry.AllocationRegistry } -func (s sessionFacade) CreateSession(session datamodel.Session) error { - err := s.validateSession(session) - if err != nil { - return err - } +func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType datamodel.SessionActionType, + getSession func() (datamodel.Session, error)) error { - // Get session lock - sessionMutex, err := s.session.GetSessionMutex(session.Name) + sessionMutex, err := s.session.GetSessionMutex(sessionName) if err != nil { - return fmt.Errorf("unable to get session mutex: %s due to: %s", session.Name, err) + return fmt.Errorf("unable to get session mutex: %s due to: %s", sessionName, err) } err = sessionMutex.Lock(context.TODO()) if err != nil { - return fmt.Errorf("unable to lock session mutex: %s due to: %s", session.Name, err) + return fmt.Errorf("unable to lock session mutex: %s due to: %s", sessionName, err) } - // Allocate bricks, and choose brick host server - session, err = s.doAllocationAndWriteSession(session) - // if no bricks allocated, no need to call CreateSessionVolume - if err != nil || session.ActualSizeBytes == 0 { + session, err := getSession() + if err != nil { sessionMutex.Unlock(context.TODO()) return err } + if session.Name == "" && err == nil { + // skip processing for this session + // e.g. its a delete and we have already been deleted + sessionMutex.Unlock(context.TODO()) + return nil + } - // Create filesystem on the brick host server - // TODO: add timeout - eventChan, createErr := s.actions.SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, session) - - // Drop mutex so the server can take it - err = sessionMutex.Unlock(context.TODO()) + // This will error out if the host is not currently up + sessionAction, err := s.actions.SendSessionAction(context.TODO(), actionType, session) + mutexErr := sessionMutex.Unlock(context.TODO()) if err != nil { - // TODO: cancel above action? return err } + if mutexErr != nil { + return mutexErr + } - // Always drop the mutex, but check error before checking the channel - if createErr != nil { - // TODO: session should go into Error State? Maybe its up the above call in this case? - return createErr + // wait for server to complete, or timeout + result := <-sessionAction + return result.Error +} + +func (s sessionFacade) CreateSession(session datamodel.Session) error { + err := s.validateSession(session) + if err != nil { + return err } - // Wait for the server to create the filesystem - sessionAction := <-eventChan - return sessionAction.Error + return s.submitJob(session.Name, datamodel.SessionCreateFilesystem, + func() (datamodel.Session, error) { + // Allocate bricks, and choose brick host server + session, err := s.doAllocationAndWriteSession(session) + if err != nil { + return session, err + } + if session.ActualSizeBytes == 0 { + // Skip creating an empty filesystem + return datamodel.Session{}, nil + } + return session, nil + }) } func (s sessionFacade) validateSession(session datamodel.Session) error { @@ -172,56 +186,20 @@ func pickBricks(bricksRequired int, poolInfo datamodel.PoolInfo) []datamodel.Bri } func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bool) error { - // Get session lock - sessionMutex, err := s.session.GetSessionMutex(sessionName) - if err != nil { - return fmt.Errorf("unable to get session mutex: %s due to: %s", sessionName, err) - } - err = sessionMutex.Lock(context.TODO()) - if err != nil { - return fmt.Errorf("unable to lock session mutex: %s due to: %s", sessionName, err) - } - - session, err := s.session.GetSession(sessionName) - if err != nil { - log.Println("Unable to find session:", sessionName) - sessionMutex.Unlock(context.TODO()) - return nil - } - - // Record we want this deleted, in case host is not alive - // can be deleted when it is next stated - session.Status.DeleteRequested = true - session.Status.DeleteSkipCopyDataOut = hurry - session, err = s.session.UpdateSession(session) - if err != nil { - sessionMutex.Unlock(context.TODO()) - return err - } - // TODO: send the hurry, i.e. request data copy out first - - if session.PrimaryBrickHost == "" { - // TODO: session has no primary brick host - // so need to do local umount - - } - - // This will error out if the host is not currently up - sessionAction, err := s.actions.SendSessionAction(context.TODO(), datamodel.SessionDelete, session) - if err != nil { - return err - } - - // Drop mutex to allow server to lock the session - err = sessionMutex.Unlock(context.TODO()) - if err != nil { - // TODO: cancel above waiting around? - return err - } + return s.submitJob(sessionName, datamodel.SessionDelete, + func() (datamodel.Session, error) { + session, err := s.session.GetSession(sessionName) + if err != nil { + log.Println("Unable to find session, skipping delete:", sessionName) + return session, nil + } - // wait for server to complete, or timeout - result := <-sessionAction - return result.Error + // Record we want this deleted, in case host is not alive + // can be deleted when it is next stated + session.Status.DeleteRequested = true + session.Status.DeleteSkipCopyDataOut = hurry + return s.session.UpdateSession(session) + }) } func (s sessionFacade) CopyDataIn(sessionName datamodel.SessionName) error { diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index a31e0c6c..d815e4e9 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -16,7 +16,7 @@ type fileSystemProvider struct { func (f *fileSystemProvider) Create(session datamodel.Session) (datamodel.FilesystemStatus, error) { log.Println("FAKE Create") - return datamodel.FilesystemStatus{}, nil + return datamodel.FilesystemStatus{InternalName: "foo", InternalData: "bar"}, nil } func (f *fileSystemProvider) Delete(session datamodel.Session) error { @@ -25,17 +25,24 @@ func (f *fileSystemProvider) Delete(session datamodel.Session) error { } func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { - panic("implement me") + log.Println("FAKE DataCopyIn") + return nil + } func (f *fileSystemProvider) DataCopyOut(session datamodel.Session) error { - panic("implement me") + log.Println("FAKE DataCopyOut") + return nil + } func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { - panic("implement me") + log.Println("FAKE Mount") + return datamodel.AttachmentSessionStatus{} + } func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { - panic("implement me") + log.Println("FAKE Unmount") + return datamodel.AttachmentSessionStatus{} } From 63d12399c060447943db3016b93f5720d3a67c36 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 22:20:18 +0100 Subject: [PATCH 107/191] Add test of slurm accounting --- tools/slurm-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/slurm-test.sh b/tools/slurm-test.sh index 79787e07..7bba5e9a 100755 --- a/tools/slurm-test.sh +++ b/tools/slurm-test.sh @@ -81,3 +81,4 @@ sleep $SLEEP_INTERVAL scontrol show burstbuffer squeue scontrol show bbstat +sacct -o jobid'%10',reqtres'%45',alloctres'%45' -X From 69fdcfa9cdbdf37b367aa33d7ebc1ba604460991 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 23:02:20 +0100 Subject: [PATCH 108/191] Try wire up all actions to filesystem_impl --- .../pkg/v2/dacctl/workflow_impl/session.go | 34 +++++++--- .../session_action_handler.go | 65 +++++++++++++++++-- .../session_action_handler_test.go | 16 ----- 3 files changed, 86 insertions(+), 29 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 087cc0bb..73eaa924 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -93,6 +93,7 @@ func (s sessionFacade) validateSession(session datamodel.Session) error { if err != nil { return fmt.Errorf("invalid session, unable to find pool %s", session.VolumeRequest.PoolName) } + // TODO: validate multi-job volumes exist // TODO: check for multi-job restrictions, etc? return nil } @@ -203,24 +204,39 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo } func (s sessionFacade) CopyDataIn(sessionName datamodel.SessionName) error { - // TODO: complete implementation of all actions - log.Println("FAKE CopyDataIn") - return nil + return s.submitJob(sessionName, datamodel.SessionCopyDataIn, + func() (datamodel.Session, error) { + return s.session.GetSession(sessionName) + }) } func (s sessionFacade) Mount(sessionName datamodel.SessionName, computeNodes []string, loginNodes []string) error { - log.Println("FAKE Mount") - return nil + return s.submitJob(sessionName, datamodel.SessionMount, + func() (datamodel.Session, error) { + session, err := s.session.GetSession(sessionName) + if err != nil { + log.Println("Unable to find session, skipping delete:", sessionName) + return session, nil + } + + // TODO: what about the login nodes? what do we want to do there? + session.RequestedAttachHosts = computeNodes + return s.session.UpdateSession(session) + }) } func (s sessionFacade) Unmount(sessionName datamodel.SessionName) error { - log.Println("FAKE Unmount") - return nil + return s.submitJob(sessionName, datamodel.SessionUnmount, + func() (datamodel.Session, error) { + return s.session.GetSession(sessionName) + }) } func (s sessionFacade) CopyDataOut(sessionName datamodel.SessionName) error { - log.Println("FAKE CopyDataOut") - return nil + return s.submitJob(sessionName, datamodel.SessionCopyDataOut, + func() (datamodel.Session, error) { + return s.session.GetSession(sessionName) + }) } func (s sessionFacade) GetPools() ([]datamodel.PoolInfo, error) { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index c27294a4..a038d225 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -35,12 +35,21 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi case datamodel.SessionDelete: // TODO... must test this better! if !s.skipActions { - go s.handleDelete(action) + s.handleDelete(action) } case datamodel.SessionCreateFilesystem: if !s.skipActions { - go s.handleCreate(action) + // TODO: really should all happen in a goroutine + s.handleCreate(action) } + case datamodel.SessionCopyDataIn: + s.handleCopyIn(action) + case datamodel.SessionMount: + s.handleMount(action) + case datamodel.SessionUnmount: + s.handleUnmount(action) + case datamodel.SessionCopyDataOut: + s.handleCopyOut(action) default: log.Panicf("not yet implemented action for %+v", action) } @@ -97,8 +106,56 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { - log.Println("delete") - // TODO: clearly need mutex here, etc + // TODO... mutex, etc? + s.doUnmount(action) + if !action.Session.Status.DeleteSkipCopyDataOut && !action.Session.Status.CopyDataOutComplete { + s.fsProvider.DataCopyOut(action.Session) + } + s.fsProvider.Delete(action.Session) s.sessionRegistry.DeleteSession(action.Session) s.actions.CompleteSessionAction(action) } + +func (s *sessionActionHandler) handleCopyIn(action datamodel.SessionAction) { + s.fsProvider.DataCopyIn(action.Session) + s.actions.CompleteSessionAction(action) +} + +func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { + if action.Session.ActualSizeBytes > 0 { + s.fsProvider.Mount(action.Session, + datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + } + for _, sessionName := range action.Session.MultiJobAttachments { + session, _ := s.sessionRegistry.GetSession(sessionName) + if session.VolumeRequest.MultiJob { + s.fsProvider.Mount(session, + datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + } + } + s.actions.CompleteSessionAction(action) +} + +func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) { + if action.Session.ActualSizeBytes > 0 { + s.fsProvider.Unmount(action.Session, + datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + } + for _, sessionName := range action.Session.MultiJobAttachments { + session, _ := s.sessionRegistry.GetSession(sessionName) + if session.VolumeRequest.MultiJob { + s.fsProvider.Mount(session, + datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + } + } +} +func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { + s.doUnmount(action) + s.actions.CompleteSessionAction(action) +} + +func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { + s.fsProvider.DataCopyOut(action.Session) + // TODO: update session.CopyDataOutComplete, mutex, etc. + s.actions.CompleteSessionAction(action) +} diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index 5242ed51..d36d4815 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -71,19 +71,3 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { handler.handleCreate(action) } - -func TestSessionActionHandler_handleDelete(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - actions := mock_registry.NewMockSessionActions(mockCtrl) - registry := mock_registry.NewMockSessionRegistry(mockCtrl) - handler := sessionActionHandler{actions: actions, sessionRegistry: registry} - action := datamodel.SessionAction{ - ActionType: datamodel.SessionDelete, - } - // TODO: need to pass session better? who deletes allocations? - registry.EXPECT().DeleteSession(action.Session) - actions.EXPECT().CompleteSessionAction(action) - - handler.handleDelete(action) -} From ef4d6bf6fa69f0cec595e51f984a51376de89d38 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 23:13:47 +0100 Subject: [PATCH 109/191] Fix session action response key --- internal/pkg/v2/registry_impl/session_actions.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index 3d885c82..b8efe72d 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -40,19 +40,14 @@ func getSessionActionRequestKey(action datamodel.SessionAction) string { const sessionActionResponsePrefix = "/session_action/response/" -func getSessionActionResponseHostPrefix(brickHost datamodel.BrickHostName) string { - if !parsers.IsValidName(string(brickHost)) { +func getSessionActionResponseKey(action datamodel.SessionAction) string { + if !parsers.IsValidName(string(action.Session.Name)) { log.Panicf("invalid session PrimaryBrickHost") } - return fmt.Sprintf("%s%s/", sessionActionResponsePrefix, brickHost) -} - -func getSessionActionResponseKey(action datamodel.SessionAction) string { - hostPrefix := getSessionActionResponseHostPrefix(action.Session.PrimaryBrickHost) if !parsers.IsValidName(action.Uuid) { log.Panicf("invalid session action uuid %s", action.Uuid) } - return fmt.Sprintf("%s%s", hostPrefix, action.Uuid) + return fmt.Sprintf("%s%s/%s", sessionActionResponsePrefix, action.Session.Name, action.Uuid) } func sessionActionToRaw(session datamodel.SessionAction) []byte { From 50d1c72b388cf8ef84f770127412d18d386cf59e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 23:34:01 +0100 Subject: [PATCH 110/191] Refactor session action hander --- .../session_action_handler.go | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index a038d225..c760e3c5 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -2,6 +2,7 @@ package brick_manager_impl import ( "context" + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" @@ -55,54 +56,66 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi } } -func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { +func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, process func() (datamodel.Session, error)) { + sessionName := action.Session.Name sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) action.Error = err - s.actions.CompleteSessionAction(action) return } err = sessionMutex.Lock(context.TODO()) if err != nil { log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) action.Error = err - s.actions.CompleteSessionAction(action) return } + + // Always complete action and drop mutex on function exit defer func() { + if err := s.actions.CompleteSessionAction(action); err != nil { + log.Printf("failed to complete action %+v\n", action) + } if err := sessionMutex.Unlock(context.TODO()); err != nil { log.Println("failed to drop mutex for:", sessionName) } }() - log.Printf("starting create for %+v\n", sessionName) - // Get latest session now we have the mutex - session, err := s.sessionRegistry.GetSession(sessionName) + log.Printf("starting action %+v\n", action) - fsStatus, err := s.fsProvider.Create(action.Session) - session.FilesystemStatus = fsStatus - session.Status.FileSystemCreated = err == nil - session.Status.Error = err - - session, err = s.sessionRegistry.UpdateSession(session) + session, err := process() if err != nil { - log.Printf("Failed to update session: %+v", session) action.Error = err + log.Printf("error during action %+v\n", action) } else { action.Session = session - action.Error = session.Status.Error + log.Printf("finished action %+v\n", action) } +} - if err := s.actions.CompleteSessionAction(action); err != nil { - log.Printf("Failed to complete action: %+v", action) - } - if action.Session.Status.Error != nil { - log.Println("error during create for", sessionName, err) - } else { - log.Printf("completed create for %+v\n", sessionName) - } +func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { + s.processWithMutex(action, func() (datamodel.Session, error) { + // Get latest session now we have the mutex + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + return action.Session, fmt.Errorf("error getting session: %s", err) + } + + fsStatus, err := s.fsProvider.Create(action.Session) + session.FilesystemStatus = fsStatus + session.Status.FileSystemCreated = err == nil + session.Status.Error = err + + session, updateErr := s.sessionRegistry.UpdateSession(session) + if updateErr != nil { + log.Println("Failed to update session:", updateErr) + if err == nil { + err = updateErr + } + } + return session, err + }) } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { From d3c7e454eb60150479699911a603cfc8d1a2ebaf Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 23:46:15 +0100 Subject: [PATCH 111/191] Add mutex to more session handler actions --- .../session_action_handler.go | 66 ++++++++++++++----- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index c760e3c5..34447a39 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -119,19 +119,54 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { - // TODO... mutex, etc? - s.doUnmount(action) - if !action.Session.Status.DeleteSkipCopyDataOut && !action.Session.Status.CopyDataOutComplete { - s.fsProvider.DataCopyOut(action.Session) - } - s.fsProvider.Delete(action.Session) - s.sessionRegistry.DeleteSession(action.Session) - s.actions.CompleteSessionAction(action) + s.processWithMutex(action, func() (datamodel.Session, error) { + if err := s.doUnmount(action); err != nil { + log.Println("failed unmount during delete", action.Session.Name) + } + if !action.Session.Status.DeleteSkipCopyDataOut && !action.Session.Status.CopyDataOutComplete { + if err := s.fsProvider.DataCopyOut(action.Session); err != nil { + log.Println("failed DataCopyOut during delete", action.Session.Name) + } + } + + if err := s.fsProvider.Delete(action.Session); err != nil { + return action.Session, err + } + + return action.Session, s.sessionRegistry.DeleteSession(action.Session) + }) } func (s *sessionActionHandler) handleCopyIn(action datamodel.SessionAction) { - s.fsProvider.DataCopyIn(action.Session) - s.actions.CompleteSessionAction(action) + s.processWithMutex(action, func() (datamodel.Session, error) { + err := s.fsProvider.DataCopyIn(action.Session) + if err != nil { + return action.Session, err + } + + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + session = action.Session + } + session.Status.CopyDataInComplete = true + return s.sessionRegistry.UpdateSession(session) + }) +} + +func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { + s.processWithMutex(action, func() (datamodel.Session, error) { + err := s.fsProvider.DataCopyOut(action.Session) + if err != nil { + return action.Session, err + } + + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + session = action.Session + } + session.Status.CopyDataOutComplete = true + return s.sessionRegistry.UpdateSession(session) + }) } func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { @@ -149,7 +184,8 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { s.actions.CompleteSessionAction(action) } -func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) { +func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) error { + var lastError error if action.Session.ActualSizeBytes > 0 { s.fsProvider.Unmount(action.Session, datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) @@ -161,14 +197,10 @@ func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) { datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) } } + return lastError } + func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { s.doUnmount(action) s.actions.CompleteSessionAction(action) } - -func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { - s.fsProvider.DataCopyOut(action.Session) - // TODO: update session.CopyDataOutComplete, mutex, etc. - s.actions.CompleteSessionAction(action) -} From 010ab8c406e49528f0a5ec1f910a8a86050182f0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 20 Aug 2019 23:58:11 +0100 Subject: [PATCH 112/191] Update session_action_hander to use a mutex And each event processed in its own goroutine --- .../session_action_handler.go | 65 +++++++++++++------ .../session_action_handler_test.go | 18 ----- internal/pkg/v2/datamodel/session.go | 4 ++ internal/pkg/v2/registry_impl/session_test.go | 10 ++- 4 files changed, 57 insertions(+), 40 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 34447a39..6e5b710b 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -31,26 +31,19 @@ type sessionActionHandler struct { } func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { - log.Printf("Started to process: %+v\n", action) switch action.ActionType { case datamodel.SessionDelete: - // TODO... must test this better! - if !s.skipActions { - s.handleDelete(action) - } + go s.handleDelete(action) case datamodel.SessionCreateFilesystem: - if !s.skipActions { - // TODO: really should all happen in a goroutine - s.handleCreate(action) - } + go s.handleCreate(action) case datamodel.SessionCopyDataIn: - s.handleCopyIn(action) + go s.handleCopyIn(action) case datamodel.SessionMount: - s.handleMount(action) + go s.handleMount(action) case datamodel.SessionUnmount: - s.handleUnmount(action) + go s.handleUnmount(action) case datamodel.SessionCopyDataOut: - s.handleCopyOut(action) + go s.handleCopyOut(action) default: log.Panicf("not yet implemented action for %+v", action) } @@ -120,8 +113,10 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - if err := s.doUnmount(action); err != nil { - log.Println("failed unmount during delete", action.Session.Name) + if !action.Session.Status.UnmountComplete { + if err := s.doUnmount(action); err != nil { + log.Println("failed unmount during delete", action.Session.Name) + } } if !action.Session.Status.DeleteSkipCopyDataOut && !action.Session.Status.CopyDataOutComplete { if err := s.fsProvider.DataCopyOut(action.Session); err != nil { @@ -169,7 +164,7 @@ func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { }) } -func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { +func (s *sessionActionHandler) doMount(action datamodel.SessionAction) error { if action.Session.ActualSizeBytes > 0 { s.fsProvider.Mount(action.Session, datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) @@ -181,11 +176,11 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) } } - s.actions.CompleteSessionAction(action) + // TODO: error handling!!! + return nil } func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) error { - var lastError error if action.Session.ActualSizeBytes > 0 { s.fsProvider.Unmount(action.Session, datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) @@ -197,10 +192,38 @@ func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) error { datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) } } - return lastError + // TODO error handling!!! + return nil +} + +func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { + s.processWithMutex(action, func() (datamodel.Session, error) { + err := s.doMount(action) + if err != nil { + return action.Session, err + } + + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + session = action.Session + } + session.Status.MountComplete = true + return s.sessionRegistry.UpdateSession(session) + }) } func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { - s.doUnmount(action) - s.actions.CompleteSessionAction(action) + s.processWithMutex(action, func() (datamodel.Session, error) { + err := s.doUnmount(action) + if err != nil { + return action.Session, err + } + + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + session = action.Session + } + session.Status.UnmountComplete = true + return s.sessionRegistry.UpdateSession(session) + }) } diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index d36d4815..bcf35e8e 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -21,24 +21,6 @@ func TestSessionActionHandler_ProcessSessionAction_Unknown(t *testing.T) { func() { handler.ProcessSessionAction(action) }) } -func TestSessionActionHandler_ProcessSessionAction_Create(t *testing.T) { - action := datamodel.SessionAction{ - ActionType: datamodel.SessionCreateFilesystem, - } - handler := sessionActionHandler{skipActions: true} - - handler.ProcessSessionAction(action) -} - -func TestSessionActionHandler_ProcessSessionAction_Delete(t *testing.T) { - action := datamodel.SessionAction{ - ActionType: datamodel.SessionDelete, - } - handler := sessionActionHandler{skipActions: true} - - handler.ProcessSessionAction(action) -} - func TestSessionActionHandler_handleCreate(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 6138f0e0..ddc2918b 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -105,6 +105,10 @@ type SessionStatus struct { // Records if we should skip copy data out on delete DeleteSkipCopyDataOut bool + + // Mount status + UnmountComplete bool + MountComplete bool } type VolumeRequest struct { diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 239519de..cce232d2 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -1,6 +1,7 @@ package registry_impl import ( + "encoding/json" "errors" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" @@ -10,9 +11,16 @@ import ( "testing" ) -var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"AllocatedBricks":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) +var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false,"UnmountComplete":false,"MountComplete":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"AllocatedBricks":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) var exampleSession = datamodel.Session{Name: "foo", PrimaryBrickHost: "host1"} +func TestExampleString(t *testing.T) { + exampleStr, err := json.Marshal(exampleSession) + assert.Nil(t, err) + // mostly here to make it easy to update the example string + assert.Equal(t, string(exampleSessionString), string(exampleStr)) +} + func TestSessionRegistry_GetSessionMutex(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() From 88f7982fd0ab07eb359aaaaa1abe418f6fb3a46e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 00:25:50 +0100 Subject: [PATCH 113/191] Copy in the old filesystem provider --- .../session_action_handler.go | 15 +- internal/pkg/v2/filesystem_impl/ansible.go | 311 ++++++++++++++++++ .../pkg/v2/filesystem_impl/ansible_test.go | 181 ++++++++++ internal/pkg/v2/filesystem_impl/copy.go | 72 ++++ internal/pkg/v2/filesystem_impl/copy_test.go | 66 ++++ internal/pkg/v2/filesystem_impl/fs_type.go | 44 +++ internal/pkg/v2/filesystem_impl/mount.go | 308 +++++++++++++++++ internal/pkg/v2/filesystem_impl/mount_test.go | 273 +++++++++++++++ 8 files changed, 1264 insertions(+), 6 deletions(-) create mode 100644 internal/pkg/v2/filesystem_impl/ansible.go create mode 100644 internal/pkg/v2/filesystem_impl/ansible_test.go create mode 100644 internal/pkg/v2/filesystem_impl/copy.go create mode 100644 internal/pkg/v2/filesystem_impl/copy_test.go create mode 100644 internal/pkg/v2/filesystem_impl/fs_type.go create mode 100644 internal/pkg/v2/filesystem_impl/mount.go create mode 100644 internal/pkg/v2/filesystem_impl/mount_test.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 6e5b710b..7873855a 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -114,11 +114,11 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { if !action.Session.Status.UnmountComplete { - if err := s.doUnmount(action); err != nil { + if err := s.doAllUnmounts(action); err != nil { log.Println("failed unmount during delete", action.Session.Name) } } - if !action.Session.Status.DeleteSkipCopyDataOut && !action.Session.Status.CopyDataOutComplete { + if !action.Session.Status.CopyDataOutComplete && !action.Session.Status.DeleteSkipCopyDataOut { if err := s.fsProvider.DataCopyOut(action.Session); err != nil { log.Println("failed DataCopyOut during delete", action.Session.Name) } @@ -164,7 +164,7 @@ func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { }) } -func (s *sessionActionHandler) doMount(action datamodel.SessionAction) error { +func (s *sessionActionHandler) doAllMounts(action datamodel.SessionAction) error { if action.Session.ActualSizeBytes > 0 { s.fsProvider.Mount(action.Session, datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) @@ -180,7 +180,7 @@ func (s *sessionActionHandler) doMount(action datamodel.SessionAction) error { return nil } -func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) error { +func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) error { if action.Session.ActualSizeBytes > 0 { s.fsProvider.Unmount(action.Session, datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) @@ -198,8 +198,11 @@ func (s *sessionActionHandler) doUnmount(action datamodel.SessionAction) error { func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.doMount(action) + err := s.doAllMounts(action) if err != nil { + if err := s.doAllUnmounts(action); err != nil { + log.Println("error while rolling back possible partial mount", action.Session.Name, err) + } return action.Session, err } @@ -214,7 +217,7 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.doUnmount(action) + err := s.doAllUnmounts(action) if err != nil { return action.Session, err } diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/v2/filesystem_impl/ansible.go new file mode 100644 index 00000000..dcbb1c9b --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/ansible.go @@ -0,0 +1,311 @@ +package filesystem_impl + +import ( + "bytes" + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "gopkg.in/yaml.v2" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "time" +) + +type HostInfo struct { + MGS string `yaml:"mgs,omitempty"` + MDTS map[string]int `yaml:"mdts,omitempty,flow"` + OSTS map[string]int `yaml:"osts,omitempty,flow"` +} + +type FSInfo struct { + Hosts map[string]HostInfo `yaml:"hosts"` + Vars map[string]string `yaml:"vars"` +} + +type FileSystems struct { + Children map[string]FSInfo `yaml:"children"` +} + +type Wrapper struct { + All FileSystems +} + +var DefaultHostGroup = "dac-prod" +var DefaultMaxMDTs uint = 24 + +func getInventory(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) string { + // NOTE: only used by lustre + mgsDevice := os.Getenv("DAC_MGS_DEV") + if mgsDevice == "" { + mgsDevice = "sdb" + } + maxMDTs := DefaultMaxMDTs + maxMDTsConf, err := strconv.ParseUint(os.Getenv("DAC_MAX_MDT_COUNT"), 10, 32) + if err == nil && maxMDTsConf > 0 { + maxMDTs = uint(maxMDTsConf) + } + + allocationsByHost := make(map[string][]registry.BrickAllocation) + for _, allocation := range brickAllocations { + allocationsByHost[allocation.Hostname] = append(allocationsByHost[allocation.Hostname], allocation) + } + + // If we have more brick allocations than maxMDTs + // assign at most one mdt per host. + // While this may give us less MDTs than max MDTs, + // but it helps spread MDTs across network connections + oneMdtPerHost := len(brickAllocations) > int(maxMDTs) + + hosts := make(map[string]HostInfo) + mgsnode := "" + for host, allocations := range allocationsByHost { + osts := make(map[string]int) + for _, allocation := range allocations { + osts[allocation.Device] = int(allocation.AllocatedIndex) + } + + mdts := make(map[string]int) + if oneMdtPerHost { + allocation := allocations[0] + mdts[allocation.Device] = int(allocation.AllocatedIndex) + } else { + for _, allocation := range allocations { + mdts[allocation.Device] = int(allocation.AllocatedIndex) + } + } + + hostInfo := HostInfo{MDTS: mdts, OSTS: osts} + + if allocations[0].AllocatedIndex == 0 { + if fsType == Lustre { + hostInfo.MGS = mgsDevice + } else { + hostInfo.MGS = allocations[0].Device + } + mgsnode = host + } + hosts[host] = hostInfo + } + + // for beegfs, mount clients via ansible + if fsType == BeegFS { + // TODO: this can't work now, as we need to also pass the job name + for _, attachment := range volume.Attachments { + hosts[attachment.Hostname] = HostInfo{} + } + } + + fsinfo := FSInfo{ + Vars: map[string]string{ + "mgsnode": mgsnode, + "client_port": fmt.Sprintf("%d", volume.ClientPort), + "lnet_suffix": getLnetSuffix(), + "mdt_size": fmt.Sprintf("%dm", getMdtSizeMB()), + }, + Hosts: hosts, + } + fsname := fmt.Sprintf("%s", volume.UUID) + data := Wrapper{All: FileSystems{Children: map[string]FSInfo{fsname: fsinfo}}} + + output, err := yaml.Marshal(data) + if err != nil { + log.Fatalln(err) + } + strOut := string(output) + strOut = strings.Replace(strOut, " mgs:", fmt.Sprintf(" %s_mgs:", fsname), -1) + strOut = strings.Replace(strOut, " mdts:", fmt.Sprintf(" %s_mdts:", fsname), -1) + strOut = strings.Replace(strOut, " osts:", fmt.Sprintf(" %s_osts:", fsname), -1) + strOut = strings.Replace(strOut, " mgsnode:", fmt.Sprintf(" %s_mgsnode:", fsname), -1) + strOut = strings.Replace(strOut, " client_port:", fmt.Sprintf(" %s_client_port:", fsname), -1) + strOut = strings.Replace(strOut, " mdt_size:", fmt.Sprintf(" %s_mdt_size:", fsname), -1) + + hostGroup := os.Getenv("DAC_HOST_GROUP") + if hostGroup == "" { + hostGroup = DefaultHostGroup + } + strOut = strings.Replace(strOut, "all:", hostGroup+":", -1) + return strOut +} + +func getPlaybook(fsType FSType, volume registry.Volume) string { + role := "lustre" + if fsType == BeegFS { + role = "beegfs" + } + return fmt.Sprintf(`--- +- name: Setup FS + hosts: %s + any_errors_fatal: true + become: yes + roles: + - role: %s + vars: + fs_name: %s`, volume.UUID, role, volume.UUID) +} + +func getAnsibleDir(suffix string) string { + ansibleDir := os.Getenv("DAC_ANSIBLE_DIR") + if ansibleDir == "" { + ansibleDir = "/var/lib/data-acc/fs-ansible/" + } + return path.Join(ansibleDir, suffix) +} + +func setupAnsible(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) (string, error) { + dir, err := ioutil.TempDir("", fmt.Sprintf("fs%s_", volume.Name)) + if err != nil { + return dir, err + } + log.Println("Using ansible tempdir:", dir) + + playbook := getPlaybook(fsType, volume) + tmpPlaybook := filepath.Join(dir, "dac.yml") + if err := ioutil.WriteFile(tmpPlaybook, bytes.NewBufferString(playbook).Bytes(), 0666); err != nil { + return dir, err + } + log.Println(playbook) + + inventory := getInventory(fsType, volume, brickAllocations) + tmpInventory := filepath.Join(dir, "inventory") + if err := ioutil.WriteFile(tmpInventory, bytes.NewBufferString(inventory).Bytes(), 0666); err != nil { + return dir, err + } + log.Println(inventory) + + cmd := exec.Command("cp", "-r", getAnsibleDir("roles"), dir) + output, err := cmd.CombinedOutput() + log.Println("copy roles", string(output)) + if err != nil { + return dir, err + } + cmd = exec.Command("cp", "-r", getAnsibleDir(".venv"), dir) + output, err = cmd.CombinedOutput() + log.Println("copy venv", string(output)) + if err != nil { + return dir, err + } + cmd = exec.Command("cp", "-r", getAnsibleDir("group_vars"), dir) + output, err = cmd.CombinedOutput() + log.Println("copy group vars", string(output)) + return dir, err +} + +func executeAnsibleSetup(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { + dir, err := setupAnsible(fsType, volume, brickAllocations) + if err != nil { + return err + } + + formatArgs := "dac.yml -i inventory --tag format" + err = executeAnsiblePlaybook(dir, formatArgs) + if err != nil { + return err + } + + startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" + err = executeAnsiblePlaybook(dir, startupArgs) + if err != nil { + return err + } + + // only delete if everything worked, to aid debugging + os.RemoveAll(dir) + return nil +} + +func executeAnsibleTeardown(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { + dir, err := setupAnsible(fsType, volume, brickAllocations) + if err != nil { + return err + } + + stopArgs := "dac.yml -i inventory --tag stop_all,unmount,client_unmount" + err = executeAnsiblePlaybook(dir, stopArgs) + if err != nil { + return err + } + + formatArgs := "dac.yml -i inventory --tag clean" + err = executeAnsiblePlaybook(dir, formatArgs) + if err != nil { + return err + } + + // only delete if everything worked, to aid debugging + os.RemoveAll(dir) + return nil +} + +func executeAnsibleMount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { + dir, err := setupAnsible(fsType, volume, brickAllocations) + if err != nil { + return err + } + + startupArgs := "dac.yml -i inventory --tag client_mount" + err = executeAnsiblePlaybook(dir, startupArgs) + if err != nil { + return err + } + + os.RemoveAll(dir) + return nil +} + +func executeAnsibleUnmount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { + dir, err := setupAnsible(fsType, volume, brickAllocations) + if err != nil { + return err + } + + stopArgs := "dac.yml -i inventory --tag client_unmount" + err = executeAnsiblePlaybook(dir, stopArgs) + if err != nil { + return err + } + + os.RemoveAll(dir) + return nil +} + +func executeAnsiblePlaybook(dir string, args string) error { + // TODO: downgrade debug log! + cmdStr := fmt.Sprintf(`cd %s; . .venv/bin/activate; ansible-playbook %s;`, dir, args) + log.Println("Requested ansible:", cmdStr) + + skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") + if skipAnsible == "True" { + log.Println("Skip as DAC_SKIP_ANSIBLE=True") + time.Sleep(time.Millisecond * 200) + return nil + } + + var err error + for i := 1; i <= 3; i++ { + log.Println("Attempt", i, "of ansible:", cmdStr) + cmd := exec.Command("bash", "-c", cmdStr) + + timer := time.AfterFunc(time.Minute*5, func() { + log.Println("Time up, waited more than 5 mins to complete.") + cmd.Process.Kill() + }) + output, currentErr := cmd.CombinedOutput() + timer.Stop() + + if currentErr == nil { + log.Println("Completed ansible run:", cmdStr) + log.Println(string(output)) + return nil + } else { + log.Println("Error in ansible run:", string(output)) + err = currentErr + time.Sleep(time.Second * 2) + } + } + return err +} diff --git a/internal/pkg/v2/filesystem_impl/ansible_test.go b/internal/pkg/v2/filesystem_impl/ansible_test.go new file mode 100644 index 00000000..863059d1 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/ansible_test.go @@ -0,0 +1,181 @@ +package filesystem_impl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPlugin_GetInventory(t *testing.T) { + volume := registry.Volume{ + Name: "1", UUID: "abcdefgh", ClientPort: 10002, + Attachments: []registry.Attachment{ + {Hostname: "cpu1"}, + {Hostname: "cpu2"}, + }, + } + brickAllocations := []registry.BrickAllocation{ + {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, + {Hostname: "dac1", Device: "nvme2n1", AllocatedIndex: 1}, + {Hostname: "dac1", Device: "nvme3n1", AllocatedIndex: 2}, + {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 3}, + {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 4}, + } + result := getInventory(BeegFS, volume, brickAllocations) + expected := `dac-prod: + children: + abcdefgh: + hosts: + cpu1: {} + cpu2: {} + dac1: + abcdefgh_mgs: nvme1n1 + abcdefgh_mdts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} + abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} + dac2: + abcdefgh_mdts: {nvme2n1: 3, nvme3n1: 4} + abcdefgh_osts: {nvme2n1: 3, nvme3n1: 4} + vars: + abcdefgh_client_port: "10002" + lnet_suffix: "" + abcdefgh_mdt_size: 20480m + abcdefgh_mgsnode: dac1 +` + assert.Equal(t, expected, result) +} + +func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { + volume := registry.Volume{Name: "1", UUID: "abcdefgh", ClientPort: 10002} + brickAllocations := []registry.BrickAllocation{ + {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, + {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 1}, + {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 2}, + } + result := getInventory(Lustre, volume, brickAllocations) + expected := `dac-prod: + children: + abcdefgh: + hosts: + dac1: + abcdefgh_mgs: sdb + abcdefgh_mdts: {nvme1n1: 0} + abcdefgh_osts: {nvme1n1: 0} + dac2: + abcdefgh_mdts: {nvme2n1: 1, nvme3n1: 2} + abcdefgh_osts: {nvme2n1: 1, nvme3n1: 2} + vars: + abcdefgh_client_port: "10002" + lnet_suffix: "" + abcdefgh_mdt_size: 20480m + abcdefgh_mgsnode: dac1 +` + assert.Equal(t, expected, result) +} + +func TestPlugin_GetPlaybook_beegfs(t *testing.T) { + volume := registry.Volume{Name: "1", UUID: "abcdefgh"} + result := getPlaybook(BeegFS, volume) + assert.Equal(t, `--- +- name: Setup FS + hosts: abcdefgh + any_errors_fatal: true + become: yes + roles: + - role: beegfs + vars: + fs_name: abcdefgh`, result) +} + +func TestPlugin_GetPlaybook_lustre(t *testing.T) { + volume := registry.Volume{Name: "1", UUID: "abcdefgh"} + result := getPlaybook(Lustre, volume) + assert.Equal(t, `--- +- name: Setup FS + hosts: abcdefgh + any_errors_fatal: true + become: yes + roles: + - role: lustre + vars: + fs_name: abcdefgh`, result) +} + +func TestPlugin_GetInventory_MaxMDT(t *testing.T) { + volume := registry.Volume{ + Name: "1", UUID: "abcdefgh", ClientPort: 10002, + Attachments: []registry.Attachment{ + {Hostname: "cpu1"}, + {Hostname: "cpu2"}, + }, + } + + var brickAllocations []registry.BrickAllocation + for i := 1; i <= 26; i = i + 2 { + brickAllocations = append(brickAllocations, registry.BrickAllocation{ + Hostname: fmt.Sprintf("dac%d", i), + Device: "nvme1n1", + AllocatedIndex: uint(i - 1), + }) + brickAllocations = append(brickAllocations, registry.BrickAllocation{ + Hostname: fmt.Sprintf("dac%d", i), + Device: "nvme2n1", + AllocatedIndex: uint(i), + }) + } + + result := getInventory(BeegFS, volume, brickAllocations) + expected := `dac-prod: + children: + abcdefgh: + hosts: + cpu1: {} + cpu2: {} + dac1: + abcdefgh_mgs: nvme1n1 + abcdefgh_mdts: {nvme1n1: 0} + abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1} + dac3: + abcdefgh_mdts: {nvme1n1: 2} + abcdefgh_osts: {nvme1n1: 2, nvme2n1: 3} + dac5: + abcdefgh_mdts: {nvme1n1: 4} + abcdefgh_osts: {nvme1n1: 4, nvme2n1: 5} + dac7: + abcdefgh_mdts: {nvme1n1: 6} + abcdefgh_osts: {nvme1n1: 6, nvme2n1: 7} + dac9: + abcdefgh_mdts: {nvme1n1: 8} + abcdefgh_osts: {nvme1n1: 8, nvme2n1: 9} + dac11: + abcdefgh_mdts: {nvme1n1: 10} + abcdefgh_osts: {nvme1n1: 10, nvme2n1: 11} + dac13: + abcdefgh_mdts: {nvme1n1: 12} + abcdefgh_osts: {nvme1n1: 12, nvme2n1: 13} + dac15: + abcdefgh_mdts: {nvme1n1: 14} + abcdefgh_osts: {nvme1n1: 14, nvme2n1: 15} + dac17: + abcdefgh_mdts: {nvme1n1: 16} + abcdefgh_osts: {nvme1n1: 16, nvme2n1: 17} + dac19: + abcdefgh_mdts: {nvme1n1: 18} + abcdefgh_osts: {nvme1n1: 18, nvme2n1: 19} + dac21: + abcdefgh_mdts: {nvme1n1: 20} + abcdefgh_osts: {nvme1n1: 20, nvme2n1: 21} + dac23: + abcdefgh_mdts: {nvme1n1: 22} + abcdefgh_osts: {nvme1n1: 22, nvme2n1: 23} + dac25: + abcdefgh_mdts: {nvme1n1: 24} + abcdefgh_osts: {nvme1n1: 24, nvme2n1: 25} + vars: + abcdefgh_client_port: "10002" + lnet_suffix: "" + abcdefgh_mdt_size: 20480m + abcdefgh_mgsnode: dac1 +` + assert.Equal(t, expected, result) +} diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/v2/filesystem_impl/copy.go new file mode 100644 index 00000000..2dcc985c --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/copy.go @@ -0,0 +1,72 @@ +package filesystem_impl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "log" + "path" + "strings" +) + +func processDataCopy(volume registry.Volume, request registry.DataCopyRequest) error { + cmd, err := generateDataCopyCmd(volume, request) + if err != nil { + return err + } + if cmd == "" { + log.Println("No files to copy for:", volume.Name) + return nil + } + + log.Printf("Doing copy: %s", cmd) + + // Make sure global dir is setup correctly + // TODO: share code with mount better + // TODO: Probably should all get setup in fs-ansible really!! + mountDir := fmt.Sprintf("/mnt/lustre/%s", volume.UUID) + sharedDir := path.Join(mountDir, "/global") + if err := mkdir("localhost", sharedDir); err != nil { + return err + } + if err := fixUpOwnership("localhost", volume.Owner, volume.Group, sharedDir); err != nil { + return err + } + + // Do the copy + return runner.Execute("localhost", cmd) +} + +func generateDataCopyCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { + rsync, err := generateRsyncCmd(volume, request) + if err != nil || rsync == "" { + return "", err + } + + cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", volume.Group, volume.Owner, rsync) + dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", volume.UUID) + cmd = fmt.Sprintf("bash -c \"export DW_JOB_STRIPED='%s' && %s\"", dacHostBufferPath, cmd) + return cmd, nil +} + +func generateRsyncCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { + if request.Source == "" && request.Destination == "" { + return "", nil + } + + var flags string + if request.SourceType == registry.Directory { + flags = "-r -ospgu --stats" + } else if request.SourceType == registry.File { + flags = "-ospgu --stats" + } else { + return "", fmt.Errorf("unsupported source type %s for volume: %s", request.SourceType, volume.Name) + } + + return fmt.Sprintf("rsync %s %s %s", flags, + escapePath(request.Source), + escapePath(request.Destination)), nil +} + +func escapePath(path string) string { + return strings.Replace(path, "$DW_JOB_STRIPED", "\\$DW_JOB_STRIPED", 1) +} diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go new file mode 100644 index 00000000..f8e60378 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -0,0 +1,66 @@ +package filesystem_impl + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_GenerateDataCopy(t *testing.T) { + testVolume := registry.Volume{ + Name: registry.VolumeName("asdf"), + Owner: 1001, + Group: 1002, + UUID: "fsuuid", + } + request := registry.DataCopyRequest{} + + cmd, err := generateDataCopyCmd(testVolume, request) + assert.Nil(t, err) + assert.Empty(t, cmd) + + request.SourceType = registry.File + request.Source = "$DW_JOB_STRIPED/source" + request.Destination = "dest" + cmd, err = generateDataCopyCmd(testVolume, request) + assert.Nil(t, err) + assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) + + request.SourceType = registry.List + request.Source = "list_filename" + cmd, err = generateDataCopyCmd(testVolume, request) + assert.Equal(t, "", cmd) + assert.Equal(t, "unsupported source type list for volume: asdf", err.Error()) + +} + +func Test_GenerateRsyncCmd(t *testing.T) { + testVolume := registry.Volume{ + Name: registry.VolumeName("asdf"), + } + request := registry.DataCopyRequest{} + + cmd, err := generateRsyncCmd(testVolume, request) + assert.Nil(t, err) + assert.Empty(t, cmd) + + request.SourceType = registry.File + request.Source = "source" + request.Destination = "dest" + cmd, err = generateRsyncCmd(testVolume, request) + assert.Nil(t, err) + assert.Equal(t, "rsync -ospgu --stats source dest", cmd) + + request.SourceType = registry.Directory + request.Source = "source" + request.Destination = "dest" + cmd, err = generateRsyncCmd(testVolume, request) + assert.Nil(t, err) + assert.Equal(t, "rsync -r -ospgu --stats source dest", cmd) + + request.SourceType = registry.List + request.Source = "list_filename" + cmd, err = generateRsyncCmd(testVolume, request) + assert.Equal(t, "", cmd) + assert.Equal(t, "unsupported source type list for volume: asdf", err.Error()) +} diff --git a/internal/pkg/v2/filesystem_impl/fs_type.go b/internal/pkg/v2/filesystem_impl/fs_type.go new file mode 100644 index 00000000..069154d8 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/fs_type.go @@ -0,0 +1,44 @@ +package filesystem_impl + +import ( + "bytes" + "encoding/json" +) + +type FSType int + +const ( + BeegFS FSType = iota + Lustre +) + +var fsTypeStrings = map[FSType]string{ + BeegFS: "BeegFS", + Lustre: "Lustre", +} +var stringToFSType = map[string]FSType{ + "": BeegFS, + "BeegFS": BeegFS, + "Lustre": Lustre, +} + +func (fsType FSType) String() string { + return fsTypeStrings[fsType] +} + +func (fsType FSType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(fsTypeStrings[fsType]) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +func (fsType *FSType) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + if err != nil { + return err + } + *fsType = stringToFSType[str] + return nil +} diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/v2/filesystem_impl/mount.go new file mode 100644 index 00000000..99130063 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/mount.go @@ -0,0 +1,308 @@ +package filesystem_impl + +import ( + "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "log" + "os" + "os/exec" + "path" + "strconv" + "time" +) + +func getMountDir(volume registry.Volume, jobName string) string { + // TODO: what about the environment variables that are being set? should share logic with here + if volume.MultiJob { + return fmt.Sprintf("/dac/%s_persistent_%s", jobName, volume.Name) + } + return fmt.Sprintf("/dac/%s_job", jobName) +} + +func getLnetSuffix() string { + return os.Getenv("DAC_LNET_SUFFIX") +} + +func getMdtSizeMB() uint { + mdtSizeGB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_GB"), 10, 32) + if err == nil && mdtSizeGB > 0 { + return uint(mdtSizeGB * 1024) + } + mdtSizeMB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_MB"), 10, 32) + if err == nil && mdtSizeMB > 0 { + return uint(mdtSizeMB) + } + return uint(20 * 1024) +} + +func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { + log.Println("Mount for:", volume.Name) + var primaryBrickHost string + for _, allocation := range brickAllocations { + if allocation.AllocatedIndex == 0 { + primaryBrickHost = allocation.Hostname + break + } + } + + if primaryBrickHost == "" { + log.Panicf("failed to find primary brick for volume: %s", volume.Name) + } + + lnetSuffix := getLnetSuffix() + + if fsType == BeegFS { + // Write out the config needed, and do the mount using ansible + // TODO: Move Lustre mount here that is done below + executeAnsibleMount(fsType, volume, brickAllocations) + } + + for _, attachment := range attachments { + if attachment.State != registry.RequestAttach { + log.Printf("Skipping volume %s attach: %+v", volume.Name, attachment) + continue + } + log.Printf("Volume %s attaching with: %+v", volume.Name, attachment) + + var mountDir = getMountDir(volume, attachment.Job) + if err := mkdir(attachment.Hostname, mountDir); err != nil { + return err + } + if err := mountRemoteFilesystem(fsType, attachment.Hostname, lnetSuffix, + primaryBrickHost, volume.UUID, mountDir); err != nil { + return err + } + + if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { + swapDir := path.Join(mountDir, "/swap") + if err := mkdir(attachment.Hostname, swapDir); err != nil { + return err + } + if err := fixUpOwnership(attachment.Hostname, 0, 0, swapDir); err != nil { + return err + } + + swapSizeMB := int(volume.AttachAsSwapBytes / (1024 * 1024)) + swapFile := path.Join(swapDir, fmt.Sprintf("/%s", attachment.Hostname)) + loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) + if err := createSwap(attachment.Hostname, swapSizeMB, swapFile, loopback); err != nil { + return err + } + + if err := swapOn(attachment.Hostname, loopback); err != nil { + return err + } + } + + if !volume.MultiJob && volume.AttachPrivateNamespace { + privateDir := path.Join(mountDir, fmt.Sprintf("/private/%s", attachment.Hostname)) + if err := mkdir(attachment.Hostname, privateDir); err != nil { + return err + } + if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, privateDir); err != nil { + return err + } + + // need a consistent symlink for shared environment variables across all hosts + privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) + if err := createSymbolicLink(attachment.Hostname, privateDir, privateSymLinkDir); err != nil { + return err + } + } + + sharedDir := path.Join(mountDir, "/global") + if err := mkdir(attachment.Hostname, sharedDir); err != nil { + return err + } + if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, sharedDir); err != nil { + return err + } + } + // TODO on error should we always call umount? maybe? + // TODO move to ansible style automation or preamble? + return nil +} + +func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { + log.Println("Umount for:", volume.Name) + + for _, attachment := range attachments { + if attachment.State != registry.RequestDetach { + log.Printf("Skipping volume %s detach for: %+v", volume.Name, attachment) + continue + } + log.Printf("Volume %s dettaching: %+v", volume.Name, attachment) + + var mountDir = getMountDir(volume, attachment.Job) + if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { + swapFile := path.Join(mountDir, fmt.Sprintf("/swap/%s", attachment.Hostname)) // TODO share? + loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) // TODO share? + if err := swapOff(attachment.Hostname, loopback); err != nil { + log.Printf("Warn: failed to swap off %+v", attachment) + } + if err := detachLoopback(attachment.Hostname, loopback); err != nil { + log.Printf("Warn: failed to detach loopback %+v", attachment) + } + if err := removeSubtree(attachment.Hostname, swapFile); err != nil { + return err + } + } + + if !volume.MultiJob && volume.AttachPrivateNamespace { + privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) + if err := removeSubtree(attachment.Hostname, privateSymLinkDir); err != nil { + return err + } + } + + if fsType == Lustre { + if err := umountLustre(attachment.Hostname, mountDir); err != nil { + return err + } + if err := removeSubtree(attachment.Hostname, mountDir); err != nil { + return err + } + + } + } + + if fsType == BeegFS { + // TODO: Move Lustre unmount here that is done below + executeAnsibleUnmount(fsType, volume, brickAllocations) + // TODO: this makes copy out much harder in its current form :( + } + + return nil +} + +func createSwap(hostname string, swapMB int, filename string, loopback string) error { + file := fmt.Sprintf("dd if=/dev/zero of=%s bs=1024 count=%d", filename, swapMB*1024) + if err := runner.Execute(hostname, file); err != nil { + return err + } + if err := runner.Execute(hostname, fmt.Sprintf("chmod 0600 %s", filename)); err != nil { + return err + } + device := fmt.Sprintf("losetup %s %s", loopback, filename) + if err := runner.Execute(hostname, device); err != nil { + return err + } + swap := fmt.Sprintf("mkswap %s", loopback) + return runner.Execute(hostname, swap) +} + +func swapOn(hostname string, loopback string) error { + return runner.Execute(hostname, fmt.Sprintf("swapon %s", loopback)) +} + +func swapOff(hostname string, loopback string) error { + return runner.Execute(hostname, fmt.Sprintf("swapoff %s", loopback)) +} + +func detachLoopback(hostname string, loopback string) error { + return runner.Execute(hostname, fmt.Sprintf("losetup -d %s", loopback)) +} + +func fixUpOwnership(hostname string, owner uint, group uint, directory string) error { + if err := runner.Execute(hostname, fmt.Sprintf("chown %d:%d %s", owner, group, directory)); err != nil { + return err + } + return runner.Execute(hostname, fmt.Sprintf("chmod 770 %s", directory)) +} + +func umountLustre(hostname string, directory string) error { + // only unmount if already mounted + if err := runner.Execute(hostname, fmt.Sprintf("grep %s /etc/mtab", directory)); err == nil { + // Don't add -l so we can spot when this fails + if err := runner.Execute(hostname, fmt.Sprintf("umount %s", directory)); err != nil { + return err + } + } else { + // TODO: we should really just avoid this being possible? + log.Println("skip umount, as not currently mounted.") + } + return nil +} + +func removeSubtree(hostname string, directory string) error { + return runner.Execute(hostname, fmt.Sprintf("rm -rf %s", directory)) +} + +func createSymbolicLink(hostname string, src string, dest string) error { + return runner.Execute(hostname, fmt.Sprintf("ln -s %s %s", src, dest)) +} + +func mountRemoteFilesystem(fsType FSType, hostname string, lnetSuffix string, mgtHost string, fsname string, directory string) error { + if fsType == Lustre { + return mountLustre(hostname, lnetSuffix, mgtHost, fsname, directory) + } else if fsType == BeegFS { + return mountBeegFS(hostname, mgtHost, fsname, directory) + } + return fmt.Errorf("mount unsuported by filesystem type %s", fsType) +} + +func mountLustre(hostname string, lnetSuffix string, mgtHost string, fsname string, directory string) error { + // We assume modprobe -v lustre is already done + // First check if we are mounted already + if err := runner.Execute(hostname, fmt.Sprintf("grep %s /etc/mtab", directory)); err != nil { + if err := runner.Execute(hostname, fmt.Sprintf( + "mount -t lustre -o flock,nodev,nosuid %s%s:/%s %s", + mgtHost, lnetSuffix, fsname, directory)); err != nil { + return err + } + } + return nil +} + +func mountBeegFS(hostname string, mgtHost string, fsname string, directory string) error { + // Ansible mounts beegfs at /mnt/beegfs/, link into above location here + // First remove the directory, then replace with a symbolic link + if err := removeSubtree(hostname, directory); err != nil { + return err + } + return runner.Execute(hostname, fmt.Sprintf("ln -s /mnt/beegfs/%s %s", fsname, directory)) +} + +func mkdir(hostname string, directory string) error { + return runner.Execute(hostname, fmt.Sprintf("mkdir -p %s", directory)) +} + +type Run interface { + Execute(name string, cmd string) error +} + +type run struct { +} + +func (*run) Execute(hostname string, cmdStr string) error { + log.Println("SSH to:", hostname, "with command:", cmdStr) + + skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") + if skipAnsible == "True" { + log.Println("Skip as DAC_SKIP_ANSIBLE=True") + time.Sleep(time.Millisecond * 200) + return nil + } + + cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", hostname, "sudo", cmdStr) + + timer := time.AfterFunc(time.Minute, func() { + log.Println("Time up, waited more than 5 mins to complete.") + cmd.Process.Kill() + }) + + output, err := cmd.CombinedOutput() + timer.Stop() + + if err == nil { + log.Println("Completed remote ssh run:", cmdStr) + log.Println(string(output)) + return nil + } else { + log.Println("Error in remove ssh run:", string(output)) + return err + } +} + +var runner Run = &run{} diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go new file mode 100644 index 00000000..e9f39b82 --- /dev/null +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -0,0 +1,273 @@ +package filesystem_impl + +import ( + "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/stretchr/testify/assert" + "testing" +) + +type fakeRunner struct { + err error + calls int + hostnames []string + cmdStrs []string +} + +func (f *fakeRunner) Execute(hostname string, cmdStr string) error { + f.calls += 1 + f.hostnames = append(f.hostnames, hostname) + f.cmdStrs = append(f.cmdStrs, cmdStr) + if cmdStr == "grep /dac/job1_job /etc/mtab" { + return errors.New("trigger mount") + } + return f.err +} + +func Test_mkdir(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + + err := mkdir("host", "dir") + assert.Nil(t, err) + assert.Equal(t, "host", fake.hostnames[0]) + assert.Equal(t, "mkdir -p dir", fake.cmdStrs[0]) + + runner = &fakeRunner{err: errors.New("expected")} + err = mkdir("", "") + assert.Equal(t, "expected", err.Error()) +} + +func Test_mountLustre(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + + err := mountLustre("host", "-opa@o2ib1", "mgt", "fs", "/dac/job1_job") + assert.Nil(t, err) + assert.Equal(t, 2, fake.calls) + assert.Equal(t, "host", fake.hostnames[0]) + assert.Equal(t, "host", fake.hostnames[1]) + assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[0]) + assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid mgt-opa@o2ib1:/fs /dac/job1_job", fake.cmdStrs[1]) + + fake = &fakeRunner{err: errors.New("expected")} + runner = fake + err = mountRemoteFilesystem(Lustre, "host", "", "mgt", "fs", "asdf") + assert.Equal(t, "expected", err.Error()) + assert.Equal(t, 2, fake.calls) + assert.Equal(t, "grep asdf /etc/mtab", fake.cmdStrs[0]) + assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid mgt:/fs asdf", fake.cmdStrs[1]) +} + +func Test_createSwap(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + + err := createSwap("host", 3, "file", "loopback") + assert.Nil(t, err) + assert.Equal(t, "host", fake.hostnames[0]) + assert.Equal(t, "host", fake.hostnames[1]) + assert.Equal(t, "host", fake.hostnames[2]) + assert.Equal(t, 4, len(fake.cmdStrs)) + assert.Equal(t, "dd if=/dev/zero of=file bs=1024 count=3072", fake.cmdStrs[0]) + assert.Equal(t, "chmod 0600 file", fake.cmdStrs[1]) + assert.Equal(t, "losetup loopback file", fake.cmdStrs[2]) + assert.Equal(t, "mkswap loopback", fake.cmdStrs[3]) +} + +func Test_fixUpOwnership(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + + err := fixUpOwnership("host", 10, 11, "dir") + assert.Nil(t, err) + + assert.Equal(t, 2, fake.calls) + assert.Equal(t, "host", fake.hostnames[0]) + assert.Equal(t, "chown 10:11 dir", fake.cmdStrs[0]) + assert.Equal(t, "host", fake.hostnames[1]) + assert.Equal(t, "chmod 770 dir", fake.cmdStrs[1]) +} + +func Test_Mount(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + attachments := []registry.Attachment{ + {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, + {Hostname: "client2", Job: "job1", State: registry.RequestAttach}, + {Hostname: "client3", Job: "job3", State: registry.Attached}, + {Hostname: "client3", Job: "job3", State: registry.RequestDetach}, + {Hostname: "client3", Job: "job3", State: registry.Detached}, + {Hostname: "client2", Job: "job2", State: registry.RequestAttach}, + } + volume := registry.Volume{ + Name: "asdf", JobName: "asdf", + AttachGlobalNamespace: true, + AttachPrivateNamespace: true, + AttachAsSwapBytes: 1024 * 1024, // 1 MiB + Attachments: attachments, + ClientPort: 42, + Owner: 1001, + Group: 1001, + } + + assert.PanicsWithValue(t, + "failed to find primary brick for volume: asdf", + func() { mount(Lustre, volume, nil, nil) }) + + bricks := []registry.BrickAllocation{ + {Hostname: "host1"}, + {Hostname: "host2"}, + } + err := mount(Lustre, volume, bricks, attachments) + assert.Nil(t, err) + assert.Equal(t, 53, fake.calls) + + assert.Equal(t, "client1", fake.hostnames[0]) + assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[0]) + assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[1]) + assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid host1:/ /dac/job1_job", fake.cmdStrs[2]) + + assert.Equal(t, "mkdir -p /dac/job1_job/swap", fake.cmdStrs[3]) + assert.Equal(t, "chown 0:0 /dac/job1_job/swap", fake.cmdStrs[4]) + assert.Equal(t, "chmod 770 /dac/job1_job/swap", fake.cmdStrs[5]) + assert.Equal(t, "dd if=/dev/zero of=/dac/job1_job/swap/client1 bs=1024 count=1024", fake.cmdStrs[6]) + assert.Equal(t, "chmod 0600 /dac/job1_job/swap/client1", fake.cmdStrs[7]) + assert.Equal(t, "losetup /dev/loop42 /dac/job1_job/swap/client1", fake.cmdStrs[8]) + assert.Equal(t, "mkswap /dev/loop42", fake.cmdStrs[9]) + assert.Equal(t, "swapon /dev/loop42", fake.cmdStrs[10]) + assert.Equal(t, "mkdir -p /dac/job1_job/private/client1", fake.cmdStrs[11]) + assert.Equal(t, "chown 1001:1001 /dac/job1_job/private/client1", fake.cmdStrs[12]) + assert.Equal(t, "chmod 770 /dac/job1_job/private/client1", fake.cmdStrs[13]) + assert.Equal(t, "ln -s /dac/job1_job/private/client1 /dac/job1_job_private", fake.cmdStrs[14]) + + assert.Equal(t, "mkdir -p /dac/job1_job/global", fake.cmdStrs[15]) + assert.Equal(t, "chown 1001:1001 /dac/job1_job/global", fake.cmdStrs[16]) + assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[17]) + + assert.Equal(t, "client2", fake.hostnames[18]) + assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[18]) + + assert.Equal(t, "client2", fake.hostnames[36]) + assert.Equal(t, "mkdir -p /dac/job2_job", fake.cmdStrs[36]) + assert.Equal(t, "client2", fake.hostnames[52]) + assert.Equal(t, "chmod 770 /dac/job2_job/global", fake.cmdStrs[52]) +} + +func Test_Umount(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + attachments := []registry.Attachment{ + {Hostname: "client1", Job: "job4", State: registry.RequestDetach}, + {Hostname: "client2", Job: "job4", State: registry.RequestDetach}, + {Hostname: "client3", Job: "job3", State: registry.Attached}, + {Hostname: "client3", Job: "job3", State: registry.RequestAttach}, + {Hostname: "client3", Job: "job3", State: registry.Detached}, + {Hostname: "client2", Job: "job1", State: registry.RequestDetach}, + } + volume := registry.Volume{ + Name: "asdf", JobName: "asdf", + AttachGlobalNamespace: true, + AttachPrivateNamespace: true, + AttachAsSwapBytes: 10000, + Attachments: attachments, + ClientPort: 42, + Owner: 1001, + Group: 1001, + } + bricks := []registry.BrickAllocation{ + {Hostname: "host1"}, + {Hostname: "host2"}, + } + err := umount(Lustre, volume, bricks, attachments) + assert.Nil(t, err) + assert.Equal(t, 20, fake.calls) + + assert.Equal(t, "client1", fake.hostnames[0]) + assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[0]) + assert.Equal(t, "losetup -d /dev/loop42", fake.cmdStrs[1]) + assert.Equal(t, "rm -rf /dac/job4_job/swap/client1", fake.cmdStrs[2]) + assert.Equal(t, "rm -rf /dac/job4_job_private", fake.cmdStrs[3]) + assert.Equal(t, "grep /dac/job4_job /etc/mtab", fake.cmdStrs[4]) + assert.Equal(t, "umount /dac/job4_job", fake.cmdStrs[5]) + assert.Equal(t, "rm -rf /dac/job4_job", fake.cmdStrs[6]) + + assert.Equal(t, "client2", fake.hostnames[7]) + assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[7]) + + assert.Equal(t, "client2", fake.hostnames[19]) + assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[19]) +} + +func Test_Umount_multi(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + attachments := []registry.Attachment{ + {Hostname: "client1", Job: "job1", State: registry.RequestDetach}, + } + volume := registry.Volume{ + MultiJob: true, + Name: "asdf", JobName: "asdf", + AttachGlobalNamespace: true, + AttachPrivateNamespace: true, + AttachAsSwapBytes: 10000, + Attachments: attachments, + ClientPort: 42, + Owner: 1001, + Group: 1001, + } + bricks := []registry.BrickAllocation{ + {Hostname: "host1"}, + {Hostname: "host2"}, + } + err := umount(Lustre, volume, bricks, attachments) + assert.Nil(t, err) + assert.Equal(t, 3, fake.calls) + + assert.Equal(t, "client1", fake.hostnames[0]) + assert.Equal(t, "grep /dac/job1_persistent_asdf /etc/mtab", fake.cmdStrs[0]) + assert.Equal(t, "umount /dac/job1_persistent_asdf", fake.cmdStrs[1]) + assert.Equal(t, "rm -rf /dac/job1_persistent_asdf", fake.cmdStrs[2]) +} + +func Test_Mount_multi(t *testing.T) { + defer func() { runner = &run{} }() + fake := &fakeRunner{} + runner = fake + attachments := []registry.Attachment{ + {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, + } + volume := registry.Volume{ + MultiJob: true, + Name: "asdf", JobName: "asdf", + AttachGlobalNamespace: true, + AttachPrivateNamespace: true, + AttachAsSwapBytes: 10000, + Attachments: attachments, + ClientPort: 42, + Owner: 1001, + Group: 1001, + UUID: "medkDfdg", + } + bricks := []registry.BrickAllocation{ + {Hostname: "host1"}, + {Hostname: "host2"}, + } + err := mount(Lustre, volume, bricks, attachments) + assert.Nil(t, err) + assert.Equal(t, 5, fake.calls) + + assert.Equal(t, "client1", fake.hostnames[0]) + assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf", fake.cmdStrs[0]) + assert.Equal(t, "grep /dac/job1_persistent_asdf /etc/mtab", fake.cmdStrs[1]) + assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf/global", fake.cmdStrs[2]) + assert.Equal(t, "chown 1001:1001 /dac/job1_persistent_asdf/global", fake.cmdStrs[3]) + assert.Equal(t, "chmod 770 /dac/job1_persistent_asdf/global", fake.cmdStrs[4]) +} From 1341a1d01344d60e50eb0214229861a200801c72 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 00:55:28 +0100 Subject: [PATCH 114/191] Add error handling for mount/unmount --- .../session_action_handler.go | 75 +++++++++++++++---- internal/pkg/v2/datamodel/session.go | 2 +- internal/pkg/v2/filesystem/provider.go | 4 +- internal/pkg/v2/filesystem_impl/provider.go | 8 +- internal/pkg/v2/mock_filesystem/provider.go | 8 +- 5 files changed, 72 insertions(+), 25 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 7873855a..d6d89506 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -165,34 +165,81 @@ func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { } func (s *sessionActionHandler) doAllMounts(action datamodel.SessionAction) error { + attachmentSession := datamodel.AttachmentSession{ + Hosts: action.Session.RequestedAttachHosts, + SessionName: action.Session.Name, + } if action.Session.ActualSizeBytes > 0 { - s.fsProvider.Mount(action.Session, - datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + return err + } + jobAttachmentStatus := datamodel.AttachmentSessionStatus{ + AttachmentSession: attachmentSession, + GlobalMount: session.VolumeRequest.Access == datamodel.Striped || session.VolumeRequest.Access == datamodel.PrivateAndStriped, + PrivateMount: session.VolumeRequest.Access == datamodel.Private || session.VolumeRequest.Access == datamodel.PrivateAndStriped, + SwapBytes: session.VolumeRequest.SwapBytes, + } + if session.CurrentAttachments == nil { + session.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ + session.Name: jobAttachmentStatus, + } + } else { + session.CurrentAttachments[session.Name] = jobAttachmentStatus + } + session, err = s.sessionRegistry.UpdateSession(session) + if err != nil { + return err + } + + if err := s.fsProvider.Mount(action.Session, jobAttachmentStatus); err != nil { + return err + } } for _, sessionName := range action.Session.MultiJobAttachments { - session, _ := s.sessionRegistry.GetSession(sessionName) - if session.VolumeRequest.MultiJob { - s.fsProvider.Mount(session, - datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + // TODO: get lock on multiJobSession? do it in order to avoid deadlock? + multiJobAttachmentStatus := datamodel.AttachmentSessionStatus{ + AttachmentSession: attachmentSession, + GlobalMount: true, + } + multiJobSession, _ := s.sessionRegistry.GetSession(sessionName) + if multiJobSession.CurrentAttachments == nil { + multiJobSession.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ + attachmentSession.SessionName: multiJobAttachmentStatus, + } + } else { + multiJobSession.CurrentAttachments[attachmentSession.SessionName] = multiJobAttachmentStatus + } + multiJobSession, err := s.sessionRegistry.UpdateSession(multiJobSession) + if err != nil { + return err + } + + if !multiJobSession.VolumeRequest.MultiJob { + log.Panicf("trying multi-job attach to non-multi job session %s", multiJobSession.Name) + } + if err := s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus); err != nil { + return err } } - // TODO: error handling!!! return nil } func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) error { if action.Session.ActualSizeBytes > 0 { - s.fsProvider.Unmount(action.Session, - datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + if err := s.fsProvider.Unmount(action.Session, action.Session.CurrentAttachments[action.Session.Name]); err != nil { + return err + } + // TODO: delete attachments? } for _, sessionName := range action.Session.MultiJobAttachments { - session, _ := s.sessionRegistry.GetSession(sessionName) - if session.VolumeRequest.MultiJob { - s.fsProvider.Mount(session, - datamodel.AttachmentSession{Hosts: action.Session.RequestedAttachHosts}) + multiJobSession, _ := s.sessionRegistry.GetSession(sessionName) + attachments := multiJobSession.CurrentAttachments[action.Session.Name] + if err := s.fsProvider.Unmount(multiJobSession, attachments); err != nil { + return err } + // TODO: delete attachments to prevent double unmount on teardown? } - // TODO error handling!!! return nil } diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index ddc2918b..c29c3f61 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -83,7 +83,7 @@ type AttachmentSessionStatus struct { PrivateMount bool SwapBytes int - DetachRequested bool + DetachRequested bool // TODO: delete this bit? Error error } diff --git a/internal/pkg/v2/filesystem/provider.go b/internal/pkg/v2/filesystem/provider.go index 93d6e6f9..aa636ad7 100644 --- a/internal/pkg/v2/filesystem/provider.go +++ b/internal/pkg/v2/filesystem/provider.go @@ -9,6 +9,6 @@ type Provider interface { DataCopyIn(session datamodel.Session) error DataCopyOut(session datamodel.Session) error - Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus - Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus + Mount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error + Unmount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error } diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index d815e4e9..c07d41f4 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -36,13 +36,13 @@ func (f *fileSystemProvider) DataCopyOut(session datamodel.Session) error { } -func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { +func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { log.Println("FAKE Mount") - return datamodel.AttachmentSessionStatus{} + return nil } -func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { +func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { log.Println("FAKE Unmount") - return datamodel.AttachmentSessionStatus{} + return nil } diff --git a/internal/pkg/v2/mock_filesystem/provider.go b/internal/pkg/v2/mock_filesystem/provider.go index 3e0da507..4a8507af 100644 --- a/internal/pkg/v2/mock_filesystem/provider.go +++ b/internal/pkg/v2/mock_filesystem/provider.go @@ -91,10 +91,10 @@ func (mr *MockProviderMockRecorder) DataCopyOut(session interface{}) *gomock.Cal } // Mount mocks base method -func (m *MockProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { +func (m *MockProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Mount", session, attachments) - ret0, _ := ret[0].(datamodel.AttachmentSessionStatus) + ret0, _ := ret[0].(error) return ret0 } @@ -105,10 +105,10 @@ func (mr *MockProviderMockRecorder) Mount(session, attachments interface{}) *gom } // Unmount mocks base method -func (m *MockProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSession) datamodel.AttachmentSessionStatus { +func (m *MockProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Unmount", session, attachments) - ret0, _ := ret[0].(datamodel.AttachmentSessionStatus) + ret0, _ := ret[0].(error) return ret0 } From 1e377d96021f2f81a7e7bd591bf03f3e395b05b3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 09:19:53 +0100 Subject: [PATCH 115/191] Add missing /global to persistent directory --- internal/pkg/v2/dacctl/actions_impl/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index c9066e70..0663734d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -100,7 +100,7 @@ func getPaths(session datamodel.Session) map[string]string { } for _, multiJobVolume := range session.MultiJobAttachments { paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf( - "/dac/%s_persistent_%s", session.Name, multiJobVolume) + "/dac/%s_persistent_%s/global", session.Name, multiJobVolume) } return paths } From 14432b5f9bd25eb1aa5238293107e09f58b3e125 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 09:46:33 +0100 Subject: [PATCH 116/191] PARTIAL: try to wire up mount, umount next --- internal/pkg/v2/filesystem_impl/ansible.go | 80 +++++++-------- internal/pkg/v2/filesystem_impl/mount.go | 102 +++++++++----------- internal/pkg/v2/filesystem_impl/provider.go | 45 ++++++--- 3 files changed, 116 insertions(+), 111 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/v2/filesystem_impl/ansible.go index dcbb1c9b..b62fb911 100644 --- a/internal/pkg/v2/filesystem_impl/ansible.go +++ b/internal/pkg/v2/filesystem_impl/ansible.go @@ -3,7 +3,7 @@ package filesystem_impl import ( "bytes" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "gopkg.in/yaml.v2" "io/ioutil" "log" @@ -38,7 +38,7 @@ type Wrapper struct { var DefaultHostGroup = "dac-prod" var DefaultMaxMDTs uint = 24 -func getInventory(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) string { +func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) string { // NOTE: only used by lustre mgsDevice := os.Getenv("DAC_MGS_DEV") if mgsDevice == "" { @@ -50,32 +50,35 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi maxMDTs = uint(maxMDTsConf) } - allocationsByHost := make(map[string][]registry.BrickAllocation) - for _, allocation := range brickAllocations { - allocationsByHost[allocation.Hostname] = append(allocationsByHost[allocation.Hostname], allocation) + allocationByHost := make(map[datamodel.BrickHostName][]datamodel.BrickAllocation) + for i, brick := range allBricks { + allocationByHost[brick.BrickHostName] = append(allocationByHost[brick.BrickHostName], datamodel.BrickAllocation{ + Brick:brick, + AllocatedIndex: uint(i), + }) } // If we have more brick allocations than maxMDTs // assign at most one mdt per host. // While this may give us less MDTs than max MDTs, // but it helps spread MDTs across network connections - oneMdtPerHost := len(brickAllocations) > int(maxMDTs) + oneMdtPerHost := len(allBricks) > int(maxMDTs) hosts := make(map[string]HostInfo) mgsnode := "" - for host, allocations := range allocationsByHost { + for host, allocations := range allocationByHost { osts := make(map[string]int) for _, allocation := range allocations { - osts[allocation.Device] = int(allocation.AllocatedIndex) + osts[allocation.Brick.Device] = int(allocation.AllocatedIndex) } mdts := make(map[string]int) if oneMdtPerHost { allocation := allocations[0] - mdts[allocation.Device] = int(allocation.AllocatedIndex) + mdts[allocation.Brick.Device] = int(allocation.AllocatedIndex) } else { for _, allocation := range allocations { - mdts[allocation.Device] = int(allocation.AllocatedIndex) + mdts[allocation.Brick.Device] = int(allocation.AllocatedIndex) } } @@ -85,31 +88,25 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi if fsType == Lustre { hostInfo.MGS = mgsDevice } else { - hostInfo.MGS = allocations[0].Device + hostInfo.MGS = allocations[0].Brick.Device } - mgsnode = host + mgsnode = string(host) } - hosts[host] = hostInfo + hosts[string(host)] = hostInfo } - // for beegfs, mount clients via ansible - if fsType == BeegFS { - // TODO: this can't work now, as we need to also pass the job name - for _, attachment := range volume.Attachments { - hosts[attachment.Hostname] = HostInfo{} - } - } + // TODO: add attachments? fsinfo := FSInfo{ Vars: map[string]string{ "mgsnode": mgsnode, - "client_port": fmt.Sprintf("%d", volume.ClientPort), + //"client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": getLnetSuffix(), "mdt_size": fmt.Sprintf("%dm", getMdtSizeMB()), }, Hosts: hosts, } - fsname := fmt.Sprintf("%s", volume.UUID) + fsname := fmt.Sprintf("%s", fsUuid) data := Wrapper{All: FileSystems{Children: map[string]FSInfo{fsname: fsinfo}}} output, err := yaml.Marshal(data) @@ -132,7 +129,7 @@ func getInventory(fsType FSType, volume registry.Volume, brickAllocations []regi return strOut } -func getPlaybook(fsType FSType, volume registry.Volume) string { +func getPlaybook(fsType FSType, fsUuid string) string { role := "lustre" if fsType == BeegFS { role = "beegfs" @@ -145,7 +142,7 @@ func getPlaybook(fsType FSType, volume registry.Volume) string { roles: - role: %s vars: - fs_name: %s`, volume.UUID, role, volume.UUID) + fs_name: %s`, fsUuid, role, fsUuid) } func getAnsibleDir(suffix string) string { @@ -156,21 +153,21 @@ func getAnsibleDir(suffix string) string { return path.Join(ansibleDir, suffix) } -func setupAnsible(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) (string, error) { - dir, err := ioutil.TempDir("", fmt.Sprintf("fs%s_", volume.Name)) +func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) (string, error) { + dir, err := ioutil.TempDir("", fmt.Sprintf("fs%s_", internalName)) if err != nil { return dir, err } log.Println("Using ansible tempdir:", dir) - playbook := getPlaybook(fsType, volume) + playbook := getPlaybook(fsType, internalName) tmpPlaybook := filepath.Join(dir, "dac.yml") if err := ioutil.WriteFile(tmpPlaybook, bytes.NewBufferString(playbook).Bytes(), 0666); err != nil { return dir, err } log.Println(playbook) - inventory := getInventory(fsType, volume, brickAllocations) + inventory := getInventory(fsType, internalName, bricks) tmpInventory := filepath.Join(dir, "inventory") if err := ioutil.WriteFile(tmpInventory, bytes.NewBufferString(inventory).Bytes(), 0666); err != nil { return dir, err @@ -195,11 +192,13 @@ func setupAnsible(fsType FSType, volume registry.Volume, brickAllocations []regi return dir, err } -func executeAnsibleSetup(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) +func executeAnsibleSetup(internalName string, bricks []datamodel.Brick) error { + // TODO: restore beegfs support + dir, err := setupAnsible(Lustre, internalName, bricks) if err != nil { return err } + defer os.RemoveAll(dir) formatArgs := "dac.yml -i inventory --tag format" err = executeAnsiblePlaybook(dir, formatArgs) @@ -208,18 +207,11 @@ func executeAnsibleSetup(fsType FSType, volume registry.Volume, brickAllocations } startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" - err = executeAnsiblePlaybook(dir, startupArgs) - if err != nil { - return err - } - - // only delete if everything worked, to aid debugging - os.RemoveAll(dir) - return nil + return executeAnsiblePlaybook(dir, startupArgs) } -func executeAnsibleTeardown(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) +func executeAnsibleTeardown(internalName string, bricks []datamodel.Brick) error { + dir, err := setupAnsible(Lustre, internalName, bricks) if err != nil { return err } @@ -241,8 +233,8 @@ func executeAnsibleTeardown(fsType FSType, volume registry.Volume, brickAllocati return nil } -func executeAnsibleMount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) +func executeAnsibleMount(internalName string, bricks []datamodel.Brick) error { + dir, err := setupAnsible(Lustre, internalName, bricks) if err != nil { return err } @@ -257,8 +249,8 @@ func executeAnsibleMount(fsType FSType, volume registry.Volume, brickAllocations return nil } -func executeAnsibleUnmount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) +func executeAnsibleUnmount(internalName string, bricks []datamodel.Brick) error { + dir, err := setupAnsible(Lustre, internalName, bricks) if err != nil { return err } diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/v2/filesystem_impl/mount.go index 99130063..cfb1bf5f 100644 --- a/internal/pkg/v2/filesystem_impl/mount.go +++ b/internal/pkg/v2/filesystem_impl/mount.go @@ -3,6 +3,7 @@ package filesystem_impl import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "os" "os/exec" @@ -11,12 +12,12 @@ import ( "time" ) -func getMountDir(volume registry.Volume, jobName string) string { +func getMountDir(sourceName datamodel.SessionName, isMultiJob bool, attachingForSession datamodel.SessionName) string { // TODO: what about the environment variables that are being set? should share logic with here - if volume.MultiJob { - return fmt.Sprintf("/dac/%s_persistent_%s", jobName, volume.Name) + if isMultiJob { + return fmt.Sprintf("/dac/%s_persistent_%s", sourceName, attachingForSession) } - return fmt.Sprintf("/dac/%s_job", jobName) + return fmt.Sprintf("/dac/%s_job", sourceName) } func getLnetSuffix() string { @@ -35,18 +36,13 @@ func getMdtSizeMB() uint { return uint(20 * 1024) } -func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Mount for:", volume.Name) - var primaryBrickHost string - for _, allocation := range brickAllocations { - if allocation.AllocatedIndex == 0 { - primaryBrickHost = allocation.Hostname - break - } - } +func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, internalName string, + primaryBrickHost datamodel.BrickHostName, attachments datamodel.AttachmentSessionStatus, + owner uint, group uint) error { + log.Println("Mount for:", sessionName) if primaryBrickHost == "" { - log.Panicf("failed to find primary brick for volume: %s", volume.Name) + log.Panicf("failed to find primary brick for volume: %s", sessionName) } lnetSuffix := getLnetSuffix() @@ -54,67 +50,61 @@ func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.Br if fsType == BeegFS { // Write out the config needed, and do the mount using ansible // TODO: Move Lustre mount here that is done below - executeAnsibleMount(fsType, volume, brickAllocations) + //executeAnsibleMount(fsType, volume, brickAllocations) } - for _, attachment := range attachments { - if attachment.State != registry.RequestAttach { - log.Printf("Skipping volume %s attach: %+v", volume.Name, attachment) - continue - } - log.Printf("Volume %s attaching with: %+v", volume.Name, attachment) + for _, attachHost := range attachments.AttachmentSession.Hosts { + log.Printf("Session %s attaching to: %+v", sessionName, attachHost) - var mountDir = getMountDir(volume, attachment.Job) - if err := mkdir(attachment.Hostname, mountDir); err != nil { + var mountDir = getMountDir(sessionName, isMultiJob, attachments.AttachmentSession.SessionName) + if err := mkdir(attachHost, mountDir); err != nil { return err } - if err := mountRemoteFilesystem(fsType, attachment.Hostname, lnetSuffix, - primaryBrickHost, volume.UUID, mountDir); err != nil { + if err := mountRemoteFilesystem(fsType, attachHost, lnetSuffix, + string(primaryBrickHost), internalName, mountDir); err != nil { return err } - - if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { - swapDir := path.Join(mountDir, "/swap") - if err := mkdir(attachment.Hostname, swapDir); err != nil { - return err - } - if err := fixUpOwnership(attachment.Hostname, 0, 0, swapDir); err != nil { - return err - } - - swapSizeMB := int(volume.AttachAsSwapBytes / (1024 * 1024)) - swapFile := path.Join(swapDir, fmt.Sprintf("/%s", attachment.Hostname)) - loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) - if err := createSwap(attachment.Hostname, swapSizeMB, swapFile, loopback); err != nil { - return err - } - - if err := swapOn(attachment.Hostname, loopback); err != nil { - return err - } - } - - if !volume.MultiJob && volume.AttachPrivateNamespace { - privateDir := path.Join(mountDir, fmt.Sprintf("/private/%s", attachment.Hostname)) - if err := mkdir(attachment.Hostname, privateDir); err != nil { + // TODO: swap! + //if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { + // swapDir := path.Join(mountDir, "/swap") + // if err := mkdir(attachment.Hostname, swapDir); err != nil { + // return err + // } + // if err := fixUpOwnership(attachment.Hostname, 0, 0, swapDir); err != nil { + // return err + // } + // swapSizeMB := int(volume.AttachAsSwapBytes / (1024 * 1024)) + // swapFile := path.Join(swapDir, fmt.Sprintf("/%s", attachment.Hostname)) + // loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) + // if err := createSwap(attachment.Hostname, swapSizeMB, swapFile, loopback); err != nil { + // return err + // } + // if err := swapOn(attachment.Hostname, loopback); err != nil { + // return err + // } + //} + + if attachments.PrivateMount { + privateDir := path.Join(mountDir, fmt.Sprintf("/private/%s", attachHost)) + if err := mkdir(attachHost, privateDir); err != nil { return err } - if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, privateDir); err != nil { + if err := fixUpOwnership(attachHost, owner, group, privateDir); err != nil { return err } // need a consistent symlink for shared environment variables across all hosts - privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) - if err := createSymbolicLink(attachment.Hostname, privateDir, privateSymLinkDir); err != nil { + privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", sessionName) + if err := createSymbolicLink(attachHost, privateDir, privateSymLinkDir); err != nil { return err } } sharedDir := path.Join(mountDir, "/global") - if err := mkdir(attachment.Hostname, sharedDir); err != nil { + if err := mkdir(attachHost, sharedDir); err != nil { return err } - if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, sharedDir); err != nil { + if err := fixUpOwnership(attachHost, owner, group, sharedDir); err != nil { return err } } @@ -168,7 +158,7 @@ func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.B if fsType == BeegFS { // TODO: Move Lustre unmount here that is done below - executeAnsibleUnmount(fsType, volume, brickAllocations) + // executeAnsibleUnmount(fsType, volume, brickAllocations) // TODO: this makes copy out much harder in its current form :( } diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index c07d41f4..83545cb8 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -4,6 +4,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" "log" + "math/rand" ) func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { @@ -12,37 +13,59 @@ func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { type fileSystemProvider struct { ansible filesystem.Ansible + // TODO: proper config object +} + +const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +func GetNewUUID() string { + b := make([]byte, 8) + for i := range b { + b[i] = letters[rand.Int63()%int64(len(letters))] + } + return string(b) } func (f *fileSystemProvider) Create(session datamodel.Session) (datamodel.FilesystemStatus, error) { - log.Println("FAKE Create") - return datamodel.FilesystemStatus{InternalName: "foo", InternalData: "bar"}, nil + session.FilesystemStatus = datamodel.FilesystemStatus{ + InternalName: GetNewUUID(), + InternalData: "", + } + err := executeAnsibleSetup(session.FilesystemStatus.InternalName, session.AllocatedBricks) + return session.FilesystemStatus, err } func (f *fileSystemProvider) Delete(session datamodel.Session) error { - log.Println("FAKE Delete") - return nil + return executeAnsibleTeardown(session.FilesystemStatus.InternalName, session.AllocatedBricks) } func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { - log.Println("FAKE DataCopyIn") + for _, dataCopy := range session.StageInRequests { + err := processDataCopy(dataCopy) + if err != nil { + return err + } + } return nil } func (f *fileSystemProvider) DataCopyOut(session datamodel.Session) error { - log.Println("FAKE DataCopyOut") + for _, dataCopy := range session.StageOutRequests { + err := processDataCopy(dataCopy) + if err != nil { + return err + } + } return nil - } func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { - log.Println("FAKE Mount") - return nil + return mount(Lustre, session.Name, session.VolumeRequest.MultiJob, session.FilesystemStatus.InternalName, + session.PrimaryBrickHost, attachments, session.Owner, session.Group) } func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { - log.Println("FAKE Unmount") - return nil + return mount(Lustre, session, attachments) } From 0646f43d4c0191a138fba658f0004d635f69294d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 11:34:12 +0100 Subject: [PATCH 117/191] Get data copy compiling --- internal/pkg/v2/filesystem_impl/copy.go | 28 ++++----- internal/pkg/v2/filesystem_impl/mount.go | 68 ++++++++++----------- internal/pkg/v2/filesystem_impl/provider.go | 8 +-- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/v2/filesystem_impl/copy.go index 2dcc985c..cc13eab2 100644 --- a/internal/pkg/v2/filesystem_impl/copy.go +++ b/internal/pkg/v2/filesystem_impl/copy.go @@ -2,19 +2,19 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "path" "strings" ) -func processDataCopy(volume registry.Volume, request registry.DataCopyRequest) error { - cmd, err := generateDataCopyCmd(volume, request) +func processDataCopy(session datamodel.Session, request datamodel.DataCopyRequest) error { + cmd, err := generateDataCopyCmd(session, request) if err != nil { return err } if cmd == "" { - log.Println("No files to copy for:", volume.Name) + log.Println("No files to copy for:", session.Name) return nil } @@ -23,12 +23,12 @@ func processDataCopy(volume registry.Volume, request registry.DataCopyRequest) e // Make sure global dir is setup correctly // TODO: share code with mount better // TODO: Probably should all get setup in fs-ansible really!! - mountDir := fmt.Sprintf("/mnt/lustre/%s", volume.UUID) + mountDir := fmt.Sprintf("/mnt/lustre/%s", session.FilesystemStatus.InternalName) sharedDir := path.Join(mountDir, "/global") if err := mkdir("localhost", sharedDir); err != nil { return err } - if err := fixUpOwnership("localhost", volume.Owner, volume.Group, sharedDir); err != nil { + if err := fixUpOwnership("localhost", session.Owner, session.Group, sharedDir); err != nil { return err } @@ -36,30 +36,30 @@ func processDataCopy(volume registry.Volume, request registry.DataCopyRequest) e return runner.Execute("localhost", cmd) } -func generateDataCopyCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { - rsync, err := generateRsyncCmd(volume, request) +func generateDataCopyCmd(session datamodel.Session, request datamodel.DataCopyRequest) (string, error) { + rsync, err := generateRsyncCmd(session, request) if err != nil || rsync == "" { return "", err } - cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", volume.Group, volume.Owner, rsync) - dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", volume.UUID) + cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", session.Group, session.Owner, rsync) + dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", session.FilesystemStatus.InternalData) cmd = fmt.Sprintf("bash -c \"export DW_JOB_STRIPED='%s' && %s\"", dacHostBufferPath, cmd) return cmd, nil } -func generateRsyncCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { +func generateRsyncCmd(session datamodel.Session, request datamodel.DataCopyRequest) (string, error) { if request.Source == "" && request.Destination == "" { return "", nil } var flags string - if request.SourceType == registry.Directory { + if request.SourceType == datamodel.Directory { flags = "-r -ospgu --stats" - } else if request.SourceType == registry.File { + } else if request.SourceType == datamodel.File { flags = "-ospgu --stats" } else { - return "", fmt.Errorf("unsupported source type %s for volume: %s", request.SourceType, volume.Name) + return "", fmt.Errorf("unsupported source type %s for volume: %s", request.SourceType, session.Name) } return fmt.Sprintf("rsync %s %s %s", flags, diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/v2/filesystem_impl/mount.go index cfb1bf5f..8e723757 100644 --- a/internal/pkg/v2/filesystem_impl/mount.go +++ b/internal/pkg/v2/filesystem_impl/mount.go @@ -2,7 +2,6 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "os" @@ -37,7 +36,7 @@ func getMdtSizeMB() uint { } func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, internalName string, - primaryBrickHost datamodel.BrickHostName, attachments datamodel.AttachmentSessionStatus, + primaryBrickHost datamodel.BrickHostName, attachment datamodel.AttachmentSessionStatus, owner uint, group uint) error { log.Println("Mount for:", sessionName) @@ -53,10 +52,11 @@ func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, in //executeAnsibleMount(fsType, volume, brickAllocations) } - for _, attachHost := range attachments.AttachmentSession.Hosts { - log.Printf("Session %s attaching to: %+v", sessionName, attachHost) + for _, attachHost := range attachment.AttachmentSession.Hosts { + log.Printf("Mounting %s on host: %s for session: %s", sessionName, attachHost, + attachment.AttachmentSession.SessionName) - var mountDir = getMountDir(sessionName, isMultiJob, attachments.AttachmentSession.SessionName) + var mountDir = getMountDir(sessionName, isMultiJob, attachment.AttachmentSession.SessionName) if err := mkdir(attachHost, mountDir); err != nil { return err } @@ -84,7 +84,7 @@ func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, in // } //} - if attachments.PrivateMount { + if attachment.PrivateMount { privateDir := path.Join(mountDir, fmt.Sprintf("/private/%s", attachHost)) if err := mkdir(attachHost, privateDir); err != nil { return err @@ -113,46 +113,43 @@ func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, in return nil } -func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Umount for:", volume.Name) +func unmount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, internalName string, + primaryBrickHost datamodel.BrickHostName, attachment datamodel.AttachmentSessionStatus) error { + log.Println("Umount for:", sessionName) - for _, attachment := range attachments { - if attachment.State != registry.RequestDetach { - log.Printf("Skipping volume %s detach for: %+v", volume.Name, attachment) - continue - } - log.Printf("Volume %s dettaching: %+v", volume.Name, attachment) - - var mountDir = getMountDir(volume, attachment.Job) - if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { - swapFile := path.Join(mountDir, fmt.Sprintf("/swap/%s", attachment.Hostname)) // TODO share? - loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) // TODO share? - if err := swapOff(attachment.Hostname, loopback); err != nil { - log.Printf("Warn: failed to swap off %+v", attachment) - } - if err := detachLoopback(attachment.Hostname, loopback); err != nil { - log.Printf("Warn: failed to detach loopback %+v", attachment) - } - if err := removeSubtree(attachment.Hostname, swapFile); err != nil { - return err - } - } + for _, attachHost := range attachment.AttachmentSession.Hosts { + log.Printf("Unmounting %s on host: %s for session: %s", sessionName, attachHost, + attachment.AttachmentSession.SessionName) - if !volume.MultiJob && volume.AttachPrivateNamespace { - privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) - if err := removeSubtree(attachment.Hostname, privateSymLinkDir); err != nil { + var mountDir = getMountDir(sessionName, isMultiJob, attachment.AttachmentSession.SessionName) + // TODO: swap! + //if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { + // swapFile := path.Join(mountDir, fmt.Sprintf("/swap/%s", attachment.Hostname)) // TODO share? + // loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) // TODO share? + // if err := swapOff(attachment.Hostname, loopback); err != nil { + // log.Printf("Warn: failed to swap off %+v", attachment) + // } + // if err := detachLoopback(attachment.Hostname, loopback); err != nil { + // log.Printf("Warn: failed to detach loopback %+v", attachment) + // } + // if err := removeSubtree(attachment.Hostname, swapFile); err != nil { + // return err + // } + //} + if attachment.PrivateMount { + privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", sessionName) + if err := removeSubtree(attachHost, privateSymLinkDir); err != nil { return err } } if fsType == Lustre { - if err := umountLustre(attachment.Hostname, mountDir); err != nil { + if err := umountLustre(attachHost, mountDir); err != nil { return err } - if err := removeSubtree(attachment.Hostname, mountDir); err != nil { + if err := removeSubtree(attachHost, mountDir); err != nil { return err } - } } @@ -161,7 +158,6 @@ func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.B // executeAnsibleUnmount(fsType, volume, brickAllocations) // TODO: this makes copy out much harder in its current form :( } - return nil } diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index 83545cb8..b45031b3 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -3,7 +3,6 @@ package filesystem_impl import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" - "log" "math/rand" ) @@ -41,7 +40,7 @@ func (f *fileSystemProvider) Delete(session datamodel.Session) error { func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { for _, dataCopy := range session.StageInRequests { - err := processDataCopy(dataCopy) + err := processDataCopy(session, dataCopy) if err != nil { return err } @@ -52,7 +51,7 @@ func (f *fileSystemProvider) DataCopyIn(session datamodel.Session) error { func (f *fileSystemProvider) DataCopyOut(session datamodel.Session) error { for _, dataCopy := range session.StageOutRequests { - err := processDataCopy(dataCopy) + err := processDataCopy(session, dataCopy) if err != nil { return err } @@ -67,5 +66,6 @@ func (f *fileSystemProvider) Mount(session datamodel.Session, attachments datamo } func (f *fileSystemProvider) Unmount(session datamodel.Session, attachments datamodel.AttachmentSessionStatus) error { - return mount(Lustre, session, attachments) + return unmount(Lustre, session.Name, session.VolumeRequest.MultiJob, session.FilesystemStatus.InternalName, + session.PrimaryBrickHost, attachments) } From 56f6096f0bef825a7d00c16477b394edbc4fd137 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 11:46:35 +0100 Subject: [PATCH 118/191] Fix test relating to adding /global to path --- internal/pkg/v2/dacctl/actions_impl/job_test.go | 4 ++-- internal/pkg/v2/filesystem_impl/ansible.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 587bf595..1143777d 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -81,8 +81,8 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { Paths: map[string]string{ "DW_JOB_PRIVATE": "/dac/token_job_private", "DW_JOB_STRIPED": "/dac/token_job/global", - "DW_PERSISTENT_STRIPED_myBBname1": "/dac/token_persistent_myBBname1", - "DW_PERSISTENT_STRIPED_myBBname2": "/dac/token_persistent_myBBname2", + "DW_PERSISTENT_STRIPED_myBBname1": "/dac/token_persistent_myBBname1/global", + "DW_PERSISTENT_STRIPED_myBBname2": "/dac/token_persistent_myBBname2/global", }, }).Return(nil) diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/v2/filesystem_impl/ansible.go index b62fb911..96b20b35 100644 --- a/internal/pkg/v2/filesystem_impl/ansible.go +++ b/internal/pkg/v2/filesystem_impl/ansible.go @@ -53,7 +53,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str allocationByHost := make(map[datamodel.BrickHostName][]datamodel.BrickAllocation) for i, brick := range allBricks { allocationByHost[brick.BrickHostName] = append(allocationByHost[brick.BrickHostName], datamodel.BrickAllocation{ - Brick:brick, + Brick: brick, AllocatedIndex: uint(i), }) } @@ -99,7 +99,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str fsinfo := FSInfo{ Vars: map[string]string{ - "mgsnode": mgsnode, + "mgsnode": mgsnode, //"client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": getLnetSuffix(), "mdt_size": fmt.Sprintf("%dm", getMdtSizeMB()), From 4ab3399b46874b3fe27a723e5f0c5fc1063ac9a8 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 12:35:30 +0100 Subject: [PATCH 119/191] Fix but in data copy Was using InternalData instead of InternalName --- .../pkg/v2/filesystem_impl/ansible_test.go | 80 ++++++----------- internal/pkg/v2/filesystem_impl/copy.go | 2 +- internal/pkg/v2/filesystem_impl/copy_test.go | 32 +++---- internal/pkg/v2/filesystem_impl/mount_test.go | 88 +++++++++++++++++-- 4 files changed, 125 insertions(+), 77 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/ansible_test.go b/internal/pkg/v2/filesystem_impl/ansible_test.go index 863059d1..54e53afc 100644 --- a/internal/pkg/v2/filesystem_impl/ansible_test.go +++ b/internal/pkg/v2/filesystem_impl/ansible_test.go @@ -2,33 +2,25 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/stretchr/testify/assert" "testing" ) func TestPlugin_GetInventory(t *testing.T) { - volume := registry.Volume{ - Name: "1", UUID: "abcdefgh", ClientPort: 10002, - Attachments: []registry.Attachment{ - {Hostname: "cpu1"}, - {Hostname: "cpu2"}, - }, + brickAllocations := []datamodel.Brick{ + {BrickHostName: "dac1", Device: "nvme1n1"}, + {BrickHostName: "dac1", Device: "nvme2n1"}, + {BrickHostName: "dac1", Device: "nvme3n1"}, + {BrickHostName: "dac2", Device: "nvme2n1"}, + {BrickHostName: "dac2", Device: "nvme3n1"}, } - brickAllocations := []registry.BrickAllocation{ - {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, - {Hostname: "dac1", Device: "nvme2n1", AllocatedIndex: 1}, - {Hostname: "dac1", Device: "nvme3n1", AllocatedIndex: 2}, - {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 3}, - {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 4}, - } - result := getInventory(BeegFS, volume, brickAllocations) + fsUuid := "abcdefgh" + result := getInventory(BeegFS, fsUuid, brickAllocations) expected := `dac-prod: children: abcdefgh: hosts: - cpu1: {} - cpu2: {} dac1: abcdefgh_mgs: nvme1n1 abcdefgh_mdts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} @@ -37,7 +29,6 @@ func TestPlugin_GetInventory(t *testing.T) { abcdefgh_mdts: {nvme2n1: 3, nvme3n1: 4} abcdefgh_osts: {nvme2n1: 3, nvme3n1: 4} vars: - abcdefgh_client_port: "10002" lnet_suffix: "" abcdefgh_mdt_size: 20480m abcdefgh_mgsnode: dac1 @@ -46,13 +37,13 @@ func TestPlugin_GetInventory(t *testing.T) { } func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh", ClientPort: 10002} - brickAllocations := []registry.BrickAllocation{ - {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, - {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 1}, - {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 2}, + brickAllocations := []datamodel.Brick{ + {BrickHostName: "dac1", Device: "nvme1n1"}, + {BrickHostName: "dac2", Device: "nvme2n1"}, + {BrickHostName: "dac2", Device: "nvme3n1"}, } - result := getInventory(Lustre, volume, brickAllocations) + fsUuid := "abcdefgh" + result := getInventory(Lustre, fsUuid, brickAllocations) expected := `dac-prod: children: abcdefgh: @@ -65,7 +56,6 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { abcdefgh_mdts: {nvme2n1: 1, nvme3n1: 2} abcdefgh_osts: {nvme2n1: 1, nvme3n1: 2} vars: - abcdefgh_client_port: "10002" lnet_suffix: "" abcdefgh_mdt_size: 20480m abcdefgh_mgsnode: dac1 @@ -74,8 +64,8 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { } func TestPlugin_GetPlaybook_beegfs(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh"} - result := getPlaybook(BeegFS, volume) + fsUuid := "abcdefgh" + result := getPlaybook(BeegFS, fsUuid) assert.Equal(t, `--- - name: Setup FS hosts: abcdefgh @@ -88,8 +78,8 @@ func TestPlugin_GetPlaybook_beegfs(t *testing.T) { } func TestPlugin_GetPlaybook_lustre(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh"} - result := getPlaybook(Lustre, volume) + fsUuid := "abcdefgh" + result := getPlaybook(Lustre, fsUuid) assert.Equal(t, `--- - name: Setup FS hosts: abcdefgh @@ -102,37 +92,26 @@ func TestPlugin_GetPlaybook_lustre(t *testing.T) { } func TestPlugin_GetInventory_MaxMDT(t *testing.T) { - volume := registry.Volume{ - Name: "1", UUID: "abcdefgh", ClientPort: 10002, - Attachments: []registry.Attachment{ - {Hostname: "cpu1"}, - {Hostname: "cpu2"}, - }, - } - - var brickAllocations []registry.BrickAllocation + var brickAllocations []datamodel.Brick for i := 1; i <= 26; i = i + 2 { - brickAllocations = append(brickAllocations, registry.BrickAllocation{ - Hostname: fmt.Sprintf("dac%d", i), - Device: "nvme1n1", - AllocatedIndex: uint(i - 1), + brickAllocations = append(brickAllocations, datamodel.Brick{ + BrickHostName: datamodel.BrickHostName(fmt.Sprintf("dac%d", i)), + Device: "nvme1n1", }) - brickAllocations = append(brickAllocations, registry.BrickAllocation{ - Hostname: fmt.Sprintf("dac%d", i), - Device: "nvme2n1", - AllocatedIndex: uint(i), + brickAllocations = append(brickAllocations, datamodel.Brick{ + BrickHostName: datamodel.BrickHostName(fmt.Sprintf("dac%d", i)), + Device: "nvme2n1", }) } - result := getInventory(BeegFS, volume, brickAllocations) + fsUuid := "abcdefgh" + result := getInventory(Lustre, fsUuid, brickAllocations) expected := `dac-prod: children: abcdefgh: hosts: - cpu1: {} - cpu2: {} dac1: - abcdefgh_mgs: nvme1n1 + abcdefgh_mgs: sdb abcdefgh_mdts: {nvme1n1: 0} abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1} dac3: @@ -172,7 +151,6 @@ func TestPlugin_GetInventory_MaxMDT(t *testing.T) { abcdefgh_mdts: {nvme1n1: 24} abcdefgh_osts: {nvme1n1: 24, nvme2n1: 25} vars: - abcdefgh_client_port: "10002" lnet_suffix: "" abcdefgh_mdt_size: 20480m abcdefgh_mgsnode: dac1 diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/v2/filesystem_impl/copy.go index cc13eab2..50698088 100644 --- a/internal/pkg/v2/filesystem_impl/copy.go +++ b/internal/pkg/v2/filesystem_impl/copy.go @@ -43,7 +43,7 @@ func generateDataCopyCmd(session datamodel.Session, request datamodel.DataCopyRe } cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", session.Group, session.Owner, rsync) - dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", session.FilesystemStatus.InternalData) + dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", session.FilesystemStatus.InternalName) cmd = fmt.Sprintf("bash -c \"export DW_JOB_STRIPED='%s' && %s\"", dacHostBufferPath, cmd) return cmd, nil } diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index f8e60378..e8abb30d 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -1,64 +1,64 @@ package filesystem_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/stretchr/testify/assert" "testing" ) func Test_GenerateDataCopy(t *testing.T) { - testVolume := registry.Volume{ - Name: registry.VolumeName("asdf"), + session := datamodel.Session{ + Name: "asdf", Owner: 1001, Group: 1002, - UUID: "fsuuid", + FilesystemStatus: datamodel.FilesystemStatus{InternalName:"fsuuid"}, } - request := registry.DataCopyRequest{} + request := datamodel.DataCopyRequest{} - cmd, err := generateDataCopyCmd(testVolume, request) + cmd, err := generateDataCopyCmd(session, request) assert.Nil(t, err) assert.Empty(t, cmd) - request.SourceType = registry.File + request.SourceType = datamodel.File request.Source = "$DW_JOB_STRIPED/source" request.Destination = "dest" - cmd, err = generateDataCopyCmd(testVolume, request) + cmd, err = generateDataCopyCmd(session, request) assert.Nil(t, err) assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) - request.SourceType = registry.List + request.SourceType = datamodel.List request.Source = "list_filename" - cmd, err = generateDataCopyCmd(testVolume, request) + cmd, err = generateDataCopyCmd(session, request) assert.Equal(t, "", cmd) assert.Equal(t, "unsupported source type list for volume: asdf", err.Error()) } func Test_GenerateRsyncCmd(t *testing.T) { - testVolume := registry.Volume{ - Name: registry.VolumeName("asdf"), + testVolume := datamodel.Session{ + Name: "asdf", } - request := registry.DataCopyRequest{} + request := datamodel.DataCopyRequest{} cmd, err := generateRsyncCmd(testVolume, request) assert.Nil(t, err) assert.Empty(t, cmd) - request.SourceType = registry.File + request.SourceType = datamodel.File request.Source = "source" request.Destination = "dest" cmd, err = generateRsyncCmd(testVolume, request) assert.Nil(t, err) assert.Equal(t, "rsync -ospgu --stats source dest", cmd) - request.SourceType = registry.Directory + request.SourceType = datamodel.Directory request.Source = "source" request.Destination = "dest" cmd, err = generateRsyncCmd(testVolume, request) assert.Nil(t, err) assert.Equal(t, "rsync -r -ospgu --stats source dest", cmd) - request.SourceType = registry.List + request.SourceType = datamodel.List request.Source = "list_filename" cmd, err = generateRsyncCmd(testVolume, request) assert.Equal(t, "", cmd) diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go index e9f39b82..8e5ef72b 100644 --- a/internal/pkg/v2/filesystem_impl/mount_test.go +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -3,7 +3,9 @@ package filesystem_impl import ( "errors" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/stretchr/testify/assert" + "log" "testing" ) @@ -115,16 +117,31 @@ func Test_Mount(t *testing.T) { Owner: 1001, Group: 1001, } - - assert.PanicsWithValue(t, - "failed to find primary brick for volume: asdf", - func() { mount(Lustre, volume, nil, nil) }) - bricks := []registry.BrickAllocation{ {Hostname: "host1"}, {Hostname: "host2"}, } - err := mount(Lustre, volume, bricks, attachments) + log.Println(bricks) + log.Println(volume) + + //err := mount(Lustre, volume, bricks, attachments) + sessionName := datamodel.SessionName("asdf") + internalName := "uuidasdf" + primaryBrickHost := datamodel.BrickHostName("host1") + owner := uint(1001) + group := uint(1002) + attachment := datamodel.AttachmentSessionStatus{ + AttachmentSession: datamodel.AttachmentSession{ + SessionName: "job1", + Hosts: []string{"client1", "client2"}, + }, + GlobalMount: true, + PrivateMount: true, + SwapBytes: 1024 * 1024, // 1 MiB + } + err := mount(Lustre, sessionName, false, + internalName, primaryBrickHost, attachment, + owner, group) assert.Nil(t, err) assert.Equal(t, 53, fake.calls) @@ -185,7 +202,23 @@ func Test_Umount(t *testing.T) { {Hostname: "host1"}, {Hostname: "host2"}, } - err := umount(Lustre, volume, bricks, attachments) + log.Println(bricks) + log.Println(volume) + + sessionName := datamodel.SessionName("asdf") + internalName := "uuidasdf" + primaryBrickHost := datamodel.BrickHostName("host1") + attachment := datamodel.AttachmentSessionStatus{ + AttachmentSession: datamodel.AttachmentSession{ + SessionName: "job1", + Hosts: []string{"client1", "client2"}, + }, + GlobalMount: true, + PrivateMount: true, + SwapBytes: 1024 * 1024, // 1 MiB + } + err := unmount(Lustre, sessionName, false, + internalName, primaryBrickHost, attachment) assert.Nil(t, err) assert.Equal(t, 20, fake.calls) @@ -227,7 +260,24 @@ func Test_Umount_multi(t *testing.T) { {Hostname: "host1"}, {Hostname: "host2"}, } - err := umount(Lustre, volume, bricks, attachments) + log.Println(bricks) + log.Println(volume) + + sessionName := datamodel.SessionName("asdf") + internalName := "uuidasdf" + primaryBrickHost := datamodel.BrickHostName("host1") + attachment := datamodel.AttachmentSessionStatus{ + AttachmentSession: datamodel.AttachmentSession{ + SessionName: "job1", + Hosts: []string{"client1", "client2"}, + }, + GlobalMount: true, + PrivateMount: false, + SwapBytes: 0, + } + err := unmount(Lustre, sessionName, true, + internalName, primaryBrickHost, attachment) + assert.Nil(t, err) assert.Equal(t, 3, fake.calls) @@ -260,7 +310,27 @@ func Test_Mount_multi(t *testing.T) { {Hostname: "host1"}, {Hostname: "host2"}, } - err := mount(Lustre, volume, bricks, attachments) + log.Println(bricks) + log.Println(volume) + + sessionName := datamodel.SessionName("asdf") + internalName := "uuidasdf" + primaryBrickHost := datamodel.BrickHostName("host1") + owner := uint(1001) + group := uint(1002) + attachment := datamodel.AttachmentSessionStatus{ + AttachmentSession: datamodel.AttachmentSession{ + SessionName: "job1", + Hosts: []string{"client1", "client2"}, + }, + GlobalMount: true, + PrivateMount: false, + SwapBytes: 0, + } + err := mount(Lustre, sessionName, true, + internalName, primaryBrickHost, attachment, + owner, group) + assert.Nil(t, err) assert.Equal(t, 5, fake.calls) From 6fcf2528b3f7a5a63d2d893efd154d109c705785 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 17:05:13 +0100 Subject: [PATCH 120/191] Fix up copy and mount tests --- internal/pkg/v2/filesystem_impl/copy_test.go | 8 +- internal/pkg/v2/filesystem_impl/mount_test.go | 89 ++++++++----------- 2 files changed, 40 insertions(+), 57 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index e8abb30d..d7367334 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -8,10 +8,10 @@ import ( func Test_GenerateDataCopy(t *testing.T) { session := datamodel.Session{ - Name: "asdf", - Owner: 1001, - Group: 1002, - FilesystemStatus: datamodel.FilesystemStatus{InternalName:"fsuuid"}, + Name: "asdf", + Owner: 1001, + Group: 1002, + FilesystemStatus: datamodel.FilesystemStatus{InternalName: "fsuuid"}, } request := datamodel.DataCopyRequest{} diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go index 8e5ef72b..734dec18 100644 --- a/internal/pkg/v2/filesystem_impl/mount_test.go +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -125,55 +125,44 @@ func Test_Mount(t *testing.T) { log.Println(volume) //err := mount(Lustre, volume, bricks, attachments) - sessionName := datamodel.SessionName("asdf") - internalName := "uuidasdf" + sessionName := datamodel.SessionName("job1") + internalName := "fsuuid" primaryBrickHost := datamodel.BrickHostName("host1") owner := uint(1001) group := uint(1002) attachment := datamodel.AttachmentSessionStatus{ AttachmentSession: datamodel.AttachmentSession{ - SessionName: "job1", + SessionName: "job2", // changed to prove this is not used Hosts: []string{"client1", "client2"}, }, - GlobalMount: true, + GlobalMount: true, PrivateMount: true, - SwapBytes: 1024 * 1024, // 1 MiB + SwapBytes: 1024 * 1024, // 1 MiB } err := mount(Lustre, sessionName, false, internalName, primaryBrickHost, attachment, owner, group) assert.Nil(t, err) - assert.Equal(t, 53, fake.calls) + assert.Equal(t, 20, fake.calls) assert.Equal(t, "client1", fake.hostnames[0]) assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[0]) assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[1]) - assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid host1:/ /dac/job1_job", fake.cmdStrs[2]) - - assert.Equal(t, "mkdir -p /dac/job1_job/swap", fake.cmdStrs[3]) - assert.Equal(t, "chown 0:0 /dac/job1_job/swap", fake.cmdStrs[4]) - assert.Equal(t, "chmod 770 /dac/job1_job/swap", fake.cmdStrs[5]) - assert.Equal(t, "dd if=/dev/zero of=/dac/job1_job/swap/client1 bs=1024 count=1024", fake.cmdStrs[6]) - assert.Equal(t, "chmod 0600 /dac/job1_job/swap/client1", fake.cmdStrs[7]) - assert.Equal(t, "losetup /dev/loop42 /dac/job1_job/swap/client1", fake.cmdStrs[8]) - assert.Equal(t, "mkswap /dev/loop42", fake.cmdStrs[9]) - assert.Equal(t, "swapon /dev/loop42", fake.cmdStrs[10]) - assert.Equal(t, "mkdir -p /dac/job1_job/private/client1", fake.cmdStrs[11]) - assert.Equal(t, "chown 1001:1001 /dac/job1_job/private/client1", fake.cmdStrs[12]) - assert.Equal(t, "chmod 770 /dac/job1_job/private/client1", fake.cmdStrs[13]) - assert.Equal(t, "ln -s /dac/job1_job/private/client1 /dac/job1_job_private", fake.cmdStrs[14]) - - assert.Equal(t, "mkdir -p /dac/job1_job/global", fake.cmdStrs[15]) - assert.Equal(t, "chown 1001:1001 /dac/job1_job/global", fake.cmdStrs[16]) - assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[17]) - - assert.Equal(t, "client2", fake.hostnames[18]) - assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[18]) - - assert.Equal(t, "client2", fake.hostnames[36]) - assert.Equal(t, "mkdir -p /dac/job2_job", fake.cmdStrs[36]) - assert.Equal(t, "client2", fake.hostnames[52]) - assert.Equal(t, "chmod 770 /dac/job2_job/global", fake.cmdStrs[52]) + assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid host1:/fsuuid /dac/job1_job", fake.cmdStrs[2]) + + assert.Equal(t, "mkdir -p /dac/job1_job/private/client1", fake.cmdStrs[3]) + assert.Equal(t, "chown 1001:1002 /dac/job1_job/private/client1", fake.cmdStrs[4]) + assert.Equal(t, "chmod 770 /dac/job1_job/private/client1", fake.cmdStrs[5]) + assert.Equal(t, "ln -s /dac/job1_job/private/client1 /dac/job1_job_private", fake.cmdStrs[6]) + + assert.Equal(t, "mkdir -p /dac/job1_job/global", fake.cmdStrs[7]) + assert.Equal(t, "chown 1001:1002 /dac/job1_job/global", fake.cmdStrs[8]) + assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[9]) + + assert.Equal(t, "client2", fake.hostnames[10]) + assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[10]) + assert.Equal(t, "client2", fake.hostnames[19]) + assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[19]) } func Test_Umount(t *testing.T) { @@ -205,37 +194,31 @@ func Test_Umount(t *testing.T) { log.Println(bricks) log.Println(volume) - sessionName := datamodel.SessionName("asdf") + sessionName := datamodel.SessionName("job1") internalName := "uuidasdf" primaryBrickHost := datamodel.BrickHostName("host1") attachment := datamodel.AttachmentSessionStatus{ AttachmentSession: datamodel.AttachmentSession{ - SessionName: "job1", + SessionName: "job2", Hosts: []string{"client1", "client2"}, }, - GlobalMount: true, + GlobalMount: true, PrivateMount: true, - SwapBytes: 1024 * 1024, // 1 MiB + SwapBytes: 1024 * 1024, // 1 MiB } err := unmount(Lustre, sessionName, false, internalName, primaryBrickHost, attachment) assert.Nil(t, err) - assert.Equal(t, 20, fake.calls) + assert.Equal(t, 6, fake.calls) assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[0]) - assert.Equal(t, "losetup -d /dev/loop42", fake.cmdStrs[1]) - assert.Equal(t, "rm -rf /dac/job4_job/swap/client1", fake.cmdStrs[2]) - assert.Equal(t, "rm -rf /dac/job4_job_private", fake.cmdStrs[3]) - assert.Equal(t, "grep /dac/job4_job /etc/mtab", fake.cmdStrs[4]) - assert.Equal(t, "umount /dac/job4_job", fake.cmdStrs[5]) - assert.Equal(t, "rm -rf /dac/job4_job", fake.cmdStrs[6]) - - assert.Equal(t, "client2", fake.hostnames[7]) - assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[7]) + assert.Equal(t, "rm -rf /dac/job1_job_private", fake.cmdStrs[1]) + assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[2]) + assert.Equal(t, "umount /dac/job1_job", fake.cmdStrs[3]) + assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[4]) - assert.Equal(t, "client2", fake.hostnames[19]) - assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[19]) + assert.Equal(t, "client2", fake.hostnames[5]) + assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[5]) } func Test_Umount_multi(t *testing.T) { @@ -271,9 +254,9 @@ func Test_Umount_multi(t *testing.T) { SessionName: "job1", Hosts: []string{"client1", "client2"}, }, - GlobalMount: true, + GlobalMount: true, PrivateMount: false, - SwapBytes: 0, + SwapBytes: 0, } err := unmount(Lustre, sessionName, true, internalName, primaryBrickHost, attachment) @@ -323,9 +306,9 @@ func Test_Mount_multi(t *testing.T) { SessionName: "job1", Hosts: []string{"client1", "client2"}, }, - GlobalMount: true, + GlobalMount: true, PrivateMount: false, - SwapBytes: 0, + SwapBytes: 0, } err := mount(Lustre, sessionName, true, internalName, primaryBrickHost, attachment, From cf42a08be61f4ea57c76fdc7ee7e6426d6946cec Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 21 Aug 2019 20:59:25 +0100 Subject: [PATCH 121/191] Fix up mount tests --- internal/pkg/v2/filesystem_impl/mount.go | 2 +- internal/pkg/v2/filesystem_impl/mount_test.go | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/v2/filesystem_impl/mount.go index 8e723757..82dddc1d 100644 --- a/internal/pkg/v2/filesystem_impl/mount.go +++ b/internal/pkg/v2/filesystem_impl/mount.go @@ -14,7 +14,7 @@ import ( func getMountDir(sourceName datamodel.SessionName, isMultiJob bool, attachingForSession datamodel.SessionName) string { // TODO: what about the environment variables that are being set? should share logic with here if isMultiJob { - return fmt.Sprintf("/dac/%s_persistent_%s", sourceName, attachingForSession) + return fmt.Sprintf("/dac/%s_persistent_%s", attachingForSession, sourceName) } return fmt.Sprintf("/dac/%s_job", sourceName) } diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go index 734dec18..4f964b75 100644 --- a/internal/pkg/v2/filesystem_impl/mount_test.go +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -161,6 +161,7 @@ func Test_Mount(t *testing.T) { assert.Equal(t, "client2", fake.hostnames[10]) assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[10]) + assert.Equal(t, "client2", fake.hostnames[19]) assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[19]) } @@ -194,8 +195,8 @@ func Test_Umount(t *testing.T) { log.Println(bricks) log.Println(volume) - sessionName := datamodel.SessionName("job1") - internalName := "uuidasdf" + sessionName := datamodel.SessionName("job4") + internalName := "fsuuid" primaryBrickHost := datamodel.BrickHostName("host1") attachment := datamodel.AttachmentSessionStatus{ AttachmentSession: datamodel.AttachmentSession{ @@ -209,16 +210,16 @@ func Test_Umount(t *testing.T) { err := unmount(Lustre, sessionName, false, internalName, primaryBrickHost, attachment) assert.Nil(t, err) - assert.Equal(t, 6, fake.calls) + assert.Equal(t, 8, fake.calls) assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "rm -rf /dac/job1_job_private", fake.cmdStrs[1]) - assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[2]) - assert.Equal(t, "umount /dac/job1_job", fake.cmdStrs[3]) - assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[4]) + assert.Equal(t, "rm -rf /dac/job4_job_private", fake.cmdStrs[0]) + assert.Equal(t, "grep /dac/job4_job /etc/mtab", fake.cmdStrs[1]) + assert.Equal(t, "umount /dac/job4_job", fake.cmdStrs[2]) + assert.Equal(t, "rm -rf /dac/job4_job", fake.cmdStrs[3]) - assert.Equal(t, "client2", fake.hostnames[5]) - assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[5]) + assert.Equal(t, "client2", fake.hostnames[7]) + assert.Equal(t, "rm -rf /dac/job4_job", fake.cmdStrs[7]) } func Test_Umount_multi(t *testing.T) { @@ -252,7 +253,7 @@ func Test_Umount_multi(t *testing.T) { attachment := datamodel.AttachmentSessionStatus{ AttachmentSession: datamodel.AttachmentSession{ SessionName: "job1", - Hosts: []string{"client1", "client2"}, + Hosts: []string{"client1"}, }, GlobalMount: true, PrivateMount: false, @@ -304,7 +305,7 @@ func Test_Mount_multi(t *testing.T) { attachment := datamodel.AttachmentSessionStatus{ AttachmentSession: datamodel.AttachmentSession{ SessionName: "job1", - Hosts: []string{"client1", "client2"}, + Hosts: []string{"client1"}, }, GlobalMount: true, PrivateMount: false, @@ -321,6 +322,6 @@ func Test_Mount_multi(t *testing.T) { assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf", fake.cmdStrs[0]) assert.Equal(t, "grep /dac/job1_persistent_asdf /etc/mtab", fake.cmdStrs[1]) assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf/global", fake.cmdStrs[2]) - assert.Equal(t, "chown 1001:1001 /dac/job1_persistent_asdf/global", fake.cmdStrs[3]) + assert.Equal(t, "chown 1001:1002 /dac/job1_persistent_asdf/global", fake.cmdStrs[3]) assert.Equal(t, "chmod 770 /dac/job1_persistent_asdf/global", fake.cmdStrs[4]) } From 3efb0b57f66bb3845095c7273397bc09fa064655 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 16:15:23 +0100 Subject: [PATCH 122/191] Add all paths from session when doing copy Give rsync access to both per job and persistent buffers. --- internal/pkg/v2/filesystem_impl/copy.go | 12 ++++++++++-- internal/pkg/v2/filesystem_impl/copy_test.go | 10 +++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/v2/filesystem_impl/copy.go index 50698088..a829bf69 100644 --- a/internal/pkg/v2/filesystem_impl/copy.go +++ b/internal/pkg/v2/filesystem_impl/copy.go @@ -42,9 +42,17 @@ func generateDataCopyCmd(session datamodel.Session, request datamodel.DataCopyRe return "", err } + if len(session.Paths) < 1 { + log.Panicf("trying to do data copy in for session with no paths %+v", session) + } + var exports []string + for name, value := range session.Paths { + exports = append(exports, fmt.Sprintf("export %s='%s'", name, value)) + } + exportString := strings.Join(exports, " && ") + cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", session.Group, session.Owner, rsync) - dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", session.FilesystemStatus.InternalName) - cmd = fmt.Sprintf("bash -c \"export DW_JOB_STRIPED='%s' && %s\"", dacHostBufferPath, cmd) + cmd = fmt.Sprintf("bash -c \"%s && %s\"", exportString, cmd) return cmd, nil } diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index d7367334..785b557f 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -12,6 +12,10 @@ func Test_GenerateDataCopy(t *testing.T) { Owner: 1001, Group: 1002, FilesystemStatus: datamodel.FilesystemStatus{InternalName: "fsuuid"}, + Paths: map[string]string{ + "DW_JOB_STRIPED": "/mnt/lustre/fsuuid/global", + "DW_PERSISTENT_other": "/mnt/lustre/other/global", + }, } request := datamodel.DataCopyRequest{} @@ -24,7 +28,11 @@ func Test_GenerateDataCopy(t *testing.T) { request.Destination = "dest" cmd, err = generateDataCopyCmd(session, request) assert.Nil(t, err) - assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) + assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && export DW_PERSISTENT_other='/mnt/lustre/other/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) + + cmd, err = generateDataCopyCmd(session, request) + assert.Nil(t, err) + assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && export DW_PERSISTENT_other='/mnt/lustre/other/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) request.SourceType = datamodel.List request.Source = "list_filename" From ec40240cece4a2050b3922efafcc01913dde9dee Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 16:16:13 +0100 Subject: [PATCH 123/191] Errors move to strings so json marshaling works --- .../brick_manager_impl/session_action_handler.go | 4 +++- internal/pkg/v2/datamodel/session.go | 8 ++++---- internal/pkg/v2/registry_impl/session_test.go | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index d6d89506..f7fb8e2f 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -98,7 +98,9 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { fsStatus, err := s.fsProvider.Create(action.Session) session.FilesystemStatus = fsStatus session.Status.FileSystemCreated = err == nil - session.Status.Error = err + if err != nil { + session.Status.Error = err.Error() + } session, updateErr := s.sessionRegistry.UpdateSession(session) if updateErr != nil { diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index c29c3f61..5a900e95 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -66,7 +66,7 @@ type Session struct { } type FilesystemStatus struct { - Error error + Error string InternalName string InternalData string } @@ -84,14 +84,14 @@ type AttachmentSessionStatus struct { SwapBytes int DetachRequested bool // TODO: delete this bit? - Error error + Error string } type SessionStatus struct { // If not nil, the session has an unresolved error // and can't be mounted by any new sessions // but it can be deleted - Error error + Error string // CreateVolume has succeeded, so other actions can now happen FileSystemCreated bool @@ -152,7 +152,7 @@ type DataCopyRequest struct { // Report if the copy has completed CopyCompleted bool // if there was problem, record it - Error error + Error string } type SourceType string diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index cce232d2..01f5f078 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -11,14 +11,26 @@ import ( "testing" ) -var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":null,"FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false,"UnmountComplete":false,"MountComplete":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"AllocatedBricks":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":null,"InternalName":"","InternalData":""},"CurrentAttachments":null}`) +var exampleSessionString = []byte(`{"Name":"foo","Revision":0,"Owner":0,"Group":0,"CreatedAt":0,"VolumeRequest":{"MultiJob":false,"Caller":"","TotalCapacityBytes":0,"PoolName":"","Access":0,"Type":0,"SwapBytes":0},"Status":{"Error":"","FileSystemCreated":false,"CopyDataInComplete":false,"CopyDataOutComplete":false,"DeleteRequested":false,"DeleteSkipCopyDataOut":false,"UnmountComplete":false,"MountComplete":false},"StageInRequests":null,"StageOutRequests":null,"MultiJobAttachments":null,"Paths":null,"ActualSizeBytes":0,"AllocatedBricks":null,"PrimaryBrickHost":"host1","RequestedAttachHosts":null,"FilesystemStatus":{"Error":"","InternalName":"","InternalData":""},"CurrentAttachments":null}`) var exampleSession = datamodel.Session{Name: "foo", PrimaryBrickHost: "host1"} func TestExampleString(t *testing.T) { exampleStr, err := json.Marshal(exampleSession) assert.Nil(t, err) - // mostly here to make it easy to update the example string assert.Equal(t, string(exampleSessionString), string(exampleStr)) + + var unmarshalSession datamodel.Session + err = json.Unmarshal(exampleStr, &unmarshalSession) + assert.Nil(t, err) + assert.Equal(t, unmarshalSession, exampleSession) + + sessionWithError := datamodel.Session{ + Name: "foo", PrimaryBrickHost: "host1", + Status:datamodel.SessionStatus{Error:"fake_error"}, + } + sessionWithErrorStr, err := json.Marshal(sessionWithError) + assert.Nil(t, err) + assert.Contains(t, string(sessionWithErrorStr), "fake_error") } func TestSessionRegistry_GetSessionMutex(t *testing.T) { From f800fb15207cf60c5fc32ff5dea6fddc5c7cf9b0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 17:21:37 +0100 Subject: [PATCH 124/191] Use ansible 2.7 in the test setup Currently our group names cause issues with ansible 2.8, simplest to pin ansible at 2.8 for now. --- dac-ansible/roles/data-acc/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dac-ansible/roles/data-acc/tasks/main.yml b/dac-ansible/roles/data-acc/tasks/main.yml index 8ca41271..c9421d02 100644 --- a/dac-ansible/roles/data-acc/tasks/main.yml +++ b/dac-ansible/roles/data-acc/tasks/main.yml @@ -59,7 +59,7 @@ virtualenv .venv source .venv/bin/activate pip install -U pip - pip install -U ansible + pip install -U 'ansible<2.8' args: creates: "{{data_acc_install_dir}}/{{data_acc_name}}/fs-ansible/.venv" From 296909526581d4c12bbd2e1f2638676a5b1e66ec Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 17:51:37 +0100 Subject: [PATCH 125/191] Make session_actions unmarshal correctly --- internal/pkg/v2/dacctl/workflow_impl/session.go | 3 ++- internal/pkg/v2/dacctl/workflow_impl/session_test.go | 4 ++-- .../v2/dacd/brick_manager_impl/session_action_handler.go | 6 +++--- internal/pkg/v2/datamodel/session.go | 2 +- internal/pkg/v2/datamodel/session_action.go | 2 +- internal/pkg/v2/filesystem_impl/copy_test.go | 2 +- internal/pkg/v2/registry_impl/session_test.go | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 73eaa924..32612e6d 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -2,6 +2,7 @@ package workflow_impl import ( "context" + "errors" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" @@ -64,7 +65,7 @@ func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType d // wait for server to complete, or timeout result := <-sessionAction - return result.Error + return errors.New(result.Error) } func (s sessionFacade) CreateSession(session datamodel.Session) error { diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 6817e44a..19b55e47 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -132,7 +132,7 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, returnedSession).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) go func() { - actionChan <- datamodel.SessionAction{Error: fakeErr} + actionChan <- datamodel.SessionAction{Error: fakeErr.Error()} close(actionChan) }() @@ -166,7 +166,7 @@ func TestSessionFacade_DeleteSession(t *testing.T) { sessionMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") go func() { - actionChan <- datamodel.SessionAction{Error: fakeErr} + actionChan <- datamodel.SessionAction{Error: fakeErr.Error()} close(actionChan) }() diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index f7fb8e2f..e0909754 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -55,13 +55,13 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) - action.Error = err + action.Error = err.Error() return } err = sessionMutex.Lock(context.TODO()) if err != nil { log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) - action.Error = err + action.Error = err.Error() return } @@ -79,7 +79,7 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, session, err := process() if err != nil { - action.Error = err + action.Error = err.Error() log.Printf("error during action %+v\n", action) } else { action.Session = session diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/v2/datamodel/session.go index 5a900e95..fadc6486 100644 --- a/internal/pkg/v2/datamodel/session.go +++ b/internal/pkg/v2/datamodel/session.go @@ -66,7 +66,7 @@ type Session struct { } type FilesystemStatus struct { - Error string + Error string InternalName string InternalData string } diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/v2/datamodel/session_action.go index e638ccae..8d50b710 100644 --- a/internal/pkg/v2/datamodel/session_action.go +++ b/internal/pkg/v2/datamodel/session_action.go @@ -4,7 +4,7 @@ type SessionAction struct { Uuid string Session Session ActionType SessionActionType - Error error + Error string } type SessionActionType int diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index 785b557f..f81bf4a3 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -13,7 +13,7 @@ func Test_GenerateDataCopy(t *testing.T) { Group: 1002, FilesystemStatus: datamodel.FilesystemStatus{InternalName: "fsuuid"}, Paths: map[string]string{ - "DW_JOB_STRIPED": "/mnt/lustre/fsuuid/global", + "DW_JOB_STRIPED": "/mnt/lustre/fsuuid/global", "DW_PERSISTENT_other": "/mnt/lustre/other/global", }, } diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 01f5f078..284491dc 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -26,7 +26,7 @@ func TestExampleString(t *testing.T) { sessionWithError := datamodel.Session{ Name: "foo", PrimaryBrickHost: "host1", - Status:datamodel.SessionStatus{Error:"fake_error"}, + Status: datamodel.SessionStatus{Error: "fake_error"}, } sessionWithErrorStr, err := json.Marshal(sessionWithError) assert.Nil(t, err) From e489697a1eb013c276d2e93356ec5d394ebc222f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 18:05:11 +0100 Subject: [PATCH 126/191] Stop trying to fine client_port It is not used, and we fail as we no longer supply it. --- fs-ansible/roles/lustre/tasks/format.yaml | 1 - fs-ansible/roles/lustre/tasks/mount.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 2e004b94..aa85e78d 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -4,7 +4,6 @@ mgsnode: "{{ vars[fs_name + '_mgsnode']}}" mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" osts: "{{ vars[fs_name + '_osts'] | default({}) }}" - client_port: "{{ vars[fs_name + '_client_port'] }}" mdt_size: "{{ vars[fs_name + '_mdt_size'] | default('10%') }}" tags: [ 'never', 'format_mgs', 'reformat_mgs', 'format', 'clean'] diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index 1e54f949..0c7bb639 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -4,7 +4,6 @@ mgsnode: "{{ vars[fs_name + '_mgsnode'] }}" mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" osts: "{{ vars[fs_name + '_osts'] | default({}) }}" - client_port: "{{ vars[fs_name + '_client_port'] }}" tags: [ 'never', 'start_mgs', 'start_mdts', 'start_osts', 'create_mgs', 'create_mdt', 'create_osts', 'stop_all', 'client_mount','client_unmount', 'stop_mgs'] - name: load lustre module From 6c7b8b16c0df9c5d85a07b60c93a4fbe75407a92 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 18:21:10 +0100 Subject: [PATCH 127/191] Fix session action error reporting --- docker-slurm/Dockerfile | 2 +- internal/pkg/v2/dacctl/workflow_impl/session.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-slurm/Dockerfile b/docker-slurm/Dockerfile index 6afecd50..186e21ca 100644 --- a/docker-slurm/Dockerfile +++ b/docker-slurm/Dockerfile @@ -8,7 +8,7 @@ LABEL org.label-schema.docker.cmd="docker-compose up -d" \ org.label-schema.description="Slurm Docker cluster on CentOS 7" \ maintainer="John Garbutt" -ARG SLURM_TAG=slurm-19-05-1-2 +ARG SLURM_TAG=slurm-19-05-2-1 ARG GOSU_VERSION=1.11 RUN set -ex \ diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 32612e6d..fa52e5fa 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -65,7 +65,10 @@ func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType d // wait for server to complete, or timeout result := <-sessionAction - return errors.New(result.Error) + if result.Error != "" { + return errors.New(result.Error) + } + return nil } func (s sessionFacade) CreateSession(session datamodel.Session) error { From f5e5de7e0133a665073e68d3089a32358483ebee Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 19:00:24 +0100 Subject: [PATCH 128/191] Clean up response keys, when response not an error --- .../pkg/v2/dacctl/workflow_impl/session.go | 21 ++++++++++++++----- internal/pkg/v2/filesystem_impl/copy_test.go | 7 +++---- .../pkg/v2/registry_impl/session_actions.go | 11 ++++++++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index fa52e5fa..27d07ca8 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -54,7 +54,7 @@ func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType d } // This will error out if the host is not currently up - sessionAction, err := s.actions.SendSessionAction(context.TODO(), actionType, session) + sessionActions, err := s.actions.SendSessionAction(context.TODO(), actionType, session) mutexErr := sessionMutex.Unlock(context.TODO()) if err != nil { return err @@ -63,10 +63,21 @@ func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType d return mutexErr } - // wait for server to complete, or timeout - result := <-sessionAction - if result.Error != "" { - return errors.New(result.Error) + // ensure we get one value, and the channel is closed + var finalResult *datamodel.SessionAction + for action := range sessionActions { + if finalResult != nil { + log.Panicf("unexpected mulitple actions") + } + finalResult = &action + } + if finalResult == nil { + log.Panicf("failed to get reponse") + } + + // report and errors in the server response + if finalResult.Error != "" { + return errors.New(finalResult.Error) } return nil } diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index f81bf4a3..98f2af2e 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -13,8 +13,7 @@ func Test_GenerateDataCopy(t *testing.T) { Group: 1002, FilesystemStatus: datamodel.FilesystemStatus{InternalName: "fsuuid"}, Paths: map[string]string{ - "DW_JOB_STRIPED": "/mnt/lustre/fsuuid/global", - "DW_PERSISTENT_other": "/mnt/lustre/other/global", + "DW_JOB_STRIPED": "/mnt/lustre/fsuuid/global", }, } request := datamodel.DataCopyRequest{} @@ -28,11 +27,11 @@ func Test_GenerateDataCopy(t *testing.T) { request.Destination = "dest" cmd, err = generateDataCopyCmd(session, request) assert.Nil(t, err) - assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && export DW_PERSISTENT_other='/mnt/lustre/other/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) + assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) cmd, err = generateDataCopyCmd(session, request) assert.Nil(t, err) - assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && export DW_PERSISTENT_other='/mnt/lustre/other/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) + assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) request.SourceType = datamodel.List request.Source = "list_filename" diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index b8efe72d..d5868f71 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -109,12 +109,19 @@ func (s *sessionActions) SendSessionAction( log.Printf("found action response %+v\n", responseSessionAction) responseChan <- responseSessionAction - close(responseChan) + + // delete response now it has been delivered, but only if it was not an error response + if responseSessionAction.Error == "" { + if count, err := s.store.DeleteAllKeysWithPrefix(responseKey); err != nil || count != 1 { + log.Panicf("failed to clean up response key: %s", responseKey) + } + } log.Printf("completed waiting for action response %+v\n", sessionAction) + close(responseChan) return } - // TODO: don't we need to stop the watch above? + // TODO: don't we need to stop the watch above? will we see the delete event? }() return responseChan, nil } From 9793eacbe9f6d9a95fc36287f00fd979ba7d4835 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 19:34:07 +0100 Subject: [PATCH 129/191] Always sort multi-job volumes --- internal/pkg/v2/dacctl/actions_impl/job.go | 7 ++++++- internal/pkg/v2/dacctl/actions_impl/job_test.go | 2 +- internal/pkg/v2/registry_impl/session_actions.go | 5 ++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index 0663734d..c0a9a548 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -6,6 +6,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" + "sort" ) func (d *dacctlActions) ValidateJob(c dacctl.CliContext) error { @@ -73,6 +74,11 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { Type: bufferType, SwapBytes: swapBytes, } + // TODO: must be a better way! + // ensure multi job volumes are sorted, to avoid deadlocks (*cough*) + sort.Slice(multiJobVolumes, func (i, j int) bool { + return multiJobVolumes[i] < multiJobVolumes[j] + }) session := datamodel.Session{ Name: datamodel.SessionName(c.String("token")), Owner: uint(c.Int("user")), @@ -84,7 +90,6 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { StageOutRequests: summary.DataOut, } session.Paths = getPaths(session) - return d.session.CreateSession(session) } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 1143777d..bea7f91e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -38,8 +38,8 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { lines := []string{ `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, - `#DW persistentdw name=myBBname1`, `#DW persistentdw name=myBBname2`, + `#DW persistentdw name=myBBname1`, `#DW swap 4MiB`, `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, `#DW stage_in source=/global/cscratch1/filelist type=list`, diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index d5868f71..78de0d9a 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -168,9 +168,12 @@ func (s *sessionActions) CompleteSessionAction(sessionAction datamodel.SessionAc // Delete the request, not it is processed requestKey := getSessionActionRequestKey(sessionAction) - err = s.store.Delete(requestKey, 0) + count, err := s.store.DeleteAllKeysWithPrefix(requestKey) if err != nil { return fmt.Errorf("unable to delete stale request message due to: %s", err) } + if count != 1 { + return fmt.Errorf("unable to delete stale request message due to: %s", err) + } return nil } From 2da532028de61db3c91668385349926adcff6ea0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 20:53:01 +0100 Subject: [PATCH 130/191] Fix up formating --- internal/pkg/v2/dacctl/actions_impl/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index c0a9a548..d351643e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -76,7 +76,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { } // TODO: must be a better way! // ensure multi job volumes are sorted, to avoid deadlocks (*cough*) - sort.Slice(multiJobVolumes, func (i, j int) bool { + sort.Slice(multiJobVolumes, func(i, j int) bool { return multiJobVolumes[i] < multiJobVolumes[j] }) session := datamodel.Session{ From 35779eb296784dcc09da7e79ed0bdc7e83551436 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 21:11:58 +0100 Subject: [PATCH 131/191] Add missing locks around multi-job volumes --- .../session_action_handler.go | 70 +++++++++++++------ 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index e0909754..82fe2c95 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -199,32 +199,62 @@ func (s *sessionActionHandler) doAllMounts(action datamodel.SessionAction) error } } for _, sessionName := range action.Session.MultiJobAttachments { - // TODO: get lock on multiJobSession? do it in order to avoid deadlock? - multiJobAttachmentStatus := datamodel.AttachmentSessionStatus{ - AttachmentSession: attachmentSession, - GlobalMount: true, - } - multiJobSession, _ := s.sessionRegistry.GetSession(sessionName) - if multiJobSession.CurrentAttachments == nil { - multiJobSession.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ - attachmentSession.SessionName: multiJobAttachmentStatus, - } - } else { - multiJobSession.CurrentAttachments[attachmentSession.SessionName] = multiJobAttachmentStatus + if err := s.doMutliJobMount(action, sessionName); err != nil { + return nil } - multiJobSession, err := s.sessionRegistry.UpdateSession(multiJobSession) - if err != nil { - return err + } + return nil +} + +func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, sessionName datamodel.SessionName) error { + sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) + if err != nil { + log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) + return err + } + if err = sessionMutex.Lock(context.TODO()); err != nil { + log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) + return err + } + defer func() { + if err := sessionMutex.Unlock(context.TODO()); err != nil { + log.Println("failed to drop mutex for:", sessionName) } + }() - if !multiJobSession.VolumeRequest.MultiJob { - log.Panicf("trying multi-job attach to non-multi job session %s", multiJobSession.Name) + multiJobSession, err := s.sessionRegistry.GetSession(sessionName) + if err != nil { + return err + } + if !multiJobSession.VolumeRequest.MultiJob { + log.Panicf("trying multi-job attach to non-multi job session %s", multiJobSession.Name) + } + + attachmentSession := datamodel.AttachmentSession{ + Hosts: action.Session.RequestedAttachHosts, + SessionName: action.Session.Name, + } + multiJobAttachmentStatus := datamodel.AttachmentSessionStatus{ + AttachmentSession: attachmentSession, + GlobalMount: true, + } + if multiJobSession.CurrentAttachments == nil { + multiJobSession.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ + attachmentSession.SessionName: multiJobAttachmentStatus, } - if err := s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus); err != nil { - return err + } else { + if _, ok := multiJobSession.CurrentAttachments[attachmentSession.SessionName]; ok { + return fmt.Errorf("already attached for session %s and multi-job %s", + attachmentSession.SessionName, sessionName) } + multiJobSession.CurrentAttachments[attachmentSession.SessionName] = multiJobAttachmentStatus } - return nil + + multiJobSession, err = s.sessionRegistry.UpdateSession(multiJobSession) + if err != nil { + return err + } + return s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus) } func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) error { From 54cae8b8b6010dc3e12469e3405511293b061343 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 22:39:47 +0100 Subject: [PATCH 132/191] Tidy ups and extra logging to chase error --- .../session_action_handler.go | 4 +-- internal/pkg/v2/datamodel/session_action.go | 17 +++++++------ .../pkg/v2/registry_impl/session_actions.go | 2 ++ internal/pkg/v2/store_impl/keystore.go | 25 ++++++------------- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 82fe2c95..c3a26529 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -68,10 +68,10 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, // Always complete action and drop mutex on function exit defer func() { if err := s.actions.CompleteSessionAction(action); err != nil { - log.Printf("failed to complete action %+v\n", action) + log.Printf("failed to complete action %+v due to: %s\n", action, err.Error()) } if err := sessionMutex.Unlock(context.TODO()); err != nil { - log.Println("failed to drop mutex for:", sessionName) + log.Printf("failed to drop mutex for: %s due to: %s\n", sessionName, err.Error()) } }() diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/v2/datamodel/session_action.go index 8d50b710..286fc17c 100644 --- a/internal/pkg/v2/datamodel/session_action.go +++ b/internal/pkg/v2/datamodel/session_action.go @@ -7,14 +7,15 @@ type SessionAction struct { Error string } -type SessionActionType int +type SessionActionType string +// TODO: probably should be an int with custom parser? const ( - UnknownSessionAction SessionActionType = iota - SessionCreateFilesystem - SessionDelete - SessionCopyDataIn - SessionMount - SessionUnmount - SessionCopyDataOut + UnknownSessionAction SessionActionType = SessionActionType("") + SessionCreateFilesystem = SessionActionType("CreateFilesystem") + SessionDelete = SessionActionType("Delete") + SessionCopyDataIn = SessionActionType("CopyDataIn") + SessionMount = SessionActionType("Mount") + SessionUnmount = SessionActionType("Unmount") + SessionCopyDataOut = SessionActionType("CopyDataOut") ) diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index 78de0d9a..c9eae34c 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -175,5 +175,7 @@ func (s *sessionActions) CompleteSessionAction(sessionAction datamodel.SessionAc if count != 1 { return fmt.Errorf("unable to delete stale request message due to: %s", err) } + + log.Printf("Completed session action %s for session %s\n", sessionAction.Uuid, sessionAction.Session.Name) return nil } diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go index daace378..4dd6d663 100644 --- a/internal/pkg/v2/store_impl/keystore.go +++ b/internal/pkg/v2/store_impl/keystore.go @@ -109,9 +109,7 @@ func (client *etcKeystore) Close() error { } func (client *etcKeystore) runTransaction(ifOps []clientv3.Cmp, thenOps []clientv3.Op) error { - kvc := clientv3.NewKV(client.Client) - kvc.Txn(context.Background()) - response, err := kvc.Txn(context.Background()).If(ifOps...).Then(thenOps...).Commit() + response, err := client.Client.Txn(context.Background()).If(ifOps...).Then(thenOps...).Commit() handleError(err) if !response.Succeeded { @@ -128,7 +126,7 @@ func (client *etcKeystore) Create(key string, value []byte) (store.KeyValueVersi thenOps = append(thenOps, clientv3.OpPut(key, string(value))) err := client.runTransaction(ifOps, thenOps) if err != nil { - return store.KeyValueVersion{}, fmt.Errorf("unable to update ke: %s", err) + return store.KeyValueVersion{}, fmt.Errorf("unable to create key: %s due to: %s", key, err) } return client.Get(key) } @@ -180,15 +178,13 @@ func getKeyValueVersion(rawKeyValue *mvccpb.KeyValue) *store.KeyValueVersion { } func (client *etcKeystore) IsExist(key string) (bool, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Get(context.Background(), key) + response, err := client.Client.Get(context.Background(), key) handleError(err) return response.Count == 1, nil } func (client *etcKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Get(context.Background(), prefix, clientv3.WithPrefix()) + response, err := client.Client.Get(context.Background(), prefix, clientv3.WithPrefix()) handleError(err) var values []store.KeyValueVersion @@ -199,8 +195,7 @@ func (client *etcKeystore) GetAll(prefix string) ([]store.KeyValueVersion, error } func (client *etcKeystore) Get(key string) (store.KeyValueVersion, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Get(context.Background(), key) + response, err := client.Client.Get(context.Background(), key) handleError(err) value := store.KeyValueVersion{} @@ -216,9 +211,8 @@ func (client *etcKeystore) Get(key string) (store.KeyValueVersion, error) { } func (client *etcKeystore) KeepAliveKey(ctxt context.Context, key string) error { - kvc := clientv3.NewKV(client.Client) - getResponse, err := kvc.Get(context.Background(), key) + getResponse, err := client.Client.Get(context.Background(), key) if getResponse.Count == 1 { // if another host seems to exist, back off for 10 seconds incase we just did a quick restart time.Sleep(time.Second * 10) @@ -232,7 +226,7 @@ func (client *etcKeystore) KeepAliveKey(ctxt context.Context, key string) error } leaseID := grantResponse.ID - txnResponse, err := kvc.Txn(ctxt). + txnResponse, err := client.Client.Txn(ctxt). If(clientv3util.KeyMissing(key)). Then(clientv3.OpPut(key, "keep-alive", clientv3.WithLease(leaseID), clientv3.WithPrevKV())). Commit() @@ -263,11 +257,8 @@ func (client *etcKeystore) KeepAliveKey(ctxt context.Context, key string) error } func (client *etcKeystore) DeleteAllKeysWithPrefix(prefix string) (int64, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Delete(context.Background(), prefix, clientv3.WithPrefix()) + response, err := client.Client.Delete(context.Background(), prefix, clientv3.WithPrefix()) handleError(err) - - log.Printf("Cleaned %d keys with prefix: '%s'.\n", response.Deleted, prefix) return response.Deleted, nil } From dd0e3b69e4659e83bf0c65a8e9512311b3ef4af5 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 22:56:21 +0100 Subject: [PATCH 133/191] Update to etcd v3.3.15 --- docker-slurm/docker-compose.yml | 2 +- go.mod | 55 +++++++--- go.sum | 138 ++++++++++++++++++++++++ internal/pkg/etcdregistry/watch_test.go | 3 + 4 files changed, 181 insertions(+), 17 deletions(-) diff --git a/docker-slurm/docker-compose.yml b/docker-slurm/docker-compose.yml index 47bb3641..93677162 100644 --- a/docker-slurm/docker-compose.yml +++ b/docker-slurm/docker-compose.yml @@ -111,7 +111,7 @@ services: - "COMPUTE_NODE=c" etcd1: - image: quay.io/coreos/etcd:v3.3.13 + image: quay.io/coreos/etcd:v3.3.15 command: etcd --name etcd1 --data-dir=/etcd-data --initial-cluster-state new --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-cluster 'etcd1=http://etcd1:2380' hostname: etcd1 container_name: etcd1 diff --git a/go.mod b/go.mod index 14fa337c..db83b47f 100644 --- a/go.mod +++ b/go.mod @@ -3,39 +3,62 @@ module github.com/RSE-Cambridge/data-acc go 1.12 require ( + cloud.google.com/go v0.44.3 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect github.com/coreos/bbolt v1.3.3 // indirect - github.com/coreos/etcd v3.3.13+incompatible + github.com/coreos/etcd v3.3.15+incompatible github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/go-kit/kit v0.9.0 // indirect + github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/mock v1.3.1 - github.com/google/btree v1.0.0 // indirect + github.com/google/go-cmp v0.3.1 // indirect + github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect github.com/google/uuid v1.1.1 - github.com/gorilla/websocket v1.4.0 // indirect + github.com/gorilla/websocket v1.4.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.9.4 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.10.0 // indirect + github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/jonboulle/clockwork v0.1.0 // indirect - github.com/prometheus/client_golang v1.0.0 // indirect - github.com/prometheus/common v0.4.1 + github.com/kisielk/errcheck v1.2.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/kr/pty v1.1.8 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/prometheus/client_golang v1.1.0 // indirect + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect + github.com/prometheus/common v0.6.0 + github.com/prometheus/procfs v0.0.4 // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.3.1 // indirect + github.com/sirupsen/logrus v1.4.2 // indirect github.com/soheilhy/cmux v0.1.4 // indirect - github.com/stretchr/testify v1.3.0 + github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/testify v1.4.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect - github.com/urfave/cli v1.20.0 + github.com/urfave/cli v1.21.0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.3 // indirect go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect - golang.org/x/text v0.3.2 // indirect - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect - google.golang.org/appengine v1.4.0 // indirect - google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532 // indirect - google.golang.org/grpc v1.22.0 // indirect + golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect + golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 // indirect + golang.org/x/image v0.0.0-20190829093649-6ea169446634 // indirect + golang.org/x/mobile v0.0.0-20190826170111-cafc553e1ac5 // indirect + golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect + golang.org/x/sys v0.0.0-20190829204830-5fe476d8906b // indirect + golang.org/x/tools v0.0.0-20190829210313-340205e581e5 // indirect + google.golang.org/api v0.9.0 // indirect + google.golang.org/appengine v1.6.2 // indirect + google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + google.golang.org/grpc v1.23.0 // indirect gopkg.in/yaml.v2 v2.2.2 + honnef.co/go/tools v0.0.1-2019.2.2 // indirect ) diff --git a/go.sum b/go.sum index f86bd382..61db8696 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,39 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.15+incompatible h1:+9RjdC18gMxNQVvSiXvObLu29mOFmkgdsB4cRTlV+EE= +github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -26,7 +42,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -37,80 +55,126 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.10.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.21.0 h1:wYSSj06510qPIzGSua9ZqsncMmWE3Zr55KBERygyrxE= +github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -120,61 +184,135 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190829093649-6ea169446634/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20190826170111-cafc553e1ac5/go.mod h1:mJOp/i0LXPxJZ9weeIadcPqKVfS05Ai7m6/t9z1Hs/Y= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190829204830-5fe476d8906b h1:GA/t9fariXOM5cIRJcMPxJHYYZmYHgXdVH0+JEzddZs= +golang.org/x/sys v0.0.0-20190829204830-5fe476d8906b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190829210313-340205e581e5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532 h1:5pOB7se0B2+IssELuQUs6uoBgYJenkU2AQlvopc2sRw= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/pkg/etcdregistry/watch_test.go b/internal/pkg/etcdregistry/watch_test.go index 8dbddaa0..bb96225e 100644 --- a/internal/pkg/etcdregistry/watch_test.go +++ b/internal/pkg/etcdregistry/watch_test.go @@ -19,6 +19,9 @@ func (fw fakeWatcher) Watch(ctx context.Context, key string, opts ...clientv3.Op assert.EqualValues(fw.t, len(fw.opts), len(opts)) // TODO: how to assert this properly? return fw.ch } +func (fw fakeWatcher) RequestProgress(ctx context.Context) error { + panic("implement me") +} func (fakeWatcher) Close() error { panic("implement me") } From e40d875cbfd9d329145dbbb0fe9aaef18e8957e3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 23:28:46 +0100 Subject: [PATCH 134/191] Simplify keystore impl Remove all the extra get() calls, and just return the revision that the transaction created. These were sometimes failing for reasons I don't quite understand when doing create(), but its not required. --- cmd/dacctl/main_test.go | 4 +-- internal/pkg/v2/mock_store/keystore.go | 8 +++--- internal/pkg/v2/registry_impl/session.go | 11 ++++---- .../v2/registry_impl/session_actions_test.go | 3 +-- internal/pkg/v2/registry_impl/session_test.go | 7 ++--- internal/pkg/v2/store/keystore.go | 4 +-- internal/pkg/v2/store_impl/keystore.go | 26 +++++++++---------- 7 files changed, 29 insertions(+), 34 deletions(-) diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 34693bd8..03bc8d5d 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -148,11 +148,11 @@ func (*stubKeystore) Close() error { return nil } -func (*stubKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { +func (*stubKeystore) Create(key string, value []byte) (int64, error) { panic("implement me") } -func (*stubKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { +func (*stubKeystore) Update(key string, value []byte, modRevision int64) (int64, error) { panic("implement me") } diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/v2/mock_store/keystore.go index 0b245307..db4f60e9 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/v2/mock_store/keystore.go @@ -49,10 +49,10 @@ func (mr *MockKeystoreMockRecorder) Close() *gomock.Call { } // Create mocks base method -func (m *MockKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { +func (m *MockKeystore) Create(key string, value []byte) (int64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", key, value) - ret0, _ := ret[0].(store.KeyValueVersion) + ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -64,10 +64,10 @@ func (mr *MockKeystoreMockRecorder) Create(key, value interface{}) *gomock.Call } // Update mocks base method -func (m *MockKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { +func (m *MockKeystore) Update(key string, value []byte, modRevision int64) (int64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Update", key, value, modRevision) - ret0, _ := ret[0].(store.KeyValueVersion) + ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 30d1680d..2a2a05b1 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -50,13 +50,13 @@ func (s *sessionRegistry) CreateSession(session datamodel.Session) (datamodel.Se // TODO: ensure not allocated to any other session? } - keyValueVersion, err := s.store.Create(sessionKey, sessionToRaw(session)) + createRevision, err := s.store.Create(sessionKey, sessionToRaw(session)) if err != nil { return session, fmt.Errorf("unable to create session due to: %s", err) } // Return the last modification revision - session.Revision = keyValueVersion.ModRevision + session.Revision = createRevision return session, nil } @@ -88,14 +88,13 @@ func (s *sessionRegistry) GetAllSessions() ([]datamodel.Session, error) { } func (s *sessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { - keyValueVersion, err := s.store.Update(getSessionKey(session.Name), sessionToRaw(session), session.Revision) + newRevision, err := s.store.Update(getSessionKey(session.Name), sessionToRaw(session), session.Revision) if err != nil { return session, fmt.Errorf("unable to update session due to: %s", err.Error()) } - newSession := sessionFromRaw(keyValueVersion.Value) - newSession.Revision = keyValueVersion.ModRevision - return newSession, nil + session.Revision = newRevision + return session, nil } func (s *sessionRegistry) DeleteSession(session datamodel.Session) error { diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/v2/registry_impl/session_actions_test.go index 8239e8b3..ce17b03b 100644 --- a/internal/pkg/v2/registry_impl/session_actions_test.go +++ b/internal/pkg/v2/registry_impl/session_actions_test.go @@ -6,7 +6,6 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -23,7 +22,7 @@ func TestSessionActions_SendSessionAction(t *testing.T) { brickHost.EXPECT().IsBrickHostAlive(session.PrimaryBrickHost).Return(true, nil) keystore.EXPECT().Watch(context.TODO(), gomock.Any(), false).Return(nil) fakeErr := errors.New("fake") - keystore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(store.KeyValueVersion{}, fakeErr) + keystore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(int64(3), fakeErr) channel, err := actions.SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, session) diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index 284491dc..ff01554e 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -55,7 +55,7 @@ func TestSessionRegistry_CreateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - keystore.EXPECT().Create("/session/foo", exampleSessionString).Return(store.KeyValueVersion{ModRevision: 42}, nil) + keystore.EXPECT().Create("/session/foo", exampleSessionString).Return(int64(42), nil) session, err := registry.CreateSession(exampleSession) assert.Nil(t, err) @@ -149,10 +149,7 @@ func TestSessionRegistry_UpdateSession(t *testing.T) { defer mockCtrl.Finish() keystore := mock_store.NewMockKeystore(mockCtrl) registry := NewSessionRegistry(keystore) - keystore.EXPECT().Update("/session/foo", exampleSessionString, int64(0)).Return(store.KeyValueVersion{ - ModRevision: 44, - Value: exampleSessionString, - }, nil) + keystore.EXPECT().Update("/session/foo", exampleSessionString, int64(0)).Return(int64(44), nil) session, err := registry.UpdateSession(datamodel.Session{Name: "foo", PrimaryBrickHost: "host1", Revision: 0}) diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/v2/store/keystore.go index faf50790..fc765a1b 100644 --- a/internal/pkg/v2/store/keystore.go +++ b/internal/pkg/v2/store/keystore.go @@ -13,7 +13,7 @@ type Keystore interface { // // If an error occurs no keyvalues are written. // Error is returned if any key already exists. - Create(key string, value []byte) (KeyValueVersion, error) + Create(key string, value []byte) (int64, error) // Update the specified key values, atomically // @@ -21,7 +21,7 @@ type Keystore interface { // Otherwise if the revisions of any key doesn't // match the current revision of that key, the update fails. // When update fails an error is returned and no keyValues are updated - Update(key string, value []byte, modRevision int64) (KeyValueVersion, error) + Update(key string, value []byte, modRevision int64) (int64, error) // Delete the specified key values, atomically // diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go index 4dd6d663..6c0854ff 100644 --- a/internal/pkg/v2/store_impl/keystore.go +++ b/internal/pkg/v2/store_impl/keystore.go @@ -108,30 +108,30 @@ func (client *etcKeystore) Close() error { return client.Client.Close() } -func (client *etcKeystore) runTransaction(ifOps []clientv3.Cmp, thenOps []clientv3.Op) error { +func (client *etcKeystore) runTransaction(ifOps []clientv3.Cmp, thenOps []clientv3.Op) (int64, error) { response, err := client.Client.Txn(context.Background()).If(ifOps...).Then(thenOps...).Commit() handleError(err) if !response.Succeeded { log.Println(ifOps) - return fmt.Errorf("transaction failed, as condition not met") + return 0, fmt.Errorf("transaction failed, as condition not met") } - return nil + return response.Header.Revision, nil } -func (client *etcKeystore) Create(key string, value []byte) (store.KeyValueVersion, error) { +func (client *etcKeystore) Create(key string, value []byte) (int64, error) { var ifOps []clientv3.Cmp var thenOps []clientv3.Op ifOps = append(ifOps, clientv3util.KeyMissing(key)) thenOps = append(thenOps, clientv3.OpPut(key, string(value))) - err := client.runTransaction(ifOps, thenOps) + revision, err := client.runTransaction(ifOps, thenOps) if err != nil { - return store.KeyValueVersion{}, fmt.Errorf("unable to create key: %s due to: %s", key, err) + return 0, fmt.Errorf("unable to create key: %s due to: %s", key, err) } - return client.Get(key) + return revision, nil } -func (client *etcKeystore) Update(key string, value []byte, modRevision int64) (store.KeyValueVersion, error) { +func (client *etcKeystore) Update(key string, value []byte, modRevision int64) (int64, error) { var ifOps []clientv3.Cmp var thenOps []clientv3.Op @@ -143,15 +143,14 @@ func (client *etcKeystore) Update(key string, value []byte, modRevision int64) ( } thenOps = append(thenOps, clientv3.OpPut(key, string(value))) - err := client.runTransaction(ifOps, thenOps) + newRevision, err := client.runTransaction(ifOps, thenOps) if err != nil { - return store.KeyValueVersion{}, fmt.Errorf("unable to update ke: %s", err) + return 0, fmt.Errorf("unable to update ke: %s", err) } - return client.Get(key) + return newRevision, nil } func (client *etcKeystore) Delete(key string, modRevision int64) error { - var ifOps []clientv3.Cmp var thenOps []clientv3.Op @@ -162,7 +161,8 @@ func (client *etcKeystore) Delete(key string, modRevision int64) error { } thenOps = append(thenOps, clientv3.OpDelete(key)) - return client.runTransaction(ifOps, thenOps) + _, err := client.runTransaction(ifOps, thenOps) + return err } func getKeyValueVersion(rawKeyValue *mvccpb.KeyValue) *store.KeyValueVersion { From ea8c05950c453f2bf557604db7267c2900ada7b1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 23:35:51 +0100 Subject: [PATCH 135/191] Start using latest release with dac-ansible --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 49640e67..1abbf52e 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,5 +1,6 @@ --- -data_acc_version: v1.3 +data_acc_version: 'v2.0-alpha.2' +data_acc_checksum: 'sha256:cbad822c54f63b6d48d1f4728f8677f0d7db44099878f0c2e4ece27652f0b8fa' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin @@ -11,7 +12,6 @@ data_acc_launch: True data_acc_name: 'data-acc-{{data_acc_version}}' data_acc_tgz: '{{data_acc_name}}.tgz' data_acc_tgz_url: '{{ data_acc_mirror }}/{{ data_acc_version }}/data-acc-{{ data_acc_version }}.tgz' -data_acc_checksum: 'sha256:0d6a8a5b9ccc30c7a7505efcf2858997c5c4487d57fd5ef45d075f25877c9485' data_acc_user: dac data_acc_group: dac From 0ab3f6797b1bea15d2ec7d84a773d2b62ecd2f91 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 29 Aug 2019 23:43:28 +0100 Subject: [PATCH 136/191] Ensuring slurm db fails sometimes Probably we are just a bit too quick, lets skip it for now, as its already in the demo script. --- dac-ansible/master.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dac-ansible/master.yml b/dac-ansible/master.yml index b7930e7b..a95b75e1 100644 --- a/dac-ansible/master.yml +++ b/dac-ansible/master.yml @@ -310,13 +310,6 @@ - docker_service: project_src: /var/lib/slurm-docker/slurm-master register: output - - name: ensure slurm cluster registered in db - shell: | - sleep 10 && docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && docker restart slurmdbd slurmctld - register: shell_result - changed_when: "shell_result.rc == 0" - failed_when: "shell_result.rc != 0 and ('already exists' not in shell_result.stdout)" - when: output.changed - hosts: slurm_workers become: true From b7e2d0cf2c999c1cd5364c19423469fe26e997e4 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 00:59:03 +0100 Subject: [PATCH 137/191] Refresh dacd.conf settings --- dac-ansible/roles/data-acc/templates/dacd.conf.j2 | 13 ++++++++----- internal/pkg/v2/dacd/config/config.go | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dac-ansible/roles/data-acc/templates/dacd.conf.j2 b/dac-ansible/roles/data-acc/templates/dacd.conf.j2 index 7c49370c..13bc311a 100644 --- a/dac-ansible/roles/data-acc/templates/dacd.conf.j2 +++ b/dac-ansible/roles/data-acc/templates/dacd.conf.j2 @@ -4,14 +4,17 @@ ETCDCTL_ENDPOINTS=https://{{ hostvars[groups['etcd_master'][0]].ansible_host }}: ETCDCTL_CERT_FILE=/etc/data-acc/pki/{{ inventory_hostname }}.pem ETCDCTL_KEY_FILE=/etc/data-acc/pki/{{ inventory_hostname }}-key.pem ETCDCTL_CA_FILE=/etc/data-acc/pki/ca.pem -DAC_ANSIBLE_DIR={{data_acc_install_dir}}/{{data_acc_name}}/fs-ansible/ + +# dacd config settings +DAC_POOL_NAME=default +DAC_BRICK_CAPACITY_GB=1600 +DAC_BRICK_COUNT=5 +DAC_BRICK_ADDRESS_PATTERN="loop%d" + #DAC_SKIP_ANSIBLE=True +DAC_ANSIBLE_DIR={{data_acc_install_dir}}/{{data_acc_name}}/fs-ansible/ DAC_HOST_GROUP=dac-fake -DEVICE_COUNT=5 -DAC_POOL_NAME=default -DAC_DEVICE_CAPACITY_GB=1600 DACCTL_LOG=/var/log/dacctl.log #DAC_LNET_SUFFIX="-opa@o2ib1" -DEVICE_TYPE="loop%d" DAC_MDT_SIZE_MB="50" DAC_MAX_MDT_COUNT=2 diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 4c8bdb7e..1f8fe70d 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -26,9 +26,9 @@ func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { config := BrickManagerConfig{ datamodel.BrickHostName(getHostname(env)), datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), - getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), - getUint(env, "DAC_BRICK_COUNT", 12), - getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), + getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB + getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT + getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), // TODO: DEVICE_TYPE getBool(env, "DAC_HOST_ENABLED", true), } log.Println("Got brick manager config:", config) From e5ff590fb8f1c02f916f6727b379d804f810d95b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 01:29:33 +0100 Subject: [PATCH 138/191] Don't call ansible for a session with no bricks --- .../brick_manager_impl/session_action_handler.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index c3a26529..fbe251e5 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -89,6 +89,12 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { + // Nothing to create, just complete the action + // TODO: why do we send the action? + if action.Session.ActualSizeBytes == 0 { + return action.Session, nil + } + // Get latest session now we have the mutex session, err := s.sessionRegistry.GetSession(action.Session.Name) if err != nil { @@ -126,8 +132,11 @@ func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { } } - if err := s.fsProvider.Delete(action.Session); err != nil { - return action.Session, err + // Only try delete if we have bricks to delete + if action.Session.ActualSizeBytes > 0 { + if err := s.fsProvider.Delete(action.Session); err != nil { + return action.Session, err + } } return action.Session, s.sessionRegistry.DeleteSession(action.Session) From acf8ed19114466ca39048e7bdcfd3a3334b775cb Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 01:33:53 +0100 Subject: [PATCH 139/191] Fix tests and formatting --- .../dacd/brick_manager_impl/session_action_handler_test.go | 7 ++++--- internal/pkg/v2/dacd/config/config.go | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index bcf35e8e..f02b7c52 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -32,7 +32,7 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { } action := datamodel.SessionAction{ ActionType: datamodel.SessionCreateFilesystem, - Session: datamodel.Session{Name: "test"}, + Session: datamodel.Session{Name: "test", ActualSizeBytes: 42}, } sessionMutex := mock_store.NewMockMutex(mockCtrl) registry.EXPECT().GetSessionMutex(action.Session.Name).Return(sessionMutex, nil) @@ -41,8 +41,9 @@ func TestSessionActionHandler_handleCreate(t *testing.T) { registry.EXPECT().GetSession(action.Session.Name).Return(action.Session, nil) fsProvider.EXPECT().Create(action.Session) updatedSession := datamodel.Session{ - Name: action.Session.Name, - Status: datamodel.SessionStatus{FileSystemCreated: true}, + Name: action.Session.Name, + Status: datamodel.SessionStatus{FileSystemCreated: true}, + ActualSizeBytes: 42, } registry.EXPECT().UpdateSession(updatedSession).Return(updatedSession, nil) updatedAction := datamodel.SessionAction{ diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 1f8fe70d..91370a5b 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -26,8 +26,8 @@ func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { config := BrickManagerConfig{ datamodel.BrickHostName(getHostname(env)), datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), - getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB - getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT + getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB + getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), // TODO: DEVICE_TYPE getBool(env, "DAC_HOST_ENABLED", true), } From e63e2801ba22954431a09cce71993180df5fe672 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:09:28 +0100 Subject: [PATCH 140/191] Ensure we don't try to create a filesystem with no bricks --- internal/pkg/v2/filesystem_impl/ansible.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/v2/filesystem_impl/ansible.go index 96b20b35..4b4c5409 100644 --- a/internal/pkg/v2/filesystem_impl/ansible.go +++ b/internal/pkg/v2/filesystem_impl/ansible.go @@ -154,6 +154,10 @@ func getAnsibleDir(suffix string) string { } func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) (string, error) { + if len(bricks) == 0 { + log.Panicf("can't create filesystem with no bricks: %s", internalName) + } + dir, err := ioutil.TempDir("", fmt.Sprintf("fs%s_", internalName)) if err != nil { return dir, err From d95015b57999701818ff25497e970e3eb5dc86f1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:13:57 +0100 Subject: [PATCH 141/191] Start building with new release --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 1abbf52e..4aebc9ae 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.2' -data_acc_checksum: 'sha256:cbad822c54f63b6d48d1f4728f8677f0d7db44099878f0c2e4ece27652f0b8fa' +data_acc_version: 'v2.0-alpha.3' +data_acc_checksum: 'sha256:797cdde7f85628c75fb86a7c2af359df57184d5162bafe6a62f2765965ce726f' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From 56952130fedcbb888db1ac5c08fdca7753a6fc79 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:31:22 +0100 Subject: [PATCH 142/191] Remove dead code --- build/rebuild_mocks.sh | 21 - cmd/dac-func-test/dacctl.go | 71 --- cmd/dac-func-test/etcdkeystore.go | 123 ----- cmd/dac-func-test/main.go | 24 - cmd/dac-func-test/pool.go | 119 ----- cmd/dac-func-test/volume.go | 112 ----- internal/pkg/dacctl/actions/actions.go | 311 ------------- internal/pkg/dacctl/actions/actions_test.go | 215 --------- internal/pkg/dacctl/buffer.go | 181 -------- internal/pkg/dacctl/buffer_test.go | 66 --- internal/pkg/dacctl/job.go | 299 ------------- internal/pkg/dacctl/job_test.go | 52 --- internal/pkg/dacctl/json.go | 14 - internal/pkg/dacctl/persistent.go | 130 ------ internal/pkg/dacctl/pools.go | 43 -- internal/pkg/dacctl/pools_test.go | 23 - internal/pkg/dacctl/show.go | 87 ---- internal/pkg/dacctl/show_test.go | 61 --- internal/pkg/etcdregistry/keystore.go | 268 ----------- internal/pkg/etcdregistry/watch.go | 52 --- internal/pkg/etcdregistry/watch_test.go | 122 ----- internal/pkg/keystoreregistry/keystore.go | 99 ---- internal/pkg/keystoreregistry/pool.go | 421 ------------------ internal/pkg/keystoreregistry/pool_test.go | 95 ---- internal/pkg/keystoreregistry/volume.go | 405 ----------------- internal/pkg/keystoreregistry/volume_test.go | 132 ------ internal/pkg/lifecycle/brickmanager/bricks.go | 313 ------------- .../pkg/lifecycle/brickmanager/manager.go | 135 ------ .../lifecycle/brickmanager/manager_test.go | 37 -- internal/pkg/lifecycle/volume.go | 247 ---------- internal/pkg/lifecycle/volume_test.go | 109 ----- internal/pkg/mocks/disk_mock.go | 62 --- internal/pkg/mocks/job_mock.go | 32 -- internal/pkg/mocks/pfsprovider_mock.go | 193 -------- internal/pkg/mocks/pool_mock.go | 180 -------- internal/pkg/mocks/volume_mock.go | 315 ------------- internal/pkg/pfsprovider/ansible/ansible.go | 311 ------------- .../pkg/pfsprovider/ansible/ansible_test.go | 181 -------- internal/pkg/pfsprovider/ansible/copy.go | 72 --- internal/pkg/pfsprovider/ansible/copy_test.go | 66 --- internal/pkg/pfsprovider/ansible/mount.go | 308 ------------- .../pkg/pfsprovider/ansible/mount_test.go | 273 ------------ internal/pkg/pfsprovider/ansible/plugin.go | 96 ---- internal/pkg/pfsprovider/fake/plugin.go | 55 --- internal/pkg/pfsprovider/interface.go | 27 -- internal/pkg/registry/pool.go | 141 ------ internal/pkg/registry/volume.go | 347 --------------- internal/pkg/v2/filesystem_impl/mount_test.go | 92 ---- 48 files changed, 7138 deletions(-) delete mode 100644 cmd/dac-func-test/dacctl.go delete mode 100644 cmd/dac-func-test/etcdkeystore.go delete mode 100644 cmd/dac-func-test/main.go delete mode 100644 cmd/dac-func-test/pool.go delete mode 100644 cmd/dac-func-test/volume.go delete mode 100644 internal/pkg/dacctl/actions/actions.go delete mode 100644 internal/pkg/dacctl/actions/actions_test.go delete mode 100644 internal/pkg/dacctl/buffer.go delete mode 100644 internal/pkg/dacctl/buffer_test.go delete mode 100644 internal/pkg/dacctl/job.go delete mode 100644 internal/pkg/dacctl/job_test.go delete mode 100644 internal/pkg/dacctl/json.go delete mode 100644 internal/pkg/dacctl/persistent.go delete mode 100644 internal/pkg/dacctl/pools.go delete mode 100644 internal/pkg/dacctl/pools_test.go delete mode 100644 internal/pkg/dacctl/show.go delete mode 100644 internal/pkg/dacctl/show_test.go delete mode 100644 internal/pkg/etcdregistry/keystore.go delete mode 100644 internal/pkg/etcdregistry/watch.go delete mode 100644 internal/pkg/etcdregistry/watch_test.go delete mode 100644 internal/pkg/keystoreregistry/keystore.go delete mode 100644 internal/pkg/keystoreregistry/pool.go delete mode 100644 internal/pkg/keystoreregistry/pool_test.go delete mode 100644 internal/pkg/keystoreregistry/volume.go delete mode 100644 internal/pkg/keystoreregistry/volume_test.go delete mode 100644 internal/pkg/lifecycle/brickmanager/bricks.go delete mode 100644 internal/pkg/lifecycle/brickmanager/manager.go delete mode 100644 internal/pkg/lifecycle/brickmanager/manager_test.go delete mode 100644 internal/pkg/lifecycle/volume.go delete mode 100644 internal/pkg/lifecycle/volume_test.go delete mode 100644 internal/pkg/mocks/disk_mock.go delete mode 100644 internal/pkg/mocks/job_mock.go delete mode 100644 internal/pkg/mocks/pfsprovider_mock.go delete mode 100644 internal/pkg/mocks/pool_mock.go delete mode 100644 internal/pkg/mocks/volume_mock.go delete mode 100644 internal/pkg/pfsprovider/ansible/ansible.go delete mode 100644 internal/pkg/pfsprovider/ansible/ansible_test.go delete mode 100644 internal/pkg/pfsprovider/ansible/copy.go delete mode 100644 internal/pkg/pfsprovider/ansible/copy_test.go delete mode 100644 internal/pkg/pfsprovider/ansible/mount.go delete mode 100644 internal/pkg/pfsprovider/ansible/mount_test.go delete mode 100644 internal/pkg/pfsprovider/ansible/plugin.go delete mode 100644 internal/pkg/pfsprovider/fake/plugin.go delete mode 100644 internal/pkg/pfsprovider/interface.go delete mode 100644 internal/pkg/registry/pool.go delete mode 100644 internal/pkg/registry/volume.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index a2d929a5..c3e98f55 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -6,33 +6,12 @@ echo "Regenerate mocks:" mkdir -p internal/pkg/mocks -items="pool volume" -for i in $items; do - mockgen -source=internal/pkg/registry/${i}.go \ - -package mocks >internal/pkg/mocks/${i}_mock.go -done - -items="job" -for i in $items; do - mockgen -source=internal/pkg/dacctl/${i}.go \ - -package mocks >internal/pkg/mocks/${i}_mock.go -done - items="disk" for i in $items; do mockgen -source=internal/pkg/fileio/${i}.go \ -package mocks >internal/pkg/mocks/${i}_mock.go done -mockgen -source=internal/pkg/pfsprovider/interface.go \ - -package mocks >internal/pkg/mocks/pfsprovider_mock.go - -items="brick_allocation brick_host session session_actions" -for i in $items; do - mockgen -source=internal/pkg/v2/registry/${i}.go \ - >internal/pkg/v2/mock_registry/${i}.go -done - items="session session_action_handler" for i in $items; do mockgen -source=internal/pkg/v2/facade/${i}.go \ diff --git a/cmd/dac-func-test/dacctl.go b/cmd/dac-func-test/dacctl.go deleted file mode 100644 index aa832054..00000000 --- a/cmd/dac-func-test/dacctl.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" -) - -func debugStatus(volumeRegistry registry.VolumeRegistry, poolRegistry registry.PoolRegistry) { - log.Println(dacctl.GetPools(poolRegistry)) - log.Println(dacctl.GetInstances(volumeRegistry)) - log.Println(dacctl.GetSessions(volumeRegistry)) - log.Println(volumeRegistry.AllVolumes()) -} - -func testPersistent(volumeRegistry registry.VolumeRegistry, poolRegistry registry.PoolRegistry) { - bufferToken := "fakebuffer1" - bufferRequest := dacctl.BufferRequest{ - Token: bufferToken, - Capacity: "b:10GiB", - Persistent: true, - Caller: "test", - } - debugStatus(volumeRegistry, poolRegistry) - - err := dacctl.CreateVolumesAndJobs(volumeRegistry, poolRegistry, bufferRequest) - if err != nil { - log.Fatal(err) - } - - bufferRequest2 := bufferRequest - bufferRequest2.Token = "fakebuffer2" - err = dacctl.CreateVolumesAndJobs(volumeRegistry, poolRegistry, bufferRequest2) - if err != nil { - log.Fatal(err) - } - - bufferRequest3 := dacctl.BufferRequest{ - Token: "fakebuffer3", - Capacity: "a:0", - Persistent: true, - Caller: "test", - } - err = dacctl.CreateVolumesAndJobs(volumeRegistry, poolRegistry, bufferRequest3) - if err != nil { - log.Fatal(err) - } - - debugStatus(volumeRegistry, poolRegistry) - - // TODO go through state machine for a given volume...? - // TODO fix up paths, real_size, etc - // TODO record all the data for fake data_in, etc - // TODO add wait for actions into volume state machine - - log.Println(dacctl.DeleteBufferComponents(volumeRegistry, poolRegistry, bufferToken)) - log.Println(dacctl.DeleteBufferComponents(volumeRegistry, poolRegistry, "fakebuffer2")) - log.Println(dacctl.DeleteBufferComponents(volumeRegistry, poolRegistry, "fakebuffer3")) - - debugStatus(volumeRegistry, poolRegistry) -} - -func TestDacctl(keystore keystoreregistry.Keystore) { - log.Println("Testing dacctl") - - volumeRegistry := keystoreregistry.NewVolumeRegistry(keystore) - poolRegistry := keystoreregistry.NewPoolRegistry(keystore) - - testPersistent(volumeRegistry, poolRegistry) -} diff --git a/cmd/dac-func-test/etcdkeystore.go b/cmd/dac-func-test/etcdkeystore.go deleted file mode 100644 index ba0496dc..00000000 --- a/cmd/dac-func-test/etcdkeystore.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "log" - "runtime" - "time" -) - -func cleanAllKeys(keystore keystoreregistry.Keystore) { - if err := keystore.CleanPrefix(""); err != nil { - log.Println("Error cleaning: ", err) - } -} - -func testAddValues(keystore keystoreregistry.Keystore) { - values := []keystoreregistry.KeyValue{ - {Key: "key1", Value: "value1"}, - {Key: "key2", Value: "value2"}, - } - - if err := keystore.Add(values); err != nil { - log.Fatalf("Error with add values") - } - - if err := keystore.Add(values); err == nil { - log.Fatalf("Expected an error") - } else { - log.Println(err) - } -} - -func testGet(keystore keystoreregistry.Keystore) { - value, _ := keystore.Get("key1") - log.Println(value) - _, err := keystore.Get("key3") - if err == nil { - log.Fatalf("failed to raise error") - } else { - log.Println(err) - } - - values, _ := keystore.GetAll("key") - log.Println(values) - _, err = keystore.GetAll("key3") - if err == nil { - log.Fatalf("failed to raise error") - } else { - log.Println(err) - } -} - -func testUpdate(keystore keystoreregistry.Keystore) { - values, err := keystore.GetAll("key") - if err != nil { - log.Fatal(err) - } - - values[0].Value = "asdf" - values[1].Value = "asdf2" - - err = keystore.Update(values) - if err != nil { - log.Fatal(err) - } - - // Error if ModVersion out of sync - err = keystore.Update(values) - if err == nil { - log.Fatal("Failed to raise error") - } else { - log.Println(err) - } - - // Ensure success if told to ignore ModRevision - values[0].ModRevision = 0 - values[1].ModRevision = 0 - values[1].Key = "key3" // add value via update - err = keystore.Update(values) - if err != nil { - log.Fatal(err) - } -} - -func testDeleteAll(keystore keystoreregistry.Keystore) { - values, err := keystore.GetAll("key") - if err != nil { - log.Fatal(err) - } - - err = keystore.DeleteAll(values) - if err != nil { - log.Fatal(err) - } -} - -func testKeepAlive(keystore keystoreregistry.Keystore) { - err := keystore.KeepAliveKey("/mytesthost") - if err != nil { - log.Fatal(err) - } - - err = keystore.KeepAliveKey("/mytesthost") - if err == nil { - log.Fatal("expected error") - } else { - log.Println(err) - } -} - -func TestEtcdKeystore(keystore keystoreregistry.Keystore) { - log.Println("Testing etcdkeystore...") - - testAddValues(keystore) - testGet(keystore) - testUpdate(keystore) - testDeleteAll(keystore) - testKeepAlive(keystore) - - // Give background things time to finish - time.Sleep(time.Millisecond * 100) - runtime.Gosched() -} diff --git a/cmd/dac-func-test/main.go b/cmd/dac-func-test/main.go deleted file mode 100644 index 9c19d22a..00000000 --- a/cmd/dac-func-test/main.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/etcdregistry" -) - -func main() { - keystore := etcdregistry.NewKeystore() - defer keystore.Close() - - cleanAllKeys(keystore) - - TestEtcdKeystore(keystore) - fmt.Println("") - - TestKeystorePoolRegistry(keystore) - fmt.Println("") - - TestKeystoreVolumeRegistry(keystore) - fmt.Println("") - - TestDacctl(keystore) -} diff --git a/cmd/dac-func-test/pool.go b/cmd/dac-func-test/pool.go deleted file mode 100644 index b2536e13..00000000 --- a/cmd/dac-func-test/pool.go +++ /dev/null @@ -1,119 +0,0 @@ -package main - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" -) - -func testGetPools(poolRegistry registry.PoolRegistry) { - if pools, err := poolRegistry.Pools(); err != nil { - log.Fatal(err) - } else { - log.Println(pools) - } -} - -func testUpdateHost(poolRegistry registry.PoolRegistry) { - brickInfo := []registry.BrickInfo{ - {Hostname: "foo", Device: "vbdb1", PoolName: "a", CapacityGB: 10}, - {Hostname: "foo", Device: "nvme3n1", PoolName: "b", CapacityGB: 20}, - {Hostname: "foo", Device: "nvme2n1", PoolName: "b", CapacityGB: 20}, - } - err := poolRegistry.UpdateHost(brickInfo) - if err != nil { - log.Fatal(err) - } else { - log.Println("added some keys") - } - - // Do not allow multiple hostnames to be updated - brickInfo = []registry.BrickInfo{ - {Hostname: "foo", Device: "vbdb1", PoolName: "a", CapacityGB: 10}, - {Hostname: "bar", Device: "nvme3n1", PoolName: "b", CapacityGB: 20}, - } - err = poolRegistry.UpdateHost(brickInfo) - if err == nil { - log.Fatal("expected error") - } else { - log.Println(err) - } -} - -func testGetBricks(poolRegistry registry.PoolRegistry) { - if raw, err := poolRegistry.GetBrickInfo("foo", "vbdb1"); err != nil { - log.Fatal(err) - } else { - log.Println(raw) - } - - if _, err := poolRegistry.GetBrickInfo("asdf", "vbdb1"); err != nil { - log.Println(err) - } else { - log.Fatal("expected error") - } -} - -func testGetAllocations(poolRegistry registry.PoolRegistry) { - allocations, err := poolRegistry.GetAllocationsForHost("foo") - if err != nil { - log.Fatal(err) - } - log.Println(allocations) - - allocations, err = poolRegistry.GetAllocationsForVolume("vol1") - if err != nil { - log.Fatal(err) - } - log.Println(allocations) - - err = poolRegistry.DeallocateBricks("vol1") - if err != nil { - log.Fatal(err) - } -} - -func testDeleteAllocations(poolRegistry registry.PoolRegistry) { - updatedAllocations, err := poolRegistry.GetAllocationsForVolume("vol1") - if err != nil { - log.Fatal(err) - } - err = poolRegistry.HardDeleteAllocations(updatedAllocations) - if err != nil { - log.Fatal(err) - } -} - -func testKeepHostAlive(poolRegistry registry.PoolRegistry) { - err := poolRegistry.KeepAliveHost("foo") - if err != nil { - log.Fatal(err) - } - err = poolRegistry.KeepAliveHost("bar") - if err != nil { - log.Fatal(err) - } - - err = poolRegistry.KeepAliveHost("foo") - if err == nil { - log.Fatal("expected error") - } else { - log.Println(err) - } -} - -func TestKeystorePoolRegistry(keystore keystoreregistry.Keystore) { - log.Println("Testing keystoreregistry.pool") - - cleanAllKeys(keystore) - - poolRegistry := keystoreregistry.NewPoolRegistry(keystore) - testUpdateHost(poolRegistry) - testGetBricks(poolRegistry) - testGetAllocations(poolRegistry) - testDeleteAllocations(poolRegistry) - testKeepHostAlive(poolRegistry) - - // TODO: update hosts first? - testGetPools(poolRegistry) -} diff --git a/cmd/dac-func-test/volume.go b/cmd/dac-func-test/volume.go deleted file mode 100644 index 138d5d56..00000000 --- a/cmd/dac-func-test/volume.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "time" -) - -func TestKeystoreVolumeRegistry(keystore keystoreregistry.Keystore) { - log.Println("Testing keystoreregistry.volume") - volumeRegistry := keystoreregistry.NewVolumeRegistry(keystore) - - testVolumeCRUD(volumeRegistry) - testJobCRUD(volumeRegistry) - - // give watches time to print - time.Sleep(time.Second) -} - -func testVolumeCRUD(volRegistry registry.VolumeRegistry) { - // TODO: test get volume changes? - - volume := registry.Volume{Name: "asdf", State: registry.Registered, JobName: "foo", SizeBricks: 2, SizeGB: 200} - volume2 := registry.Volume{Name: "asdf2", JobName: "foo", SizeBricks: 3, SizeGB: 300} - if err := volRegistry.AddVolume(volume); err != nil { - log.Fatal(err) - } - if err := volRegistry.AddVolume(volume); err == nil { - log.Fatal("expected an error") - } else { - log.Println(err) - } - - if volume, err := volRegistry.Volume(volume.Name); err != nil { - log.Fatal(err) - } else { - log.Println(volume) - } - - if err := volRegistry.DeleteVolume(volume.Name); err != nil { - log.Fatal(err) - } - if err := volRegistry.DeleteVolume(volume.Name); err == nil { - log.Fatal("expected error") - } else { - log.Println(err) - } - - // leave around for following tests - volRegistry.AddVolume(volume) - volRegistry.AddVolume(volume2) - - if err := volRegistry.UpdateState(volume.Name, registry.BricksProvisioned); err != nil { - log.Fatal(err) - } - if err := volRegistry.UpdateState("badname", registry.BricksProvisioned); err == nil { - log.Fatal("expected error") - } - if err := volRegistry.UpdateState(volume.Name, registry.BricksProvisioned); err == nil { - log.Fatal("expected error with repeated update") - } - if err := volRegistry.UpdateState(volume.Name, registry.Unknown); err == nil { - log.Fatal("expected error with out of order update") - } - volRegistry.UpdateState(volume2.Name, registry.Registered) - - if volumes, err := volRegistry.AllVolumes(); err != nil { - log.Fatal(err) - } else { - log.Println(volumes) - } - // testJobCRUD uses volume and volume1 -} - -func testJobCRUD(volRegistry registry.VolumeRegistry) { - job := registry.Job{Name: "foo", - MultiJobVolumes: []registry.VolumeName{"asdf", "asdf2"}, - Owner: 1001, - CreatedAt: uint(time.Now().Unix()), - } - if err := volRegistry.AddJob(job); err != nil { - log.Fatal(err) - } - - if err := volRegistry.AddJob(job); err == nil { - log.Fatal("expected an error adding duplicate job") - } - badJob := registry.Job{Name: "bar", MultiJobVolumes: []registry.VolumeName{"asdf", "asdf3"}} - if err := volRegistry.AddJob(badJob); err == nil { - log.Fatal("expected an error for invalid volume name") - } - - jobs, err := volRegistry.Jobs() - if err != nil { - log.Fatal(err) - } - log.Println(jobs) - - err = volRegistry.DeleteJob("foo") - if err != nil { - panic(err) - } - err = volRegistry.DeleteJob("foo") - if err == nil { - panic(err) - } - - // remove volumes now we are done with them - volRegistry.DeleteVolume(registry.VolumeName("asdf")) - volRegistry.DeleteVolume(registry.VolumeName("asdf2")) -} diff --git a/internal/pkg/dacctl/actions/actions.go b/internal/pkg/dacctl/actions/actions.go deleted file mode 100644 index e336ac1a..00000000 --- a/internal/pkg/dacctl/actions/actions.go +++ /dev/null @@ -1,311 +0,0 @@ -package actions - -import ( - "errors" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/lifecycle" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "strings" -) - -type CliContext interface { - String(name string) string - Int(name string) int -} - -type DacctlActions interface { - CreatePersistentBuffer(c CliContext) error - DeleteBuffer(c CliContext) error - CreatePerJobBuffer(c CliContext) error - ShowInstances() error - ShowSessions() error - ListPools() error - ShowConfigurations() error - ValidateJob(c CliContext) error - RealSize(c CliContext) error - DataIn(c CliContext) error - Paths(c CliContext) error - PreRun(c CliContext) error - PostRun(c CliContext) error - DataOut(c CliContext) error -} - -func NewDacctlActions( - poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, disk fileio.Disk) DacctlActions { - - return &dacctlActions{poolRegistry, volumeRegistry, disk} -} - -type dacctlActions struct { - poolRegistry registry.PoolRegistry - volumeRegistry registry.VolumeRegistry - disk fileio.Disk -} - -func (fwa *dacctlActions) CreatePersistentBuffer(c CliContext) error { - checkRequiredStrings(c, "token", "caller", "capacity", "user", "access", "type") - request := dacctl.BufferRequest{Token: c.String("token"), Caller: c.String("caller"), - Capacity: c.String("capacity"), User: c.Int("user"), - Group: c.Int("groupid"), Access: dacctl.AccessModeFromString(c.String("access")), - Type: dacctl.BufferTypeFromString(c.String("type")), Persistent: true} - if request.Group == 0 { - request.Group = request.User - } - err := dacctl.CreateVolumesAndJobs(fwa.volumeRegistry, fwa.poolRegistry, request) - if err == nil { - // Slurm is looking for the string "created" to know this worked - fmt.Printf("created %s\n", request.Token) - } - return err -} - -func checkRequiredStrings(c CliContext, flags ...string) { - errs := []string{} - for _, flag := range flags { - if str := c.String(flag); str == "" { - errs = append(errs, flag) - } - } - if len(errs) > 0 { - log.Fatalf("Please provide these required parameters: %s", strings.Join(errs, ", ")) - } -} - -func (fwa *dacctlActions) DeleteBuffer(c CliContext) error { - checkRequiredStrings(c, "token") - token := c.String("token") - return dacctl.DeleteBufferComponents(fwa.volumeRegistry, fwa.poolRegistry, token) -} - -func (fwa *dacctlActions) CreatePerJobBuffer(c CliContext) error { - checkRequiredStrings(c, "token", "job", "caller", "capacity") - return dacctl.CreatePerJobBuffer(fwa.volumeRegistry, fwa.poolRegistry, fwa.disk, - c.String("token"), c.Int("user"), c.Int("group"), c.String("capacity"), - c.String("caller"), c.String("job"), c.String("nodehostnamefile")) -} - -func (fwa *dacctlActions) ShowInstances() error { - instances, err := dacctl.GetInstances(fwa.volumeRegistry) - if err != nil { - return err - } - fmt.Println(instances) - return nil -} - -func (fwa *dacctlActions) ShowSessions() error { - sessions, err := dacctl.GetSessions(fwa.volumeRegistry) - if err != nil { - return err - } - fmt.Println(sessions) - return nil -} - -func (fwa *dacctlActions) ListPools() error { - pools, err := dacctl.GetPools(fwa.poolRegistry) - if err != nil { - return err - } - fmt.Println(pools) - return nil -} - -func (fwa *dacctlActions) ShowConfigurations() error { - fmt.Print(dacctl.GetConfigurations()) - return nil -} - -func (fwa *dacctlActions) ValidateJob(c CliContext) error { - checkRequiredStrings(c, "job") - if summary, err := dacctl.ParseJobFile(fwa.disk, c.String("job")); err != nil { - return err - } else { - // TODO check valid pools, etc, etc. - log.Println("Summary of job file:", summary) - } - return nil -} - -func (fwa *dacctlActions) RealSize(c CliContext) error { - checkRequiredStrings(c, "token") - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - if job.JobVolume == "" { - return fmt.Errorf("no volume to report the size of: %s", job.Name) - } - - volume, err := fwa.volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - // TODO get GiB vs GB correct here! - fmt.Printf(`{"token":"%s", "capacity":%d, "units":"bytes"}`, volume.Name, volume.SizeGB*1073741824) - return nil -} - -func (fwa *dacctlActions) DataIn(c CliContext) error { - checkRequiredStrings(c, "token") - fmt.Printf("--token %s --job %s\n", c.String("token"), c.String("job")) - - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - if job.JobVolume == "" { - log.Print("No data in required") - return nil - } - - volume, err := fwa.volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - - vlm := lifecycle.NewVolumeLifecycleManager(fwa.volumeRegistry, fwa.poolRegistry, volume) - return vlm.DataIn() -} - -func (fwa *dacctlActions) Paths(c CliContext) error { - checkRequiredStrings(c, "token", "pathfile") - fmt.Printf("--token %s --job %s --pathfile %s\n", - c.String("token"), c.String("job"), c.String("pathfile")) - - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - paths := []string{} - for key, value := range job.Paths { - paths = append(paths, fmt.Sprintf("%s=%s", key, value)) - } - return fwa.disk.Write(c.String("pathfile"), paths) -} - -var testVLM lifecycle.VolumeLifecycleManager - -func (fwa *dacctlActions) getVolumeLifecycleManger(volume registry.Volume) lifecycle.VolumeLifecycleManager { - if testVLM != nil { - return testVLM - } - return lifecycle.NewVolumeLifecycleManager(fwa.volumeRegistry, fwa.poolRegistry, volume) -} - -func (fwa *dacctlActions) PreRun(c CliContext) error { - checkRequiredStrings(c, "token", "nodehostnamefile") - fmt.Printf("--token %s --job %s --nodehostnamefile %s\n", - c.String("token"), c.String("job"), c.String("nodehostnamefile")) - - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - hosts, err := fwa.disk.Lines(c.String("nodehostnamefile")) - if err != nil { - return err - } - if len(hosts) < 1 { - return errors.New("unable to mount to zero compute hosts") - } - - err = fwa.volumeRegistry.JobAttachHosts(job.Name, hosts) - if err != nil { - return err - } - - if job.JobVolume == "" { - log.Print("No job volume to mount") - } else { - volume, err := fwa.volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - vlm := fwa.getVolumeLifecycleManger(volume) - if err := vlm.Mount(hosts, job.Name); err != nil { - return err - } - } - - for _, volumeName := range job.MultiJobVolumes { - volume, err := fwa.volumeRegistry.Volume(volumeName) - if err != nil { - return err - } - vlm := fwa.getVolumeLifecycleManger(volume) - if err := vlm.Mount(hosts, job.Name); err != nil { - return err - } - } - - return nil -} - -func (fwa *dacctlActions) PostRun(c CliContext) error { - checkRequiredStrings(c, "token") - fmt.Printf("--token %s --job %s\n", - c.String("token"), c.String("job")) - - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - if job.JobVolume == "" { - log.Print("No job volume to unmount") - } else { - volume, err := fwa.volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - vlm := lifecycle.NewVolumeLifecycleManager(fwa.volumeRegistry, fwa.poolRegistry, volume) - if err := vlm.Unmount(job.AttachHosts, job.Name); err != nil { - return err - } - } - - for _, volumeName := range job.MultiJobVolumes { - volume, err := fwa.volumeRegistry.Volume(volumeName) - if err != nil { - return err - } - vlm := lifecycle.NewVolumeLifecycleManager(fwa.volumeRegistry, fwa.poolRegistry, volume) - if err := vlm.Unmount(job.AttachHosts, job.Name); err != nil { - return err - } - } - - return nil -} - -func (fwa *dacctlActions) DataOut(c CliContext) error { - checkRequiredStrings(c, "token") - fmt.Printf("--token %s --job %s\n", - c.String("token"), c.String("job")) - - job, err := fwa.volumeRegistry.Job(c.String("token")) - if err != nil { - return err - } - - if job.JobVolume == "" { - log.Print("No data out required") - return nil - } - - volume, err := fwa.volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - - vlm := lifecycle.NewVolumeLifecycleManager(fwa.volumeRegistry, fwa.poolRegistry, volume) - return vlm.DataOut() -} diff --git a/internal/pkg/dacctl/actions/actions_test.go b/internal/pkg/dacctl/actions/actions_test.go deleted file mode 100644 index 4660388c..00000000 --- a/internal/pkg/dacctl/actions/actions_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package actions - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "testing" - "time" -) - -type mockCliContext struct { - capacity int -} - -func (c *mockCliContext) String(name string) string { - switch name { - case "capacity": - return fmt.Sprintf("pool1:%dGB", c.capacity) - case "token": - return "token" - case "caller": - return "caller" - case "user": - return "user" - case "access": - return "access" - case "type": - return "type" - case "job": - return "jobfile" - case "nodehostnamefile": - return "nodehostnamefile1" - case "pathfile": - return "pathfile1" - default: - return "" - } -} - -func (c *mockCliContext) Int(name string) int { - switch name { - case "user": - return 1001 - case "group": - return 1001 - default: - return 42 + len(name) - } -} - -func TestCreatePersistentBufferReturnsError(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockObj := mocks.NewMockVolumeRegistry(mockCtrl) - mockObj.EXPECT().AddVolume(gomock.Any()) // TODO - mockObj.EXPECT().AddJob(gomock.Any()) - mockPool := mocks.NewMockPoolRegistry(mockCtrl) - mockPool.EXPECT().Pools().DoAndReturn(func() ([]registry.Pool, error) { - return []registry.Pool{{Name: "pool1", GranularityGB: 1}}, nil - }) - - mockPool.EXPECT().AllocateBricksForVolume(gomock.Any()) - mockCtxt := &mockCliContext{} - - actions := NewDacctlActions(mockPool, mockObj, nil) - - err := actions.CreatePersistentBuffer(mockCtxt) - assert.Nil(t, err) -} - -func TestDacctlActions_PreRun(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - mockDisk := mocks.NewMockDisk(mockCtrl) - mockCtxt := &mockCliContext{} - actions := NewDacctlActions(nil, mockVolReg, mockDisk) - testVLM = &mockVLM{} - defer func() { testVLM = nil }() - - mockDisk.EXPECT().Lines("nodehostnamefile1").DoAndReturn(func(string) ([]string, error) { - return []string{"host1", "host2"}, nil - }) - mockVolReg.EXPECT().Job("token").DoAndReturn( - func(name string) (registry.Job, error) { - return registry.Job{ - Name: "token", - JobVolume: registry.VolumeName("token"), - MultiJobVolumes: []registry.VolumeName{registry.VolumeName("othervolume")}, - }, nil - }) - mockVolReg.EXPECT().JobAttachHosts("token", []string{"host1", "host2"}) - mockVolReg.EXPECT().Volume(registry.VolumeName("token")) - mockVolReg.EXPECT().Volume(registry.VolumeName("othervolume")).DoAndReturn( - func(name registry.VolumeName) (registry.Volume, error) { - return registry.Volume{Name: name}, nil - }) - - err := actions.PreRun(mockCtxt) - assert.Nil(t, err) -} - -func TestDacctlActions_Paths(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - mockDisk := mocks.NewMockDisk(mockCtrl) - mockCtxt := &mockCliContext{} - actions := NewDacctlActions(nil, mockVolReg, mockDisk) - testVLM = &mockVLM{} - defer func() { testVLM = nil }() - - mockVolReg.EXPECT().Job("token").DoAndReturn( - func(name string) (registry.Job, error) { - return registry.Job{JobVolume: registry.VolumeName("token"), Paths: map[string]string{"a": "A"}}, nil - }) - mockDisk.EXPECT().Write("pathfile1", []string{"a=A"}) - - err := actions.Paths(mockCtxt) - assert.Nil(t, err) -} - -func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockPoolReg := mocks.NewMockPoolRegistry(mockCtrl) - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - mockDisk := mocks.NewMockDisk(mockCtrl) - mockCtxt := &mockCliContext{capacity: 300} - actions := NewDacctlActions(mockPoolReg, mockVolReg, mockDisk) - - mockDisk.EXPECT().Lines("jobfile").DoAndReturn(func(string) ([]string, error) { - return []string{ - "#DW persistentdw name=mybuffer", - "#DW jobdw capacity=2GB access_mode=striped,private type=scratch", - }, nil - }) - - mockPoolReg.EXPECT().Pools().DoAndReturn(func() ([]registry.Pool, error) { - return []registry.Pool{{Name: "pool1", GranularityGB: 200}}, nil - }) - mockVolReg.EXPECT().Volume(registry.VolumeName("mybuffer")).DoAndReturn( - func(name registry.VolumeName) (registry.Volume, error) { - return registry.Volume{Name: name, MultiJob: true}, nil - }) - expectedVolume := registry.Volume{ - Name: "token", - MultiJob: false, - State: registry.Registered, - Pool: "pool1", - SizeBricks: 2, - SizeGB: 400, - JobName: "token", - Owner: 1001, - Group: 1001, - CreatedBy: "caller", - CreatedAt: uint(time.Now().Unix()), // TODO this is racey! - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 0, - } - mockVolReg.EXPECT().AddVolume(expectedVolume) - mockVolReg.EXPECT().AddJob(registry.Job{ - Name: "token", - Owner: 1001, - CreatedAt: uint(time.Now().Unix()), - Paths: map[string]string{ - "DW_PERSISTENT_STRIPED_mybuffer": "/dac/token_persistent_mybuffer", - "DW_JOB_PRIVATE": "/dac/token_job_private", - "DW_JOB_STRIPED": "/dac/token_job/global", - }, - JobVolume: registry.VolumeName("token"), - MultiJobVolumes: []registry.VolumeName{"mybuffer"}, - }) - mockVolReg.EXPECT().Volume(registry.VolumeName("token")).DoAndReturn( - func(name registry.VolumeName) (registry.Volume, error) { - return registry.Volume{ - Name: name, - SizeBricks: 0, // TODO: skips ProvisionBricks logic - }, nil - }) - // TODO: sort out the volume passed here! - mockPoolReg.EXPECT().AllocateBricksForVolume(gomock.Any()) - - err := actions.CreatePerJobBuffer(mockCtxt) - assert.Nil(t, err) -} - -type mockVLM struct{} - -func (*mockVLM) ProvisionBricks() error { - panic("implement me") -} - -func (*mockVLM) DataIn() error { - panic("implement me") -} - -func (*mockVLM) Mount(hosts []string, jobName string) error { - return nil -} - -func (*mockVLM) Unmount(hosts []string, jobName string) error { - panic("implement me") -} - -func (*mockVLM) DataOut() error { - panic("implement me") -} - -func (*mockVLM) Delete() error { - panic("implement me") -} diff --git a/internal/pkg/dacctl/buffer.go b/internal/pkg/dacctl/buffer.go deleted file mode 100644 index b1b8b472..00000000 --- a/internal/pkg/dacctl/buffer.go +++ /dev/null @@ -1,181 +0,0 @@ -package dacctl - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/lifecycle" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "strings" - "time" -) - -func DeleteBufferComponents(volumeRegistry registry.VolumeRegistry, poolRegistry registry.PoolRegistry, - token string) error { - - job, err := volumeRegistry.Job(token) - if err != nil { - if strings.Contains(err.Error(), "unable to find any values for key") { - log.Println("Unable to find job, must be deleted already or never created.") - return nil - } - return err - } - - if job.JobVolume != "" { - volume, err := volumeRegistry.Volume(job.JobVolume) - if err != nil { - return err - } else { - vlm := lifecycle.NewVolumeLifecycleManager(volumeRegistry, poolRegistry, volume) - if err := vlm.Delete(); err != nil { - return err - } - } - } - - return volumeRegistry.DeleteJob(token) -} - -func CreatePerJobBuffer(volumeRegistry registry.VolumeRegistry, poolRegistry registry.PoolRegistry, disk fileio.Disk, - token string, user int, group int, capacity string, caller string, jobFile string, nodeFile string) error { - summary, err := ParseJobFile(disk, jobFile) - if err != nil { - return err - } - - if nodeFile != "" { - // TODO we could add this into the volume as a scheduling hint, when its available? - log.Printf("Ignoring nodeFile in setup: %s", nodeFile) - } - - pool, bricksRequired, err := getPoolAndBrickCount(poolRegistry, capacity) - if err != nil { - return err - } - - createdAt := uint(time.Now().Unix()) - job := registry.Job{ - Name: token, - Owner: uint(user), - CreatedAt: createdAt, - } - - var perJobVolume *registry.Volume - if bricksRequired > 0 && summary.PerJobBuffer != nil { - perJobVolume = getPerJobVolume(token, pool, bricksRequired, - user, group, caller, createdAt, summary) - - err := volumeRegistry.AddVolume(*perJobVolume) - if err != nil { - return err - } - - job.JobVolume = perJobVolume.Name - } - - for _, attachment := range summary.Attachments { - name := registry.VolumeName(attachment.Name) - volume, err := volumeRegistry.Volume(name) - if err != nil { - return err - } - // TODO: need to check permissions and not just go for it! - if !volume.MultiJob { - return fmt.Errorf("%s is not a multijob volume", volume.Name) - } - job.MultiJobVolumes = append(job.MultiJobVolumes, volume.Name) - } - - job.Paths = setPaths(perJobVolume, job) - - err = volumeRegistry.AddJob(job) - if err != nil { - if job.JobVolume != "" { - volumeRegistry.DeleteVolume(job.JobVolume) - } - return err - } - - if job.JobVolume != "" { - volume, err := volumeRegistry.Volume(job.JobVolume) - vlm := lifecycle.NewVolumeLifecycleManager(volumeRegistry, poolRegistry, volume) - err = vlm.ProvisionBricks() - if err != nil { - log.Println("Bricks may be left behnd, not deleting volume due to: ", err) - return err - } - } - return nil -} - -func setPaths(perJobVolume *registry.Volume, job registry.Job) map[string]string { - paths := make(map[string]string) - if perJobVolume != nil { - if perJobVolume.AttachPrivateNamespace { - paths["DW_JOB_PRIVATE"] = fmt.Sprintf("/dac/%s_job_private", job.Name) - } - if perJobVolume.AttachGlobalNamespace { - paths["DW_JOB_STRIPED"] = fmt.Sprintf("/dac/%s_job/global", job.Name) - } - } - for _, multiJobVolume := range job.MultiJobVolumes { - paths[fmt.Sprintf("DW_PERSISTENT_STRIPED_%s", multiJobVolume)] = fmt.Sprintf( - "/dac/%s_persistent_%s", job.Name, multiJobVolume) - } - return paths -} - -func getPerJobVolume(token string, pool *registry.Pool, bricksRequired uint, - user int, group int, caller string, createdAt uint, summary jobSummary) *registry.Volume { - adjustedSizeGB := bricksRequired * pool.GranularityGB - perJobVolume := registry.Volume{ - Name: registry.VolumeName(token), - MultiJob: false, - State: registry.Registered, - Pool: pool.Name, - SizeBricks: bricksRequired, - SizeGB: adjustedSizeGB, - JobName: token, - Owner: uint(user), - Group: uint(group), - CreatedBy: caller, - CreatedAt: createdAt, - } - jobBuffer := summary.PerJobBuffer - if jobBuffer.BufferType == scratch && - (jobBuffer.AccessMode == private || jobBuffer.AccessMode == privateAndStriped) { - perJobVolume.AttachPrivateNamespace = true - } - if jobBuffer.BufferType == scratch && - (jobBuffer.AccessMode == striped || jobBuffer.AccessMode == privateAndStriped) { - perJobVolume.AttachGlobalNamespace = true - } - if jobBuffer.BufferType == scratch && summary.Swap != nil { - perJobVolume.AttachAsSwapBytes = uint(summary.Swap.SizeBytes) - } - // TODO that can be many data_in and data_out, we only allow one relating to striped job buffer - if summary.DataIn != nil && summary.DataIn.Source != "" { - // TODO check destination includes striped buffer path? - perJobVolume.StageIn.Source = summary.DataIn.Source - perJobVolume.StageIn.Destination = summary.DataIn.Destination - switch summary.DataIn.StageType { - case file: - perJobVolume.StageIn.SourceType = registry.File - case directory: - perJobVolume.StageIn.SourceType = registry.Directory - } - } - if summary.DataOut != nil && summary.DataOut.Source != "" { - // TODO check source includes striped buffer path? - perJobVolume.StageOut.Source = summary.DataOut.Source - perJobVolume.StageOut.Destination = summary.DataOut.Destination - switch summary.DataOut.StageType { - case file: - perJobVolume.StageOut.SourceType = registry.File - case directory: - perJobVolume.StageOut.SourceType = registry.Directory - } - } - return &perJobVolume -} diff --git a/internal/pkg/dacctl/buffer_test.go b/internal/pkg/dacctl/buffer_test.go deleted file mode 100644 index 1a99f82b..00000000 --- a/internal/pkg/dacctl/buffer_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package dacctl - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestCreatePerJobBuffer(t *testing.T) { - mockCtrl := gomock.NewController(t) - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - mockPoolReg := mocks.NewMockPoolRegistry(mockCtrl) - mockDisk := mocks.NewMockDisk(mockCtrl) - mockDisk.EXPECT().Lines("jobfile") - - err := CreatePerJobBuffer(mockVolReg, mockPoolReg, mockDisk, "token", - 2, 2, "", "test", "jobfile", "nodefile") - assert.Equal(t, "must format capacity correctly and include pool", err.Error()) -} - -func TestGetPerJobVolume(t *testing.T) { - pool := registry.Pool{} - summary := jobSummary{ - PerJobBuffer: &cmdPerJobBuffer{}, - } - volume := getPerJobVolume("token", &pool, 3, 42, 42, - "test", 20, summary) - // TODO: lots more work to do here! - assert.Equal(t, registry.VolumeName("token"), volume.Name) - assert.True(t, volume.AttachGlobalNamespace) - assert.False(t, volume.AttachPrivateNamespace) -} - -func TestSetPaths(t *testing.T) { - volume := registry.Volume{ - Name: "job1", - UUID: "uuid1", - AttachPrivateNamespace: true, - AttachGlobalNamespace: true, - } - job := registry.Job{ - Name: "job1", - MultiJobVolumes: []registry.VolumeName{ - registry.VolumeName("multi1"), - registry.VolumeName("multi2"), - }, - } - - paths := setPaths(&volume, job) - - assert.Equal(t, 4, len(paths)) - assert.Equal(t, - "/dac/job1_job_private", - paths["DW_JOB_PRIVATE"]) - assert.Equal(t, - "/dac/job1_job/global", - paths["DW_JOB_STRIPED"]) - assert.Equal(t, - "/dac/job1_persistent_multi1", - paths["DW_PERSISTENT_STRIPED_multi1"]) - assert.Equal(t, - "/dac/job1_persistent_multi2", - paths["DW_PERSISTENT_STRIPED_multi2"]) -} diff --git a/internal/pkg/dacctl/job.go b/internal/pkg/dacctl/job.go deleted file mode 100644 index 992e3efa..00000000 --- a/internal/pkg/dacctl/job.go +++ /dev/null @@ -1,299 +0,0 @@ -package dacctl - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "log" - "strconv" - "strings" -) - -type jobSummary struct { - PerJobBuffer *cmdPerJobBuffer - Swap *cmdAttachPerJobSwap - Attachments []cmdAttachPersistent - DataIn *cmdStageInData - DataOut *cmdStageOutData - //createPersistent *cmdCreatePersistent - //destroyPersistent *cmdDestroyPersistent -} - -func (s jobSummary) String() string { - return toJson(s) -} - -// Parse a given job file -func ParseJobFile(disk fileio.Disk, filename string) (jobSummary, error) { - lines, err := disk.Lines(filename) - if err != nil { - return jobSummary{}, err - } - return getJobSummary(lines) -} - -func getJobSummary(lines []string) (jobSummary, error) { - var summary jobSummary - jobCommands, err := parseJobRequest(lines) - if err != nil { - return summary, err - } - - for _, cmd := range jobCommands { - switch c := cmd.(type) { - case cmdPerJobBuffer: - if summary.PerJobBuffer == nil { - summary.PerJobBuffer = &c - } else { - return summary, fmt.Errorf("only one per job buffer allowed") - } - case cmdAttachPersistent: - summary.Attachments = append(summary.Attachments, c) - case cmdAttachPerJobSwap: - if summary.Swap != nil { - // TODO check amount isn't too big for per job buffer - return summary, fmt.Errorf("only one swap request allowed") - } - summary.Swap = &c - case cmdStageOutData: - if summary.DataOut != nil { - // TODO really should check if data out matches one of the requested buffers - return summary, fmt.Errorf("only one per data out requested allowed") - } - summary.DataOut = &c - case cmdStageInData: - if summary.DataIn != nil { - // TODO really should check if data in matches one of the requested buffers - return summary, fmt.Errorf("only one per data in requested allowed") - } - summary.DataIn = &c - default: - // do nothing - } - } - return summary, nil -} - -type jobCommand interface{} - -type AccessMode int - -const ( - striped AccessMode = 0 - private = 1 - privateAndStriped = 2 -) - -var stringToAccessMode = map[string]AccessMode{ - "": striped, - "striped": striped, - "private": private, - "private,striped": privateAndStriped, - "striped,private": privateAndStriped, -} - -func AccessModeFromString(raw string) AccessMode { - return stringToAccessMode[strings.ToLower(raw)] -} - -type BufferType int - -const ( - scratch BufferType = iota - cache -) - -var stringToBufferType = map[string]BufferType{ - "": scratch, - "scratch": scratch, - "cache": cache, -} - -type cmdCreatePersistent struct { - Name string - CapacityBytes int - AccessMode AccessMode - BufferType BufferType - GenericCmd bool -} - -func BufferTypeFromString(raw string) BufferType { - return stringToBufferType[strings.ToLower(raw)] -} - -type cmdDestroyPersistent struct { - Name string -} - -type cmdAttachPersistent struct { - Name string -} - -type cmdPerJobBuffer struct { - CapacityBytes int - AccessMode AccessMode - BufferType BufferType - GenericCmd bool -} - -type cmdAttachPerJobSwap struct { - SizeBytes int -} - -type StageType int - -const ( - directory StageType = iota - file // TODO there is also list, but we ignore that for now -) - -var stringToStageType = map[string]StageType{ - "": directory, - "directory": directory, - "file": file, -} - -func stageTypeFromString(raw string) StageType { - return stringToStageType[strings.ToLower(raw)] -} - -type cmdStageInData struct { - Source string - Destination string - StageType StageType -} - -type cmdStageOutData struct { - Source string - Destination string - StageType StageType -} - -var sizeSuffixMulitiplyer = map[string]int{ - "TiB": 1099511627776, - "TB": 1000000000000, - "GiB": 1073741824, - "GB": 1000000000, - "MiB": 1048576, - "MB": 1000000, -} - -func parseSize(raw string) (int, error) { - intVal, err := strconv.Atoi(raw) - if err == nil { - // specified raw bytes - return intVal, nil - } - for suffix, multiplyer := range sizeSuffixMulitiplyer { - if strings.HasSuffix(raw, suffix) { - rawInt := strings.TrimSuffix(raw, suffix) - intVal, err := strconv.Atoi(rawInt) - if err != nil { - return 0, err - } - return intVal * multiplyer, nil - } - } - return 0, fmt.Errorf("unable to parse size: %s", raw) -} - -func parseArgs(rawArgs []string) (map[string]string, error) { - args := make(map[string]string, len(rawArgs)) - for _, arg := range rawArgs { - parts := strings.Split(arg, "=") - if len(parts) != 2 { - return args, fmt.Errorf("unable to parse arg: %s", arg) - } - args[strings.ToLower(parts[0])] = parts[1] - } - return args, nil -} - -func parseJobRequest(lines []string) ([]jobCommand, error) { - var commands []jobCommand - for _, line := range lines { - tokens := strings.Split(line, " ") - if len(tokens) < 3 { - if line != "" && line != "#!/bin/bash" { - log.Println("Skip badly formatted line:", line) - } - continue - } - - cmdType := tokens[0] - cmd := tokens[1] - args := tokens[2:] - - var isGeneric bool - switch cmdType { - case "#DW": - isGeneric = false - case "#BB": - isGeneric = true - default: - log.Println("unrecognised command type:", cmdType) - continue - } - - argKeyPair, _ := parseArgs(args) // TODO deal with errors when not swap - - var command jobCommand - switch cmd { - case "create_persistent": - size, err := parseSize(argKeyPair["capacity"]) - if err != nil { - log.Println(err) - continue - } - command = cmdCreatePersistent{ - Name: argKeyPair["name"], - CapacityBytes: size, - GenericCmd: isGeneric, - AccessMode: AccessModeFromString(argKeyPair["access_mode"]), - BufferType: BufferTypeFromString(argKeyPair["type"]), - } - case "destroy_persistent": - command = cmdDestroyPersistent{Name: argKeyPair["name"]} - case "persistentdw": - command = cmdAttachPersistent{Name: argKeyPair["name"]} - case "jobdw": - size, err := parseSize(argKeyPair["capacity"]) - if err != nil { - log.Println(err) - continue - } - command = cmdPerJobBuffer{ - CapacityBytes: size, - GenericCmd: isGeneric, - AccessMode: AccessModeFromString(argKeyPair["access_mode"]), - BufferType: BufferTypeFromString(argKeyPair["type"]), - } - case "swap": - if len(args) != 1 { - log.Println("Unable to parse swap command:", line) - } - if size, err := parseSize(args[0]); err != nil { - log.Println(err) - continue - } else { - command = cmdAttachPerJobSwap{SizeBytes: size} - } - case "stage_in": - command = cmdStageInData{ - Source: argKeyPair["source"], - Destination: argKeyPair["destination"], - StageType: stageTypeFromString(argKeyPair["type"]), - } - case "stage_out": - command = cmdStageOutData{ - Source: argKeyPair["source"], - Destination: argKeyPair["destination"], - StageType: stageTypeFromString(argKeyPair["type"]), - } - default: - log.Println("unrecognised command:", cmd, "with argument length", len(args)) - continue - } - commands = append(commands, command) - } - return commands, nil -} diff --git a/internal/pkg/dacctl/job_test.go b/internal/pkg/dacctl/job_test.go deleted file mode 100644 index 3b590211..00000000 --- a/internal/pkg/dacctl/job_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package dacctl - -import ( - "github.com/stretchr/testify/assert" - "log" - "testing" -) - -func TestParseJobRequest(t *testing.T) { - jobRequest := []string{ - `#BB create_persistent name=myBBname capacity=100GB access_mode=striped type=scratch`, - `#BB create_persistent name=myBBname capacity=1073741824 access_mode=striped type=cache`, - `#BB destroy_persistent name=myBBname`, - `#DW persistentdw name=myBBname1`, - `#DW persistentdw name=myBBname2`, - `#DW persistentdw name=myBBname2`, - `#DW jobdw capacity=10GB access_mode=striped type=scratch`, - `#DW jobdw capacity=2TB access_mode=private type=scratch`, - `#DW jobdw capacity=4TiB access_mode=striped,private type=scratch`, - `#BB jobdw capacity=42GiB access_mode=ldbalance type=cache pfs=/global/scratch1/john`, - `#DW swap 3TiB`, - `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, - `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, - } - if cmds, err := parseJobRequest(jobRequest); err != nil { - log.Fatal(err) - } else { - assert.Equal(t, 13, len(jobRequest)) // TODO should check returned values!! - for _, cmd := range cmds { - log.Printf("Cmd: %T Args: %s\n", cmd, cmd) - } - } -} - -func TestGetJobSummary(t *testing.T) { - lines := []string{ - `#DW persistentdw name=myBBname1`, - `#DW persistentdw name=myBBname2`, - `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, - `#DW swap 4MB`, - `#DW stage_in source=/global/cscratch1/filename1 destination=$DW_JOB_STRIPED/filename1 type=file`, - `#DW stage_out source=$DW_JOB_STRIPED/outdir destination=/global/scratch1/outdir type=directory`, - } - result, err := getJobSummary(lines) - - assert.Nil(t, err) - assert.EqualValues(t, "/global/cscratch1/filename1", result.DataIn.Source) - assert.EqualValues(t, "$DW_JOB_STRIPED/outdir", result.DataOut.Source) - - assert.Equal(t, 4194304, result.PerJobBuffer.CapacityBytes) - assert.Equal(t, 4000000, result.Swap.SizeBytes) -} diff --git a/internal/pkg/dacctl/json.go b/internal/pkg/dacctl/json.go deleted file mode 100644 index f79eaf44..00000000 --- a/internal/pkg/dacctl/json.go +++ /dev/null @@ -1,14 +0,0 @@ -package dacctl - -import ( - "encoding/json" - "log" -) - -func toJson(message interface{}) string { - b, error := json.Marshal(message) - if error != nil { - log.Fatal(error) - } - return string(b) -} diff --git a/internal/pkg/dacctl/persistent.go b/internal/pkg/dacctl/persistent.go deleted file mode 100644 index 71dc1e79..00000000 --- a/internal/pkg/dacctl/persistent.go +++ /dev/null @@ -1,130 +0,0 @@ -package dacctl - -import ( - "errors" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/lifecycle" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "math" - "strings" - "time" -) - -type BufferRequest struct { - Token string - Caller string - Capacity string - User int - Group int - Access AccessMode - Type BufferType - Persistent bool -} - -// Creates a persistent buffer. -// If it works, we return the name of the buffer, otherwise an error is returned - -func parseCapacity(raw string) (string, int, error) { - parts := strings.Split(raw, ":") - if len(parts) != 2 { - return "", 0, errors.New("must format capacity correctly and include pool") - } - pool := parts[0] - rawCapacity := parts[1] - sizeBytes, err := parseSize(rawCapacity) - if err != nil { - return "", 0, err - } - capacityInt := int(sizeBytes / bytesInGB) - return pool, capacityInt, nil -} - -func findPool(poolRegistry registry.PoolRegistry, poolName string) (pool *registry.Pool, err error) { - pools, err := poolRegistry.Pools() - if err != nil { - return - } - - for _, p := range pools { - if p.Name == poolName { - pool = &p - } - } - - if pool == nil { - err = fmt.Errorf("unable to find pool: %s", poolName) - return - } - return -} - -func getPoolAndBrickCount(poolRegistry registry.PoolRegistry, capacity string) (pool *registry.Pool, - bricksRequired uint, err error) { - - poolName, capacityGB, err := parseCapacity(capacity) - if err != nil { - return - } - - pool, err = findPool(poolRegistry, poolName) - if err != nil { - return - } - - bricksRequired = uint(math.Ceil(float64(capacityGB) / float64(pool.GranularityGB))) - return -} - -// TODO: ideally this would be private, if not for testing -func CreateVolumesAndJobs(volReg registry.VolumeRegistry, poolRegistry registry.PoolRegistry, - request BufferRequest) error { - - createdAt := uint(time.Now().Unix()) - - pool, bricksRequired, err := getPoolAndBrickCount(poolRegistry, request.Capacity) - if err != nil { - return err - } - adjustedSizeGB := bricksRequired * pool.GranularityGB - - volume := registry.Volume{ - Name: registry.VolumeName(request.Token), - JobName: request.Token, - Owner: uint(request.User), - CreatedAt: createdAt, - CreatedBy: request.Caller, - Group: uint(request.Group), - SizeGB: adjustedSizeGB, - SizeBricks: bricksRequired, - Pool: pool.Name, - State: registry.Registered, - MultiJob: request.Persistent, - } - err = volReg.AddVolume(volume) - if err != nil { - return err - } - - job := registry.Job{ - Name: request.Token, - Owner: uint(request.User), - CreatedAt: createdAt, - JobVolume: volume.Name, // Even though its a persistent buffer, we add it here to ensure we delete buffer - Paths: make(map[string]string), - } - - err = volReg.AddJob(job) - if err != nil { - delErr := volReg.DeleteVolume(volume.Name) - log.Println("volume deleted: ", delErr) // TODO: remove debug logs later, once understood - return err - } - - vlm := lifecycle.NewVolumeLifecycleManager(volReg, poolRegistry, volume) - err = vlm.ProvisionBricks() - if err != nil { - log.Println("Bricks may be left behnd, not deleting volume due to: ", err) - } - return err -} diff --git a/internal/pkg/dacctl/pools.go b/internal/pkg/dacctl/pools.go deleted file mode 100644 index b4a37e44..00000000 --- a/internal/pkg/dacctl/pools.go +++ /dev/null @@ -1,43 +0,0 @@ -package dacctl - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" -) - -type pool struct { - Id string `json:"id"` - Units string `json:"units"` - Granularity uint `json:"granularity"` - Quantity uint `json:"quantity"` - Free uint `json:"free"` -} - -type pools []pool - -func (list *pools) String() string { - message := map[string]pools{"pools": *list} - return toJson(message) -} - -const GbInBytes = 1073741824 - -func GetPools(registry registry.PoolRegistry) (*pools, error) { - pools := pools{} - regPools, err := registry.Pools() - if err != nil { - return &pools, err - } - - for _, regPool := range regPools { - free := len(regPool.AvailableBricks) - quantity := free + len(regPool.AllocatedBricks) - pools = append(pools, pool{ - Id: regPool.Name, - Units: "bytes", - Granularity: regPool.GranularityGB * GbInBytes, - Quantity: uint(quantity), - Free: uint(free), - }) - } - return &pools, nil -} diff --git a/internal/pkg/dacctl/pools_test.go b/internal/pkg/dacctl/pools_test.go deleted file mode 100644 index c47b644c..00000000 --- a/internal/pkg/dacctl/pools_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package dacctl - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestGetPools(t *testing.T) { - mockCtrl := gomock.NewController(t) - mockReg := mocks.NewMockPoolRegistry(mockCtrl) - fakePools := func() ([]registry.Pool, error) { - return []registry.Pool{{Name: "fake", GranularityGB: 1}}, nil - } - mockReg.EXPECT().Pools().DoAndReturn(fakePools) - - pools, _ := GetPools(mockReg) - actual := pools.String() - expected := `{"pools":[{"id":"fake","units":"bytes","granularity":1073741824,"quantity":0,"free":0}]}` - assert.EqualValues(t, expected, actual) -} diff --git a/internal/pkg/dacctl/show.go b/internal/pkg/dacctl/show.go deleted file mode 100644 index b0df92b4..00000000 --- a/internal/pkg/dacctl/show.go +++ /dev/null @@ -1,87 +0,0 @@ -package dacctl - -import "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - -type instanceCapacity struct { - Bytes uint `json:"bytes"` - Nodes uint `json:"nodes"` -} - -type instanceLinks struct { - Session string `json:"session"` -} - -type instance struct { - Id string `json:"id"` - Capacity instanceCapacity `json:"capacity"` - Links instanceLinks `json:"links"` -} - -type instances []instance - -func (list *instances) String() string { - message := map[string]instances{"instances": *list} - return toJson(message) -} - -const bytesInGB = 1073741824 - -func GetInstances(volRegistry registry.VolumeRegistry) (*instances, error) { - instances := instances{} - volumes, err := volRegistry.AllVolumes() - if err != nil { - // TODO... normally means there are no instances - } - - for _, volume := range volumes { - instances = append(instances, instance{ - Id: string(volume.Name), - Capacity: instanceCapacity{Bytes: volume.SizeGB * bytesInGB, Nodes: volume.SizeBricks}, - Links: instanceLinks{volume.JobName}, - }) - } - return &instances, nil -} - -type session struct { - Id string `json:"id"` - Created uint `json:"created"` - Owner uint `json:"owner"` - Token string `json:"token"` -} - -type sessions []session - -func (list *sessions) String() string { - message := map[string]sessions{"sessions": *list} - return toJson(message) -} - -func GetSessions(volRegistry registry.VolumeRegistry) (*sessions, error) { - jobs, err := volRegistry.Jobs() - if err != nil { - // TODO: error is usually about there not being any jobs - jobs = []registry.Job{} - } - sessions := sessions{} - for _, job := range jobs { - sessions = append(sessions, session{ - Id: job.Name, - Created: job.CreatedAt, - Owner: job.Owner, - Token: job.Name, - }) - } - return &sessions, nil -} - -type configurations []string - -func (list *configurations) String() string { - message := map[string]configurations{"configurations": *list} - return toJson(message) -} - -func GetConfigurations() *configurations { - return &configurations{} -} diff --git a/internal/pkg/dacctl/show_test.go b/internal/pkg/dacctl/show_test.go deleted file mode 100644 index 3b881e11..00000000 --- a/internal/pkg/dacctl/show_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package dacctl - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "log" - "testing" -) - -func assertNewline(t *testing.T, actual string) { - assert.EqualValues(t, "\n", actual[len(actual)-1:]) -} - -func TestGetInstances(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockReg := mocks.NewMockVolumeRegistry(mockCtrl) - fakeGetVolumes := func() ([]registry.Volume, error) { - return []registry.Volume{ - {Name: "fake1", Pool: "pool1", SizeGB: 2}, - }, nil - } - mockReg.EXPECT().AllVolumes().DoAndReturn(fakeGetVolumes) - - instances, err := GetInstances(mockReg) - if err != nil { - log.Fatal(err) - } - actual := instances.String() - - // TODO need to return sessions correctly... i.e. job - expected := `{"instances":[{"id":"fake1","capacity":{"bytes":2147483648,"nodes":0},"links":{"session":""}}]}` - assert.EqualValues(t, expected, actual) -} - -func TestGetSessions(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockReg := mocks.NewMockVolumeRegistry(mockCtrl) - mockJobs := func() ([]registry.Job, error) { - return []registry.Job{{Name: "fake1", CreatedAt: 42, Owner: 1001}}, nil - - } - mockReg.EXPECT().Jobs().DoAndReturn(mockJobs) - sessions, err := GetSessions(mockReg) - if err != nil { - log.Fatal(err) - } - actual := sessions.String() - - expected := `{"sessions":[{"id":"fake1","created":42,"owner":1001,"token":"fake1"}]}` - assert.EqualValues(t, expected, actual) -} - -func TestGetConfigurations(t *testing.T) { - actual := GetConfigurations().String() - expected := `{"configurations":[]}` - assert.EqualValues(t, expected, actual) -} diff --git a/internal/pkg/etcdregistry/keystore.go b/internal/pkg/etcdregistry/keystore.go deleted file mode 100644 index b4ecaed1..00000000 --- a/internal/pkg/etcdregistry/keystore.go +++ /dev/null @@ -1,268 +0,0 @@ -package etcdregistry - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/clientv3/clientv3util" - "github.com/coreos/etcd/clientv3/concurrency" - "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" - "github.com/coreos/etcd/mvcc/mvccpb" - "github.com/coreos/etcd/pkg/transport" - "log" - "os" - "strings" - "time" -) - -func getTLSConfig() *tls.Config { - certFile := os.Getenv("ETCDCTL_CERT_FILE") - keyFile := os.Getenv("ETCDCTL_KEY_FILE") - caFile := os.Getenv("ETCDCTL_CA_FILE") - - if certFile == "" || keyFile == "" || caFile == "" { - return nil - } - - tlsInfo := transport.TLSInfo{ - CertFile: certFile, - KeyFile: keyFile, - TrustedCAFile: caFile, - } - tlsConfig, err := tlsInfo.ClientConfig() - if err != nil { - log.Fatal(err) - } - return tlsConfig -} - -func getEndpoints() []string { - endpoints := os.Getenv("ETCDCTL_ENDPOINTS") - if endpoints == "" { - endpoints = os.Getenv("ETCD_ENDPOINTS") - } - if endpoints == "" { - log.Fatalf("Must set ETCDCTL_ENDPOINTS environemnt variable, e.g. export ETCDCTL_ENDPOINTS=127.0.0.1:2379") - } - return strings.Split(endpoints, ",") -} - -func newEtcdClient() *clientv3.Client { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: getEndpoints(), - DialTimeout: 10 * time.Second, - TLS: getTLSConfig(), - }) - if err != nil { - fmt.Println("failed to create client") - log.Fatal(err) - } - return cli -} - -func NewKeystore() keystoreregistry.Keystore { - cli := newEtcdClient() - return &etcKeystore{ - Watcher: cli.Watcher, - KV: cli.KV, - Lease: cli.Lease, - Client: cli, - } -} - -type etcKeystore struct { - Watcher clientv3.Watcher - KV clientv3.KV - Lease clientv3.Lease - Client *clientv3.Client -} - -func (client *etcKeystore) NewMutex(lockKey string) (keystoreregistry.Mutex, error) { - session, err := concurrency.NewSession(client.Client) - if err != nil { - return nil, err - } - key := fmt.Sprintf("/locks/%s", lockKey) - return concurrency.NewMutex(session, key), nil -} - -func handleError(err error) { - if err != nil { - switch err { - case context.Canceled: - log.Printf("ctx is canceled by another routine: %v", err) - case context.DeadlineExceeded: - log.Printf("ctx is attached with a deadline is exceeded: %v", err) - case rpctypes.ErrEmptyKey: - log.Printf("client-side error: %v", err) - default: - log.Printf("bad cluster endpoints, which are not etcd servers: %v", err) - } - log.Fatal(err) - } -} - -func (client *etcKeystore) Close() error { - return client.Client.Close() -} - -func (client *etcKeystore) runTransaction(ifOps []clientv3.Cmp, thenOps []clientv3.Op) error { - kvc := clientv3.NewKV(client.Client) - kvc.Txn(context.Background()) - response, err := kvc.Txn(context.Background()).If(ifOps...).Then(thenOps...).Commit() - handleError(err) - - if !response.Succeeded { - log.Println(ifOps) - return fmt.Errorf("transaction failed, as condition not met") - } - return nil -} - -func (client *etcKeystore) Add(keyValues []keystoreregistry.KeyValue) error { - var ifOps []clientv3.Cmp - var thenOps []clientv3.Op - for _, keyValue := range keyValues { - ifOps = append(ifOps, clientv3util.KeyMissing(keyValue.Key)) - thenOps = append(thenOps, clientv3.OpPut(keyValue.Key, keyValue.Value)) - } - return client.runTransaction(ifOps, thenOps) -} - -func (client *etcKeystore) Update(keyValues []keystoreregistry.KeyValueVersion) error { - var ifOps []clientv3.Cmp - var thenOps []clientv3.Op - for _, keyValue := range keyValues { - if keyValue.ModRevision > 0 { - ifOps = append(ifOps, clientv3util.KeyExists(keyValue.Key)) // only add new keys if ModRevision == 0 - checkModRev := clientv3.Compare(clientv3.ModRevision(keyValue.Key), "=", keyValue.ModRevision) - ifOps = append(ifOps, checkModRev) - } - thenOps = append(thenOps, clientv3.OpPut(keyValue.Key, keyValue.Value)) - } - return client.runTransaction(ifOps, thenOps) -} - -func (client *etcKeystore) DeleteAll(keyValues []keystoreregistry.KeyValueVersion) error { - var ifOps []clientv3.Cmp - var thenOps []clientv3.Op - for _, keyValue := range keyValues { - ifOps = append(ifOps, clientv3util.KeyExists(keyValue.Key)) - if keyValue.ModRevision > 0 { - checkModRev := clientv3.Compare(clientv3.ModRevision(keyValue.Key), "=", keyValue.ModRevision) - ifOps = append(ifOps, checkModRev) - } - thenOps = append(thenOps, clientv3.OpDelete(keyValue.Key)) - } - return client.runTransaction(ifOps, thenOps) -} - -func getKeyValueVersion(rawKeyValue *mvccpb.KeyValue) *keystoreregistry.KeyValueVersion { - if rawKeyValue == nil { - return nil - } - return &keystoreregistry.KeyValueVersion{ - Key: string(rawKeyValue.Key), - Value: string(rawKeyValue.Value), - ModRevision: rawKeyValue.ModRevision, - CreateRevision: rawKeyValue.CreateRevision, - } -} - -func (client *etcKeystore) GetAll(prefix string) ([]keystoreregistry.KeyValueVersion, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Get(context.Background(), prefix, clientv3.WithPrefix()) - handleError(err) - - if response.Count == 0 { - return []keystoreregistry.KeyValueVersion{}, - fmt.Errorf("unable to find any values for prefix: %s", prefix) - } - var values []keystoreregistry.KeyValueVersion - for _, rawKeyValue := range response.Kvs { - values = append(values, *getKeyValueVersion(rawKeyValue)) - } - return values, nil -} - -func (client *etcKeystore) Get(key string) (keystoreregistry.KeyValueVersion, error) { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Get(context.Background(), key) - handleError(err) - - value := keystoreregistry.KeyValueVersion{} - - if response.Count == 0 { - return value, fmt.Errorf("unable to find any values for key: %s", key) - } - if response.Count > 1 { - panic(errors.New("should never get more than one value for get")) - } - - return *getKeyValueVersion(response.Kvs[0]), nil -} - -func (client *etcKeystore) KeepAliveKey(key string) error { - kvc := clientv3.NewKV(client.Client) - - getResponse, err := kvc.Get(context.Background(), key) - if getResponse.Count == 1 { - // if another host seems to exist, back off for 10 seconds incase we just did a quick restart - time.Sleep(time.Second * 10) - } - - // TODO what about configure timeout and ttl? - var ttl int64 = 10 - grantResponse, err := client.Client.Grant(context.Background(), ttl) - if err != nil { - log.Fatal(err) - } - leaseID := grantResponse.ID - - txnResponse, err := kvc.Txn(context.Background()). - If(clientv3util.KeyMissing(key)). - Then(clientv3.OpPut(key, "keep-alive", clientv3.WithLease(leaseID), clientv3.WithPrevKV())). - Commit() - handleError(err) - if !txnResponse.Succeeded { - return fmt.Errorf("unable to create keep-alive key: %s", key) - } - - ch, err := client.Client.KeepAlive(context.Background(), leaseID) - if err != nil { - log.Fatal(err) - } - - counter := 9 - go func() { - for range ch { - if counter >= 9 { - counter = 0 - log.Println("Still refreshing key:", key) - } else { - counter++ - } - } - log.Panicf("Unable to refresh key: %s", key) - }() - return nil -} - -// TODO... old methods may need removing.... - -func (client *etcKeystore) CleanPrefix(prefix string) error { - kvc := clientv3.NewKV(client.Client) - response, err := kvc.Delete(context.Background(), prefix, clientv3.WithPrefix()) - handleError(err) - - if response.Deleted == 0 { - return fmt.Errorf("no keys with prefix: %s", prefix) - } - - log.Printf("Cleaned %d keys with prefix: '%s'.\n", response.Deleted, prefix) - // TODO return deleted count - return nil -} diff --git a/internal/pkg/etcdregistry/watch.go b/internal/pkg/etcdregistry/watch.go deleted file mode 100644 index 78b4b0c9..00000000 --- a/internal/pkg/etcdregistry/watch.go +++ /dev/null @@ -1,52 +0,0 @@ -package etcdregistry - -import ( - "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/keystoreregistry" - "github.com/coreos/etcd/clientv3" -) - -func (client *etcKeystore) Watch(ctxt context.Context, key string, withPrefix bool) keystoreregistry.KeyValueUpdateChan { - options := []clientv3.OpOption{clientv3.WithPrevKV()} - if withPrefix { - options = append(options, clientv3.WithPrefix()) - } - rch := client.Watcher.Watch(ctxt, key, options...) - - c := make(chan keystoreregistry.KeyValueUpdate) - - go processWatchEvents(rch, c) - - return c -} - -func processWatchEvents(watchChan clientv3.WatchChan, c chan keystoreregistry.KeyValueUpdate) { - for watchResponse := range watchChan { - // if error, send empty update with an error - err := watchResponse.Err() - if err != nil { - c <- keystoreregistry.KeyValueUpdate{Err: err} - } - - // send all events in this watch response - for _, ev := range watchResponse.Events { - update := keystoreregistry.KeyValueUpdate{ - IsCreate: ev.IsCreate(), - IsModify: ev.IsModify(), - IsDelete: ev.Type == clientv3.EventTypeDelete, - } - if update.IsCreate || update.IsModify { - update.New = getKeyValueVersion(ev.Kv) - } - if update.IsDelete || update.IsModify { - update.Old = getKeyValueVersion(ev.PrevKv) - } - - c <- update - } - } - - // Assuming we get here when the context is cancelled or hits its timeout - // i.e. there are no more events, so we close the channel - close(c) -} diff --git a/internal/pkg/etcdregistry/watch_test.go b/internal/pkg/etcdregistry/watch_test.go deleted file mode 100644 index bb96225e..00000000 --- a/internal/pkg/etcdregistry/watch_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package etcdregistry - -import ( - "context" - "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/mvcc/mvccpb" - "github.com/stretchr/testify/assert" - "testing" -) - -type fakeWatcher struct { - t *testing.T - ch clientv3.WatchChan - opts []clientv3.OpOption -} - -func (fw fakeWatcher) Watch(ctx context.Context, key string, opts ...clientv3.OpOption) clientv3.WatchChan { - assert.Equal(fw.t, "key", key) - assert.EqualValues(fw.t, len(fw.opts), len(opts)) // TODO: how to assert this properly? - return fw.ch -} -func (fw fakeWatcher) RequestProgress(ctx context.Context) error { - panic("implement me") -} -func (fakeWatcher) Close() error { - panic("implement me") -} - -func TestEtcKeystore_Watch_Nil(t *testing.T) { - keystore := etcKeystore{ - Watcher: fakeWatcher{ - t: t, ch: nil, - opts: []clientv3.OpOption{clientv3.WithPrevKV()}, - }, - } - - response := keystore.Watch(context.TODO(), "key", false) - - assert.Empty(t, response) -} - -func TestEtcKeystore_Watch(t *testing.T) { - ch := make(chan clientv3.WatchResponse) - - keystore := etcKeystore{ - Watcher: fakeWatcher{ - t: t, ch: ch, - opts: []clientv3.OpOption{clientv3.WithPrefix(), clientv3.WithPrevKV()}, - }, - } - - go func() { - ch <- clientv3.WatchResponse{ - Events: []*clientv3.Event{ - {Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{Key: []byte("key1")}}, - {Type: clientv3.EventTypePut, Kv: &mvccpb.KeyValue{Key: []byte("key2")}}, - }} - ch <- clientv3.WatchResponse{ - Events: []*clientv3.Event{ - { - Type: clientv3.EventTypePut, - Kv: &mvccpb.KeyValue{ModRevision: 1, Key: []byte("key2")}, - PrevKv: &mvccpb.KeyValue{ModRevision: 1, Key: []byte("key2")}, - }, - }} - ch <- clientv3.WatchResponse{ - Events: []*clientv3.Event{ - {Type: clientv3.EventTypeDelete, PrevKv: &mvccpb.KeyValue{Key: []byte("key2")}}, - {Type: clientv3.EventTypeDelete, PrevKv: &mvccpb.KeyValue{Key: []byte("key1")}}, - }} - ch <- clientv3.WatchResponse{Canceled: true} - close(ch) - }() - - response := keystore.Watch(context.TODO(), "key", true) - - ev1 := <-response - assert.True(t, ev1.IsCreate) - assert.False(t, ev1.IsModify) - assert.False(t, ev1.IsDelete) - assert.Nil(t, ev1.Old) - assert.EqualValues(t, "key1", ev1.New.Key) - - ev2 := <-response - assert.True(t, ev2.IsCreate) - assert.False(t, ev2.IsModify) - assert.False(t, ev2.IsDelete) - assert.Nil(t, ev2.Old) - assert.EqualValues(t, "key2", ev2.New.Key) - - ev3 := <-response - assert.False(t, ev3.IsCreate) - assert.True(t, ev3.IsModify) - assert.False(t, ev3.IsDelete) - assert.EqualValues(t, "key2", ev3.New.Key) - assert.EqualValues(t, "key2", ev3.Old.Key) - - ev4 := <-response - assert.False(t, ev4.IsCreate) - assert.False(t, ev4.IsModify) - assert.True(t, ev4.IsDelete) - assert.Nil(t, ev4.New) - assert.EqualValues(t, "key2", ev4.Old.Key) - - ev5 := <-response - assert.False(t, ev5.IsCreate) - assert.False(t, ev5.IsModify) - assert.True(t, ev5.IsDelete) - assert.Nil(t, ev5.New) - assert.EqualValues(t, "key1", ev5.Old.Key) - - ev6 := <-response - assert.Equal(t, - "etcdserver: mvcc: required revision is a future revision", - ev6.Err.Error()) - - // Check channels are closed - _, ok := <-response - assert.False(t, ok) - _, ok = <-ch - assert.False(t, ok) -} diff --git a/internal/pkg/keystoreregistry/keystore.go b/internal/pkg/keystoreregistry/keystore.go deleted file mode 100644 index c92c191a..00000000 --- a/internal/pkg/keystoreregistry/keystore.go +++ /dev/null @@ -1,99 +0,0 @@ -package keystoreregistry - -import ( - "context" - "encoding/json" - "log" -) - -type Keystore interface { - // Used to clean up any resources being used - // ... such as a connection to etcd. - Close() error - - // Removes any key starting with the given prefix. - // An error is returned if nothing was deleted, - // which some users may choose to safely ignore. - CleanPrefix(prefix string) error - - // Atomically add all the key value pairs - // - // If an error occurs no keyvalues are written. - // Error is returned if any key already exists. - Add(keyValues []KeyValue) error - - // Update the specifed key values, atomically - // - // If ModRevision is 0, it is ignored. - // Otherwise if the revisions of any key doesn't - // match the current revision of that key, the update fails. - // When update fails an error is returned and no keyValues are updated - Update(keyValues []KeyValueVersion) error - - // Delete the specifed key values, atomically - // - // Similar to update, checks ModRevision matches current key, - // ignores ModRevision if not zero. - // If any keys are not currently present, the request fails. - // Deletes no keys if an error is returned - DeleteAll(keyValues []KeyValueVersion) error - - // Get all key values for a given prefix. - GetAll(prefix string) ([]KeyValueVersion, error) - - // Get all keys for a given prefix. - Get(key string) (KeyValueVersion, error) - - // Get a channel containing all KeyValueUpdate events - // - // Use the context to control if you watch forever, or if you choose to cancel when a key - // is deleted, or you stop watching after some timeout. - Watch(ctxt context.Context, key string, withPrefix bool) KeyValueUpdateChan - - // Add a key, and remove it when calling process dies - // Error is returned if the key already exists - KeepAliveKey(key string) error - - // Get a new mutex associated with the specified key - NewMutex(lockKey string) (Mutex, error) -} - -type KeyValueUpdateChan <-chan KeyValueUpdate - -type KeyValue struct { - Key string - Value string // TODO: should this be []byte? Or have a json parsed version? -} - -type KeyValueVersion struct { - Key string - Value string - CreateRevision int64 - ModRevision int64 -} - -type KeyValueUpdate struct { - Old *KeyValueVersion - New *KeyValueVersion - IsCreate bool - IsModify bool - IsDelete bool - Err error -} - -func (kvv KeyValueVersion) String() string { - return toJson(kvv) -} - -func toJson(message interface{}) string { - b, error := json.Marshal(message) - if error != nil { - log.Fatal(error) - } - return string(b) -} - -type Mutex interface { - Lock(ctx context.Context) error - Unlock(ctx context.Context) error -} diff --git a/internal/pkg/keystoreregistry/pool.go b/internal/pkg/keystoreregistry/pool.go deleted file mode 100644 index 541b43e8..00000000 --- a/internal/pkg/keystoreregistry/pool.go +++ /dev/null @@ -1,421 +0,0 @@ -package keystoreregistry - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "math/rand" - "strings" - "time" -) - -func NewPoolRegistry(keystore Keystore) registry.PoolRegistry { - return &poolRegistry{keystore} -} - -type poolRegistry struct { - keystore Keystore -} - -const registeredBricksPrefix = "/bricks/registered/" - -func getBrickInfoKey(hostname string, device string) string { - return fmt.Sprintf("%s%s/%s/", registeredBricksPrefix, hostname, device) -} - -const allocatedBricksPrefix = "/bricks/allocated/host/" - -func getPrefixAllocationHost(hostname string) string { - return fmt.Sprintf("%s%s/", allocatedBricksPrefix, hostname) -} - -func getBrickAllocationKeyHost(allocation registry.BrickAllocation) string { - prefix := getPrefixAllocationHost(allocation.Hostname) - return fmt.Sprintf("%s%s", prefix, allocation.Device) -} - -func getPrefixAllocationVolume(volume registry.VolumeName) string { - return fmt.Sprintf("/bricks/allocated/volume/%s/", volume) -} -func getBrickAllocationKeyVolume(allocation registry.BrickAllocation) string { - prefix := getPrefixAllocationVolume(allocation.AllocatedVolume) - return fmt.Sprintf("%s%d/%s/%s", - prefix, allocation.AllocatedIndex, allocation.Hostname, allocation.Device) -} - -func (poolRegistry *poolRegistry) UpdateHost(bricks []registry.BrickInfo) error { - var values []KeyValueVersion - var problems []string - var hostname string - for _, brickInfo := range bricks { - if hostname == "" { - hostname = brickInfo.Hostname - } - if hostname != brickInfo.Hostname { - problems = append(problems, "Only one host to be updated at once") - } - // TODO: lots more error handing needed, like pool consistency, valid keys for etcd - values = append(values, KeyValueVersion{ - Key: getBrickInfoKey(brickInfo.Hostname, brickInfo.Device), - Value: toJson(brickInfo), - }) - } - if len(problems) > 0 { - return fmt.Errorf("can't update host because: %s", strings.Join(problems, ", ")) - } - return poolRegistry.keystore.Update(values) -} - -func getKeepAliveKey(hostname string) string { - return fmt.Sprintf("/host/keepalive/%s", hostname) -} - -func (poolRegistry *poolRegistry) KeepAliveHost(hostname string) error { - return poolRegistry.keystore.KeepAliveKey(getKeepAliveKey(hostname)) -} - -func (poolRegistry *poolRegistry) HostAlive(hostname string) (bool, error) { - keyValue, err := poolRegistry.keystore.Get(getKeepAliveKey(hostname)) - return keyValue.Key != "", err -} - -func (poolRegistry *poolRegistry) AllocateBricksForVolume(volume registry.Volume) ([]registry.BrickAllocation, error) { - // No bricks requested, so return right away - if volume.SizeBricks == 0 { - return nil, nil - } - - // TODO: would retries on clashes be better? this seems simpler for now - // lock the pool to stop races - mutex, err := poolRegistry.keystore.NewMutex(fmt.Sprintf("allocation/%s", volume.Pool)) - if err != nil { - return nil, err - } - if err := mutex.Lock(context.TODO()); err != nil { - return nil, err - } - defer mutex.Unlock(context.TODO()) - - pools, err := poolRegistry.Pools() - if err != nil { - return nil, err - } - - var pool *registry.Pool - for _, candidate := range pools { - if candidate.Name == volume.Pool { - pool = &candidate - } - } - - if pool == nil { - return nil, fmt.Errorf("unable to find pool %s", volume.Pool) - } - - allocations, err := getBricksForBuffer(pool, &volume) - if err != nil { - return nil, err - } - - // Note that this call triggers dacd of the first allocation to provision the bricks - // TODO: probably better to make the provision a spearate state change? - if err := poolRegistry.allocateBricks(allocations); err != nil { - return nil, err - } - - return allocations, nil -} - -func getBricksForBuffer(pool *registry.Pool, volume *registry.Volume) ([]registry.BrickAllocation, error) { - - availableBricks := pool.AvailableBricks - var chosenBricks []registry.BrickInfo - - // pick some of the available bricks - s := rand.NewSource(time.Now().Unix()) - r := rand.New(s) // initialize local pseudorandom generator - - randomWalk := r.Perm(len(availableBricks)) - for _, i := range randomWalk { - candidateBrick := availableBricks[i] - - // TODO: should not the random walk mean this isn't needed! - goodCandidate := true - for _, brick := range chosenBricks { - if brick == candidateBrick { - goodCandidate = false - break - } - } - if goodCandidate { - chosenBricks = append(chosenBricks, candidateBrick) - } - if uint(len(chosenBricks)) >= volume.SizeBricks { - break - } - } - - if uint(len(chosenBricks)) != volume.SizeBricks { - return nil, fmt.Errorf( - "unable to get number of requested bricks (%d) for given pool (%s)", - volume.SizeBricks, pool.Name) - } - - var allocations []registry.BrickAllocation - for _, brick := range chosenBricks { - allocations = append(allocations, registry.BrickAllocation{ - Device: brick.Device, - Hostname: brick.Hostname, - AllocatedVolume: volume.Name, - DeallocateRequested: false, - }) - } - return allocations, nil -} - -func (poolRegistry *poolRegistry) allocateBricks(allocations []registry.BrickAllocation) error { - var bricks []registry.BrickInfo - var volume registry.VolumeName - var raw []KeyValue - for i, allocation := range allocations { - brick, err := poolRegistry.GetBrickInfo(allocation.Hostname, allocation.Device) - if err != nil { - return fmt.Errorf("unable to find brick for: %+v", allocation) - } - bricks = append(bricks, brick) - - if allocation.DeallocateRequested { - return fmt.Errorf("should not requeste deallocated: %+v", allocation) - } - if allocation.AllocatedIndex != 0 { - return fmt.Errorf("should not specify the allocated index") - } - if volume == "" { - volume = allocation.AllocatedVolume - } - if volume != allocation.AllocatedVolume { - return fmt.Errorf("all allocations must be for same volume") - } - // TODO: this error checking suggest we specify the wrong format here! - - allocation.AllocatedIndex = uint(i) - raw = append(raw, KeyValue{ - Key: getBrickAllocationKeyHost(allocation), - Value: toJson(allocation), - }) - // TODO: maybe just point to the other key?? this duplication is terrible - raw = append(raw, KeyValue{ - Key: getBrickAllocationKeyVolume(allocation), - Value: toJson(allocation), - }) - } - return poolRegistry.keystore.Add(raw) -} - -func (poolRegistry *poolRegistry) deallocate(raw []KeyValueVersion, - updated []KeyValueVersion) ([]KeyValueVersion, []string) { - var keys []string - for _, entry := range raw { - rawValue := entry.Value - var allocation registry.BrickAllocation - json.Unmarshal(bytes.NewBufferString(rawValue).Bytes(), &allocation) - - allocation.DeallocateRequested = true - entry.Value = toJson(&allocation) - updated = append(updated, entry) - keys = append(keys, getBrickAllocationKeyHost(allocation)) - } - return updated, keys -} - -func (poolRegistry *poolRegistry) DeallocateBricks(volume registry.VolumeName) error { - var updated []KeyValueVersion - - volPrefix := getPrefixAllocationVolume(volume) - raw, err := poolRegistry.keystore.GetAll(volPrefix) - if err != nil { - return nil - } - updated, keys := poolRegistry.deallocate(raw, updated) - - raw = []KeyValueVersion{} - for _, key := range keys { - entry, err := poolRegistry.keystore.Get(key) - if err != nil { - return err - } - raw = append(raw, entry) - } - updated, _ = poolRegistry.deallocate(raw, updated) - return poolRegistry.keystore.Update(updated) -} - -func (poolRegistry *poolRegistry) HardDeleteAllocations(allocations []registry.BrickAllocation) error { - var keys []string - for _, allocation := range allocations { - keys = append(keys, getBrickAllocationKeyHost(allocation)) - keys = append(keys, getBrickAllocationKeyVolume(allocation)) - if !allocation.DeallocateRequested { - return fmt.Errorf("must first call deallocate on: %+v", allocation) - } - } - - var keyValues []KeyValueVersion - for _, key := range keys { - keyValue, err := poolRegistry.keystore.Get(key) - if err != nil { - return err - } - // TODO check we have already called deallocate properly - keyValues = append(keyValues, keyValue) - } - return poolRegistry.keystore.DeleteAll(keyValues) -} - -func (poolRegistry *poolRegistry) getAllocations(prefix string) ([]registry.BrickAllocation, error) { - raw, err := poolRegistry.keystore.GetAll(prefix) - if err != nil { - return nil, err - } - var allocations []registry.BrickAllocation - for _, entry := range raw { - rawValue := entry.Value - var allocation registry.BrickAllocation - json.Unmarshal(bytes.NewBufferString(rawValue).Bytes(), &allocation) - allocations = append(allocations, allocation) - } - return allocations, nil -} - -func (poolRegistry *poolRegistry) GetAllocationsForHost(hostname string) ([]registry.BrickAllocation, error) { - return poolRegistry.getAllocations(getPrefixAllocationHost(hostname)) -} - -func (poolRegistry *poolRegistry) GetAllocationsForVolume(volume registry.VolumeName) ([]registry.BrickAllocation, error) { - return poolRegistry.getAllocations(getPrefixAllocationVolume(volume)) -} - -func (poolRegistry *poolRegistry) GetBrickInfo(hostname string, device string) (registry.BrickInfo, error) { - raw, error := poolRegistry.keystore.Get(getBrickInfoKey(hostname, device)) - var value registry.BrickInfo - json.Unmarshal(bytes.NewBufferString(raw.Value).Bytes(), &value) - return value, error -} - -func (poolRegistry *poolRegistry) GetNewHostBrickAllocations( - ctxt context.Context, hostname string) <-chan registry.BrickAllocation { - - events := make(chan registry.BrickAllocation) - - key := getPrefixAllocationHost(hostname) - rawEvents := poolRegistry.keystore.Watch(ctxt, key, true) - - go func() { - defer close(events) - if rawEvents == nil { - return - } - for raw := range rawEvents { - if raw.Err != nil { - // consider sending error back to the listener? For now force process restart - log.Panicf("Error when watching %s for new brick hosts: %+v", hostname, raw) - } - if raw.IsCreate { - newBrick := registry.BrickAllocation{} - err := json.Unmarshal(bytes.NewBufferString(raw.New.Value).Bytes(), &newBrick) - if err != nil { - log.Panicf("error parsing create brick host %s event %+v %s", hostname, raw, err) - } else { - events <- newBrick - } - } - } - // we get here if the context is canceled, etc - }() - - return events -} - -func (poolRegistry *poolRegistry) getBricks(prefix string) ([]registry.BrickInfo, error) { - raw, err := poolRegistry.keystore.GetAll(prefix) - if err != nil { - return nil, err - } - var allocations []registry.BrickInfo - for _, entry := range raw { - rawValue := entry.Value - var allocation registry.BrickInfo - json.Unmarshal(bytes.NewBufferString(rawValue).Bytes(), &allocation) - allocations = append(allocations, allocation) - } - return allocations, nil -} - -func (poolRegistry *poolRegistry) Pools() ([]registry.Pool, error) { - allBricks, _ := poolRegistry.getBricks(registeredBricksPrefix) - allAllocations, _ := poolRegistry.getAllocations(allocatedBricksPrefix) - - allocationLookup := make(map[string]registry.BrickAllocation) - for _, allocation := range allAllocations { - key := fmt.Sprintf("%s/%s", allocation.Hostname, allocation.Device) - allocationLookup[key] = allocation - } - - pools := make(map[string]*registry.Pool) - hosts := make(map[string]*registry.HostInfo) - for _, brick := range allBricks { - pool, ok := pools[brick.PoolName] - if !ok { - pool = ®istry.Pool{ - Name: brick.PoolName, - GranularityGB: brick.CapacityGB, - AllocatedBricks: []registry.BrickAllocation{}, - AvailableBricks: []registry.BrickInfo{}, - Hosts: make(map[string]registry.HostInfo), - } - } - - if brick.CapacityGB != pool.GranularityGB { - log.Printf("brick doesn't match pool granularity: %+v\n", brick) - if brick.CapacityGB < pool.GranularityGB { - pool.GranularityGB = brick.CapacityGB - } - } - - host, ok := hosts[brick.Hostname] - if !ok { - hostAlive, _ := poolRegistry.HostAlive(brick.Hostname) - host = ®istry.HostInfo{ - Hostname: brick.Hostname, - Alive: hostAlive, - } - } - - if _, ok := pool.Hosts[brick.Hostname]; !ok { - pool.Hosts[brick.Hostname] = *host - } - - key := fmt.Sprintf("%s/%s", brick.Hostname, brick.Device) - allocation, ok := allocationLookup[key] - if ok { - pool.AllocatedBricks = append(pool.AllocatedBricks, allocation) - hosts[brick.Hostname] = host - pools[brick.PoolName] = pool - } else { - if host.Alive { - pool.AvailableBricks = append(pool.AvailableBricks, brick) - hosts[brick.Hostname] = host - pools[brick.PoolName] = pool - } - } - } - - var poolList []registry.Pool - for _, value := range pools { - poolList = append(poolList, *value) - } - return poolList, nil -} diff --git a/internal/pkg/keystoreregistry/pool_test.go b/internal/pkg/keystoreregistry/pool_test.go deleted file mode 100644 index 7a71de58..00000000 --- a/internal/pkg/keystoreregistry/pool_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package keystoreregistry - -import ( - "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -type fakeKeystore struct { - watchChan KeyValueUpdateChan - t *testing.T - key string - withPrefix bool -} - -func (fakeKeystore) Close() error { - panic("implement me") -} -func (fakeKeystore) CleanPrefix(prefix string) error { - panic("implement me") -} -func (fakeKeystore) Add(keyValues []KeyValue) error { - panic("implement me") -} -func (fakeKeystore) Update(keyValues []KeyValueVersion) error { - panic("implement me") -} -func (fakeKeystore) DeleteAll(keyValues []KeyValueVersion) error { - panic("implement me") -} -func (fakeKeystore) GetAll(prefix string) ([]KeyValueVersion, error) { - panic("implement me") -} -func (fakeKeystore) Get(key string) (KeyValueVersion, error) { - panic("implement me") -} -func (fakeKeystore) WatchPrefix(prefix string, onUpdate func(old *KeyValueVersion, new *KeyValueVersion)) { - panic("implement me") -} -func (fakeKeystore) WatchKey(ctxt context.Context, key string, onUpdate func(old *KeyValueVersion, new *KeyValueVersion)) { - panic("implement me") -} -func (fk fakeKeystore) Watch(ctxt context.Context, key string, withPrefix bool) KeyValueUpdateChan { - assert.Equal(fk.t, fk.key, key) - assert.Equal(fk.t, fk.withPrefix, withPrefix) - return fk.watchChan -} -func (fakeKeystore) KeepAliveKey(key string) error { - panic("implement me") -} -func (fakeKeystore) NewMutex(lockKey string) (Mutex, error) { - panic("implement me") -} - -func TestPoolRegistry_GetNewHostBrickAllocations(t *testing.T) { - rawEvents := make(chan KeyValueUpdate) - reg := poolRegistry{keystore: &fakeKeystore{ - watchChan: rawEvents, t: t, key: "/bricks/allocated/host/host1/", withPrefix: true, - }} - - events := reg.GetNewHostBrickAllocations(context.TODO(), "host1") - - go func() { - rawEvents <- KeyValueUpdate{IsCreate: false} - rawEvents <- KeyValueUpdate{ - IsCreate: true, - New: &KeyValueVersion{Value: toJson(registry.BrickAllocation{ - Hostname: "host1", Device: "sdb", - })}, - } - rawEvents <- KeyValueUpdate{IsCreate: false} - close(rawEvents) - }() - - ev1 := <-events - assert.Equal(t, "host1", ev1.Hostname) - assert.Equal(t, "sdb", ev1.Device) - - _, ok := <-events - assert.False(t, ok) - _, ok = <-rawEvents - assert.False(t, ok) -} - -func TestPoolRegistry_GetNewHostBrickAllocations_nil(t *testing.T) { - reg := poolRegistry{keystore: &fakeKeystore{ - watchChan: nil, t: t, key: "/bricks/allocated/host/host2/", withPrefix: true, - }} - - events := reg.GetNewHostBrickAllocations(context.TODO(), "host2") - - _, ok := <-events - assert.False(t, ok) -} diff --git a/internal/pkg/keystoreregistry/volume.go b/internal/pkg/keystoreregistry/volume.go deleted file mode 100644 index 82ca4177..00000000 --- a/internal/pkg/keystoreregistry/volume.go +++ /dev/null @@ -1,405 +0,0 @@ -package keystoreregistry - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "math/rand" - "time" -) - -func NewVolumeRegistry(keystore Keystore) registry.VolumeRegistry { - return &volumeRegistry{keystore} -} - -type volumeRegistry struct { - keystore Keystore -} - -func (volRegistry *volumeRegistry) AllVolumes() ([]registry.Volume, error) { - var volumes []registry.Volume - keyValues, err := volRegistry.keystore.GetAll(volumeKeyPrefix) - if err != nil { - return volumes, err - } - for _, keyValue := range keyValues { - var volume registry.Volume - err = volumeFromKeyValue(keyValue, &volume) - if err != nil { - return volumes, nil - } - volumes = append(volumes, volume) - } - return volumes, nil -} - -func (volRegistry *volumeRegistry) Jobs() ([]registry.Job, error) { - var jobs []registry.Job - keyValues, err := volRegistry.keystore.GetAll(jobPrefix) - for _, keyValue := range keyValues { - var job registry.Job - err := json.Unmarshal(bytes.NewBufferString(keyValue.Value).Bytes(), &job) - if err != nil { - return jobs, err - } - jobs = append(jobs, job) - } - return jobs, err -} - -const jobPrefix = "/job/" - -func getJobKey(jobName string) string { - return fmt.Sprintf("%s%s/", jobPrefix, jobName) -} - -func (volRegistry *volumeRegistry) Job(jobName string) (registry.Job, error) { - var job registry.Job // TODO return a pointer instead? - keyValue, err := volRegistry.keystore.Get(getJobKey(jobName)) - if err != nil { - return job, err - } - err = json.Unmarshal(bytes.NewBufferString(keyValue.Value).Bytes(), &job) - if err != nil { - return job, err - } - return job, nil -} - -func (volRegistry *volumeRegistry) AddJob(job registry.Job) error { - for _, volumeName := range job.MultiJobVolumes { - volume, err := volRegistry.Volume(volumeName) - if err != nil { - return err - } - // TODO: what other checks are required? - if volume.State < registry.Registered { - return fmt.Errorf("must register volume: %s", volume.Name) - } - } - if job.JobVolume != "" { - volume, err := volRegistry.Volume(job.JobVolume) - if err != nil { - return err - } - // TODO: what other checks are required? - if volume.State < registry.Registered { - return fmt.Errorf("must register volume: %s", volume.Name) - } - } - return volRegistry.keystore.Add([]KeyValue{ - {Key: getJobKey(job.Name), Value: toJson(job)}, - }) -} - -func (volRegistry *volumeRegistry) DeleteJob(jobName string) error { - keyValue, err := volRegistry.keystore.Get(getJobKey(jobName)) - if err != nil { - return err - } - return volRegistry.keystore.DeleteAll([]KeyValueVersion{keyValue}) -} - -func (volRegistry *volumeRegistry) JobAttachHosts(jobName string, hosts []string) error { - keyValue, err := volRegistry.keystore.Get(getJobKey(jobName)) - if err != nil { - return err - } - var job registry.Job - err = json.Unmarshal(bytes.NewBufferString(keyValue.Value).Bytes(), &job) - if err != nil { - return err - } - - // TODO validate hostnames? - job.AttachHosts = hosts - keyValue.Value = toJson(job) - - return volRegistry.keystore.Update([]KeyValueVersion{keyValue}) -} - -func findAttachment(attachments []registry.Attachment, - hostname string, jobName string) (*registry.Attachment, bool) { - for _, candidate := range attachments { - if candidate.Hostname == hostname && candidate.Job == jobName { - // TODO: double check for duplicate match? - return &candidate, true - } - } - return nil, false -} - -func mergeAttachments(oldAttachments []registry.Attachment, updates []registry.Attachment) []registry.Attachment { - var newAttachments []registry.Attachment - for _, update := range updates { - newAttachments = append(newAttachments, update) - } - - // add any existing attachments that don't match an update - for _, oldAttachment := range oldAttachments { - _, ok := findAttachment( - updates, oldAttachment.Hostname, oldAttachment.Job) - if !ok { - newAttachments = append(newAttachments, oldAttachment) - } - } - return newAttachments -} - -func (volRegistry *volumeRegistry) UpdateVolumeAttachments(name registry.VolumeName, - updates []registry.Attachment) error { - update := func(volume *registry.Volume) error { - volume.Attachments = mergeAttachments(volume.Attachments, updates) - return nil - } - return volRegistry.updateVolume(name, update) -} - -func (volRegistry *volumeRegistry) DeleteVolumeAttachments(name registry.VolumeName, hostnames []string, jobName string) error { - - update := func(volume *registry.Volume) error { - if volume.Attachments == nil { - return errors.New("no attachments to delete") - } else { - numberRemoved := removeAttachments(volume, jobName, hostnames) - if numberRemoved != len(hostnames) { - return fmt.Errorf("unable to find all attachments for volume %s", name) - } - } - return nil - } - return volRegistry.updateVolume(name, update) -} - -func removeAttachments(volume *registry.Volume, jobName string, hostnames []string) int { - var newAttachments []registry.Attachment - for _, attachment := range volume.Attachments { - remove := false - if attachment.Job == jobName { - for _, host := range hostnames { - if attachment.Hostname == host { - remove = true - break - } - } - } - if !remove { - newAttachments = append(newAttachments, attachment) - } - } - numberRemoved := len(volume.Attachments) - len(newAttachments) - volume.Attachments = newAttachments - return numberRemoved -} - -func (volRegistry *volumeRegistry) updateVolume(name registry.VolumeName, - update func(volume *registry.Volume) error) error { - - // TODO: if we restructure attachments into separate keys, we can probably ditch this mutex - mutex, err := volRegistry.keystore.NewMutex(getVolumeKey(string(name))) - if err != nil { - return err - } - if err := mutex.Lock(context.TODO()); err != nil { - return err - } - defer mutex.Unlock(context.TODO()) - - keyValue, err := volRegistry.keystore.Get(getVolumeKey(string(name))) - if err != nil { - return err - } - - volume := registry.Volume{} - err = volumeFromKeyValue(keyValue, &volume) - if err != nil { - return nil - } - if err := update(&volume); err != nil { - return err - } - - keyValue.Value = toJson(volume) - return volRegistry.keystore.Update([]KeyValueVersion{keyValue}) -} - -func (volRegistry *volumeRegistry) VolumeOperationMutex(name registry.VolumeName) (registry.Mutex, error) { - return volRegistry.keystore.NewMutex(fmt.Sprintf("operation_%s", name)) -} - -func (volRegistry *volumeRegistry) UpdateState(name registry.VolumeName, state registry.VolumeState) error { - updateState := func(volume *registry.Volume) error { - stateDifference := state - volume.State - if stateDifference != 1 && state != registry.Error && state != registry.DeleteRequested { - return fmt.Errorf("must update volume %s to the next state, current state: %s", - volume.Name, volume.State) - } - volume.State = state - if state == registry.BricksAllocated { - // From this point onwards, we know bricks might need to be cleaned up - volume.HadBricksAssigned = true - } - return nil - } - return volRegistry.updateVolume(name, updateState) -} - -const volumeKeyPrefix = "/volume/" - -func getVolumeKey(volumeName string) string { - return fmt.Sprintf("%s%s/", volumeKeyPrefix, volumeName) -} - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - -func GetNewUUID() string { - b := make([]byte, 8) - for i := range b { - b[i] = letters[rand.Int63()%int64(len(letters))] - } - return string(b) -} - -func (volRegistry *volumeRegistry) AddVolume(volume registry.Volume) error { - // TODO: both uuid and client port might clash, need to check they don't!! - volume.UUID = GetNewUUID() - volume.ClientPort = rand.Intn(50000) + 10000 - return volRegistry.keystore.Add([]KeyValue{{ - Key: getVolumeKey(string(volume.Name)), - Value: toJson(volume), - }}) -} - -func volumeFromKeyValue(keyValue KeyValueVersion, volume *registry.Volume) error { - return json.Unmarshal(bytes.NewBufferString(keyValue.Value).Bytes(), &volume) -} - -func (volRegistry *volumeRegistry) Volume(name registry.VolumeName) (registry.Volume, error) { - volume := registry.Volume{} - keyValue, err := volRegistry.keystore.Get(getVolumeKey(string(name))) - if err != nil { - return volume, err - } - err = volumeFromKeyValue(keyValue, &volume) - if err != nil { - return volume, nil - } - return volume, nil -} - -func (volRegistry *volumeRegistry) DeleteVolume(name registry.VolumeName) error { - keyValue, err := volRegistry.keystore.Get(getVolumeKey(string(name))) - if err != nil { - return err - } - return volRegistry.keystore.DeleteAll([]KeyValueVersion{keyValue}) -} - -func (volRegistry *volumeRegistry) GetVolumeChanges(ctx context.Context, volume registry.Volume) registry.VolumeChangeChan { - // TODO: we should watch from the version of the passed in volume - key := getVolumeKey(string(volume.Name)) - rawEvents := volRegistry.keystore.Watch(ctx, key, false) - - events := make(chan registry.VolumeChange) - - go func() { - defer close(events) - if rawEvents == nil { - return - } - for rawEvent := range rawEvents { - if rawEvent.Err != nil { - events <- registry.VolumeChange{Err: rawEvent.Err} - continue - } - - event := registry.VolumeChange{ - IsDelete: rawEvent.IsDelete, - Old: nil, - New: nil, - } - if rawEvent.Old != nil { - oldVolume := ®istry.Volume{} - if err := volumeFromKeyValue(*rawEvent.Old, oldVolume); err != nil { - event.Err = err - } else { - event.Old = oldVolume - } - } - if rawEvent.New != nil { - newVolume := ®istry.Volume{} - if err := volumeFromKeyValue(*rawEvent.New, newVolume); err != nil { - event.Err = err - } else { - event.New = newVolume - } - } - events <- event - } - }() - - return events -} - -func (volRegistry *volumeRegistry) WaitForState(volumeName registry.VolumeName, state registry.VolumeState) error { - log.Println("Start waiting for volume", volumeName, "to reach state", state) - err := volRegistry.WaitForCondition(volumeName, func(event *registry.VolumeChange) bool { - if event.New == nil { - log.Panicf("unable to process event %+v", event) - } - return event.New.State == state || event.New.State == registry.Error - }) - log.Println("Stopped waiting for volume", volumeName, "to reach state", state, err) - if err != nil { - return err - } - - // return error if we went to an error state - volume, err := volRegistry.Volume(volumeName) - if err == nil && volume.State == registry.Error { - return fmt.Errorf("stopped waiting as volume %s in error state", volumeName) - } - return err -} - -// TODO: maybe have environment variable to tune this wait time? -var defaultTimeout = time.Minute * 10 - -func (volRegistry *volumeRegistry) WaitForCondition(volumeName registry.VolumeName, - condition func(event *registry.VolumeChange) bool) error { - - volume, err := volRegistry.Volume(volumeName) - if err != nil { - return err - } - - ctxt, cancelFunc := context.WithTimeout(context.Background(), defaultTimeout) - events := volRegistry.GetVolumeChanges(ctxt, volume) - defer cancelFunc() - - log.Printf("About to wait for condition on volume: %+v", volume) - - for event := range events { - if event.Err != nil { - return event.Err - } - if event.IsDelete { - return fmt.Errorf("stopped waiting as volume %s is deleted", volume.Name) - } - - conditionMet := condition(&event) - if conditionMet { - return nil - } - } - - return fmt.Errorf("stopped waiting for volume %s to meet supplied condition", volume.Name) -} diff --git a/internal/pkg/keystoreregistry/volume_test.go b/internal/pkg/keystoreregistry/volume_test.go deleted file mode 100644 index 3af42647..00000000 --- a/internal/pkg/keystoreregistry/volume_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package keystoreregistry - -import ( - "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestVolumeRegistry_DeleteVolumeAttachments(t *testing.T) { - vol := registry.Volume{ - Attachments: []registry.Attachment{ - {Hostname: "host1", Job: "job1"}, - {Hostname: "host2", Job: "job1"}, - {Hostname: "host2", Job: "job2"}, - }, - } - numRemoved := removeAttachments(&vol, "job1", []string{"host1", "host2"}) - assert.Equal(t, 2, numRemoved) - assert.Equal(t, 1, len(vol.Attachments)) -} - -func TestVolumeRegistry_FindAttachment(t *testing.T) { - attachments := []registry.Attachment{ - {Job: "job1", Hostname: "foo1"}, {Job: "job2", Hostname: "foo1"}, {Job: "job2", Hostname: "foo2"}, - } - - attachment, ok := findAttachment(nil, "", "") - assert.Nil(t, attachment) - assert.False(t, ok) - - attachment, ok = findAttachment(attachments, "foo2", "job1") - assert.Nil(t, attachment) - assert.False(t, ok) - - attachment, ok = findAttachment(attachments, "foo1", "job1") - assert.True(t, ok) - assert.Equal(t, registry.Attachment{Job: "job1", Hostname: "foo1"}, *attachment) - - attachment, ok = findAttachment(attachments, "foo1", "job2") - assert.True(t, ok) - assert.Equal(t, registry.Attachment{Job: "job2", Hostname: "foo1"}, *attachment) -} - -func TestVolumeRegistry_MergeAttachments(t *testing.T) { - oldAttachments := []registry.Attachment{ - {Job: "job1", Hostname: "foo1"}, {Job: "job2", Hostname: "foo1"}, {Job: "job2", Hostname: "foo2"}, - } - - assert.Nil(t, mergeAttachments(nil, nil)) - assert.Equal(t, oldAttachments, mergeAttachments(oldAttachments, nil)) - assert.Equal(t, oldAttachments, mergeAttachments(nil, oldAttachments)) - - // add new - result := mergeAttachments(oldAttachments, []registry.Attachment{{Job: "foo", Hostname: "foo"}}) - assert.Equal(t, 4, len(result)) - assert.Equal(t, registry.Attachment{Job: "foo", Hostname: "foo"}, result[0]) - assert.Equal(t, oldAttachments[0], result[1]) - assert.Equal(t, oldAttachments[1], result[2]) - assert.Equal(t, oldAttachments[2], result[3]) - - // in place update - updates := []registry.Attachment{ - {Job: "job2", Hostname: "foo1", State: registry.Attached}, - {Job: "job2", Hostname: "foo2", State: registry.Attached}, - } - result = mergeAttachments(oldAttachments, updates) - assert.Equal(t, 3, len(result)) - assert.Equal(t, updates[0], result[0]) - assert.Equal(t, updates[1], result[1]) - assert.Equal(t, oldAttachments[0], result[2]) -} - -func TestVolumeRegistry_GetVolumeChanges_nil(t *testing.T) { - volReg := volumeRegistry{keystore: fakeKeystore{watchChan: nil, t: t, key: "/volume/vol1/"}} - - changes := volReg.GetVolumeChanges(context.TODO(), registry.Volume{Name: "vol1"}) - - _, ok := <-changes - assert.False(t, ok) -} - -func TestVolumeRegistry_GetVolumeChanges(t *testing.T) { - raw := make(chan KeyValueUpdate) - volReg := volumeRegistry{keystore: fakeKeystore{ - watchChan: raw, t: t, key: "/volume/vol1/", withPrefix: false, - }} - - changes := volReg.GetVolumeChanges(context.TODO(), registry.Volume{Name: "vol1"}) - - vol := ®istry.Volume{Name: "test1"} - - go func() { - raw <- KeyValueUpdate{ - New: &KeyValueVersion{Key: "asdf", Value: toJson(vol)}, - Old: &KeyValueVersion{Key: "asdf", Value: toJson(vol)}, - } - raw <- KeyValueUpdate{ - IsDelete: true, - New: nil, - Old: &KeyValueVersion{Key: "asdf", Value: toJson(vol)}, - } - raw <- KeyValueUpdate{ - New: &KeyValueVersion{Key: "asdf", Value: "asdf"}, - Old: nil, - } - close(raw) - }() - - ch1 := <-changes - assert.Nil(t, ch1.Err) - assert.False(t, ch1.IsDelete) - assert.Equal(t, vol, ch1.Old) - assert.Equal(t, vol, ch1.New) - - ch2 := <-changes - assert.Nil(t, ch2.Err) - assert.True(t, ch2.IsDelete) - assert.Nil(t, ch2.New) - assert.Equal(t, vol, ch1.Old) - - ch3 := <-changes - assert.Equal(t, "invalid character 'a' looking for beginning of value", ch3.Err.Error()) - assert.False(t, ch3.IsDelete) - assert.Nil(t, ch3.Old) - assert.Nil(t, ch3.New) - - _, ok := <-changes - assert.False(t, ok) - _, ok = <-raw - assert.False(t, ok) -} diff --git a/internal/pkg/lifecycle/brickmanager/bricks.go b/internal/pkg/lifecycle/brickmanager/bricks.go deleted file mode 100644 index 18a41c7f..00000000 --- a/internal/pkg/lifecycle/brickmanager/bricks.go +++ /dev/null @@ -1,313 +0,0 @@ -package brickmanager - -import ( - "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/pfsprovider/ansible" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "strings" -) - -func setupBrickEventHandlers(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, - hostname string) { - - newBricks := poolRegistry.GetNewHostBrickAllocations(context.Background(), hostname) - go func() { - for brick := range newBricks { - if brick.AllocatedIndex == 0 { - log.Printf("found new primary brick %+v", brick) - go processNewPrimaryBlock(poolRegistry, volumeRegistry, &brick) - } else { - log.Printf("ignore brick create, as it is not a primary brick %+v", brick) - } - } - log.Panic("we appear to have stopped watching for new bricks") - }() - - allocations, err := poolRegistry.GetAllocationsForHost(hostname) - if err != nil { - if !strings.Contains(err.Error(), "unable to find any values") { - log.Panic(err) - } - } - - for _, allocation := range allocations { - volume, err := volumeRegistry.Volume(allocation.AllocatedVolume) - if err != nil { - log.Panicf("unable to find volume for allocation %+v", allocation) - } - if allocation.AllocatedIndex == 0 { - log.Printf("Start watching again, as we host a primary brick for: %+v", volume) - // TODO: do we finish watching correctly? - watchForVolumeChanges(poolRegistry, volumeRegistry, volume) - - // TODO: trigger events if we missed the "edge" already - if volume.State == registry.DeleteRequested { - log.Println("Complete pending delete request for volume:", volume.Name) - processDelete(poolRegistry, volumeRegistry, volume) - } - } - } - - // TODO what about catching up with changes while we were down, make sure system in correct state!! -} - -func processNewPrimaryBlock(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, - new *registry.BrickAllocation) { - volume, err := volumeRegistry.Volume(new.AllocatedVolume) - if err != nil { - log.Printf("Could not file volume: %s because: %s\n", new.AllocatedVolume, err) - return - } - log.Println("Found new volume to watch:", volume.Name) - log.Println(volume) - - watchForVolumeChanges(poolRegistry, volumeRegistry, volume) - - // Move to new state, ignored by above watch - provisionNewVolume(poolRegistry, volumeRegistry, volume) -} - -func watchForVolumeChanges(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, - volume registry.Volume) { - - ctxt, cancelFunc := context.WithCancel(context.Background()) - changes := volumeRegistry.GetVolumeChanges(ctxt, volume) - - go func() { - defer cancelFunc() - - for change := range changes { - old := change.Old - new := change.New - - if change.IsDelete { - log.Printf("Stop watching volume: %s", volume.Name) - return - } - - if old == nil || new == nil { - log.Printf("nil volume seen, unable to process volume event: %+v", change) - } - - if change.Err != nil { - log.Printf("Error while waiting for volume %s saw error: %s with: %+v", - volume.Name, change.Err.Error(), change) - } - - if new.State != old.State { - log.Printf("volume:%s state move: %s -> %s", new.Name, old.State, new.State) - switch new.State { - case registry.DataInRequested: - processDataIn(volumeRegistry, *new) - case registry.DataOutRequested: - processDataOut(volumeRegistry, *new) - case registry.DeleteRequested: - processDelete(poolRegistry, volumeRegistry, *new) - case registry.BricksDeleted: - log.Println("Volume", new.Name, "has had bricks deleted.") - default: - // Ignore the state changes we triggered - log.Printf("ignore volume state move %+v", change) - } - } - - if len(new.Attachments) > len(old.Attachments) { - var attachRequested []registry.Attachment - for _, newAttachment := range new.Attachments { - isNew := false - if old.Attachments == nil { - isNew = true - } else { - _, ok := old.FindMatchingAttachment(newAttachment) - isNew = !ok - } - if isNew && newAttachment.State == registry.RequestAttach { - attachRequested = append(attachRequested, newAttachment) - } - } - if attachRequested != nil && len(attachRequested) > 0 { - processAttach(poolRegistry, volumeRegistry, *new, attachRequested) - } - } - - if len(new.Attachments) == len(old.Attachments) && new.Attachments != nil && old.Attachments != nil { - var detachRequested []registry.Attachment - for _, newAttachment := range new.Attachments { - oldAttachment, ok := old.FindMatchingAttachment(newAttachment) - if ok && newAttachment.State == registry.RequestDetach && oldAttachment.State == registry.Attached { - detachRequested = append(detachRequested, newAttachment) - } - } - if len(detachRequested) > 0 { - processDetach(poolRegistry, volumeRegistry, *new, detachRequested) - } - } - } - }() -} - -func handleError(volumeRegistry registry.VolumeRegistry, volume registry.Volume, err error) { - if err != nil { - log.Println("Error provisioning", volume.Name, err) - err = volumeRegistry.UpdateState(volume.Name, registry.Error) // TODO record an error string? - if err != nil { - log.Println("Unable to move volume", volume.Name, "to Error state") - } - } -} - -// TODO: should not be hardcoded here -var FSType = ansible.Lustre -var plugin = ansible.GetPlugin(FSType) - -func provisionNewVolume(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, volume registry.Volume) { - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - // Note: this blocks all delete requests till we are finished, which stops nasty races in ansible - - // TODO: fetch fresh copy of volume now we have aquired the lock? Ensure no delete has been processed already - if volume.State != registry.Registered { - log.Println("Volume in bad initial state:", volume.Name) - return - } - - bricks, err := poolRegistry.GetAllocationsForVolume(volume.Name) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - // Before we provision the bricks, notify that we have seen the volume - err = volumeRegistry.UpdateState(volume.Name, registry.BricksAllocated) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - err = plugin.VolumeProvider().SetupVolume(volume, bricks) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - err = volumeRegistry.UpdateState(volume.Name, registry.BricksProvisioned) - handleError(volumeRegistry, volume, err) -} - -func processDataIn(volumeRegistry registry.VolumeRegistry, volume registry.Volume) { - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - - // TODO: check volume is not deleted already, etc. - - err = plugin.VolumeProvider().CopyDataIn(volume) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - err = volumeRegistry.UpdateState(volume.Name, registry.DataInComplete) - handleError(volumeRegistry, volume, err) -} - -func processAttach(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, volume registry.Volume, - attachments []registry.Attachment) { - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - - bricks, err := poolRegistry.GetAllocationsForVolume(volume.Name) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - err = plugin.Mounter().Mount(volume, bricks, attachments) // TODO pass down specific attachments? - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - // TODO: this is really odd, mount should probably do this? - updates := []registry.Attachment{} - for _, attachment := range attachments { - if attachment.State == registry.RequestAttach { - attachment.State = registry.Attached - updates = append(updates, attachment) - } - - } - // TODO: what can we do if we hit an error here? - volumeRegistry.UpdateVolumeAttachments(volume.Name, updates) -} - -func processDetach(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, volume registry.Volume, - attachments []registry.Attachment) { - - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - - bricks, err := poolRegistry.GetAllocationsForVolume(volume.Name) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - err = plugin.Mounter().Unmount(volume, bricks, attachments) // TODO pass down specific attachments? - if err != nil { - // TODO: update specific attachment into an error state? - handleError(volumeRegistry, volume, err) - } - - var updates []registry.Attachment - for _, attachment := range attachments { - if attachment.State == registry.RequestDetach { - attachment.State = registry.Detached - updates = append(updates, attachment) - } - } - volumeRegistry.UpdateVolumeAttachments(volume.Name, updates) -} - -func processDataOut(volumeRegistry registry.VolumeRegistry, volume registry.Volume) { - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - - err = plugin.VolumeProvider().CopyDataOut(volume) - if err != nil { - handleError(volumeRegistry, volume, err) - } - - err = volumeRegistry.UpdateState(volume.Name, registry.DataOutComplete) - handleError(volumeRegistry, volume, err) -} - -func processDelete(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry, volume registry.Volume) { - // TODO: error handling!! - mutex, err := volumeRegistry.VolumeOperationMutex(volume.Name) - mutex.Lock(context.TODO()) - defer mutex.Unlock(context.TODO()) - - bricks, err := poolRegistry.GetAllocationsForVolume(volume.Name) - if err != nil { - handleError(volumeRegistry, volume, err) - return - } - - err = plugin.VolumeProvider().TeardownVolume(volume, bricks) - if err != nil { - handleError(volumeRegistry, volume, err) - } - - err = volumeRegistry.UpdateState(volume.Name, registry.BricksDeleted) - handleError(volumeRegistry, volume, err) -} diff --git a/internal/pkg/lifecycle/brickmanager/manager.go b/internal/pkg/lifecycle/brickmanager/manager.go deleted file mode 100644 index 679adfb3..00000000 --- a/internal/pkg/lifecycle/brickmanager/manager.go +++ /dev/null @@ -1,135 +0,0 @@ -package brickmanager - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "os" - "strconv" - "strings" -) - -type BrickManager interface { - Start() error - Hostname() string -} - -func NewBrickManager(poolRegistry registry.PoolRegistry, volumeRegistry registry.VolumeRegistry) BrickManager { - return &brickManager{poolRegistry, volumeRegistry, getHostname()} -} - -func getHostname() string { - // TODO: make this configurable? - hostname, err := os.Hostname() - if err != nil { - log.Fatal(err) - } - return hostname -} - -type brickManager struct { - poolRegistry registry.PoolRegistry - volumeRegistry registry.VolumeRegistry - hostname string -} - -func (bm *brickManager) Hostname() string { - return bm.hostname -} - -func (bm *brickManager) Start() error { - updateBricks(bm.poolRegistry, bm.hostname) - - // TODO, on startup see what existing allocations there are, and watch those volumes - setupBrickEventHandlers(bm.poolRegistry, bm.volumeRegistry, bm.hostname) - - // Do this after registering all bricks and their handlers, and any required tidy up - notifyStarted(bm.poolRegistry, bm.hostname) - - // Check after the processes have started up - outputDebugLogs(bm.poolRegistry, bm.hostname) - - return nil -} - -const DefaultDeviceAddress = "nvme%dn1" -const DefaultDeviceCapacityGB = 1400 -const DefaultPoolName = "default" - -func getDevices(devicesStr string, devType string) []string { - // TODO: check for real devices! - count, err := strconv.Atoi(devicesStr) - if err != nil { - count = 12 - } - - if devType == "" || !strings.Contains(devType, "%d") { - devType = DefaultDeviceAddress - } - - var bricks []string - for i := 0; i < count; i++ { - device := fmt.Sprintf(devType, i) - bricks = append(bricks, device) - } - return bricks -} - -func getBricks(devices []string, hostname string, capacityStr string, poolName string) []registry.BrickInfo { - capacity, ok := strconv.Atoi(capacityStr) - if ok != nil || capacityStr == "" || capacity <= 0 { - capacity = DefaultDeviceCapacityGB - } - if poolName == "" { - poolName = DefaultPoolName - } - - var bricks []registry.BrickInfo - for _, device := range devices { - bricks = append(bricks, registry.BrickInfo{ - Device: device, - Hostname: hostname, - CapacityGB: uint(capacity), - PoolName: poolName, - }) - } - return bricks -} - -func updateBricks(poolRegistry registry.PoolRegistry, hostname string) { - devicesStr := os.Getenv("DEVICE_COUNT") - devType := os.Getenv("DEVICE_TYPE") - devices := getDevices(devicesStr, devType) - - capacityStr := os.Getenv("DAC_DEVICE_CAPACITY_GB") - poolName := os.Getenv("DAC_POOL_NAME") - bricks := getBricks(devices, hostname, capacityStr, poolName) - - err := poolRegistry.UpdateHost(bricks) - if err != nil { - log.Fatalln(err) - } -} - -func outputDebugLogs(poolRegistry registry.PoolRegistry, hostname string) { - allocations, err := poolRegistry.GetAllocationsForHost(hostname) - if err != nil { - // Ignore errors, we may not have any results when there are no allocations - // TODO: maybe stop returing an error for the empty case? - log.Println(err) - } - log.Println("Current allocations:", allocations) - - pools, err := poolRegistry.Pools() - if err != nil { - log.Fatalln(err) - } - log.Println("Current pools:", pools) -} - -func notifyStarted(poolRegistry registry.PoolRegistry, hostname string) { - err := poolRegistry.KeepAliveHost(hostname) - if err != nil { - log.Fatalln(err) - } -} diff --git a/internal/pkg/lifecycle/brickmanager/manager_test.go b/internal/pkg/lifecycle/brickmanager/manager_test.go deleted file mode 100644 index b16a8a75..00000000 --- a/internal/pkg/lifecycle/brickmanager/manager_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package brickmanager - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestGetDevices(t *testing.T) { - devices := getDevices("5", "") - assert.Equal(t, 5, len(devices)) - assert.Equal(t, "nvme0n1", devices[0]) - assert.Equal(t, "nvme4n1", devices[4]) - - devices = getDevices("asdf", "loop%d") - assert.Equal(t, 12, len(devices)) - assert.Equal(t, "loop0", devices[0]) - assert.Equal(t, "loop11", devices[11]) -} - -func TestGetBricks(t *testing.T) { - devices := []string{"a", "b"} - bricks := getBricks(devices, "host", "-1", "") - - assert.Equal(t, 2, len(bricks)) - assert.Equal(t, registry.BrickInfo{ - Device: "a", Hostname: "host", PoolName: "default", CapacityGB: 1400, - }, bricks[0]) - assert.Equal(t, registry.BrickInfo{ - Device: "b", Hostname: "host", PoolName: "default", CapacityGB: 1400, - }, bricks[1]) - - bricks = getBricks(devices, "host", "20", "foo") - assert.Equal(t, registry.BrickInfo{ - Device: "b", Hostname: "host", PoolName: "foo", CapacityGB: 20, - }, bricks[1]) -} diff --git a/internal/pkg/lifecycle/volume.go b/internal/pkg/lifecycle/volume.go deleted file mode 100644 index 02ce3853..00000000 --- a/internal/pkg/lifecycle/volume.go +++ /dev/null @@ -1,247 +0,0 @@ -package lifecycle - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" -) - -type VolumeLifecycleManager interface { - ProvisionBricks() error - DataIn() error - Mount(hosts []string, jobName string) error - Unmount(hosts []string, jobName string) error - DataOut() error - Delete() error // TODO allow context for timeout and cancel? -} - -func NewVolumeLifecycleManager(volumeRegistry registry.VolumeRegistry, poolRegistry registry.PoolRegistry, - volume registry.Volume) VolumeLifecycleManager { - return &volumeLifecycleManager{volumeRegistry, poolRegistry, volume} -} - -type volumeLifecycleManager struct { - volumeRegistry registry.VolumeRegistry - poolRegistry registry.PoolRegistry - volume registry.Volume -} - -func (vlm *volumeLifecycleManager) ProvisionBricks() error { - _, err := vlm.poolRegistry.AllocateBricksForVolume(vlm.volume) - if err != nil { - return err - } - - // if there are no bricks requested, don't wait for a provision that will never happen - if vlm.volume.SizeBricks != 0 { - err = vlm.volumeRegistry.WaitForState(vlm.volume.Name, registry.BricksProvisioned) - } - return err -} - -func (vlm *volumeLifecycleManager) Delete() error { - // TODO convert errors into volume related errors, somewhere? - log.Println("Deleting volume:", vlm.volume.Name, vlm.volume) - - if vlm.volume.SizeBricks == 0 { - log.Println("No bricks to delete, skipping request delete bricks for:", vlm.volume.Name) - } else if vlm.volume.HadBricksAssigned == false { - allocations, _ := vlm.poolRegistry.GetAllocationsForVolume(vlm.volume.Name) - if len(allocations) == 0 { - // TODO should we be holding a lock here? - log.Println("No bricks yet assigned, skip delete bricks.") - } else { - return fmt.Errorf("bricks assigned but dacd hasn't noticed them yet for: %+v", vlm.volume) - } - } else { - log.Printf("Requested delete of %d bricks for %s", vlm.volume.SizeBricks, vlm.volume.Name) - err := vlm.volumeRegistry.UpdateState(vlm.volume.Name, registry.DeleteRequested) - if err != nil { - return err - } - err = vlm.volumeRegistry.WaitForState(vlm.volume.Name, registry.BricksDeleted) - if err != nil { - return err - } - log.Println("Bricks deleted by brick manager for:", vlm.volume.Name) - - // TODO should we error out here when one of these steps fail? - err = vlm.poolRegistry.DeallocateBricks(vlm.volume.Name) - if err != nil { - return err - } - allocations, err := vlm.poolRegistry.GetAllocationsForVolume(vlm.volume.Name) - if err != nil { - return err - } - // TODO we should really wait for the brick manager to call this API - err = vlm.poolRegistry.HardDeleteAllocations(allocations) - if err != nil { - return err - } - log.Println("Allocations all deleted, count:", len(allocations)) - } - - // TODO: what about any pending mounts that might get left behind for job? - - log.Println("Deleting volume record in registry for:", vlm.volume.Name) - return vlm.volumeRegistry.DeleteVolume(vlm.volume.Name) -} - -func (vlm *volumeLifecycleManager) DataIn() error { - if vlm.volume.SizeBricks == 0 { - log.Println("skipping datain for:", vlm.volume.Name) - return nil - } - - err := vlm.volumeRegistry.UpdateState(vlm.volume.Name, registry.DataInRequested) - if err != nil { - return err - } - return vlm.volumeRegistry.WaitForState(vlm.volume.Name, registry.DataInComplete) -} - -func (vlm *volumeLifecycleManager) Mount(hosts []string, jobName string) error { - if vlm.volume.SizeBricks == 0 { - log.Println("skipping mount for:", vlm.volume.Name) // TODO: should never happen now? - return nil - } - - if vlm.volume.State != registry.BricksProvisioned && vlm.volume.State != registry.DataInComplete { - return fmt.Errorf("unable to mount volume: %s in state: %s", vlm.volume.Name, vlm.volume.State) - } - - var attachments []registry.Attachment - for _, host := range hosts { - attachments = append(attachments, registry.Attachment{ - Hostname: host, State: registry.RequestAttach, Job: jobName, - }) - } - - if err := vlm.volumeRegistry.UpdateVolumeAttachments(vlm.volume.Name, attachments); err != nil { - return err - } - - // TODO: should share code with Unmount!! - var volumeInErrorState bool - err := vlm.volumeRegistry.WaitForCondition(vlm.volume.Name, func(event *registry.VolumeChange) bool { - if event.New.State == registry.Error { - volumeInErrorState = true - return true - } - allAttached := false - for _, host := range hosts { - - var isAttached bool - for _, attachment := range event.New.Attachments { - if attachment.Job == jobName && attachment.Hostname == host { - if attachment.State == registry.Attached { - isAttached = true - } else if attachment.State == registry.AttachmentError { - // found an error bail out early - volumeInErrorState = true - return true // Return true to stop the waiting - } else { - isAttached = false - } - break - } - } - - if isAttached { - allAttached = true - } else { - allAttached = false - break - } - } - return allAttached - }) - if volumeInErrorState { - return fmt.Errorf("unable to mount volume: %s", vlm.volume.Name) - } - return err -} - -func (vlm *volumeLifecycleManager) Unmount(hosts []string, jobName string) error { - if vlm.volume.SizeBricks == 0 { - log.Println("skipping postrun for:", vlm.volume.Name) // TODO return error type and handle outside? - return nil - } - - if vlm.volume.State != registry.BricksProvisioned && vlm.volume.State != registry.DataInComplete { - return fmt.Errorf("unable to unmount volume: %s in state: %s", vlm.volume.Name, vlm.volume.State) - } - - var updates []registry.Attachment - for _, host := range hosts { - attachment, ok := vlm.volume.FindAttachment(host, jobName) - if !ok { - return fmt.Errorf( - "can't find attachment for volume: %s host: %s job: %s", - vlm.volume.Name, host, jobName) - } - - if attachment.State != registry.Attached { - return fmt.Errorf("attachment must be attached to do unmount for volume: %s", vlm.volume.Name) - } - attachment.State = registry.RequestDetach - updates = append(updates, *attachment) - } - // TODO: I think we need to split attachments out of the volume object to avoid the races - if err := vlm.volumeRegistry.UpdateVolumeAttachments(vlm.volume.Name, updates); err != nil { - return err - } - - // TODO: must share way more code and do more tests on this logic!! - var volumeInErrorState error - err := vlm.volumeRegistry.WaitForCondition(vlm.volume.Name, func(event *registry.VolumeChange) bool { - if event.New.State == registry.Error { - volumeInErrorState = fmt.Errorf("volume %s now in error state", event.New.Name) - return true - } - allDettached := false - for _, host := range hosts { - newAttachment, ok := event.New.FindAttachment(host, jobName) - if !ok { - // TODO: debug log or something? - volumeInErrorState = fmt.Errorf("unable to find attachment for host: %s", host) - return true - } - - if newAttachment.State == registry.AttachmentError { - // found an error bail out early - volumeInErrorState = fmt.Errorf("attachment for host %s in error state", host) - return true - } - - if newAttachment.State == registry.Detached { - allDettached = true - } else { - allDettached = false - break - } - } - return allDettached - }) - if volumeInErrorState != nil { - return fmt.Errorf("unable to unmount volume: %s because: %s", vlm.volume.Name, volumeInErrorState) - } - if err != nil { - return err - } - return vlm.volumeRegistry.DeleteVolumeAttachments(vlm.volume.Name, hosts, jobName) -} - -func (vlm *volumeLifecycleManager) DataOut() error { - if vlm.volume.SizeBricks == 0 { - log.Println("skipping data_out for:", vlm.volume.Name) - return nil - } - - err := vlm.volumeRegistry.UpdateState(vlm.volume.Name, registry.DataOutRequested) - if err != nil { - return err - } - return vlm.volumeRegistry.WaitForState(vlm.volume.Name, registry.DataOutComplete) -} diff --git a/internal/pkg/lifecycle/volume_test.go b/internal/pkg/lifecycle/volume_test.go deleted file mode 100644 index fb18ce82..00000000 --- a/internal/pkg/lifecycle/volume_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package lifecycle - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestVolumeLifecycleManager_Mount(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - - volume := registry.Volume{ - Name: "vol1", SizeBricks: 3, State: registry.BricksProvisioned, JobName: "job1"} - vlm := NewVolumeLifecycleManager(mockVolReg, nil, volume) - hosts := []string{"host1", "host2"} - - mockVolReg.EXPECT().UpdateVolumeAttachments(volume.Name, []registry.Attachment{ - {Hostname: "host1", State: registry.RequestAttach, Job: "job1"}, - {Hostname: "host2", State: registry.RequestAttach, Job: "job1"}, - }) - fakeWait := func(volumeName registry.VolumeName, condition func(event *registry.VolumeChange) bool) error { - event := ®istry.VolumeChange{New: ®istry.Volume{}} - assert.False(t, condition(event)) - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2", State: registry.Detached}, - {Hostname: "host1", Job: "job1", State: registry.Attached}, - {Hostname: "host2", Job: "job1", State: registry.Attached}, - } - assert.True(t, condition(event)) - - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2", State: registry.AttachmentError}, - {Hostname: "host1", Job: "job1", State: registry.Detached}, - {Hostname: "host2", Job: "job1", State: registry.Attached}, - } - assert.False(t, condition(event)) - - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2", State: registry.Attached}, - {Hostname: "host1", Job: "job1", State: registry.AttachmentError}, - {Hostname: "host2", Job: "job1", State: registry.Attached}, - } - assert.True(t, condition(event)) - return nil - } - mockVolReg.EXPECT().WaitForCondition(volume.Name, gomock.Any()).DoAndReturn(fakeWait) - - err := vlm.Mount(hosts, "job1") - assert.Equal(t, "unable to mount volume: vol1", err.Error()) -} - -func TestVolumeLifecycleManager_Unmount(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockVolReg := mocks.NewMockVolumeRegistry(mockCtrl) - - volume := registry.Volume{ - Name: "vol1", SizeBricks: 3, State: registry.BricksProvisioned, JobName: "job1", - Attachments: []registry.Attachment{ - {Hostname: "host1", Job: "job1", State: registry.Attached}, - {Hostname: "host2", Job: "job1", State: registry.Attached}, - {Hostname: "host1", Job: "job2"}, - }} - vlm := NewVolumeLifecycleManager(mockVolReg, nil, volume) - hosts := []string{"host1", "host2"} - - mockVolReg.EXPECT().UpdateVolumeAttachments(volume.Name, []registry.Attachment{ - {Hostname: "host1", State: registry.RequestDetach, Job: "job1"}, - {Hostname: "host2", State: registry.RequestDetach, Job: "job1"}, - }) - fakeWait := func(volumeName registry.VolumeName, condition func(event *registry.VolumeChange) bool) error { - event := ®istry.VolumeChange{New: ®istry.Volume{}} - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2"}, - {Hostname: "host1", Job: "job1", State: registry.Detached}, - {Hostname: "host2", Job: "job1", State: registry.Detached}, - } - assert.True(t, condition(event)) - - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2", State: registry.AttachmentError}, - {Hostname: "host1", Job: "job1", State: registry.Detached}, - {Hostname: "host2", Job: "job1", State: registry.Attached}, - } - assert.False(t, condition(event)) - - event.New.Attachments = []registry.Attachment{ - {Hostname: "host1", Job: "job2"}, - {Hostname: "host1", Job: "job1", State: registry.AttachmentError}, - {Hostname: "host2", Job: "job1", State: registry.Detached}, - } - assert.True(t, condition(event)) - return nil - } - mockVolReg.EXPECT().WaitForCondition(volume.Name, gomock.Any()).DoAndReturn(fakeWait) - - err := vlm.Unmount(hosts, "job2") - assert.Equal(t, "attachment must be attached to do unmount for volume: vol1", err.Error()) - - err = vlm.Unmount(hosts, "job3") - assert.Equal(t, "can't find attachment for volume: vol1 host: host1 job: job3", err.Error()) - - err = vlm.Unmount(hosts, "job1") - assert.Equal(t, "unable to unmount volume: vol1 because: attachment for host host1 in error state", err.Error()) -} diff --git a/internal/pkg/mocks/disk_mock.go b/internal/pkg/mocks/disk_mock.go deleted file mode 100644 index 115e16e3..00000000 --- a/internal/pkg/mocks/disk_mock.go +++ /dev/null @@ -1,62 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/fileio/disk.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockDisk is a mock of Disk interface -type MockDisk struct { - ctrl *gomock.Controller - recorder *MockDiskMockRecorder -} - -// MockDiskMockRecorder is the mock recorder for MockDisk -type MockDiskMockRecorder struct { - mock *MockDisk -} - -// NewMockDisk creates a new mock instance -func NewMockDisk(ctrl *gomock.Controller) *MockDisk { - mock := &MockDisk{ctrl: ctrl} - mock.recorder = &MockDiskMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockDisk) EXPECT() *MockDiskMockRecorder { - return m.recorder -} - -// Lines mocks base method -func (m *MockDisk) Lines(filename string) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Lines", filename) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Lines indicates an expected call of Lines -func (mr *MockDiskMockRecorder) Lines(filename interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lines", reflect.TypeOf((*MockDisk)(nil).Lines), filename) -} - -// Write mocks base method -func (m *MockDisk) Write(filename string, lines []string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Write", filename, lines) - ret0, _ := ret[0].(error) - return ret0 -} - -// Write indicates an expected call of Write -func (mr *MockDiskMockRecorder) Write(filename, lines interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockDisk)(nil).Write), filename, lines) -} diff --git a/internal/pkg/mocks/job_mock.go b/internal/pkg/mocks/job_mock.go deleted file mode 100644 index 478a1b56..00000000 --- a/internal/pkg/mocks/job_mock.go +++ /dev/null @@ -1,32 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/dacctl/job.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" -) - -// MockjobCommand is a mock of jobCommand interface -type MockjobCommand struct { - ctrl *gomock.Controller - recorder *MockjobCommandMockRecorder -} - -// MockjobCommandMockRecorder is the mock recorder for MockjobCommand -type MockjobCommandMockRecorder struct { - mock *MockjobCommand -} - -// NewMockjobCommand creates a new mock instance -func NewMockjobCommand(ctrl *gomock.Controller) *MockjobCommand { - mock := &MockjobCommand{ctrl: ctrl} - mock.recorder = &MockjobCommandMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockjobCommand) EXPECT() *MockjobCommandMockRecorder { - return m.recorder -} diff --git a/internal/pkg/mocks/pfsprovider_mock.go b/internal/pkg/mocks/pfsprovider_mock.go deleted file mode 100644 index 26fa23ca..00000000 --- a/internal/pkg/mocks/pfsprovider_mock.go +++ /dev/null @@ -1,193 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/pfsprovider/interface.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - pfsprovider "github.com/RSE-Cambridge/data-acc/internal/pkg/pfsprovider" - registry "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockPlugin is a mock of Plugin interface -type MockPlugin struct { - ctrl *gomock.Controller - recorder *MockPluginMockRecorder -} - -// MockPluginMockRecorder is the mock recorder for MockPlugin -type MockPluginMockRecorder struct { - mock *MockPlugin -} - -// NewMockPlugin creates a new mock instance -func NewMockPlugin(ctrl *gomock.Controller) *MockPlugin { - mock := &MockPlugin{ctrl: ctrl} - mock.recorder = &MockPluginMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { - return m.recorder -} - -// Mounter mocks base method -func (m *MockPlugin) Mounter() pfsprovider.Mounter { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Mounter") - ret0, _ := ret[0].(pfsprovider.Mounter) - return ret0 -} - -// Mounter indicates an expected call of Mounter -func (mr *MockPluginMockRecorder) Mounter() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mounter", reflect.TypeOf((*MockPlugin)(nil).Mounter)) -} - -// VolumeProvider mocks base method -func (m *MockPlugin) VolumeProvider() pfsprovider.VolumeProvider { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "VolumeProvider") - ret0, _ := ret[0].(pfsprovider.VolumeProvider) - return ret0 -} - -// VolumeProvider indicates an expected call of VolumeProvider -func (mr *MockPluginMockRecorder) VolumeProvider() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeProvider", reflect.TypeOf((*MockPlugin)(nil).VolumeProvider)) -} - -// MockVolumeProvider is a mock of VolumeProvider interface -type MockVolumeProvider struct { - ctrl *gomock.Controller - recorder *MockVolumeProviderMockRecorder -} - -// MockVolumeProviderMockRecorder is the mock recorder for MockVolumeProvider -type MockVolumeProviderMockRecorder struct { - mock *MockVolumeProvider -} - -// NewMockVolumeProvider creates a new mock instance -func NewMockVolumeProvider(ctrl *gomock.Controller) *MockVolumeProvider { - mock := &MockVolumeProvider{ctrl: ctrl} - mock.recorder = &MockVolumeProviderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockVolumeProvider) EXPECT() *MockVolumeProviderMockRecorder { - return m.recorder -} - -// SetupVolume mocks base method -func (m *MockVolumeProvider) SetupVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupVolume", volume, brickAllocations) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetupVolume indicates an expected call of SetupVolume -func (mr *MockVolumeProviderMockRecorder) SetupVolume(volume, brickAllocations interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupVolume", reflect.TypeOf((*MockVolumeProvider)(nil).SetupVolume), volume, brickAllocations) -} - -// TeardownVolume mocks base method -func (m *MockVolumeProvider) TeardownVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TeardownVolume", volume, brickAllocations) - ret0, _ := ret[0].(error) - return ret0 -} - -// TeardownVolume indicates an expected call of TeardownVolume -func (mr *MockVolumeProviderMockRecorder) TeardownVolume(volume, brickAllocations interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownVolume", reflect.TypeOf((*MockVolumeProvider)(nil).TeardownVolume), volume, brickAllocations) -} - -// CopyDataIn mocks base method -func (m *MockVolumeProvider) CopyDataIn(volume registry.Volume) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CopyDataIn", volume) - ret0, _ := ret[0].(error) - return ret0 -} - -// CopyDataIn indicates an expected call of CopyDataIn -func (mr *MockVolumeProviderMockRecorder) CopyDataIn(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataIn", reflect.TypeOf((*MockVolumeProvider)(nil).CopyDataIn), volume) -} - -// CopyDataOut mocks base method -func (m *MockVolumeProvider) CopyDataOut(volume registry.Volume) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CopyDataOut", volume) - ret0, _ := ret[0].(error) - return ret0 -} - -// CopyDataOut indicates an expected call of CopyDataOut -func (mr *MockVolumeProviderMockRecorder) CopyDataOut(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyDataOut", reflect.TypeOf((*MockVolumeProvider)(nil).CopyDataOut), volume) -} - -// MockMounter is a mock of Mounter interface -type MockMounter struct { - ctrl *gomock.Controller - recorder *MockMounterMockRecorder -} - -// MockMounterMockRecorder is the mock recorder for MockMounter -type MockMounterMockRecorder struct { - mock *MockMounter -} - -// NewMockMounter creates a new mock instance -func NewMockMounter(ctrl *gomock.Controller) *MockMounter { - mock := &MockMounter{ctrl: ctrl} - mock.recorder = &MockMounterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockMounter) EXPECT() *MockMounterMockRecorder { - return m.recorder -} - -// Mount mocks base method -func (m *MockMounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Mount", volume, brickAllocations, attachments) - ret0, _ := ret[0].(error) - return ret0 -} - -// Mount indicates an expected call of Mount -func (mr *MockMounterMockRecorder) Mount(volume, brickAllocations, attachments interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mount", reflect.TypeOf((*MockMounter)(nil).Mount), volume, brickAllocations, attachments) -} - -// Unmount mocks base method -func (m *MockMounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Unmount", volume, brickAllocations, attachments) - ret0, _ := ret[0].(error) - return ret0 -} - -// Unmount indicates an expected call of Unmount -func (mr *MockMounterMockRecorder) Unmount(volume, brickAllocations, attachments interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unmount", reflect.TypeOf((*MockMounter)(nil).Unmount), volume, brickAllocations, attachments) -} diff --git a/internal/pkg/mocks/pool_mock.go b/internal/pkg/mocks/pool_mock.go deleted file mode 100644 index f01738e6..00000000 --- a/internal/pkg/mocks/pool_mock.go +++ /dev/null @@ -1,180 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/registry/pool.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - registry "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockPoolRegistry is a mock of PoolRegistry interface -type MockPoolRegistry struct { - ctrl *gomock.Controller - recorder *MockPoolRegistryMockRecorder -} - -// MockPoolRegistryMockRecorder is the mock recorder for MockPoolRegistry -type MockPoolRegistryMockRecorder struct { - mock *MockPoolRegistry -} - -// NewMockPoolRegistry creates a new mock instance -func NewMockPoolRegistry(ctrl *gomock.Controller) *MockPoolRegistry { - mock := &MockPoolRegistry{ctrl: ctrl} - mock.recorder = &MockPoolRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockPoolRegistry) EXPECT() *MockPoolRegistryMockRecorder { - return m.recorder -} - -// Pools mocks base method -func (m *MockPoolRegistry) Pools() ([]registry.Pool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Pools") - ret0, _ := ret[0].([]registry.Pool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Pools indicates an expected call of Pools -func (mr *MockPoolRegistryMockRecorder) Pools() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pools", reflect.TypeOf((*MockPoolRegistry)(nil).Pools)) -} - -// UpdateHost mocks base method -func (m *MockPoolRegistry) UpdateHost(bricks []registry.BrickInfo) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateHost", bricks) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateHost indicates an expected call of UpdateHost -func (mr *MockPoolRegistryMockRecorder) UpdateHost(bricks interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHost", reflect.TypeOf((*MockPoolRegistry)(nil).UpdateHost), bricks) -} - -// KeepAliveHost mocks base method -func (m *MockPoolRegistry) KeepAliveHost(hostname string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KeepAliveHost", hostname) - ret0, _ := ret[0].(error) - return ret0 -} - -// KeepAliveHost indicates an expected call of KeepAliveHost -func (mr *MockPoolRegistryMockRecorder) KeepAliveHost(hostname interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockPoolRegistry)(nil).KeepAliveHost), hostname) -} - -// AllocateBricksForVolume mocks base method -func (m *MockPoolRegistry) AllocateBricksForVolume(volume registry.Volume) ([]registry.BrickAllocation, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllocateBricksForVolume", volume) - ret0, _ := ret[0].([]registry.BrickAllocation) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AllocateBricksForVolume indicates an expected call of AllocateBricksForVolume -func (mr *MockPoolRegistryMockRecorder) AllocateBricksForVolume(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocateBricksForVolume", reflect.TypeOf((*MockPoolRegistry)(nil).AllocateBricksForVolume), volume) -} - -// DeallocateBricks mocks base method -func (m *MockPoolRegistry) DeallocateBricks(volume registry.VolumeName) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeallocateBricks", volume) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeallocateBricks indicates an expected call of DeallocateBricks -func (mr *MockPoolRegistryMockRecorder) DeallocateBricks(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocateBricks", reflect.TypeOf((*MockPoolRegistry)(nil).DeallocateBricks), volume) -} - -// HardDeleteAllocations mocks base method -func (m *MockPoolRegistry) HardDeleteAllocations(allocations []registry.BrickAllocation) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HardDeleteAllocations", allocations) - ret0, _ := ret[0].(error) - return ret0 -} - -// HardDeleteAllocations indicates an expected call of HardDeleteAllocations -func (mr *MockPoolRegistryMockRecorder) HardDeleteAllocations(allocations interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HardDeleteAllocations", reflect.TypeOf((*MockPoolRegistry)(nil).HardDeleteAllocations), allocations) -} - -// GetAllocationsForHost mocks base method -func (m *MockPoolRegistry) GetAllocationsForHost(hostname string) ([]registry.BrickAllocation, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllocationsForHost", hostname) - ret0, _ := ret[0].([]registry.BrickAllocation) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllocationsForHost indicates an expected call of GetAllocationsForHost -func (mr *MockPoolRegistryMockRecorder) GetAllocationsForHost(hostname interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationsForHost", reflect.TypeOf((*MockPoolRegistry)(nil).GetAllocationsForHost), hostname) -} - -// GetAllocationsForVolume mocks base method -func (m *MockPoolRegistry) GetAllocationsForVolume(volume registry.VolumeName) ([]registry.BrickAllocation, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllocationsForVolume", volume) - ret0, _ := ret[0].([]registry.BrickAllocation) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAllocationsForVolume indicates an expected call of GetAllocationsForVolume -func (mr *MockPoolRegistryMockRecorder) GetAllocationsForVolume(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationsForVolume", reflect.TypeOf((*MockPoolRegistry)(nil).GetAllocationsForVolume), volume) -} - -// GetBrickInfo mocks base method -func (m *MockPoolRegistry) GetBrickInfo(hostname, device string) (registry.BrickInfo, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBrickInfo", hostname, device) - ret0, _ := ret[0].(registry.BrickInfo) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBrickInfo indicates an expected call of GetBrickInfo -func (mr *MockPoolRegistryMockRecorder) GetBrickInfo(hostname, device interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrickInfo", reflect.TypeOf((*MockPoolRegistry)(nil).GetBrickInfo), hostname, device) -} - -// GetNewHostBrickAllocations mocks base method -func (m *MockPoolRegistry) GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan registry.BrickAllocation { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetNewHostBrickAllocations", ctxt, hostname) - ret0, _ := ret[0].(<-chan registry.BrickAllocation) - return ret0 -} - -// GetNewHostBrickAllocations indicates an expected call of GetNewHostBrickAllocations -func (mr *MockPoolRegistryMockRecorder) GetNewHostBrickAllocations(ctxt, hostname interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNewHostBrickAllocations", reflect.TypeOf((*MockPoolRegistry)(nil).GetNewHostBrickAllocations), ctxt, hostname) -} diff --git a/internal/pkg/mocks/volume_mock.go b/internal/pkg/mocks/volume_mock.go deleted file mode 100644 index 49ffb6bb..00000000 --- a/internal/pkg/mocks/volume_mock.go +++ /dev/null @@ -1,315 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/registry/volume.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - registry "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockVolumeRegistry is a mock of VolumeRegistry interface -type MockVolumeRegistry struct { - ctrl *gomock.Controller - recorder *MockVolumeRegistryMockRecorder -} - -// MockVolumeRegistryMockRecorder is the mock recorder for MockVolumeRegistry -type MockVolumeRegistryMockRecorder struct { - mock *MockVolumeRegistry -} - -// NewMockVolumeRegistry creates a new mock instance -func NewMockVolumeRegistry(ctrl *gomock.Controller) *MockVolumeRegistry { - mock := &MockVolumeRegistry{ctrl: ctrl} - mock.recorder = &MockVolumeRegistryMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockVolumeRegistry) EXPECT() *MockVolumeRegistryMockRecorder { - return m.recorder -} - -// Jobs mocks base method -func (m *MockVolumeRegistry) Jobs() ([]registry.Job, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Jobs") - ret0, _ := ret[0].([]registry.Job) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Jobs indicates an expected call of Jobs -func (mr *MockVolumeRegistryMockRecorder) Jobs() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Jobs", reflect.TypeOf((*MockVolumeRegistry)(nil).Jobs)) -} - -// Job mocks base method -func (m *MockVolumeRegistry) Job(jobName string) (registry.Job, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Job", jobName) - ret0, _ := ret[0].(registry.Job) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Job indicates an expected call of Job -func (mr *MockVolumeRegistryMockRecorder) Job(jobName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Job", reflect.TypeOf((*MockVolumeRegistry)(nil).Job), jobName) -} - -// AddJob mocks base method -func (m *MockVolumeRegistry) AddJob(job registry.Job) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddJob", job) - ret0, _ := ret[0].(error) - return ret0 -} - -// AddJob indicates an expected call of AddJob -func (mr *MockVolumeRegistryMockRecorder) AddJob(job interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddJob", reflect.TypeOf((*MockVolumeRegistry)(nil).AddJob), job) -} - -// JobAttachHosts mocks base method -func (m *MockVolumeRegistry) JobAttachHosts(jobName string, hosts []string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "JobAttachHosts", jobName, hosts) - ret0, _ := ret[0].(error) - return ret0 -} - -// JobAttachHosts indicates an expected call of JobAttachHosts -func (mr *MockVolumeRegistryMockRecorder) JobAttachHosts(jobName, hosts interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "JobAttachHosts", reflect.TypeOf((*MockVolumeRegistry)(nil).JobAttachHosts), jobName, hosts) -} - -// DeleteJob mocks base method -func (m *MockVolumeRegistry) DeleteJob(jobName string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteJob", jobName) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteJob indicates an expected call of DeleteJob -func (mr *MockVolumeRegistryMockRecorder) DeleteJob(jobName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteJob", reflect.TypeOf((*MockVolumeRegistry)(nil).DeleteJob), jobName) -} - -// AddVolume mocks base method -func (m *MockVolumeRegistry) AddVolume(volume registry.Volume) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddVolume", volume) - ret0, _ := ret[0].(error) - return ret0 -} - -// AddVolume indicates an expected call of AddVolume -func (mr *MockVolumeRegistryMockRecorder) AddVolume(volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddVolume", reflect.TypeOf((*MockVolumeRegistry)(nil).AddVolume), volume) -} - -// Volume mocks base method -func (m *MockVolumeRegistry) Volume(name registry.VolumeName) (registry.Volume, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Volume", name) - ret0, _ := ret[0].(registry.Volume) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Volume indicates an expected call of Volume -func (mr *MockVolumeRegistryMockRecorder) Volume(name interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Volume", reflect.TypeOf((*MockVolumeRegistry)(nil).Volume), name) -} - -// AllVolumes mocks base method -func (m *MockVolumeRegistry) AllVolumes() ([]registry.Volume, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AllVolumes") - ret0, _ := ret[0].([]registry.Volume) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AllVolumes indicates an expected call of AllVolumes -func (mr *MockVolumeRegistryMockRecorder) AllVolumes() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllVolumes", reflect.TypeOf((*MockVolumeRegistry)(nil).AllVolumes)) -} - -// DeleteVolume mocks base method -func (m *MockVolumeRegistry) DeleteVolume(name registry.VolumeName) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteVolume", name) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteVolume indicates an expected call of DeleteVolume -func (mr *MockVolumeRegistryMockRecorder) DeleteVolume(name interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVolume", reflect.TypeOf((*MockVolumeRegistry)(nil).DeleteVolume), name) -} - -// UpdateState mocks base method -func (m *MockVolumeRegistry) UpdateState(name registry.VolumeName, state registry.VolumeState) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateState", name, state) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateState indicates an expected call of UpdateState -func (mr *MockVolumeRegistryMockRecorder) UpdateState(name, state interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateState", reflect.TypeOf((*MockVolumeRegistry)(nil).UpdateState), name, state) -} - -// UpdateVolumeAttachments mocks base method -func (m *MockVolumeRegistry) UpdateVolumeAttachments(name registry.VolumeName, attachments []registry.Attachment) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateVolumeAttachments", name, attachments) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateVolumeAttachments indicates an expected call of UpdateVolumeAttachments -func (mr *MockVolumeRegistryMockRecorder) UpdateVolumeAttachments(name, attachments interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateVolumeAttachments", reflect.TypeOf((*MockVolumeRegistry)(nil).UpdateVolumeAttachments), name, attachments) -} - -// DeleteVolumeAttachments mocks base method -func (m *MockVolumeRegistry) DeleteVolumeAttachments(name registry.VolumeName, hostnames []string, jobName string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteVolumeAttachments", name, hostnames, jobName) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteVolumeAttachments indicates an expected call of DeleteVolumeAttachments -func (mr *MockVolumeRegistryMockRecorder) DeleteVolumeAttachments(name, hostnames, jobName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVolumeAttachments", reflect.TypeOf((*MockVolumeRegistry)(nil).DeleteVolumeAttachments), name, hostnames, jobName) -} - -// WaitForState mocks base method -func (m *MockVolumeRegistry) WaitForState(name registry.VolumeName, state registry.VolumeState) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitForState", name, state) - ret0, _ := ret[0].(error) - return ret0 -} - -// WaitForState indicates an expected call of WaitForState -func (mr *MockVolumeRegistryMockRecorder) WaitForState(name, state interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForState", reflect.TypeOf((*MockVolumeRegistry)(nil).WaitForState), name, state) -} - -// WaitForCondition mocks base method -func (m *MockVolumeRegistry) WaitForCondition(volumeName registry.VolumeName, condition func(*registry.VolumeChange) bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "WaitForCondition", volumeName, condition) - ret0, _ := ret[0].(error) - return ret0 -} - -// WaitForCondition indicates an expected call of WaitForCondition -func (mr *MockVolumeRegistryMockRecorder) WaitForCondition(volumeName, condition interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForCondition", reflect.TypeOf((*MockVolumeRegistry)(nil).WaitForCondition), volumeName, condition) -} - -// GetVolumeChanges mocks base method -func (m *MockVolumeRegistry) GetVolumeChanges(ctx context.Context, volume registry.Volume) registry.VolumeChangeChan { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVolumeChanges", ctx, volume) - ret0, _ := ret[0].(registry.VolumeChangeChan) - return ret0 -} - -// GetVolumeChanges indicates an expected call of GetVolumeChanges -func (mr *MockVolumeRegistryMockRecorder) GetVolumeChanges(ctx, volume interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumeChanges", reflect.TypeOf((*MockVolumeRegistry)(nil).GetVolumeChanges), ctx, volume) -} - -// VolumeOperationMutex mocks base method -func (m *MockVolumeRegistry) VolumeOperationMutex(volumeName registry.VolumeName) (registry.Mutex, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "VolumeOperationMutex", volumeName) - ret0, _ := ret[0].(registry.Mutex) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// VolumeOperationMutex indicates an expected call of VolumeOperationMutex -func (mr *MockVolumeRegistryMockRecorder) VolumeOperationMutex(volumeName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeOperationMutex", reflect.TypeOf((*MockVolumeRegistry)(nil).VolumeOperationMutex), volumeName) -} - -// MockMutex is a mock of Mutex interface -type MockMutex struct { - ctrl *gomock.Controller - recorder *MockMutexMockRecorder -} - -// MockMutexMockRecorder is the mock recorder for MockMutex -type MockMutexMockRecorder struct { - mock *MockMutex -} - -// NewMockMutex creates a new mock instance -func NewMockMutex(ctrl *gomock.Controller) *MockMutex { - mock := &MockMutex{ctrl: ctrl} - mock.recorder = &MockMutexMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockMutex) EXPECT() *MockMutexMockRecorder { - return m.recorder -} - -// Lock mocks base method -func (m *MockMutex) Lock(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Lock", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// Lock indicates an expected call of Lock -func (mr *MockMutexMockRecorder) Lock(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lock", reflect.TypeOf((*MockMutex)(nil).Lock), ctx) -} - -// Unlock mocks base method -func (m *MockMutex) Unlock(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Unlock", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// Unlock indicates an expected call of Unlock -func (mr *MockMutexMockRecorder) Unlock(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unlock", reflect.TypeOf((*MockMutex)(nil).Unlock), ctx) -} diff --git a/internal/pkg/pfsprovider/ansible/ansible.go b/internal/pkg/pfsprovider/ansible/ansible.go deleted file mode 100644 index 27fcb359..00000000 --- a/internal/pkg/pfsprovider/ansible/ansible.go +++ /dev/null @@ -1,311 +0,0 @@ -package ansible - -import ( - "bytes" - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "gopkg.in/yaml.v2" - "io/ioutil" - "log" - "os" - "os/exec" - "path" - "path/filepath" - "strconv" - "strings" - "time" -) - -type HostInfo struct { - MGS string `yaml:"mgs,omitempty"` - MDTS map[string]int `yaml:"mdts,omitempty,flow"` - OSTS map[string]int `yaml:"osts,omitempty,flow"` -} - -type FSInfo struct { - Hosts map[string]HostInfo `yaml:"hosts"` - Vars map[string]string `yaml:"vars"` -} - -type FileSystems struct { - Children map[string]FSInfo `yaml:"children"` -} - -type Wrapper struct { - All FileSystems -} - -var DefaultHostGroup = "dac-prod" -var DefaultMaxMDTs uint = 24 - -func getInventory(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) string { - // NOTE: only used by lustre - mgsDevice := os.Getenv("DAC_MGS_DEV") - if mgsDevice == "" { - mgsDevice = "sdb" - } - maxMDTs := DefaultMaxMDTs - maxMDTsConf, err := strconv.ParseUint(os.Getenv("DAC_MAX_MDT_COUNT"), 10, 32) - if err == nil && maxMDTsConf > 0 { - maxMDTs = uint(maxMDTsConf) - } - - allocationsByHost := make(map[string][]registry.BrickAllocation) - for _, allocation := range brickAllocations { - allocationsByHost[allocation.Hostname] = append(allocationsByHost[allocation.Hostname], allocation) - } - - // If we have more brick allocations than maxMDTs - // assign at most one mdt per host. - // While this may give us less MDTs than max MDTs, - // but it helps spread MDTs across network connections - oneMdtPerHost := len(brickAllocations) > int(maxMDTs) - - hosts := make(map[string]HostInfo) - mgsnode := "" - for host, allocations := range allocationsByHost { - osts := make(map[string]int) - for _, allocation := range allocations { - osts[allocation.Device] = int(allocation.AllocatedIndex) - } - - mdts := make(map[string]int) - if oneMdtPerHost { - allocation := allocations[0] - mdts[allocation.Device] = int(allocation.AllocatedIndex) - } else { - for _, allocation := range allocations { - mdts[allocation.Device] = int(allocation.AllocatedIndex) - } - } - - hostInfo := HostInfo{MDTS: mdts, OSTS: osts} - - if allocations[0].AllocatedIndex == 0 { - if fsType == Lustre { - hostInfo.MGS = mgsDevice - } else { - hostInfo.MGS = allocations[0].Device - } - mgsnode = host - } - hosts[host] = hostInfo - } - - // for beegfs, mount clients via ansible - if fsType == BeegFS { - // TODO: this can't work now, as we need to also pass the job name - for _, attachment := range volume.Attachments { - hosts[attachment.Hostname] = HostInfo{} - } - } - - fsinfo := FSInfo{ - Vars: map[string]string{ - "mgsnode": mgsnode, - "client_port": fmt.Sprintf("%d", volume.ClientPort), - "lnet_suffix": getLnetSuffix(), - "mdt_size": fmt.Sprintf("%dm", getMdtSizeMB()), - }, - Hosts: hosts, - } - fsname := fmt.Sprintf("%s", volume.UUID) - data := Wrapper{All: FileSystems{Children: map[string]FSInfo{fsname: fsinfo}}} - - output, err := yaml.Marshal(data) - if err != nil { - log.Fatalln(err) - } - strOut := string(output) - strOut = strings.Replace(strOut, " mgs:", fmt.Sprintf(" %s_mgs:", fsname), -1) - strOut = strings.Replace(strOut, " mdts:", fmt.Sprintf(" %s_mdts:", fsname), -1) - strOut = strings.Replace(strOut, " osts:", fmt.Sprintf(" %s_osts:", fsname), -1) - strOut = strings.Replace(strOut, " mgsnode:", fmt.Sprintf(" %s_mgsnode:", fsname), -1) - strOut = strings.Replace(strOut, " client_port:", fmt.Sprintf(" %s_client_port:", fsname), -1) - strOut = strings.Replace(strOut, " mdt_size:", fmt.Sprintf(" %s_mdt_size:", fsname), -1) - - hostGroup := os.Getenv("DAC_HOST_GROUP") - if hostGroup == "" { - hostGroup = DefaultHostGroup - } - strOut = strings.Replace(strOut, "all:", hostGroup+":", -1) - return strOut -} - -func getPlaybook(fsType FSType, volume registry.Volume) string { - role := "lustre" - if fsType == BeegFS { - role = "beegfs" - } - return fmt.Sprintf(`--- -- name: Setup FS - hosts: %s - any_errors_fatal: true - become: yes - roles: - - role: %s - vars: - fs_name: %s`, volume.UUID, role, volume.UUID) -} - -func getAnsibleDir(suffix string) string { - ansibleDir := os.Getenv("DAC_ANSIBLE_DIR") - if ansibleDir == "" { - ansibleDir = "/var/lib/data-acc/fs-ansible/" - } - return path.Join(ansibleDir, suffix) -} - -func setupAnsible(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) (string, error) { - dir, err := ioutil.TempDir("", fmt.Sprintf("fs%s_", volume.Name)) - if err != nil { - return dir, err - } - log.Println("Using ansible tempdir:", dir) - - playbook := getPlaybook(fsType, volume) - tmpPlaybook := filepath.Join(dir, "dac.yml") - if err := ioutil.WriteFile(tmpPlaybook, bytes.NewBufferString(playbook).Bytes(), 0666); err != nil { - return dir, err - } - log.Println(playbook) - - inventory := getInventory(fsType, volume, brickAllocations) - tmpInventory := filepath.Join(dir, "inventory") - if err := ioutil.WriteFile(tmpInventory, bytes.NewBufferString(inventory).Bytes(), 0666); err != nil { - return dir, err - } - log.Println(inventory) - - cmd := exec.Command("cp", "-r", getAnsibleDir("roles"), dir) - output, err := cmd.CombinedOutput() - log.Println("copy roles", string(output)) - if err != nil { - return dir, err - } - cmd = exec.Command("cp", "-r", getAnsibleDir(".venv"), dir) - output, err = cmd.CombinedOutput() - log.Println("copy venv", string(output)) - if err != nil { - return dir, err - } - cmd = exec.Command("cp", "-r", getAnsibleDir("group_vars"), dir) - output, err = cmd.CombinedOutput() - log.Println("copy group vars", string(output)) - return dir, err -} - -func executeAnsibleSetup(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) - if err != nil { - return err - } - - formatArgs := "dac.yml -i inventory --tag format" - err = executeAnsiblePlaybook(dir, formatArgs) - if err != nil { - return err - } - - startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" - err = executeAnsiblePlaybook(dir, startupArgs) - if err != nil { - return err - } - - // only delete if everything worked, to aid debugging - os.RemoveAll(dir) - return nil -} - -func executeAnsibleTeardown(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) - if err != nil { - return err - } - - stopArgs := "dac.yml -i inventory --tag stop_all,unmount,client_unmount" - err = executeAnsiblePlaybook(dir, stopArgs) - if err != nil { - return err - } - - formatArgs := "dac.yml -i inventory --tag clean" - err = executeAnsiblePlaybook(dir, formatArgs) - if err != nil { - return err - } - - // only delete if everything worked, to aid debugging - os.RemoveAll(dir) - return nil -} - -func executeAnsibleMount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) - if err != nil { - return err - } - - startupArgs := "dac.yml -i inventory --tag client_mount" - err = executeAnsiblePlaybook(dir, startupArgs) - if err != nil { - return err - } - - os.RemoveAll(dir) - return nil -} - -func executeAnsibleUnmount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - dir, err := setupAnsible(fsType, volume, brickAllocations) - if err != nil { - return err - } - - stopArgs := "dac.yml -i inventory --tag client_unmount" - err = executeAnsiblePlaybook(dir, stopArgs) - if err != nil { - return err - } - - os.RemoveAll(dir) - return nil -} - -func executeAnsiblePlaybook(dir string, args string) error { - // TODO: downgrade debug log! - cmdStr := fmt.Sprintf(`cd %s; . .venv/bin/activate; ansible-playbook %s;`, dir, args) - log.Println("Requested ansible:", cmdStr) - - skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") - if skipAnsible == "True" { - log.Println("Skip as DAC_SKIP_ANSIBLE=True") - time.Sleep(time.Millisecond * 200) - return nil - } - - var err error - for i := 1; i <= 3; i++ { - log.Println("Attempt", i, "of ansible:", cmdStr) - cmd := exec.Command("bash", "-c", cmdStr) - - timer := time.AfterFunc(time.Minute*5, func() { - log.Println("Time up, waited more than 5 mins to complete.") - cmd.Process.Kill() - }) - output, currentErr := cmd.CombinedOutput() - timer.Stop() - - if currentErr == nil { - log.Println("Completed ansible run:", cmdStr) - log.Println(string(output)) - return nil - } else { - log.Println("Error in ansible run:", string(output)) - err = currentErr - time.Sleep(time.Second * 2) - } - } - return err -} diff --git a/internal/pkg/pfsprovider/ansible/ansible_test.go b/internal/pkg/pfsprovider/ansible/ansible_test.go deleted file mode 100644 index 9d81d2dd..00000000 --- a/internal/pkg/pfsprovider/ansible/ansible_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package ansible - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestPlugin_GetInventory(t *testing.T) { - volume := registry.Volume{ - Name: "1", UUID: "abcdefgh", ClientPort: 10002, - Attachments: []registry.Attachment{ - {Hostname: "cpu1"}, - {Hostname: "cpu2"}, - }, - } - brickAllocations := []registry.BrickAllocation{ - {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, - {Hostname: "dac1", Device: "nvme2n1", AllocatedIndex: 1}, - {Hostname: "dac1", Device: "nvme3n1", AllocatedIndex: 2}, - {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 3}, - {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 4}, - } - result := getInventory(BeegFS, volume, brickAllocations) - expected := `dac-prod: - children: - abcdefgh: - hosts: - cpu1: {} - cpu2: {} - dac1: - abcdefgh_mgs: nvme1n1 - abcdefgh_mdts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} - abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} - dac2: - abcdefgh_mdts: {nvme2n1: 3, nvme3n1: 4} - abcdefgh_osts: {nvme2n1: 3, nvme3n1: 4} - vars: - abcdefgh_client_port: "10002" - lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 -` - assert.Equal(t, expected, result) -} - -func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh", ClientPort: 10002} - brickAllocations := []registry.BrickAllocation{ - {Hostname: "dac1", Device: "nvme1n1", AllocatedIndex: 0}, - {Hostname: "dac2", Device: "nvme2n1", AllocatedIndex: 1}, - {Hostname: "dac2", Device: "nvme3n1", AllocatedIndex: 2}, - } - result := getInventory(Lustre, volume, brickAllocations) - expected := `dac-prod: - children: - abcdefgh: - hosts: - dac1: - abcdefgh_mgs: sdb - abcdefgh_mdts: {nvme1n1: 0} - abcdefgh_osts: {nvme1n1: 0} - dac2: - abcdefgh_mdts: {nvme2n1: 1, nvme3n1: 2} - abcdefgh_osts: {nvme2n1: 1, nvme3n1: 2} - vars: - abcdefgh_client_port: "10002" - lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 -` - assert.Equal(t, expected, result) -} - -func TestPlugin_GetPlaybook_beegfs(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh"} - result := getPlaybook(BeegFS, volume) - assert.Equal(t, `--- -- name: Setup FS - hosts: abcdefgh - any_errors_fatal: true - become: yes - roles: - - role: beegfs - vars: - fs_name: abcdefgh`, result) -} - -func TestPlugin_GetPlaybook_lustre(t *testing.T) { - volume := registry.Volume{Name: "1", UUID: "abcdefgh"} - result := getPlaybook(Lustre, volume) - assert.Equal(t, `--- -- name: Setup FS - hosts: abcdefgh - any_errors_fatal: true - become: yes - roles: - - role: lustre - vars: - fs_name: abcdefgh`, result) -} - -func TestPlugin_GetInventory_MaxMDT(t *testing.T) { - volume := registry.Volume{ - Name: "1", UUID: "abcdefgh", ClientPort: 10002, - Attachments: []registry.Attachment{ - {Hostname: "cpu1"}, - {Hostname: "cpu2"}, - }, - } - - var brickAllocations []registry.BrickAllocation - for i := 1; i <= 26; i = i + 2 { - brickAllocations = append(brickAllocations, registry.BrickAllocation{ - Hostname: fmt.Sprintf("dac%d", i), - Device: "nvme1n1", - AllocatedIndex: uint(i - 1), - }) - brickAllocations = append(brickAllocations, registry.BrickAllocation{ - Hostname: fmt.Sprintf("dac%d", i), - Device: "nvme2n1", - AllocatedIndex: uint(i), - }) - } - - result := getInventory(BeegFS, volume, brickAllocations) - expected := `dac-prod: - children: - abcdefgh: - hosts: - cpu1: {} - cpu2: {} - dac1: - abcdefgh_mgs: nvme1n1 - abcdefgh_mdts: {nvme1n1: 0} - abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1} - dac3: - abcdefgh_mdts: {nvme1n1: 2} - abcdefgh_osts: {nvme1n1: 2, nvme2n1: 3} - dac5: - abcdefgh_mdts: {nvme1n1: 4} - abcdefgh_osts: {nvme1n1: 4, nvme2n1: 5} - dac7: - abcdefgh_mdts: {nvme1n1: 6} - abcdefgh_osts: {nvme1n1: 6, nvme2n1: 7} - dac9: - abcdefgh_mdts: {nvme1n1: 8} - abcdefgh_osts: {nvme1n1: 8, nvme2n1: 9} - dac11: - abcdefgh_mdts: {nvme1n1: 10} - abcdefgh_osts: {nvme1n1: 10, nvme2n1: 11} - dac13: - abcdefgh_mdts: {nvme1n1: 12} - abcdefgh_osts: {nvme1n1: 12, nvme2n1: 13} - dac15: - abcdefgh_mdts: {nvme1n1: 14} - abcdefgh_osts: {nvme1n1: 14, nvme2n1: 15} - dac17: - abcdefgh_mdts: {nvme1n1: 16} - abcdefgh_osts: {nvme1n1: 16, nvme2n1: 17} - dac19: - abcdefgh_mdts: {nvme1n1: 18} - abcdefgh_osts: {nvme1n1: 18, nvme2n1: 19} - dac21: - abcdefgh_mdts: {nvme1n1: 20} - abcdefgh_osts: {nvme1n1: 20, nvme2n1: 21} - dac23: - abcdefgh_mdts: {nvme1n1: 22} - abcdefgh_osts: {nvme1n1: 22, nvme2n1: 23} - dac25: - abcdefgh_mdts: {nvme1n1: 24} - abcdefgh_osts: {nvme1n1: 24, nvme2n1: 25} - vars: - abcdefgh_client_port: "10002" - lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 -` - assert.Equal(t, expected, result) -} diff --git a/internal/pkg/pfsprovider/ansible/copy.go b/internal/pkg/pfsprovider/ansible/copy.go deleted file mode 100644 index 3fc8a84b..00000000 --- a/internal/pkg/pfsprovider/ansible/copy.go +++ /dev/null @@ -1,72 +0,0 @@ -package ansible - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "path" - "strings" -) - -func processDataCopy(volume registry.Volume, request registry.DataCopyRequest) error { - cmd, err := generateDataCopyCmd(volume, request) - if err != nil { - return err - } - if cmd == "" { - log.Println("No files to copy for:", volume.Name) - return nil - } - - log.Printf("Doing copy: %s", cmd) - - // Make sure global dir is setup correctly - // TODO: share code with mount better - // TODO: Probably should all get setup in fs-ansible really!! - mountDir := fmt.Sprintf("/mnt/lustre/%s", volume.UUID) - sharedDir := path.Join(mountDir, "/global") - if err := mkdir("localhost", sharedDir); err != nil { - return err - } - if err := fixUpOwnership("localhost", volume.Owner, volume.Group, sharedDir); err != nil { - return err - } - - // Do the copy - return runner.Execute("localhost", cmd) -} - -func generateDataCopyCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { - rsync, err := generateRsyncCmd(volume, request) - if err != nil || rsync == "" { - return "", err - } - - cmd := fmt.Sprintf("sudo -g '#%d' -u '#%d' %s", volume.Group, volume.Owner, rsync) - dacHostBufferPath := fmt.Sprintf("/mnt/lustre/%s/global", volume.UUID) - cmd = fmt.Sprintf("bash -c \"export DW_JOB_STRIPED='%s' && %s\"", dacHostBufferPath, cmd) - return cmd, nil -} - -func generateRsyncCmd(volume registry.Volume, request registry.DataCopyRequest) (string, error) { - if request.Source == "" && request.Destination == "" { - return "", nil - } - - var flags string - if request.SourceType == registry.Directory { - flags = "-r -ospgu --stats" - } else if request.SourceType == registry.File { - flags = "-ospgu --stats" - } else { - return "", fmt.Errorf("unsupported source type %s for volume: %s", request.SourceType, volume.Name) - } - - return fmt.Sprintf("rsync %s %s %s", flags, - escapePath(request.Source), - escapePath(request.Destination)), nil -} - -func escapePath(path string) string { - return strings.Replace(path, "$DW_JOB_STRIPED", "\\$DW_JOB_STRIPED", 1) -} diff --git a/internal/pkg/pfsprovider/ansible/copy_test.go b/internal/pkg/pfsprovider/ansible/copy_test.go deleted file mode 100644 index 71aa8d57..00000000 --- a/internal/pkg/pfsprovider/ansible/copy_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package ansible - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -func Test_GenerateDataCopy(t *testing.T) { - testVolume := registry.Volume{ - Name: registry.VolumeName("asdf"), - Owner: 1001, - Group: 1002, - UUID: "fsuuid", - } - request := registry.DataCopyRequest{} - - cmd, err := generateDataCopyCmd(testVolume, request) - assert.Nil(t, err) - assert.Empty(t, cmd) - - request.SourceType = registry.File - request.Source = "$DW_JOB_STRIPED/source" - request.Destination = "dest" - cmd, err = generateDataCopyCmd(testVolume, request) - assert.Nil(t, err) - assert.Equal(t, "bash -c \"export DW_JOB_STRIPED='/mnt/lustre/fsuuid/global' && sudo -g '#1002' -u '#1001' rsync -ospgu --stats \\$DW_JOB_STRIPED/source dest\"", cmd) - - request.SourceType = registry.List - request.Source = "list_filename" - cmd, err = generateDataCopyCmd(testVolume, request) - assert.Equal(t, "", cmd) - assert.Equal(t, "unsupported source type list for volume: asdf", err.Error()) - -} - -func Test_GenerateRsyncCmd(t *testing.T) { - testVolume := registry.Volume{ - Name: registry.VolumeName("asdf"), - } - request := registry.DataCopyRequest{} - - cmd, err := generateRsyncCmd(testVolume, request) - assert.Nil(t, err) - assert.Empty(t, cmd) - - request.SourceType = registry.File - request.Source = "source" - request.Destination = "dest" - cmd, err = generateRsyncCmd(testVolume, request) - assert.Nil(t, err) - assert.Equal(t, "rsync -ospgu --stats source dest", cmd) - - request.SourceType = registry.Directory - request.Source = "source" - request.Destination = "dest" - cmd, err = generateRsyncCmd(testVolume, request) - assert.Nil(t, err) - assert.Equal(t, "rsync -r -ospgu --stats source dest", cmd) - - request.SourceType = registry.List - request.Source = "list_filename" - cmd, err = generateRsyncCmd(testVolume, request) - assert.Equal(t, "", cmd) - assert.Equal(t, "unsupported source type list for volume: asdf", err.Error()) -} diff --git a/internal/pkg/pfsprovider/ansible/mount.go b/internal/pkg/pfsprovider/ansible/mount.go deleted file mode 100644 index ce3cd0b0..00000000 --- a/internal/pkg/pfsprovider/ansible/mount.go +++ /dev/null @@ -1,308 +0,0 @@ -package ansible - -import ( - "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" - "os" - "os/exec" - "path" - "strconv" - "time" -) - -func getMountDir(volume registry.Volume, jobName string) string { - // TODO: what about the environment variables that are being set? should share logic with here - if volume.MultiJob { - return fmt.Sprintf("/dac/%s_persistent_%s", jobName, volume.Name) - } - return fmt.Sprintf("/dac/%s_job", jobName) -} - -func getLnetSuffix() string { - return os.Getenv("DAC_LNET_SUFFIX") -} - -func getMdtSizeMB() uint { - mdtSizeGB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_GB"), 10, 32) - if err == nil && mdtSizeGB > 0 { - return uint(mdtSizeGB * 1024) - } - mdtSizeMB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_MB"), 10, 32) - if err == nil && mdtSizeMB > 0 { - return uint(mdtSizeMB) - } - return uint(20 * 1024) -} - -func mount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Mount for:", volume.Name) - var primaryBrickHost string - for _, allocation := range brickAllocations { - if allocation.AllocatedIndex == 0 { - primaryBrickHost = allocation.Hostname - break - } - } - - if primaryBrickHost == "" { - log.Panicf("failed to find primary brick for volume: %s", volume.Name) - } - - lnetSuffix := getLnetSuffix() - - if fsType == BeegFS { - // Write out the config needed, and do the mount using ansible - // TODO: Move Lustre mount here that is done below - executeAnsibleMount(fsType, volume, brickAllocations) - } - - for _, attachment := range attachments { - if attachment.State != registry.RequestAttach { - log.Printf("Skipping volume %s attach: %+v", volume.Name, attachment) - continue - } - log.Printf("Volume %s attaching with: %+v", volume.Name, attachment) - - var mountDir = getMountDir(volume, attachment.Job) - if err := mkdir(attachment.Hostname, mountDir); err != nil { - return err - } - if err := mountRemoteFilesystem(fsType, attachment.Hostname, lnetSuffix, - primaryBrickHost, volume.UUID, mountDir); err != nil { - return err - } - - if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { - swapDir := path.Join(mountDir, "/swap") - if err := mkdir(attachment.Hostname, swapDir); err != nil { - return err - } - if err := fixUpOwnership(attachment.Hostname, 0, 0, swapDir); err != nil { - return err - } - - swapSizeMB := int(volume.AttachAsSwapBytes / (1024 * 1024)) - swapFile := path.Join(swapDir, fmt.Sprintf("/%s", attachment.Hostname)) - loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) - if err := createSwap(attachment.Hostname, swapSizeMB, swapFile, loopback); err != nil { - return err - } - - if err := swapOn(attachment.Hostname, loopback); err != nil { - return err - } - } - - if !volume.MultiJob && volume.AttachPrivateNamespace { - privateDir := path.Join(mountDir, fmt.Sprintf("/private/%s", attachment.Hostname)) - if err := mkdir(attachment.Hostname, privateDir); err != nil { - return err - } - if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, privateDir); err != nil { - return err - } - - // need a consistent symlink for shared environment variables across all hosts - privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) - if err := createSymbolicLink(attachment.Hostname, privateDir, privateSymLinkDir); err != nil { - return err - } - } - - sharedDir := path.Join(mountDir, "/global") - if err := mkdir(attachment.Hostname, sharedDir); err != nil { - return err - } - if err := fixUpOwnership(attachment.Hostname, volume.Owner, volume.Group, sharedDir); err != nil { - return err - } - } - // TODO on error should we always call umount? maybe? - // TODO move to ansible style automation or preamble? - return nil -} - -func umount(fsType FSType, volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Umount for:", volume.Name) - - for _, attachment := range attachments { - if attachment.State != registry.RequestDetach { - log.Printf("Skipping volume %s detach for: %+v", volume.Name, attachment) - continue - } - log.Printf("Volume %s dettaching: %+v", volume.Name, attachment) - - var mountDir = getMountDir(volume, attachment.Job) - if !volume.MultiJob && volume.AttachAsSwapBytes > 0 { - swapFile := path.Join(mountDir, fmt.Sprintf("/swap/%s", attachment.Hostname)) // TODO share? - loopback := fmt.Sprintf("/dev/loop%d", volume.ClientPort) // TODO share? - if err := swapOff(attachment.Hostname, loopback); err != nil { - log.Printf("Warn: failed to swap off %+v", attachment) - } - if err := detachLoopback(attachment.Hostname, loopback); err != nil { - log.Printf("Warn: failed to detach loopback %+v", attachment) - } - if err := removeSubtree(attachment.Hostname, swapFile); err != nil { - return err - } - } - - if !volume.MultiJob && volume.AttachPrivateNamespace { - privateSymLinkDir := fmt.Sprintf("/dac/%s_job_private", attachment.Job) - if err := removeSubtree(attachment.Hostname, privateSymLinkDir); err != nil { - return err - } - } - - if fsType == Lustre { - if err := umountLustre(attachment.Hostname, mountDir); err != nil { - return err - } - if err := removeSubtree(attachment.Hostname, mountDir); err != nil { - return err - } - - } - } - - if fsType == BeegFS { - // TODO: Move Lustre unmount here that is done below - executeAnsibleUnmount(fsType, volume, brickAllocations) - // TODO: this makes copy out much harder in its current form :( - } - - return nil -} - -func createSwap(hostname string, swapMB int, filename string, loopback string) error { - file := fmt.Sprintf("dd if=/dev/zero of=%s bs=1024 count=%d", filename, swapMB*1024) - if err := runner.Execute(hostname, file); err != nil { - return err - } - if err := runner.Execute(hostname, fmt.Sprintf("chmod 0600 %s", filename)); err != nil { - return err - } - device := fmt.Sprintf("losetup %s %s", loopback, filename) - if err := runner.Execute(hostname, device); err != nil { - return err - } - swap := fmt.Sprintf("mkswap %s", loopback) - return runner.Execute(hostname, swap) -} - -func swapOn(hostname string, loopback string) error { - return runner.Execute(hostname, fmt.Sprintf("swapon %s", loopback)) -} - -func swapOff(hostname string, loopback string) error { - return runner.Execute(hostname, fmt.Sprintf("swapoff %s", loopback)) -} - -func detachLoopback(hostname string, loopback string) error { - return runner.Execute(hostname, fmt.Sprintf("losetup -d %s", loopback)) -} - -func fixUpOwnership(hostname string, owner uint, group uint, directory string) error { - if err := runner.Execute(hostname, fmt.Sprintf("chown %d:%d %s", owner, group, directory)); err != nil { - return err - } - return runner.Execute(hostname, fmt.Sprintf("chmod 770 %s", directory)) -} - -func umountLustre(hostname string, directory string) error { - // only unmount if already mounted - if err := runner.Execute(hostname, fmt.Sprintf("grep %s /etc/mtab", directory)); err == nil { - // Don't add -l so we can spot when this fails - if err := runner.Execute(hostname, fmt.Sprintf("umount %s", directory)); err != nil { - return err - } - } else { - // TODO: we should really just avoid this being possible? - log.Println("skip umount, as not currently mounted.") - } - return nil -} - -func removeSubtree(hostname string, directory string) error { - return runner.Execute(hostname, fmt.Sprintf("rm -rf %s", directory)) -} - -func createSymbolicLink(hostname string, src string, dest string) error { - return runner.Execute(hostname, fmt.Sprintf("ln -s %s %s", src, dest)) -} - -func mountRemoteFilesystem(fsType FSType, hostname string, lnetSuffix string, mgtHost string, fsname string, directory string) error { - if fsType == Lustre { - return mountLustre(hostname, lnetSuffix, mgtHost, fsname, directory) - } else if fsType == BeegFS { - return mountBeegFS(hostname, mgtHost, fsname, directory) - } - return fmt.Errorf("mount unsuported by filesystem type %s", fsType) -} - -func mountLustre(hostname string, lnetSuffix string, mgtHost string, fsname string, directory string) error { - // We assume modprobe -v lustre is already done - // First check if we are mounted already - if err := runner.Execute(hostname, fmt.Sprintf("grep %s /etc/mtab", directory)); err != nil { - if err := runner.Execute(hostname, fmt.Sprintf( - "mount -t lustre -o flock,nodev,nosuid %s%s:/%s %s", - mgtHost, lnetSuffix, fsname, directory)); err != nil { - return err - } - } - return nil -} - -func mountBeegFS(hostname string, mgtHost string, fsname string, directory string) error { - // Ansible mounts beegfs at /mnt/beegfs/, link into above location here - // First remove the directory, then replace with a symbolic link - if err := removeSubtree(hostname, directory); err != nil { - return err - } - return runner.Execute(hostname, fmt.Sprintf("ln -s /mnt/beegfs/%s %s", fsname, directory)) -} - -func mkdir(hostname string, directory string) error { - return runner.Execute(hostname, fmt.Sprintf("mkdir -p %s", directory)) -} - -type Run interface { - Execute(name string, cmd string) error -} - -type run struct { -} - -func (*run) Execute(hostname string, cmdStr string) error { - log.Println("SSH to:", hostname, "with command:", cmdStr) - - skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") - if skipAnsible == "True" { - log.Println("Skip as DAC_SKIP_ANSIBLE=True") - time.Sleep(time.Millisecond * 200) - return nil - } - - cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", hostname, "sudo", cmdStr) - - timer := time.AfterFunc(time.Minute, func() { - log.Println("Time up, waited more than 5 mins to complete.") - cmd.Process.Kill() - }) - - output, err := cmd.CombinedOutput() - timer.Stop() - - if err == nil { - log.Println("Completed remote ssh run:", cmdStr) - log.Println(string(output)) - return nil - } else { - log.Println("Error in remove ssh run:", string(output)) - return err - } -} - -var runner Run = &run{} diff --git a/internal/pkg/pfsprovider/ansible/mount_test.go b/internal/pkg/pfsprovider/ansible/mount_test.go deleted file mode 100644 index 8dd45f74..00000000 --- a/internal/pkg/pfsprovider/ansible/mount_test.go +++ /dev/null @@ -1,273 +0,0 @@ -package ansible - -import ( - "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "github.com/stretchr/testify/assert" - "testing" -) - -type fakeRunner struct { - err error - calls int - hostnames []string - cmdStrs []string -} - -func (f *fakeRunner) Execute(hostname string, cmdStr string) error { - f.calls += 1 - f.hostnames = append(f.hostnames, hostname) - f.cmdStrs = append(f.cmdStrs, cmdStr) - if cmdStr == "grep /dac/job1_job /etc/mtab" { - return errors.New("trigger mount") - } - return f.err -} - -func Test_mkdir(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - - err := mkdir("host", "dir") - assert.Nil(t, err) - assert.Equal(t, "host", fake.hostnames[0]) - assert.Equal(t, "mkdir -p dir", fake.cmdStrs[0]) - - runner = &fakeRunner{err: errors.New("expected")} - err = mkdir("", "") - assert.Equal(t, "expected", err.Error()) -} - -func Test_mountLustre(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - - err := mountLustre("host", "-opa@o2ib1", "mgt", "fs", "/dac/job1_job") - assert.Nil(t, err) - assert.Equal(t, 2, fake.calls) - assert.Equal(t, "host", fake.hostnames[0]) - assert.Equal(t, "host", fake.hostnames[1]) - assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[0]) - assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid mgt-opa@o2ib1:/fs /dac/job1_job", fake.cmdStrs[1]) - - fake = &fakeRunner{err: errors.New("expected")} - runner = fake - err = mountRemoteFilesystem(Lustre, "host", "", "mgt", "fs", "asdf") - assert.Equal(t, "expected", err.Error()) - assert.Equal(t, 2, fake.calls) - assert.Equal(t, "grep asdf /etc/mtab", fake.cmdStrs[0]) - assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid mgt:/fs asdf", fake.cmdStrs[1]) -} - -func Test_createSwap(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - - err := createSwap("host", 3, "file", "loopback") - assert.Nil(t, err) - assert.Equal(t, "host", fake.hostnames[0]) - assert.Equal(t, "host", fake.hostnames[1]) - assert.Equal(t, "host", fake.hostnames[2]) - assert.Equal(t, 4, len(fake.cmdStrs)) - assert.Equal(t, "dd if=/dev/zero of=file bs=1024 count=3072", fake.cmdStrs[0]) - assert.Equal(t, "chmod 0600 file", fake.cmdStrs[1]) - assert.Equal(t, "losetup loopback file", fake.cmdStrs[2]) - assert.Equal(t, "mkswap loopback", fake.cmdStrs[3]) -} - -func Test_fixUpOwnership(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - - err := fixUpOwnership("host", 10, 11, "dir") - assert.Nil(t, err) - - assert.Equal(t, 2, fake.calls) - assert.Equal(t, "host", fake.hostnames[0]) - assert.Equal(t, "chown 10:11 dir", fake.cmdStrs[0]) - assert.Equal(t, "host", fake.hostnames[1]) - assert.Equal(t, "chmod 770 dir", fake.cmdStrs[1]) -} - -func Test_Mount(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, - {Hostname: "client2", Job: "job1", State: registry.RequestAttach}, - {Hostname: "client3", Job: "job3", State: registry.Attached}, - {Hostname: "client3", Job: "job3", State: registry.RequestDetach}, - {Hostname: "client3", Job: "job3", State: registry.Detached}, - {Hostname: "client2", Job: "job2", State: registry.RequestAttach}, - } - volume := registry.Volume{ - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 1024 * 1024, // 1 MiB - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - - assert.PanicsWithValue(t, - "failed to find primary brick for volume: asdf", - func() { mount(Lustre, volume, nil, nil) }) - - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - err := mount(Lustre, volume, bricks, attachments) - assert.Nil(t, err) - assert.Equal(t, 53, fake.calls) - - assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[0]) - assert.Equal(t, "grep /dac/job1_job /etc/mtab", fake.cmdStrs[1]) - assert.Equal(t, "mount -t lustre -o flock,nodev,nosuid host1:/ /dac/job1_job", fake.cmdStrs[2]) - - assert.Equal(t, "mkdir -p /dac/job1_job/swap", fake.cmdStrs[3]) - assert.Equal(t, "chown 0:0 /dac/job1_job/swap", fake.cmdStrs[4]) - assert.Equal(t, "chmod 770 /dac/job1_job/swap", fake.cmdStrs[5]) - assert.Equal(t, "dd if=/dev/zero of=/dac/job1_job/swap/client1 bs=1024 count=1024", fake.cmdStrs[6]) - assert.Equal(t, "chmod 0600 /dac/job1_job/swap/client1", fake.cmdStrs[7]) - assert.Equal(t, "losetup /dev/loop42 /dac/job1_job/swap/client1", fake.cmdStrs[8]) - assert.Equal(t, "mkswap /dev/loop42", fake.cmdStrs[9]) - assert.Equal(t, "swapon /dev/loop42", fake.cmdStrs[10]) - assert.Equal(t, "mkdir -p /dac/job1_job/private/client1", fake.cmdStrs[11]) - assert.Equal(t, "chown 1001:1001 /dac/job1_job/private/client1", fake.cmdStrs[12]) - assert.Equal(t, "chmod 770 /dac/job1_job/private/client1", fake.cmdStrs[13]) - assert.Equal(t, "ln -s /dac/job1_job/private/client1 /dac/job1_job_private", fake.cmdStrs[14]) - - assert.Equal(t, "mkdir -p /dac/job1_job/global", fake.cmdStrs[15]) - assert.Equal(t, "chown 1001:1001 /dac/job1_job/global", fake.cmdStrs[16]) - assert.Equal(t, "chmod 770 /dac/job1_job/global", fake.cmdStrs[17]) - - assert.Equal(t, "client2", fake.hostnames[18]) - assert.Equal(t, "mkdir -p /dac/job1_job", fake.cmdStrs[18]) - - assert.Equal(t, "client2", fake.hostnames[36]) - assert.Equal(t, "mkdir -p /dac/job2_job", fake.cmdStrs[36]) - assert.Equal(t, "client2", fake.hostnames[52]) - assert.Equal(t, "chmod 770 /dac/job2_job/global", fake.cmdStrs[52]) -} - -func Test_Umount(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job4", State: registry.RequestDetach}, - {Hostname: "client2", Job: "job4", State: registry.RequestDetach}, - {Hostname: "client3", Job: "job3", State: registry.Attached}, - {Hostname: "client3", Job: "job3", State: registry.RequestAttach}, - {Hostname: "client3", Job: "job3", State: registry.Detached}, - {Hostname: "client2", Job: "job1", State: registry.RequestDetach}, - } - volume := registry.Volume{ - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - err := umount(Lustre, volume, bricks, attachments) - assert.Nil(t, err) - assert.Equal(t, 20, fake.calls) - - assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[0]) - assert.Equal(t, "losetup -d /dev/loop42", fake.cmdStrs[1]) - assert.Equal(t, "rm -rf /dac/job4_job/swap/client1", fake.cmdStrs[2]) - assert.Equal(t, "rm -rf /dac/job4_job_private", fake.cmdStrs[3]) - assert.Equal(t, "grep /dac/job4_job /etc/mtab", fake.cmdStrs[4]) - assert.Equal(t, "umount /dac/job4_job", fake.cmdStrs[5]) - assert.Equal(t, "rm -rf /dac/job4_job", fake.cmdStrs[6]) - - assert.Equal(t, "client2", fake.hostnames[7]) - assert.Equal(t, "swapoff /dev/loop42", fake.cmdStrs[7]) - - assert.Equal(t, "client2", fake.hostnames[19]) - assert.Equal(t, "rm -rf /dac/job1_job", fake.cmdStrs[19]) -} - -func Test_Umount_multi(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestDetach}, - } - volume := registry.Volume{ - MultiJob: true, - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - err := umount(Lustre, volume, bricks, attachments) - assert.Nil(t, err) - assert.Equal(t, 3, fake.calls) - - assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "grep /dac/job1_persistent_asdf /etc/mtab", fake.cmdStrs[0]) - assert.Equal(t, "umount /dac/job1_persistent_asdf", fake.cmdStrs[1]) - assert.Equal(t, "rm -rf /dac/job1_persistent_asdf", fake.cmdStrs[2]) -} - -func Test_Mount_multi(t *testing.T) { - defer func() { runner = &run{} }() - fake := &fakeRunner{} - runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, - } - volume := registry.Volume{ - MultiJob: true, - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - UUID: "medkDfdg", - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - err := mount(Lustre, volume, bricks, attachments) - assert.Nil(t, err) - assert.Equal(t, 5, fake.calls) - - assert.Equal(t, "client1", fake.hostnames[0]) - assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf", fake.cmdStrs[0]) - assert.Equal(t, "grep /dac/job1_persistent_asdf /etc/mtab", fake.cmdStrs[1]) - assert.Equal(t, "mkdir -p /dac/job1_persistent_asdf/global", fake.cmdStrs[2]) - assert.Equal(t, "chown 1001:1001 /dac/job1_persistent_asdf/global", fake.cmdStrs[3]) - assert.Equal(t, "chmod 770 /dac/job1_persistent_asdf/global", fake.cmdStrs[4]) -} diff --git a/internal/pkg/pfsprovider/ansible/plugin.go b/internal/pkg/pfsprovider/ansible/plugin.go deleted file mode 100644 index a55f0678..00000000 --- a/internal/pkg/pfsprovider/ansible/plugin.go +++ /dev/null @@ -1,96 +0,0 @@ -package ansible - -import ( - "bytes" - "encoding/json" - "github.com/RSE-Cambridge/data-acc/internal/pkg/pfsprovider" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" -) - -func GetPlugin(fsType FSType) pfsprovider.Plugin { - return &plugin{FSType: fsType} -} - -type FSType int - -const ( - BeegFS FSType = iota - Lustre -) - -var fsTypeStrings = map[FSType]string{ - BeegFS: "BeegFS", - Lustre: "Lustre", -} -var stringToFSType = map[string]FSType{ - "": BeegFS, - "BeegFS": BeegFS, - "Lustre": Lustre, -} - -func (fsType FSType) String() string { - return fsTypeStrings[fsType] -} - -func (fsType FSType) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString(`"`) - buffer.WriteString(fsTypeStrings[fsType]) - buffer.WriteString(`"`) - return buffer.Bytes(), nil -} - -func (fsType *FSType) UnmarshalJSON(b []byte) error { - var str string - err := json.Unmarshal(b, &str) - if err != nil { - return err - } - *fsType = stringToFSType[str] - return nil -} - -type plugin struct { - FSType FSType -} - -func (plugin *plugin) Mounter() pfsprovider.Mounter { - return &mounter{FSType: plugin.FSType} -} - -func (plugin *plugin) VolumeProvider() pfsprovider.VolumeProvider { - return &volumeProvider{FSType: plugin.FSType} -} - -type volumeProvider struct { - FSType FSType -} - -func (volProvider *volumeProvider) SetupVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - return executeAnsibleSetup(volProvider.FSType, volume, brickAllocations) -} - -func (volProvider *volumeProvider) TeardownVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - return executeAnsibleTeardown(volProvider.FSType, volume, brickAllocations) -} - -func (*volumeProvider) CopyDataIn(volume registry.Volume) error { - // TODO we should support multiple stagein commands! oops! - return processDataCopy(volume, volume.StageIn) -} - -func (*volumeProvider) CopyDataOut(volume registry.Volume) error { - // TODO we should support multiple stageout commands too! oops! - return processDataCopy(volume, volume.StageOut) -} - -type mounter struct { - FSType FSType -} - -func (mounter *mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - return mount(mounter.FSType, volume, brickAllocations, attachments) -} - -func (mounter *mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - return umount(mounter.FSType, volume, brickAllocations, attachments) -} diff --git a/internal/pkg/pfsprovider/fake/plugin.go b/internal/pkg/pfsprovider/fake/plugin.go deleted file mode 100644 index 1583c43c..00000000 --- a/internal/pkg/pfsprovider/fake/plugin.go +++ /dev/null @@ -1,55 +0,0 @@ -package fake - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/pfsprovider" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" - "log" -) - -func GetPlugin() pfsprovider.Plugin { - return &plugin{} -} - -type plugin struct{} - -func (*plugin) Mounter() pfsprovider.Mounter { - return &mounter{} -} - -func (*plugin) VolumeProvider() pfsprovider.VolumeProvider { - return &volumeProvider{} -} - -type volumeProvider struct{} - -func (*volumeProvider) SetupVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - log.Println("SetupVolume for:", volume.Name) - return nil -} - -func (*volumeProvider) TeardownVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error { - log.Println("TeardownVolume for:", volume.Name) - return nil -} - -func (*volumeProvider) CopyDataIn(volume registry.Volume) error { - log.Println("CopyDataIn for:", volume.Name) - return nil -} - -func (*volumeProvider) CopyDataOut(volume registry.Volume) error { - log.Println("CopyDataOut for:", volume.Name) - return nil -} - -type mounter struct{} - -func (*mounter) Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Mount for:", volume.Name) - return nil -} - -func (*mounter) Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error { - log.Println("Umount for:", volume.Name) - return nil -} diff --git a/internal/pkg/pfsprovider/interface.go b/internal/pkg/pfsprovider/interface.go deleted file mode 100644 index 4bbc85d0..00000000 --- a/internal/pkg/pfsprovider/interface.go +++ /dev/null @@ -1,27 +0,0 @@ -package pfsprovider - -import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" -) - -// A plugin must provide implementations for both interfaces -type Plugin interface { - Mounter() Mounter - VolumeProvider() VolumeProvider -} - -// Actions on the host assigned to the primary brick -type VolumeProvider interface { - SetupVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error - TeardownVolume(volume registry.Volume, brickAllocations []registry.BrickAllocation) error - - CopyDataIn(volume registry.Volume) error - CopyDataOut(volume registry.Volume) error -} - -// Actions that are sent to remote hosts, -// typically compute nodes and primary brick hosts -type Mounter interface { - Mount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error - Unmount(volume registry.Volume, brickAllocations []registry.BrickAllocation, attachments []registry.Attachment) error -} diff --git a/internal/pkg/registry/pool.go b/internal/pkg/registry/pool.go deleted file mode 100644 index 09acd4d8..00000000 --- a/internal/pkg/registry/pool.go +++ /dev/null @@ -1,141 +0,0 @@ -package registry - -import ( - "context" - "encoding/json" - "log" -) - -type PoolRegistry interface { - // Returns a summary of the current state of all pools, including the bricks in each pool - Pools() ([]Pool, error) - - // TODO: Pool(name string) (Pool, error) - - // Update (or add) information on what bricks are present - // - // Note: it is possible to have bricks from multiple pools on a single host - // If any bricks that were previously registered have gone away, - // they will be removed, unless there is an associated BrickAllocation which will - // cause the update to fail and returns an error. - // If any bricks in the same pool have a different capacity, - // the update fails and returns an error. - UpdateHost(bricks []BrickInfo) error - - // While the process is still running this notifies others the host is up - // - // When a host is dead non of its bricks will get new volumes assigned, - // and no bricks will get cleaned up until the next service start. - // Error will be returned if the host info has not yet been written. - KeepAliveHost(hostname string) error - - // Update a brick with allocation information. - // - // No update is made and an error is returned if: - // any brick already has an allocation, - // or any volume a brick is being assigned to already has an allocation, - // or if any of the volumes do not exist - // or if there is not exactly one primary brick. - // - // Note: you may assign multiple volumes in a single call, but all bricks - // for a particular volume must be set in a single call - AllocateBricksForVolume(volume Volume) ([]BrickAllocation, error) - - // Deallocate all bricks associated with the given volume - // - // No update is made and an error is returned if any of brick allocations don't match the current state. - // If any host associated with one of the bricks is down, an error is returned and the deallocate is - // recorded as requested and not executed. - // Note: this returns as soon as deallocate is requested, doesn't wait for cleanup completion - DeallocateBricks(volume VolumeName) error - - // This is called after DeallocateBricks has been processed - HardDeleteAllocations(allocations []BrickAllocation) error - - // Get all the allocations for bricks associated with the specified hostname - GetAllocationsForHost(hostname string) ([]BrickAllocation, error) - - // Get all the allocations for bricks associated with the specific volume - GetAllocationsForVolume(volume VolumeName) ([]BrickAllocation, error) - - // Get information on a specific brick - GetBrickInfo(hostname string, device string) (BrickInfo, error) - - // Returns a channel that reports all new brick allocations for given hostname - // - // The channel is closed when the context is cancelled or timeout. - // Any errors in the watching log the issue and panic - GetNewHostBrickAllocations(ctxt context.Context, hostname string) <-chan BrickAllocation -} - -type Pool struct { - // The pool is derived from all the reported bricks - // It must only contain the characters A-Za-z0-9 - Name string // TODO: should we create PoolName type? - - // Returns all unallocated bricks in this pool associated with a live host - AvailableBricks []BrickInfo - - // Returns all brick allocations for this pool - AllocatedBricks []BrickAllocation - - // This is the allocation unit for the pool - // It is the minimum size of any registered brick - GranularityGB uint - - // List of all hosts that report bricks in this pool - Hosts map[string]HostInfo -} - -func (pool Pool) String() string { - poolString, err := json.Marshal(pool) - if err != nil { - log.Fatal(err) - } - return string(poolString) -} - -type HostInfo struct { - // It must only contain the characters "A-Za-z0-9." - Hostname string - - // True if data accelerator process is thought to be running - Alive bool -} - -type BrickInfo struct { - // Bricks are identified by device and hostname - // It must only contain the characters A-Za-z0-9 - Device string - - // It must only contain the characters "A-Za-z0-9." - Hostname string - - // The bool a brick is associated with - // It must only contain the characters A-Za-z0-9 - PoolName string - - // Size of the brick, defines the pool granularity - CapacityGB uint -} - -type BrickAllocation struct { - // Bricks are identified by device and hostname - // It must only contain the characters A-Za-z0-9 - Device string - - // It must only contain the characters "A-Za-z0-9." - Hostname string - - // Name of the volume that owns the brick - AllocatedVolume VolumeName - - // 0 index allocation is the primary brick, - // which is responsible for provisioning the associated volume - AllocatedIndex uint - - // If any allocation sent to deallocate has a host that isn't - // alive, this flag is set rather than have allocations removed. - // A host should check for any allocations - DeallocateRequested bool -} diff --git a/internal/pkg/registry/volume.go b/internal/pkg/registry/volume.go deleted file mode 100644 index 03855174..00000000 --- a/internal/pkg/registry/volume.go +++ /dev/null @@ -1,347 +0,0 @@ -package registry - -import ( - "bytes" - "context" - "encoding/json" -) - -type VolumeRegistry interface { - // Get all registered jobs and their volumes - Jobs() ([]Job, error) - - // Get a specific job - Job(jobName string) (Job, error) - - // Add job and associated volumes - // Fails to add job if volumes are in a bad state - AddJob(job Job) error - - // Update specified job with given hosts - // Fails if the job already has any hosts associated with it - JobAttachHosts(jobName string, hosts []string) error - - // Remove job from the system - // TODO: fails if volumes are not in the deleted state? - DeleteJob(jobName string) error - - // Get information about specific volume - // TODO: remove add/detele and only - AddVolume(volume Volume) error - - // Get information about a specific volume - Volume(name VolumeName) (Volume, error) - - // Get information about all volumes - AllVolumes() ([]Volume, error) - - // TODO: this should error if volume is not in correct state? - DeleteVolume(name VolumeName) error - - // Move between volume states, but only one by one - UpdateState(name VolumeName, state VolumeState) error - - // Update all the specified attachments - // if attachment doesn't exist, attachment is added - UpdateVolumeAttachments(name VolumeName, attachments []Attachment) error - - // Delete all the specified attachments - DeleteVolumeAttachments(name VolumeName, hostnames []string, jobName string) error - - // Wait for a specific state, error returned if not possible - WaitForState(name VolumeName, state VolumeState) error - - // Wait for a given condition - // - // Blocks until condition returns true, or returns error if things timeout - WaitForCondition(volumeName VolumeName, condition func(event *VolumeChange) bool) error - - // Gets all changes that happen to the given volume - // - // To stop watching cancel or timeout the context, this will close the channel. - GetVolumeChanges(ctx context.Context, volume Volume) VolumeChangeChan - - // Get a new mutex associated with the specified key - VolumeOperationMutex(volumeName VolumeName) (Mutex, error) -} - -type Mutex interface { - Lock(ctx context.Context) error - Unlock(ctx context.Context) error -} - -type VolumeChangeChan <-chan VolumeChange - -type VolumeChange struct { - New *Volume - Old *Volume - IsDelete bool - Err error -} - -// TODO: Attachment request, or session is probably a better name here... -type Job struct { - // Name of the job - Name string // TODO: should we make a JobName type? - Owner uint - CreatedAt uint - - // The hosts that want to mount the storage - // Note: to allow for copy in/out the brick hosts are assumed to have an attachment - AttachHosts []string - - // If non-zero capacity requested, a volume is created for this job - // It may be exposed to the attach hosts in a variety of ways, as defined by the volume - JobVolume VolumeName - - // There maybe be attachments to multiple shared volumes - MultiJobVolumes []VolumeName - - // Environment variables for each volume associated with the job - Paths map[string]string -} - -type VolumeName string - -// Volume information -// To get assigned bricks see PoolRegistry -type Volume struct { - // e.g. job1 or Foo - Name VolumeName - // its 8 characters long, so works nicely with lustre - UUID string - // True if multiple jobs can attach to this volume - MultiJob bool - - // Message requested actions to primary brick host - // TODO: move mount and data copy actions to other parts of the volume state - State VolumeState - - // Requested pool of bricks for volume - Pool string // TODO: PoolName? - // Number of bricks requested, calculated from requested capacity - SizeBricks uint - // Actual size of the volume - SizeGB uint - - // Back reference to what job created this volume - JobName string - // e.g. 1001 - Owner uint - // If empty defaults to User - Group uint - // e.g. SLURM or Manila - CreatedBy string - // The unix (utc) timestamp of when this volume was created - CreatedAt uint - - // TODO: need to fill these in... - // They all related to how the volume is attached - - // All current attachments - Attachments []Attachment - // Attach all attachments to a shared global namespace - // Allowed for any volume type - AttachGlobalNamespace bool - // Have an attachment specific namespace mounted, only for non multi job - AttachPrivateNamespace bool - // If not zero, swap of the requested amount mounted for each attachment - // Not allowed for multi job - AttachAsSwapBytes uint - // Add attachment specific cache for each given filesystem path - // Not allowed for multi job - // Note: assumes the same path is cached for all attachments - AttachPrivateCache []string - - // TODO: maybe data copy should be a slice associated with the job? - // Request certain files to be staged in - // Not currently allowed for multi job volumes - StageIn DataCopyRequest - // Request certain files to be staged in - // Not currently allowed for multi job volumes - StageOut DataCopyRequest - - // BeeGFS wants each fs to be assigned a unique port number - ClientPort int - - // Track if we have had bricks assigned - // if we request delete, no bricks ever assigned, don't ait for dacd! - HadBricksAssigned bool - - // TODO: data datamodel currently does not do these things well: - // 1. correctly track multiple jobs at the same time attach to the same persistent buffer - // 2. data in/out requests for persistent buffer - // 3. track amount of space used by swap and/or metadata -} - -func (volume Volume) String() string { - rawVolume, _ := json.Marshal(volume) - return string(rawVolume) -} - -type VolumeState int - -const ( - Unknown VolumeState = iota - Registered - BricksAllocated - BricksProvisioned // setup waits for this, updated by host manager, paths should be setup, or gone to ERROR - DataInRequested - DataInComplete // data_in waits for host manager to data in, or gone to ERROR - DataOutRequested - DataOutComplete // data copied out by host manager, or gone to ERROR - DeleteRequested VolumeState = 399 - BricksDeleted VolumeState = 400 // all bricks correctly deprovisioned unless host down or gone to ERROR - Error VolumeState = 500 -) - -var volumeStateStrings = map[VolumeState]string{ - Unknown: "", - Registered: "Registered", - BricksAllocated: "BricksAllocated", - BricksProvisioned: "BricksProvisioned", - DataInRequested: "DataInRequested", - DataInComplete: "DataInComplete", - DataOutRequested: "DataOutRequested", - DataOutComplete: "DataOutComplete", - DeleteRequested: "DeleteRequested", - BricksDeleted: "BricksDeleted", - Error: "Error", -} -var stringToVolumeState = map[string]VolumeState{ - "": Unknown, - "Registered": Registered, - "BricksAllocated": BricksAllocated, - "BricksProvisioned": BricksProvisioned, - "DataInRequested": DataInRequested, - "DataInComplete": DataInComplete, - "DataOutRequested": DataOutRequested, - "DataOutComplete": DataOutComplete, - "DeleteRequested": DeleteRequested, - "BricksDeleted": BricksDeleted, - "Error": Error, -} - -func (volumeState VolumeState) String() string { - return volumeStateStrings[volumeState] -} - -func (volumeState VolumeState) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString(`"`) - buffer.WriteString(volumeStateStrings[volumeState]) - buffer.WriteString(`"`) - return buffer.Bytes(), nil -} - -func (volumeState *VolumeState) UnmarshalJSON(b []byte) error { - var str string - err := json.Unmarshal(b, &str) - if err != nil { - return err - } - *volumeState = stringToVolumeState[str] - return nil -} - -type DataCopyRequest struct { - // Source points to a File or a Directory, - // or a file that contains a list of source and destinations, - // with each pair on a new line - SourceType SourceType - // The path is either to a file or a directory or a - Source string - // Must be empty string for type list, otherwise specifes location - Destination string - // Used to notify if copy in has been requested - // TODO: remove volume states and update this instead - RequestCopyIn bool - // Report if the copy has completed - CopyCompleted bool - // if there was problem, record it - Error error -} - -type SourceType string - -const ( - File SourceType = "file" - Directory SourceType = "directory" - // Provide a file that has source and destination file space separated pairs, each on a new line - List SourceType = "list" -) - -type Attachment struct { - // BrickHostName, Job and Volume name uniquely identify an attachment - Hostname string - - // Associated jobName - Job string - - State AttachmentState - - // If any error happened, it is reported here - Error error -} - -type AttachmentState int - -const ( - UnknownAttachmentState AttachmentState = iota - RequestAttach - Attached - RequestDetach - Detached AttachmentState = 400 // all bricks correctly deprovisioned unless host down or gone to ERROR - AttachmentError AttachmentState = 500 -) - -var attachStateStrings = map[AttachmentState]string{ - UnknownAttachmentState: "", - RequestAttach: "RequestAttach", - Attached: "Attached", - RequestDetach: "RequestDetach", - Detached: "Detached", - AttachmentError: "AttachmentError", -} -var stringToAttachmentState = map[string]AttachmentState{ - "": UnknownAttachmentState, - "RequestAttach": RequestAttach, - "Attached": Attached, - "RequestDetach": RequestDetach, - "Detached": Detached, - "AttachmentError": AttachmentError, -} - -func (attachmentState AttachmentState) String() string { - return attachStateStrings[attachmentState] -} - -func (attachmentState AttachmentState) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString(`"`) - buffer.WriteString(attachStateStrings[attachmentState]) - buffer.WriteString(`"`) - return buffer.Bytes(), nil -} - -func (attachmentState *AttachmentState) UnmarshalJSON(b []byte) error { - var str string - err := json.Unmarshal(b, &str) - if err != nil { - return err - } - *attachmentState = stringToAttachmentState[str] - return nil -} - -func (volume Volume) FindMatchingAttachment(attachment Attachment) (*Attachment, bool) { - return volume.FindAttachment(attachment.Hostname, attachment.Job) -} - -func (volume Volume) FindAttachment(hostname string, jobName string) (*Attachment, bool) { - for _, candidate := range volume.Attachments { - if candidate.Hostname == hostname && candidate.Job == jobName { - // TODO: double check for duplicate match? - return &candidate, true - } - } - return nil, false -} diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go index 4f964b75..ff828a54 100644 --- a/internal/pkg/v2/filesystem_impl/mount_test.go +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -2,10 +2,8 @@ package filesystem_impl import ( "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/stretchr/testify/assert" - "log" "testing" ) @@ -99,32 +97,7 @@ func Test_Mount(t *testing.T) { defer func() { runner = &run{} }() fake := &fakeRunner{} runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, - {Hostname: "client2", Job: "job1", State: registry.RequestAttach}, - {Hostname: "client3", Job: "job3", State: registry.Attached}, - {Hostname: "client3", Job: "job3", State: registry.RequestDetach}, - {Hostname: "client3", Job: "job3", State: registry.Detached}, - {Hostname: "client2", Job: "job2", State: registry.RequestAttach}, - } - volume := registry.Volume{ - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 1024 * 1024, // 1 MiB - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - log.Println(bricks) - log.Println(volume) - //err := mount(Lustre, volume, bricks, attachments) sessionName := datamodel.SessionName("job1") internalName := "fsuuid" primaryBrickHost := datamodel.BrickHostName("host1") @@ -170,30 +143,6 @@ func Test_Umount(t *testing.T) { defer func() { runner = &run{} }() fake := &fakeRunner{} runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job4", State: registry.RequestDetach}, - {Hostname: "client2", Job: "job4", State: registry.RequestDetach}, - {Hostname: "client3", Job: "job3", State: registry.Attached}, - {Hostname: "client3", Job: "job3", State: registry.RequestAttach}, - {Hostname: "client3", Job: "job3", State: registry.Detached}, - {Hostname: "client2", Job: "job1", State: registry.RequestDetach}, - } - volume := registry.Volume{ - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - log.Println(bricks) - log.Println(volume) sessionName := datamodel.SessionName("job4") internalName := "fsuuid" @@ -226,26 +175,6 @@ func Test_Umount_multi(t *testing.T) { defer func() { runner = &run{} }() fake := &fakeRunner{} runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestDetach}, - } - volume := registry.Volume{ - MultiJob: true, - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - log.Println(bricks) - log.Println(volume) sessionName := datamodel.SessionName("asdf") internalName := "uuidasdf" @@ -275,27 +204,6 @@ func Test_Mount_multi(t *testing.T) { defer func() { runner = &run{} }() fake := &fakeRunner{} runner = fake - attachments := []registry.Attachment{ - {Hostname: "client1", Job: "job1", State: registry.RequestAttach}, - } - volume := registry.Volume{ - MultiJob: true, - Name: "asdf", JobName: "asdf", - AttachGlobalNamespace: true, - AttachPrivateNamespace: true, - AttachAsSwapBytes: 10000, - Attachments: attachments, - ClientPort: 42, - Owner: 1001, - Group: 1001, - UUID: "medkDfdg", - } - bricks := []registry.BrickAllocation{ - {Hostname: "host1"}, - {Hostname: "host2"}, - } - log.Println(bricks) - log.Println(volume) sessionName := datamodel.SessionName("asdf") internalName := "uuidasdf" From be5a0d8bfb730e30aa3a79c20bd7ae76df88fd0e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:33:31 +0100 Subject: [PATCH 143/191] Fix up fileio mocks --- build/rebuild_mocks.sh | 2 +- internal/pkg/mock_fileio/disk_mock.go | 62 +++++++++++++++++++ .../v2/dacctl/actions_impl/actions_test.go | 12 ++-- .../pkg/v2/dacctl/actions_impl/job_test.go | 6 +- .../actions_impl/parsers/hostnames_test.go | 8 +-- .../dacctl/actions_impl/parsers/job_test.go | 6 +- .../pkg/v2/dacctl/actions_impl/show_test.go | 4 +- .../pkg/v2/mock_registry/brick_allocation.go | 10 +-- internal/pkg/v2/mock_registry/brick_host.go | 8 +-- internal/pkg/v2/mock_registry/session.go | 12 ++-- .../pkg/v2/mock_registry/session_actions.go | 6 +- 11 files changed, 99 insertions(+), 37 deletions(-) create mode 100644 internal/pkg/mock_fileio/disk_mock.go diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index c3e98f55..27c620e8 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -9,7 +9,7 @@ mkdir -p internal/pkg/mocks items="disk" for i in $items; do mockgen -source=internal/pkg/fileio/${i}.go \ - -package mocks >internal/pkg/mocks/${i}_mock.go + >internal/pkg/mock_fileio/${i}_mock.go done items="session session_action_handler" diff --git a/internal/pkg/mock_fileio/disk_mock.go b/internal/pkg/mock_fileio/disk_mock.go new file mode 100644 index 00000000..78beb650 --- /dev/null +++ b/internal/pkg/mock_fileio/disk_mock.go @@ -0,0 +1,62 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/pkg/fileio/disk.go + +// Package mock_fileio is a generated GoMock package. +package mock_fileio + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockDisk is a mock of Disk interface +type MockDisk struct { + ctrl *gomock.Controller + recorder *MockDiskMockRecorder +} + +// MockDiskMockRecorder is the mock recorder for MockDisk +type MockDiskMockRecorder struct { + mock *MockDisk +} + +// NewMockDisk creates a new mock instance +func NewMockDisk(ctrl *gomock.Controller) *MockDisk { + mock := &MockDisk{ctrl: ctrl} + mock.recorder = &MockDiskMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockDisk) EXPECT() *MockDiskMockRecorder { + return m.recorder +} + +// Lines mocks base method +func (m *MockDisk) Lines(filename string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Lines", filename) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Lines indicates an expected call of Lines +func (mr *MockDiskMockRecorder) Lines(filename interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Lines", reflect.TypeOf((*MockDisk)(nil).Lines), filename) +} + +// Write mocks base method +func (m *MockDisk) Write(filename string, lines []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", filename, lines) + ret0, _ := ret[0].(error) + return ret0 +} + +// Write indicates an expected call of Write +func (mr *MockDiskMockRecorder) Write(filename, lines interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockDisk)(nil).Write), filename, lines) +} diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index a58b44c7..c6e66144 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -3,7 +3,7 @@ package actions_impl import ( "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" @@ -115,7 +115,7 @@ func TestDacctlActions_PreRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} loginHosts := []string{"login"} @@ -146,7 +146,7 @@ func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) @@ -167,7 +167,7 @@ func TestDacctlActions_PreRun_NoLoginHosts(t *testing.T) { func TestDacctlActions_PreRun_BadHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host/2"} disk.EXPECT().Lines("computehostfile").Return(computeHosts, nil) @@ -187,7 +187,7 @@ func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) computeHosts := []string{"host1", "host2"} loginHosts := []string{"login/asdf"} @@ -209,7 +209,7 @@ func TestDacctlActions_PreRun_BadLoginHosts(t *testing.T) { func TestDacctlActions_PreRun_NoHosts(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) disk.EXPECT().Lines("computehostfile").Return(nil, nil) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index bea7f91e..7a2100f2 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -1,7 +1,7 @@ package actions_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" @@ -13,7 +13,7 @@ func TestDacctlActions_ValidateJob_BadInput(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) lines := []string{`#DW bad cmd`} disk.EXPECT().Lines("jobfile").Return(lines, nil) @@ -34,7 +34,7 @@ func TestDacctlActions_CreatePerJobBuffer(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) lines := []string{ `#DW jobdw capacity=4MiB access_mode=striped,private type=scratch`, diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go index da163da1..f6bfa94f 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go @@ -2,7 +2,7 @@ package parsers import ( "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" @@ -11,7 +11,7 @@ import ( func TestGetHostnamesFromFile(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) fakeHosts := []string{"test1", "test2"} disk.EXPECT().Lines("file").Return(fakeHosts, nil) @@ -24,7 +24,7 @@ func TestGetHostnamesFromFile(t *testing.T) { func TestGetHostnamesFromFile_Empty(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) disk.EXPECT().Lines("file").Return(nil, nil) @@ -43,7 +43,7 @@ func TestGetHostnamesFromFile_Empty(t *testing.T) { func TestGetHostnamesFromFile_ErrorOnBadHostname(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) fakeHosts := []string{"Test", "test", "test1", "test2.com", "bad hostname", "foo/bar", ""} disk.EXPECT().Lines("file").Return(fakeHosts, nil) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index d823eb42..35f49655 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -2,7 +2,7 @@ package parsers import ( "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -14,14 +14,14 @@ func TestParseJobFile(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - mockFileIO := mocks.NewMockDisk(mockCtrl) + mockFileIO := mock_fileio.NewMockDisk(mockCtrl) mockFileIO.EXPECT().Lines("testfile").Return(nil, errors.New("asdf")) lines, err := ParseJobFile(mockFileIO, "testfile") assert.Equal(t, jobSummary{}, lines) assert.Equal(t, "asdf", err.Error()) - mockFileIO = mocks.NewMockDisk(mockCtrl) + mockFileIO = mock_fileio.NewMockDisk(mockCtrl) mockFileIO.EXPECT().Lines("testfile").Return([]string{`#DW swap asdf`}, nil) lines, err = ParseJobFile(mockFileIO, "testfile") diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index f1004cc1..deb860ee 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -2,7 +2,7 @@ package actions_impl import ( "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/mocks" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" @@ -35,7 +35,7 @@ func TestDacctlActions_Paths(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() session := mock_facade.NewMockSession(mockCtrl) - disk := mocks.NewMockDisk(mockCtrl) + disk := mock_fileio.NewMockDisk(mockCtrl) session.EXPECT().GetSession(datamodel.SessionName("bar")).Return(datamodel.Session{ Name: datamodel.SessionName("bar"), diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/v2/mock_registry/brick_allocation.go index 900a1b79..d187f7c8 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/v2/mock_registry/brick_allocation.go @@ -34,7 +34,7 @@ func (m *MockAllocationRegistry) EXPECT() *MockAllocationRegistryMockRecorder { return m.recorder } -// GetAllocationMutex mocks base method +// GetAllocationMutex mock_fileio base method func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllocationMutex") @@ -49,7 +49,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetAllocationMutex() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllocationMutex)) } -// GetPool mocks base method +// GetPool mock_fileio base method func (m *MockAllocationRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPool", name) @@ -64,7 +64,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPool), name) } -// EnsurePoolCreated mocks base method +// EnsurePoolCreated mock_fileio base method func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityBytes uint) (datamodel.Pool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityBytes) @@ -79,7 +79,7 @@ func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granul return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityBytes) } -// GetAllPoolInfos mocks base method +// GetAllPoolInfos mock_fileio base method func (m *MockAllocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllPoolInfos") @@ -94,7 +94,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetAllPoolInfos() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPoolInfos", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllPoolInfos)) } -// GetPoolInfo mocks base method +// GetPoolInfo mock_fileio base method func (m *MockAllocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPoolInfo", poolName) diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/v2/mock_registry/brick_host.go index a42ead05..9f863f59 100644 --- a/internal/pkg/v2/mock_registry/brick_host.go +++ b/internal/pkg/v2/mock_registry/brick_host.go @@ -34,7 +34,7 @@ func (m *MockBrickHostRegistry) EXPECT() *MockBrickHostRegistryMockRecorder { return m.recorder } -// UpdateBrickHost mocks base method +// UpdateBrickHost mock_fileio base method func (m *MockBrickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateBrickHost", brickHostInfo) @@ -48,7 +48,7 @@ func (mr *MockBrickHostRegistryMockRecorder) UpdateBrickHost(brickHostInfo inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).UpdateBrickHost), brickHostInfo) } -// GetAllBrickHosts mocks base method +// GetAllBrickHosts mock_fileio base method func (m *MockBrickHostRegistry) GetAllBrickHosts() ([]datamodel.BrickHost, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllBrickHosts") @@ -63,7 +63,7 @@ func (mr *MockBrickHostRegistryMockRecorder) GetAllBrickHosts() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrickHosts", reflect.TypeOf((*MockBrickHostRegistry)(nil).GetAllBrickHosts)) } -// KeepAliveHost mocks base method +// KeepAliveHost mock_fileio base method func (m *MockBrickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "KeepAliveHost", ctxt, brickHostName) @@ -77,7 +77,7 @@ func (mr *MockBrickHostRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).KeepAliveHost), ctxt, brickHostName) } -// IsBrickHostAlive mocks base method +// IsBrickHostAlive mock_fileio base method func (m *MockBrickHostRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "IsBrickHostAlive", brickHostName) diff --git a/internal/pkg/v2/mock_registry/session.go b/internal/pkg/v2/mock_registry/session.go index ad60e97d..256aff8d 100644 --- a/internal/pkg/v2/mock_registry/session.go +++ b/internal/pkg/v2/mock_registry/session.go @@ -34,7 +34,7 @@ func (m *MockSessionRegistry) EXPECT() *MockSessionRegistryMockRecorder { return m.recorder } -// GetSessionMutex mocks base method +// GetSessionMutex mock_fileio base method func (m *MockSessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSessionMutex", sessionName) @@ -49,7 +49,7 @@ func (mr *MockSessionRegistryMockRecorder) GetSessionMutex(sessionName interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionMutex", reflect.TypeOf((*MockSessionRegistry)(nil).GetSessionMutex), sessionName) } -// CreateSession mocks base method +// CreateSession mock_fileio base method func (m *MockSessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSession", session) @@ -64,7 +64,7 @@ func (mr *MockSessionRegistryMockRecorder) CreateSession(session interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockSessionRegistry)(nil).CreateSession), session) } -// GetSession mocks base method +// GetSession mock_fileio base method func (m *MockSessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSession", sessionName) @@ -79,7 +79,7 @@ func (mr *MockSessionRegistryMockRecorder) GetSession(sessionName interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockSessionRegistry)(nil).GetSession), sessionName) } -// GetAllSessions mocks base method +// GetAllSessions mock_fileio base method func (m *MockSessionRegistry) GetAllSessions() ([]datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllSessions") @@ -94,7 +94,7 @@ func (mr *MockSessionRegistryMockRecorder) GetAllSessions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockSessionRegistry)(nil).GetAllSessions)) } -// UpdateSession mocks base method +// UpdateSession mock_fileio base method func (m *MockSessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateSession", session) @@ -109,7 +109,7 @@ func (mr *MockSessionRegistryMockRecorder) UpdateSession(session interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSession", reflect.TypeOf((*MockSessionRegistry)(nil).UpdateSession), session) } -// DeleteSession mocks base method +// DeleteSession mock_fileio base method func (m *MockSessionRegistry) DeleteSession(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteSession", session) diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/v2/mock_registry/session_actions.go index 122dfb04..a849793f 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/v2/mock_registry/session_actions.go @@ -34,7 +34,7 @@ func (m *MockSessionActions) EXPECT() *MockSessionActionsMockRecorder { return m.recorder } -// SendSessionAction mocks base method +// SendSessionAction mock_fileio base method func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SendSessionAction", ctxt, actionType, session) @@ -49,7 +49,7 @@ func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, se return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSessionAction", reflect.TypeOf((*MockSessionActions)(nil).SendSessionAction), ctxt, actionType, session) } -// GetSessionActionRequests mocks base method +// GetSessionActionRequests mock_fileio base method func (m *MockSessionActions) GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSessionActionRequests", ctxt, brickHostName) @@ -64,7 +64,7 @@ func (mr *MockSessionActionsMockRecorder) GetSessionActionRequests(ctxt, brickHo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActionRequests", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActionRequests), ctxt, brickHostName) } -// CompleteSessionAction mocks base method +// CompleteSessionAction mock_fileio base method func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CompleteSessionAction", action) From 414359164152deed2a6bc956a254203cd574641e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:50:03 +0100 Subject: [PATCH 144/191] Remame the majority of the v2 code --- build/rebuild_mocks.sh | 24 ++++++++++++------- cmd/dacctl/actions.go | 2 +- cmd/dacctl/main_test.go | 2 +- .../{v2 => }/datamodel/brick_allocation.go | 0 internal/pkg/{v2 => }/datamodel/brick_host.go | 0 internal/pkg/{v2 => }/datamodel/pool.go | 0 internal/pkg/{v2 => }/datamodel/session.go | 0 .../pkg/{v2 => }/datamodel/session_action.go | 0 internal/pkg/{v2 => }/facade/session.go | 2 +- .../{v2 => }/facade/session_action_handler.go | 2 +- internal/pkg/{v2 => }/filesystem/ansible.go | 2 +- internal/pkg/{v2 => }/filesystem/provider.go | 2 +- internal/pkg/{v2 => }/mock_facade/session.go | 4 ++-- .../mock_facade/session_action_handler.go | 4 ++-- .../pkg/{v2 => }/mock_filesystem/ansible.go | 6 ++--- .../pkg/{v2 => }/mock_filesystem/provider.go | 4 ++-- .../mock_registry/brick_allocation.go | 16 ++++++------- .../pkg/{v2 => }/mock_registry/brick_host.go | 12 +++++----- .../pkg/{v2 => }/mock_registry/session.go | 18 +++++++------- .../{v2 => }/mock_registry/session_actions.go | 10 ++++---- internal/pkg/{v2 => }/mock_store/keystore.go | 4 ++-- .../pkg/{v2 => }/registry/brick_allocation.go | 4 ++-- internal/pkg/{v2 => }/registry/brick_host.go | 2 +- internal/pkg/{v2 => }/registry/session.go | 4 ++-- .../pkg/{v2 => }/registry/session_actions.go | 2 +- internal/pkg/{v2 => }/store/keystore.go | 0 .../pkg/v2/dacctl/actions_impl/actions.go | 6 ++--- .../v2/dacctl/actions_impl/actions_test.go | 4 ++-- internal/pkg/v2/dacctl/actions_impl/job.go | 2 +- .../pkg/v2/dacctl/actions_impl/job_test.go | 4 ++-- .../pkg/v2/dacctl/actions_impl/parsers/job.go | 2 +- .../dacctl/actions_impl/parsers/job_test.go | 2 +- .../pkg/v2/dacctl/actions_impl/persistent.go | 2 +- .../v2/dacctl/actions_impl/persistent_test.go | 4 ++-- internal/pkg/v2/dacctl/actions_impl/show.go | 2 +- .../pkg/v2/dacctl/actions_impl/show_test.go | 4 ++-- .../pkg/v2/dacctl/workflow_impl/session.go | 8 +++---- .../v2/dacctl/workflow_impl/session_test.go | 6 ++--- .../dacd/brick_manager_impl/brick_manager.go | 6 ++--- .../brick_manager_impl/brick_manager_test.go | 6 ++--- .../v2/dacd/brick_manager_impl/host_bricks.go | 2 +- .../session_action_handler.go | 10 ++++---- .../session_action_handler_test.go | 8 +++---- internal/pkg/v2/dacd/config/config.go | 2 +- internal/pkg/v2/dacd/config/config_test.go | 2 +- internal/pkg/v2/filesystem_impl/ansible.go | 2 +- .../pkg/v2/filesystem_impl/ansible_test.go | 2 +- internal/pkg/v2/filesystem_impl/copy.go | 2 +- internal/pkg/v2/filesystem_impl/copy_test.go | 2 +- internal/pkg/v2/filesystem_impl/mount.go | 2 +- internal/pkg/v2/filesystem_impl/mount_test.go | 2 +- internal/pkg/v2/filesystem_impl/provider.go | 4 ++-- .../pkg/v2/registry_impl/brick_allocation.go | 6 ++--- internal/pkg/v2/registry_impl/brick_host.go | 6 ++--- internal/pkg/v2/registry_impl/session.go | 6 ++--- .../pkg/v2/registry_impl/session_actions.go | 6 ++--- .../v2/registry_impl/session_actions_test.go | 6 ++--- internal/pkg/v2/registry_impl/session_test.go | 6 ++--- internal/pkg/v2/store_impl/keystore.go | 2 +- 59 files changed, 134 insertions(+), 128 deletions(-) rename internal/pkg/{v2 => }/datamodel/brick_allocation.go (100%) rename internal/pkg/{v2 => }/datamodel/brick_host.go (100%) rename internal/pkg/{v2 => }/datamodel/pool.go (100%) rename internal/pkg/{v2 => }/datamodel/session.go (100%) rename internal/pkg/{v2 => }/datamodel/session_action.go (100%) rename internal/pkg/{v2 => }/facade/session.go (94%) rename internal/pkg/{v2 => }/facade/session_action_handler.go (61%) rename internal/pkg/{v2 => }/filesystem/ansible.go (88%) rename internal/pkg/{v2 => }/filesystem/provider.go (86%) rename internal/pkg/{v2 => }/mock_facade/session.go (97%) rename internal/pkg/{v2 => }/mock_facade/session_action_handler.go (92%) rename internal/pkg/{v2 => }/mock_filesystem/ansible.go (93%) rename internal/pkg/{v2 => }/mock_filesystem/provider.go (97%) rename internal/pkg/{v2 => }/mock_registry/brick_allocation.go (90%) rename internal/pkg/{v2 => }/mock_registry/brick_host.go (91%) rename internal/pkg/{v2 => }/mock_registry/session.go (90%) rename internal/pkg/{v2 => }/mock_registry/session_actions.go (91%) rename internal/pkg/{v2 => }/mock_store/keystore.go (98%) rename internal/pkg/{v2 => }/registry/brick_allocation.go (85%) rename internal/pkg/{v2 => }/registry/brick_host.go (93%) rename internal/pkg/{v2 => }/registry/session.go (91%) rename internal/pkg/{v2 => }/registry/session_actions.go (92%) rename internal/pkg/{v2 => }/store/keystore.go (100%) diff --git a/build/rebuild_mocks.sh b/build/rebuild_mocks.sh index 27c620e8..b92a41db 100755 --- a/build/rebuild_mocks.sh +++ b/build/rebuild_mocks.sh @@ -6,26 +6,32 @@ echo "Regenerate mocks:" mkdir -p internal/pkg/mocks +items="session session_action_handler" +for i in $items; do + mockgen -source=internal/pkg/facade/${i}.go \ + >internal/pkg/mock_facade/${i}.go +done + items="disk" for i in $items; do mockgen -source=internal/pkg/fileio/${i}.go \ >internal/pkg/mock_fileio/${i}_mock.go done -items="session session_action_handler" +items="provider ansible" for i in $items; do - mockgen -source=internal/pkg/v2/facade/${i}.go \ - >internal/pkg/v2/mock_facade/${i}.go + mockgen -source=internal/pkg/filesystem/${i}.go \ + >internal/pkg/mock_filesystem/${i}.go done -items="keystore" +items="brick_allocation brick_host session session_actions" for i in $items; do - mockgen -source=internal/pkg/v2/store/${i}.go \ - >internal/pkg/v2/mock_store/${i}.go + mockgen -source=internal/pkg/registry/${i}.go \ + >internal/pkg/mock_registry/${i}.go done -items="provider ansible" +items="keystore" for i in $items; do - mockgen -source=internal/pkg/v2/filesystem/${i}.go \ - >internal/pkg/v2/mock_filesystem/${i}.go + mockgen -source=internal/pkg/store/${i}.go \ + >internal/pkg/mock_store/${i}.go done diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index ffd493d1..542ef1de 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -3,9 +3,9 @@ package main import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store_impl" "github.com/urfave/cli" ) diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 03bc8d5d..aaf525ea 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/stretchr/testify/assert" "log" "strings" diff --git a/internal/pkg/v2/datamodel/brick_allocation.go b/internal/pkg/datamodel/brick_allocation.go similarity index 100% rename from internal/pkg/v2/datamodel/brick_allocation.go rename to internal/pkg/datamodel/brick_allocation.go diff --git a/internal/pkg/v2/datamodel/brick_host.go b/internal/pkg/datamodel/brick_host.go similarity index 100% rename from internal/pkg/v2/datamodel/brick_host.go rename to internal/pkg/datamodel/brick_host.go diff --git a/internal/pkg/v2/datamodel/pool.go b/internal/pkg/datamodel/pool.go similarity index 100% rename from internal/pkg/v2/datamodel/pool.go rename to internal/pkg/datamodel/pool.go diff --git a/internal/pkg/v2/datamodel/session.go b/internal/pkg/datamodel/session.go similarity index 100% rename from internal/pkg/v2/datamodel/session.go rename to internal/pkg/datamodel/session.go diff --git a/internal/pkg/v2/datamodel/session_action.go b/internal/pkg/datamodel/session_action.go similarity index 100% rename from internal/pkg/v2/datamodel/session_action.go rename to internal/pkg/datamodel/session_action.go diff --git a/internal/pkg/v2/facade/session.go b/internal/pkg/facade/session.go similarity index 94% rename from internal/pkg/v2/facade/session.go rename to internal/pkg/facade/session.go index af61127f..09f83dcf 100644 --- a/internal/pkg/v2/facade/session.go +++ b/internal/pkg/facade/session.go @@ -1,6 +1,6 @@ package facade -import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" // Each volume has an associated primary brick // that is responsible for responding to any actions diff --git a/internal/pkg/v2/facade/session_action_handler.go b/internal/pkg/facade/session_action_handler.go similarity index 61% rename from internal/pkg/v2/facade/session_action_handler.go rename to internal/pkg/facade/session_action_handler.go index 0727d9af..3e765bef 100644 --- a/internal/pkg/v2/facade/session_action_handler.go +++ b/internal/pkg/facade/session_action_handler.go @@ -1,6 +1,6 @@ package facade -import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type SessionActionHandler interface { ProcessSessionAction(action datamodel.SessionAction) diff --git a/internal/pkg/v2/filesystem/ansible.go b/internal/pkg/filesystem/ansible.go similarity index 88% rename from internal/pkg/v2/filesystem/ansible.go rename to internal/pkg/filesystem/ansible.go index 682d3455..d4cc9431 100644 --- a/internal/pkg/v2/filesystem/ansible.go +++ b/internal/pkg/filesystem/ansible.go @@ -1,6 +1,6 @@ package filesystem -import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type AnsibleEnv struct { directory string diff --git a/internal/pkg/v2/filesystem/provider.go b/internal/pkg/filesystem/provider.go similarity index 86% rename from internal/pkg/v2/filesystem/provider.go rename to internal/pkg/filesystem/provider.go index aa636ad7..4d502382 100644 --- a/internal/pkg/v2/filesystem/provider.go +++ b/internal/pkg/filesystem/provider.go @@ -1,6 +1,6 @@ package filesystem -import "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" +import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type Provider interface { Create(session datamodel.Session) (datamodel.FilesystemStatus, error) diff --git a/internal/pkg/v2/mock_facade/session.go b/internal/pkg/mock_facade/session.go similarity index 97% rename from internal/pkg/v2/mock_facade/session.go rename to internal/pkg/mock_facade/session.go index c19048a7..0009481b 100644 --- a/internal/pkg/v2/mock_facade/session.go +++ b/internal/pkg/mock_facade/session.go @@ -1,11 +1,11 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/facade/session.go +// Source: internal/pkg/facade/session.go // Package mock_facade is a generated GoMock package. package mock_facade import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/internal/pkg/v2/mock_facade/session_action_handler.go b/internal/pkg/mock_facade/session_action_handler.go similarity index 92% rename from internal/pkg/v2/mock_facade/session_action_handler.go rename to internal/pkg/mock_facade/session_action_handler.go index bf6745fa..e5e4b6db 100644 --- a/internal/pkg/v2/mock_facade/session_action_handler.go +++ b/internal/pkg/mock_facade/session_action_handler.go @@ -1,11 +1,11 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/facade/session_action_handler.go +// Source: internal/pkg/facade/session_action_handler.go // Package mock_facade is a generated GoMock package. package mock_facade import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/internal/pkg/v2/mock_filesystem/ansible.go b/internal/pkg/mock_filesystem/ansible.go similarity index 93% rename from internal/pkg/v2/mock_filesystem/ansible.go rename to internal/pkg/mock_filesystem/ansible.go index 52a0a151..febe4f80 100644 --- a/internal/pkg/v2/mock_filesystem/ansible.go +++ b/internal/pkg/mock_filesystem/ansible.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/filesystem/ansible.go +// Source: internal/pkg/filesystem/ansible.go // Package mock_filesystem is a generated GoMock package. package mock_filesystem import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - filesystem "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + filesystem "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/internal/pkg/v2/mock_filesystem/provider.go b/internal/pkg/mock_filesystem/provider.go similarity index 97% rename from internal/pkg/v2/mock_filesystem/provider.go rename to internal/pkg/mock_filesystem/provider.go index 4a8507af..062773f0 100644 --- a/internal/pkg/v2/mock_filesystem/provider.go +++ b/internal/pkg/mock_filesystem/provider.go @@ -1,11 +1,11 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/filesystem/provider.go +// Source: internal/pkg/filesystem/provider.go // Package mock_filesystem is a generated GoMock package. package mock_filesystem import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/internal/pkg/v2/mock_registry/brick_allocation.go b/internal/pkg/mock_registry/brick_allocation.go similarity index 90% rename from internal/pkg/v2/mock_registry/brick_allocation.go rename to internal/pkg/mock_registry/brick_allocation.go index d187f7c8..74cfebbd 100644 --- a/internal/pkg/v2/mock_registry/brick_allocation.go +++ b/internal/pkg/mock_registry/brick_allocation.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/brick_allocation.go +// Source: internal/pkg/registry/brick_allocation.go // Package mock_registry is a generated GoMock package. package mock_registry import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/store" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -34,7 +34,7 @@ func (m *MockAllocationRegistry) EXPECT() *MockAllocationRegistryMockRecorder { return m.recorder } -// GetAllocationMutex mock_fileio base method +// GetAllocationMutex mocks base method func (m *MockAllocationRegistry) GetAllocationMutex() (store.Mutex, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllocationMutex") @@ -49,7 +49,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetAllocationMutex() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllocationMutex", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllocationMutex)) } -// GetPool mock_fileio base method +// GetPool mocks base method func (m *MockAllocationRegistry) GetPool(name datamodel.PoolName) (datamodel.Pool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPool", name) @@ -64,7 +64,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetPool(name interface{}) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPool", reflect.TypeOf((*MockAllocationRegistry)(nil).GetPool), name) } -// EnsurePoolCreated mock_fileio base method +// EnsurePoolCreated mocks base method func (m *MockAllocationRegistry) EnsurePoolCreated(poolName datamodel.PoolName, granularityBytes uint) (datamodel.Pool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "EnsurePoolCreated", poolName, granularityBytes) @@ -79,7 +79,7 @@ func (mr *MockAllocationRegistryMockRecorder) EnsurePoolCreated(poolName, granul return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsurePoolCreated", reflect.TypeOf((*MockAllocationRegistry)(nil).EnsurePoolCreated), poolName, granularityBytes) } -// GetAllPoolInfos mock_fileio base method +// GetAllPoolInfos mocks base method func (m *MockAllocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllPoolInfos") @@ -94,7 +94,7 @@ func (mr *MockAllocationRegistryMockRecorder) GetAllPoolInfos() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllPoolInfos", reflect.TypeOf((*MockAllocationRegistry)(nil).GetAllPoolInfos)) } -// GetPoolInfo mock_fileio base method +// GetPoolInfo mocks base method func (m *MockAllocationRegistry) GetPoolInfo(poolName datamodel.PoolName) (datamodel.PoolInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetPoolInfo", poolName) diff --git a/internal/pkg/v2/mock_registry/brick_host.go b/internal/pkg/mock_registry/brick_host.go similarity index 91% rename from internal/pkg/v2/mock_registry/brick_host.go rename to internal/pkg/mock_registry/brick_host.go index 9f863f59..a1251a7f 100644 --- a/internal/pkg/v2/mock_registry/brick_host.go +++ b/internal/pkg/mock_registry/brick_host.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/brick_host.go +// Source: internal/pkg/registry/brick_host.go // Package mock_registry is a generated GoMock package. package mock_registry import ( context "context" - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -34,7 +34,7 @@ func (m *MockBrickHostRegistry) EXPECT() *MockBrickHostRegistryMockRecorder { return m.recorder } -// UpdateBrickHost mock_fileio base method +// UpdateBrickHost mocks base method func (m *MockBrickHostRegistry) UpdateBrickHost(brickHostInfo datamodel.BrickHost) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateBrickHost", brickHostInfo) @@ -48,7 +48,7 @@ func (mr *MockBrickHostRegistryMockRecorder) UpdateBrickHost(brickHostInfo inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateBrickHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).UpdateBrickHost), brickHostInfo) } -// GetAllBrickHosts mock_fileio base method +// GetAllBrickHosts mocks base method func (m *MockBrickHostRegistry) GetAllBrickHosts() ([]datamodel.BrickHost, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllBrickHosts") @@ -63,7 +63,7 @@ func (mr *MockBrickHostRegistryMockRecorder) GetAllBrickHosts() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrickHosts", reflect.TypeOf((*MockBrickHostRegistry)(nil).GetAllBrickHosts)) } -// KeepAliveHost mock_fileio base method +// KeepAliveHost mocks base method func (m *MockBrickHostRegistry) KeepAliveHost(ctxt context.Context, brickHostName datamodel.BrickHostName) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "KeepAliveHost", ctxt, brickHostName) @@ -77,7 +77,7 @@ func (mr *MockBrickHostRegistryMockRecorder) KeepAliveHost(ctxt, brickHostName i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KeepAliveHost", reflect.TypeOf((*MockBrickHostRegistry)(nil).KeepAliveHost), ctxt, brickHostName) } -// IsBrickHostAlive mock_fileio base method +// IsBrickHostAlive mocks base method func (m *MockBrickHostRegistry) IsBrickHostAlive(brickHostName datamodel.BrickHostName) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "IsBrickHostAlive", brickHostName) diff --git a/internal/pkg/v2/mock_registry/session.go b/internal/pkg/mock_registry/session.go similarity index 90% rename from internal/pkg/v2/mock_registry/session.go rename to internal/pkg/mock_registry/session.go index 256aff8d..cfd289ad 100644 --- a/internal/pkg/v2/mock_registry/session.go +++ b/internal/pkg/mock_registry/session.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/session.go +// Source: internal/pkg/registry/session.go // Package mock_registry is a generated GoMock package. package mock_registry import ( - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/store" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -34,7 +34,7 @@ func (m *MockSessionRegistry) EXPECT() *MockSessionRegistryMockRecorder { return m.recorder } -// GetSessionMutex mock_fileio base method +// GetSessionMutex mocks base method func (m *MockSessionRegistry) GetSessionMutex(sessionName datamodel.SessionName) (store.Mutex, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSessionMutex", sessionName) @@ -49,7 +49,7 @@ func (mr *MockSessionRegistryMockRecorder) GetSessionMutex(sessionName interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionMutex", reflect.TypeOf((*MockSessionRegistry)(nil).GetSessionMutex), sessionName) } -// CreateSession mock_fileio base method +// CreateSession mocks base method func (m *MockSessionRegistry) CreateSession(session datamodel.Session) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateSession", session) @@ -64,7 +64,7 @@ func (mr *MockSessionRegistryMockRecorder) CreateSession(session interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSession", reflect.TypeOf((*MockSessionRegistry)(nil).CreateSession), session) } -// GetSession mock_fileio base method +// GetSession mocks base method func (m *MockSessionRegistry) GetSession(sessionName datamodel.SessionName) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSession", sessionName) @@ -79,7 +79,7 @@ func (mr *MockSessionRegistryMockRecorder) GetSession(sessionName interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSession", reflect.TypeOf((*MockSessionRegistry)(nil).GetSession), sessionName) } -// GetAllSessions mock_fileio base method +// GetAllSessions mocks base method func (m *MockSessionRegistry) GetAllSessions() ([]datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllSessions") @@ -94,7 +94,7 @@ func (mr *MockSessionRegistryMockRecorder) GetAllSessions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockSessionRegistry)(nil).GetAllSessions)) } -// UpdateSession mock_fileio base method +// UpdateSession mocks base method func (m *MockSessionRegistry) UpdateSession(session datamodel.Session) (datamodel.Session, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateSession", session) @@ -109,7 +109,7 @@ func (mr *MockSessionRegistryMockRecorder) UpdateSession(session interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateSession", reflect.TypeOf((*MockSessionRegistry)(nil).UpdateSession), session) } -// DeleteSession mock_fileio base method +// DeleteSession mocks base method func (m *MockSessionRegistry) DeleteSession(session datamodel.Session) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteSession", session) diff --git a/internal/pkg/v2/mock_registry/session_actions.go b/internal/pkg/mock_registry/session_actions.go similarity index 91% rename from internal/pkg/v2/mock_registry/session_actions.go rename to internal/pkg/mock_registry/session_actions.go index a849793f..5bdc2e47 100644 --- a/internal/pkg/v2/mock_registry/session_actions.go +++ b/internal/pkg/mock_registry/session_actions.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/registry/session_actions.go +// Source: internal/pkg/registry/session_actions.go // Package mock_registry is a generated GoMock package. package mock_registry import ( context "context" - datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -34,7 +34,7 @@ func (m *MockSessionActions) EXPECT() *MockSessionActionsMockRecorder { return m.recorder } -// SendSessionAction mock_fileio base method +// SendSessionAction mocks base method func (m *MockSessionActions) SendSessionAction(ctxt context.Context, actionType datamodel.SessionActionType, session datamodel.Session) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SendSessionAction", ctxt, actionType, session) @@ -49,7 +49,7 @@ func (mr *MockSessionActionsMockRecorder) SendSessionAction(ctxt, actionType, se return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendSessionAction", reflect.TypeOf((*MockSessionActions)(nil).SendSessionAction), ctxt, actionType, session) } -// GetSessionActionRequests mock_fileio base method +// GetSessionActionRequests mocks base method func (m *MockSessionActions) GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSessionActionRequests", ctxt, brickHostName) @@ -64,7 +64,7 @@ func (mr *MockSessionActionsMockRecorder) GetSessionActionRequests(ctxt, brickHo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActionRequests", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActionRequests), ctxt, brickHostName) } -// CompleteSessionAction mock_fileio base method +// CompleteSessionAction mocks base method func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CompleteSessionAction", action) diff --git a/internal/pkg/v2/mock_store/keystore.go b/internal/pkg/mock_store/keystore.go similarity index 98% rename from internal/pkg/v2/mock_store/keystore.go rename to internal/pkg/mock_store/keystore.go index db4f60e9..b3ff0e4a 100644 --- a/internal/pkg/v2/mock_store/keystore.go +++ b/internal/pkg/mock_store/keystore.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: internal/pkg/v2/store/keystore.go +// Source: internal/pkg/store/keystore.go // Package mock_store is a generated GoMock package. package mock_store import ( context "context" - store "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + store "github.com/RSE-Cambridge/data-acc/internal/pkg/store" gomock "github.com/golang/mock/gomock" reflect "reflect" ) diff --git a/internal/pkg/v2/registry/brick_allocation.go b/internal/pkg/registry/brick_allocation.go similarity index 85% rename from internal/pkg/v2/registry/brick_allocation.go rename to internal/pkg/registry/brick_allocation.go index e53485f9..13b7780c 100644 --- a/internal/pkg/v2/registry/brick_allocation.go +++ b/internal/pkg/registry/brick_allocation.go @@ -1,8 +1,8 @@ package registry import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" ) type AllocationRegistry interface { diff --git a/internal/pkg/v2/registry/brick_host.go b/internal/pkg/registry/brick_host.go similarity index 93% rename from internal/pkg/v2/registry/brick_host.go rename to internal/pkg/registry/brick_host.go index eb863202..ebff7ce0 100644 --- a/internal/pkg/v2/registry/brick_host.go +++ b/internal/pkg/registry/brick_host.go @@ -2,7 +2,7 @@ package registry import ( "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" ) type BrickHostRegistry interface { diff --git a/internal/pkg/v2/registry/session.go b/internal/pkg/registry/session.go similarity index 91% rename from internal/pkg/v2/registry/session.go rename to internal/pkg/registry/session.go index 4344a685..9bde9b0a 100644 --- a/internal/pkg/v2/registry/session.go +++ b/internal/pkg/registry/session.go @@ -1,8 +1,8 @@ package registry import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" ) // TODO: rename to instance? or filesystem? or just in the object model? diff --git a/internal/pkg/v2/registry/session_actions.go b/internal/pkg/registry/session_actions.go similarity index 92% rename from internal/pkg/v2/registry/session_actions.go rename to internal/pkg/registry/session_actions.go index efc80aab..c1aed9ae 100644 --- a/internal/pkg/v2/registry/session_actions.go +++ b/internal/pkg/registry/session_actions.go @@ -2,7 +2,7 @@ package registry import ( "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" ) type SessionActions interface { diff --git a/internal/pkg/v2/store/keystore.go b/internal/pkg/store/keystore.go similarity index 100% rename from internal/pkg/v2/store/keystore.go rename to internal/pkg/store/keystore.go diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/v2/dacctl/actions_impl/actions.go index 1c83f49e..711d1d87 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions.go @@ -3,13 +3,13 @@ package actions_impl import ( "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/workflow_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" "strings" ) diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/v2/dacctl/actions_impl/actions_test.go index c6e66144..92eece11 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/actions_test.go @@ -3,9 +3,9 @@ package actions_impl import ( "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/v2/dacctl/actions_impl/job.go index d351643e..c1fa81a1 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/job.go @@ -2,9 +2,9 @@ package actions_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "sort" ) diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/v2/dacctl/actions_impl/job_test.go index 7a2100f2..7be98364 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/job_test.go @@ -1,9 +1,9 @@ package actions_impl import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go index 797552a2..a6526f40 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job.go @@ -2,8 +2,8 @@ package parsers import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "log" "strings" ) diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go index 35f49655..d58e623a 100644 --- a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go @@ -2,8 +2,8 @@ package parsers import ( "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "log" diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/v2/dacctl/actions_impl/persistent.go index 43cee938..7dcf4645 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent.go @@ -1,9 +1,9 @@ package actions_impl import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" "strings" "time" ) diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go index 240fb733..6c61e65e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/persistent_test.go @@ -1,8 +1,8 @@ package actions_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/v2/dacctl/actions_impl/show.go index 48eaed6c..136fc108 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/v2/dacctl/actions_impl/show.go @@ -2,8 +2,8 @@ package actions_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" ) func (d *dacctlActions) getSession(c dacctl.CliContext) (datamodel.Session, error) { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/v2/dacctl/actions_impl/show_test.go index deb860ee..b8ab91b4 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show_test.go +++ b/internal/pkg/v2/dacctl/actions_impl/show_test.go @@ -2,9 +2,9 @@ package actions_impl import ( "errors" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_fileio" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 27d07ca8..64a2372d 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -4,11 +4,11 @@ import ( "context" "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" "math" "math/rand" diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/v2/dacctl/workflow_impl/session_test.go index 19b55e47..7f77f2ec 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session_test.go @@ -3,9 +3,9 @@ package workflow_impl import ( "context" "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index fd61c9d2..7e6e63a5 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -2,12 +2,12 @@ package brick_manager_impl import ( "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go index b5dce218..43602dd8 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go @@ -2,10 +2,10 @@ package brick_manager_impl import ( "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_facade" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "os" diff --git a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go index f3d8e8f3..63176408 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go @@ -2,8 +2,8 @@ package brick_manager_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" ) func getDevices(brickManagerConfig config.BrickManagerConfig) []string { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index fbe251e5..6dd7f01d 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -3,13 +3,13 @@ package brick_manager_impl import ( "context" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/facade" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go index f02b7c52..25f7f1a6 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go @@ -3,10 +3,10 @@ package brick_manager_impl import ( "context" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_filesystem" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/v2/dacd/config/config.go index 91370a5b..ca5180ce 100644 --- a/internal/pkg/v2/dacd/config/config.go +++ b/internal/pkg/v2/dacd/config/config.go @@ -1,7 +1,7 @@ package config import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "log" "os" "strconv" diff --git a/internal/pkg/v2/dacd/config/config_test.go b/internal/pkg/v2/dacd/config/config_test.go index ccfbd7c6..1b824e8b 100644 --- a/internal/pkg/v2/dacd/config/config_test.go +++ b/internal/pkg/v2/dacd/config/config_test.go @@ -1,7 +1,7 @@ package config import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/stretchr/testify/assert" "os" "testing" diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/v2/filesystem_impl/ansible.go index 4b4c5409..cf6f9a76 100644 --- a/internal/pkg/v2/filesystem_impl/ansible.go +++ b/internal/pkg/v2/filesystem_impl/ansible.go @@ -3,7 +3,7 @@ package filesystem_impl import ( "bytes" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "gopkg.in/yaml.v2" "io/ioutil" "log" diff --git a/internal/pkg/v2/filesystem_impl/ansible_test.go b/internal/pkg/v2/filesystem_impl/ansible_test.go index 54e53afc..a5c106fe 100644 --- a/internal/pkg/v2/filesystem_impl/ansible_test.go +++ b/internal/pkg/v2/filesystem_impl/ansible_test.go @@ -2,7 +2,7 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/stretchr/testify/assert" "testing" ) diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/v2/filesystem_impl/copy.go index a829bf69..848f5a26 100644 --- a/internal/pkg/v2/filesystem_impl/copy.go +++ b/internal/pkg/v2/filesystem_impl/copy.go @@ -2,7 +2,7 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "log" "path" "strings" diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/v2/filesystem_impl/copy_test.go index 98f2af2e..5f6d14d5 100644 --- a/internal/pkg/v2/filesystem_impl/copy_test.go +++ b/internal/pkg/v2/filesystem_impl/copy_test.go @@ -1,7 +1,7 @@ package filesystem_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/stretchr/testify/assert" "testing" ) diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/v2/filesystem_impl/mount.go index 82dddc1d..3d3b154d 100644 --- a/internal/pkg/v2/filesystem_impl/mount.go +++ b/internal/pkg/v2/filesystem_impl/mount.go @@ -2,7 +2,7 @@ package filesystem_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "log" "os" "os/exec" diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/v2/filesystem_impl/mount_test.go index ff828a54..8ca775ee 100644 --- a/internal/pkg/v2/filesystem_impl/mount_test.go +++ b/internal/pkg/v2/filesystem_impl/mount_test.go @@ -2,7 +2,7 @@ package filesystem_impl import ( "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/stretchr/testify/assert" "testing" ) diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/v2/filesystem_impl/provider.go index b45031b3..1a38aca9 100644 --- a/internal/pkg/v2/filesystem_impl/provider.go +++ b/internal/pkg/v2/filesystem_impl/provider.go @@ -1,8 +1,8 @@ package filesystem_impl import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" "math/rand" ) diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/v2/registry_impl/brick_allocation.go index 9f917a1a..f0a15001 100644 --- a/internal/pkg/v2/registry_impl/brick_allocation.go +++ b/internal/pkg/v2/registry_impl/brick_allocation.go @@ -3,10 +3,10 @@ package registry_impl import ( "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/v2/registry_impl/brick_host.go index efb84d83..7903b27a 100644 --- a/internal/pkg/v2/registry_impl/brick_host.go +++ b/internal/pkg/v2/registry_impl/brick_host.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/v2/registry_impl/session.go index 2a2a05b1..15f786ff 100644 --- a/internal/pkg/v2/registry_impl/session.go +++ b/internal/pkg/v2/registry_impl/session.go @@ -3,10 +3,10 @@ package registry_impl import ( "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "log" ) diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/v2/registry_impl/session_actions.go index c9eae34c..07ee77be 100644 --- a/internal/pkg/v2/registry_impl/session_actions.go +++ b/internal/pkg/v2/registry_impl/session_actions.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" "github.com/google/uuid" "log" ) diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/v2/registry_impl/session_actions_test.go index ce17b03b..305e7034 100644 --- a/internal/pkg/v2/registry_impl/session_actions_test.go +++ b/internal/pkg/v2/registry_impl/session_actions_test.go @@ -3,9 +3,9 @@ package registry_impl import ( "context" "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/v2/registry_impl/session_test.go index ff01554e..33abdfc2 100644 --- a/internal/pkg/v2/registry_impl/session_test.go +++ b/internal/pkg/v2/registry_impl/session_test.go @@ -3,9 +3,9 @@ package registry_impl import ( "encoding/json" "errors" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/mock_store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "testing" diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/v2/store_impl/keystore.go index 6c0854ff..7904b75d 100644 --- a/internal/pkg/v2/store_impl/keystore.go +++ b/internal/pkg/v2/store_impl/keystore.go @@ -5,7 +5,7 @@ import ( "crypto/tls" "errors" "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/clientv3util" "github.com/coreos/etcd/clientv3/concurrency" From 3450c152e199e681b57f58acb9f11584dd5b3590 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 08:54:21 +0100 Subject: [PATCH 145/191] Move majority of impl packages --- cmd/dacctl/actions.go | 2 +- cmd/dacd/main.go | 2 +- internal/pkg/{v2 => }/filesystem_impl/ansible.go | 0 internal/pkg/{v2 => }/filesystem_impl/ansible_test.go | 0 internal/pkg/{v2 => }/filesystem_impl/copy.go | 0 internal/pkg/{v2 => }/filesystem_impl/copy_test.go | 0 internal/pkg/{v2 => }/filesystem_impl/fs_type.go | 0 internal/pkg/{v2 => }/filesystem_impl/mount.go | 0 internal/pkg/{v2 => }/filesystem_impl/mount_test.go | 0 internal/pkg/{v2 => }/filesystem_impl/provider.go | 0 internal/pkg/{v2 => }/registry_impl/brick_allocation.go | 0 internal/pkg/{v2 => }/registry_impl/brick_host.go | 0 internal/pkg/{v2 => }/registry_impl/session.go | 0 internal/pkg/{v2 => }/registry_impl/session_actions.go | 0 internal/pkg/{v2 => }/registry_impl/session_actions_test.go | 0 internal/pkg/{v2 => }/registry_impl/session_test.go | 0 internal/pkg/{v2 => }/store_impl/keystore.go | 0 internal/pkg/v2/dacctl/workflow_impl/session.go | 2 +- internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go | 2 +- .../pkg/v2/dacd/brick_manager_impl/session_action_handler.go | 4 ++-- 20 files changed, 6 insertions(+), 6 deletions(-) rename internal/pkg/{v2 => }/filesystem_impl/ansible.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/ansible_test.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/copy.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/copy_test.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/fs_type.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/mount.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/mount_test.go (100%) rename internal/pkg/{v2 => }/filesystem_impl/provider.go (100%) rename internal/pkg/{v2 => }/registry_impl/brick_allocation.go (100%) rename internal/pkg/{v2 => }/registry_impl/brick_host.go (100%) rename internal/pkg/{v2 => }/registry_impl/session.go (100%) rename internal/pkg/{v2 => }/registry_impl/session_actions.go (100%) rename internal/pkg/{v2 => }/registry_impl/session_actions_test.go (100%) rename internal/pkg/{v2 => }/registry_impl/session_test.go (100%) rename internal/pkg/{v2 => }/store_impl/keystore.go (100%) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index 542ef1de..213d67ad 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" + "github.com/RSE-Cambridge/data-acc/internal/pkg/store_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store_impl" "github.com/urfave/cli" ) diff --git a/cmd/dacd/main.go b/cmd/dacd/main.go index 46bdbd0d..3cf4b5a4 100644 --- a/cmd/dacd/main.go +++ b/cmd/dacd/main.go @@ -1,9 +1,9 @@ package main import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/store_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/store_impl" "log" "os" "os/signal" diff --git a/internal/pkg/v2/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/ansible.go rename to internal/pkg/filesystem_impl/ansible.go diff --git a/internal/pkg/v2/filesystem_impl/ansible_test.go b/internal/pkg/filesystem_impl/ansible_test.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/ansible_test.go rename to internal/pkg/filesystem_impl/ansible_test.go diff --git a/internal/pkg/v2/filesystem_impl/copy.go b/internal/pkg/filesystem_impl/copy.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/copy.go rename to internal/pkg/filesystem_impl/copy.go diff --git a/internal/pkg/v2/filesystem_impl/copy_test.go b/internal/pkg/filesystem_impl/copy_test.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/copy_test.go rename to internal/pkg/filesystem_impl/copy_test.go diff --git a/internal/pkg/v2/filesystem_impl/fs_type.go b/internal/pkg/filesystem_impl/fs_type.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/fs_type.go rename to internal/pkg/filesystem_impl/fs_type.go diff --git a/internal/pkg/v2/filesystem_impl/mount.go b/internal/pkg/filesystem_impl/mount.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/mount.go rename to internal/pkg/filesystem_impl/mount.go diff --git a/internal/pkg/v2/filesystem_impl/mount_test.go b/internal/pkg/filesystem_impl/mount_test.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/mount_test.go rename to internal/pkg/filesystem_impl/mount_test.go diff --git a/internal/pkg/v2/filesystem_impl/provider.go b/internal/pkg/filesystem_impl/provider.go similarity index 100% rename from internal/pkg/v2/filesystem_impl/provider.go rename to internal/pkg/filesystem_impl/provider.go diff --git a/internal/pkg/v2/registry_impl/brick_allocation.go b/internal/pkg/registry_impl/brick_allocation.go similarity index 100% rename from internal/pkg/v2/registry_impl/brick_allocation.go rename to internal/pkg/registry_impl/brick_allocation.go diff --git a/internal/pkg/v2/registry_impl/brick_host.go b/internal/pkg/registry_impl/brick_host.go similarity index 100% rename from internal/pkg/v2/registry_impl/brick_host.go rename to internal/pkg/registry_impl/brick_host.go diff --git a/internal/pkg/v2/registry_impl/session.go b/internal/pkg/registry_impl/session.go similarity index 100% rename from internal/pkg/v2/registry_impl/session.go rename to internal/pkg/registry_impl/session.go diff --git a/internal/pkg/v2/registry_impl/session_actions.go b/internal/pkg/registry_impl/session_actions.go similarity index 100% rename from internal/pkg/v2/registry_impl/session_actions.go rename to internal/pkg/registry_impl/session_actions.go diff --git a/internal/pkg/v2/registry_impl/session_actions_test.go b/internal/pkg/registry_impl/session_actions_test.go similarity index 100% rename from internal/pkg/v2/registry_impl/session_actions_test.go rename to internal/pkg/registry_impl/session_actions_test.go diff --git a/internal/pkg/v2/registry_impl/session_test.go b/internal/pkg/registry_impl/session_test.go similarity index 100% rename from internal/pkg/v2/registry_impl/session_test.go rename to internal/pkg/registry_impl/session_test.go diff --git a/internal/pkg/v2/store_impl/keystore.go b/internal/pkg/store_impl/keystore.go similarity index 100% rename from internal/pkg/v2/store_impl/keystore.go rename to internal/pkg/store_impl/keystore.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/v2/dacctl/workflow_impl/session.go index 64a2372d..00fd049a 100644 --- a/internal/pkg/v2/dacctl/workflow_impl/session.go +++ b/internal/pkg/v2/dacctl/workflow_impl/session.go @@ -7,8 +7,8 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" "log" "math" "math/rand" diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go index 7e6e63a5..5d905f5b 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go @@ -4,10 +4,10 @@ import ( "context" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" "log" ) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go index 6dd7f01d..8f4712d2 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go @@ -6,10 +6,10 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" + "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/filesystem_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/registry_impl" "log" ) From df5e3384adf745588f6e5050b21c07eec8ab8285 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 09:04:06 +0100 Subject: [PATCH 146/191] Remove final trace of "v2" --- cmd/dacctl/actions.go | 4 ++-- cmd/dacctl/main_test.go | 2 +- cmd/dacd/main.go | 4 ++-- internal/pkg/{v2 => }/dacctl/actions_impl/actions.go | 12 ++++++------ .../pkg/{v2 => }/dacctl/actions_impl/actions_test.go | 0 .../{v2 => }/dacctl/actions_impl/configurations.go | 0 .../pkg/{v2 => }/dacctl/actions_impl/instances.go | 0 internal/pkg/{v2 => }/dacctl/actions_impl/job.go | 10 +++++----- .../pkg/{v2 => }/dacctl/actions_impl/job_test.go | 0 .../{v2 => }/dacctl/actions_impl/parsers/capacity.go | 0 .../dacctl/actions_impl/parsers/capacity_test.go | 0 .../dacctl/actions_impl/parsers/hostnames.go | 0 .../dacctl/actions_impl/parsers/hostnames_test.go | 0 .../pkg/{v2 => }/dacctl/actions_impl/parsers/job.go | 0 .../{v2 => }/dacctl/actions_impl/parsers/job_test.go | 0 .../pkg/{v2 => }/dacctl/actions_impl/persistent.go | 6 +++--- .../{v2 => }/dacctl/actions_impl/persistent_test.go | 0 internal/pkg/{v2 => }/dacctl/actions_impl/pools.go | 0 .../pkg/{v2 => }/dacctl/actions_impl/sessions.go | 0 internal/pkg/{v2 => }/dacctl/actions_impl/show.go | 2 +- .../pkg/{v2 => }/dacctl/actions_impl/show_test.go | 0 internal/pkg/{v2 => }/dacctl/interface.go | 0 .../pkg/{v2 => }/dacctl/workflow_impl/session.go | 0 .../{v2 => }/dacctl/workflow_impl/session_test.go | 0 .../dacd/brick_manager_impl/brick_manager.go | 4 ++-- .../dacd/brick_manager_impl/brick_manager_test.go | 2 +- .../{v2 => }/dacd/brick_manager_impl/host_bricks.go | 2 +- .../brick_manager_impl/session_action_handler.go | 0 .../session_action_handler_test.go | 0 internal/pkg/{v2 => }/dacd/config/config.go | 0 internal/pkg/{v2 => }/dacd/config/config_test.go | 0 internal/pkg/{v2 => }/dacd/interface.go | 0 internal/pkg/registry_impl/brick_allocation.go | 2 +- internal/pkg/registry_impl/brick_host.go | 2 +- internal/pkg/registry_impl/session.go | 2 +- internal/pkg/registry_impl/session_actions.go | 2 +- 36 files changed, 28 insertions(+), 28 deletions(-) rename internal/pkg/{v2 => }/dacctl/actions_impl/actions.go (86%) rename internal/pkg/{v2 => }/dacctl/actions_impl/actions_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/configurations.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/instances.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/job.go (90%) rename internal/pkg/{v2 => }/dacctl/actions_impl/job_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/capacity.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/capacity_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/hostnames.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/hostnames_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/job.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/parsers/job_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/persistent.go (88%) rename internal/pkg/{v2 => }/dacctl/actions_impl/persistent_test.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/pools.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/sessions.go (100%) rename internal/pkg/{v2 => }/dacctl/actions_impl/show.go (97%) rename internal/pkg/{v2 => }/dacctl/actions_impl/show_test.go (100%) rename internal/pkg/{v2 => }/dacctl/interface.go (100%) rename internal/pkg/{v2 => }/dacctl/workflow_impl/session.go (100%) rename internal/pkg/{v2 => }/dacctl/workflow_impl/session_test.go (100%) rename internal/pkg/{v2 => }/dacd/brick_manager_impl/brick_manager.go (94%) rename internal/pkg/{v2 => }/dacd/brick_manager_impl/brick_manager_test.go (95%) rename internal/pkg/{v2 => }/dacd/brick_manager_impl/host_bricks.go (93%) rename internal/pkg/{v2 => }/dacd/brick_manager_impl/session_action_handler.go (100%) rename internal/pkg/{v2 => }/dacd/brick_manager_impl/session_action_handler_test.go (100%) rename internal/pkg/{v2 => }/dacd/config/config.go (100%) rename internal/pkg/{v2 => }/dacd/config/config_test.go (100%) rename internal/pkg/{v2 => }/dacd/interface.go (100%) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index 213d67ad..01fe276d 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -2,11 +2,11 @@ package main import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/RSE-Cambridge/data-acc/internal/pkg/store_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl" "github.com/urfave/cli" ) diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index aaf525ea..0ef12733 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" "github.com/stretchr/testify/assert" "log" "strings" diff --git a/cmd/dacd/main.go b/cmd/dacd/main.go index 3cf4b5a4..8a087c44 100644 --- a/cmd/dacd/main.go +++ b/cmd/dacd/main.go @@ -1,9 +1,9 @@ package main import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/brick_manager_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store_impl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/brick_manager_impl" "log" "os" "os/signal" diff --git a/internal/pkg/v2/dacctl/actions_impl/actions.go b/internal/pkg/dacctl/actions_impl/actions.go similarity index 86% rename from internal/pkg/v2/dacctl/actions_impl/actions.go rename to internal/pkg/dacctl/actions_impl/actions.go index 711d1d87..d5d7b283 100644 --- a/internal/pkg/v2/dacctl/actions_impl/actions.go +++ b/internal/pkg/dacctl/actions_impl/actions.go @@ -3,13 +3,13 @@ package actions_impl import ( "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" + parsers2 "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/workflow_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/fileio" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/workflow_impl" "log" "strings" ) @@ -48,7 +48,7 @@ func (d *dacctlActions) getSessionName(c dacctl.CliContext) (datamodel.SessionNa } token := c.String("token") - if !parsers.IsValidName(token) { + if !parsers2.IsValidName(token) { return "", fmt.Errorf("badly formatted session name: %s", token) } @@ -82,7 +82,7 @@ func (d *dacctlActions) PreRun(c dacctl.CliContext) error { return err } - computeHosts, err := parsers.GetHostnamesFromFile(d.disk, c.String("nodehostnamefile")) + computeHosts, err := parsers2.GetHostnamesFromFile(d.disk, c.String("nodehostnamefile")) if err != nil { return err } @@ -93,7 +93,7 @@ func (d *dacctlActions) PreRun(c dacctl.CliContext) error { loginNodeFilename := c.String("jobexecutionnodefile") var loginNodeHosts []string if loginNodeFilename != "" { - loginNodeHosts, err = parsers.GetHostnamesFromFile(d.disk, loginNodeFilename) + loginNodeHosts, err = parsers2.GetHostnamesFromFile(d.disk, loginNodeFilename) if err != nil { return err } diff --git a/internal/pkg/v2/dacctl/actions_impl/actions_test.go b/internal/pkg/dacctl/actions_impl/actions_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/actions_test.go rename to internal/pkg/dacctl/actions_impl/actions_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/configurations.go b/internal/pkg/dacctl/actions_impl/configurations.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/configurations.go rename to internal/pkg/dacctl/actions_impl/configurations.go diff --git a/internal/pkg/v2/dacctl/actions_impl/instances.go b/internal/pkg/dacctl/actions_impl/instances.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/instances.go rename to internal/pkg/dacctl/actions_impl/instances.go diff --git a/internal/pkg/v2/dacctl/actions_impl/job.go b/internal/pkg/dacctl/actions_impl/job.go similarity index 90% rename from internal/pkg/v2/dacctl/actions_impl/job.go rename to internal/pkg/dacctl/actions_impl/job.go index c1fa81a1..d96f1b4e 100644 --- a/internal/pkg/v2/dacctl/actions_impl/job.go +++ b/internal/pkg/dacctl/actions_impl/job.go @@ -2,9 +2,9 @@ package actions_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" + parsers2 "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "log" "sort" ) @@ -16,7 +16,7 @@ func (d *dacctlActions) ValidateJob(c dacctl.CliContext) error { } jobFile := c.String("job") - summary, err := parsers.ParseJobFile(d.disk, jobFile) + summary, err := parsers2.ParseJobFile(d.disk, jobFile) if err != nil { return err } else { @@ -31,7 +31,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { // TODO: need to specify user and group too jobFile := c.String("job") - summary, err := parsers.ParseJobFile(d.disk, jobFile) + summary, err := parsers2.ParseJobFile(d.disk, jobFile) if err != nil { return err } @@ -42,7 +42,7 @@ func (d *dacctlActions) CreatePerJobBuffer(c dacctl.CliContext) error { log.Printf("Ignoring nodeFile in setup: %s", nodeFile) } - pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) + pool, capacityBytes, err := parsers2.ParseCapacityBytes(c.String("capacity")) if err != nil { return err } diff --git a/internal/pkg/v2/dacctl/actions_impl/job_test.go b/internal/pkg/dacctl/actions_impl/job_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/job_test.go rename to internal/pkg/dacctl/actions_impl/job_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go b/internal/pkg/dacctl/actions_impl/parsers/capacity.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/capacity.go rename to internal/pkg/dacctl/actions_impl/parsers/capacity.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go b/internal/pkg/dacctl/actions_impl/parsers/capacity_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/capacity_test.go rename to internal/pkg/dacctl/actions_impl/parsers/capacity_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go b/internal/pkg/dacctl/actions_impl/parsers/hostnames.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/hostnames.go rename to internal/pkg/dacctl/actions_impl/parsers/hostnames.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go b/internal/pkg/dacctl/actions_impl/parsers/hostnames_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/hostnames_test.go rename to internal/pkg/dacctl/actions_impl/parsers/hostnames_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job.go b/internal/pkg/dacctl/actions_impl/parsers/job.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/job.go rename to internal/pkg/dacctl/actions_impl/parsers/job.go diff --git a/internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go b/internal/pkg/dacctl/actions_impl/parsers/job_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/parsers/job_test.go rename to internal/pkg/dacctl/actions_impl/parsers/job_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent.go b/internal/pkg/dacctl/actions_impl/persistent.go similarity index 88% rename from internal/pkg/v2/dacctl/actions_impl/persistent.go rename to internal/pkg/dacctl/actions_impl/persistent.go index 7dcf4645..182455f5 100644 --- a/internal/pkg/v2/dacctl/actions_impl/persistent.go +++ b/internal/pkg/dacctl/actions_impl/persistent.go @@ -1,9 +1,9 @@ package actions_impl import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" + parsers2 "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "strings" "time" ) @@ -44,7 +44,7 @@ func (d *dacctlActions) CreatePersistentBuffer(c dacctl.CliContext) error { if err != nil { return err } - pool, capacityBytes, err := parsers.ParseCapacityBytes(c.String("capacity")) + pool, capacityBytes, err := parsers2.ParseCapacityBytes(c.String("capacity")) if err != nil { return err } diff --git a/internal/pkg/v2/dacctl/actions_impl/persistent_test.go b/internal/pkg/dacctl/actions_impl/persistent_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/persistent_test.go rename to internal/pkg/dacctl/actions_impl/persistent_test.go diff --git a/internal/pkg/v2/dacctl/actions_impl/pools.go b/internal/pkg/dacctl/actions_impl/pools.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/pools.go rename to internal/pkg/dacctl/actions_impl/pools.go diff --git a/internal/pkg/v2/dacctl/actions_impl/sessions.go b/internal/pkg/dacctl/actions_impl/sessions.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/sessions.go rename to internal/pkg/dacctl/actions_impl/sessions.go diff --git a/internal/pkg/v2/dacctl/actions_impl/show.go b/internal/pkg/dacctl/actions_impl/show.go similarity index 97% rename from internal/pkg/v2/dacctl/actions_impl/show.go rename to internal/pkg/dacctl/actions_impl/show.go index 136fc108..f98e4525 100644 --- a/internal/pkg/v2/dacctl/actions_impl/show.go +++ b/internal/pkg/dacctl/actions_impl/show.go @@ -2,8 +2,8 @@ package actions_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl" ) func (d *dacctlActions) getSession(c dacctl.CliContext) (datamodel.Session, error) { diff --git a/internal/pkg/v2/dacctl/actions_impl/show_test.go b/internal/pkg/dacctl/actions_impl/show_test.go similarity index 100% rename from internal/pkg/v2/dacctl/actions_impl/show_test.go rename to internal/pkg/dacctl/actions_impl/show_test.go diff --git a/internal/pkg/v2/dacctl/interface.go b/internal/pkg/dacctl/interface.go similarity index 100% rename from internal/pkg/v2/dacctl/interface.go rename to internal/pkg/dacctl/interface.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session.go b/internal/pkg/dacctl/workflow_impl/session.go similarity index 100% rename from internal/pkg/v2/dacctl/workflow_impl/session.go rename to internal/pkg/dacctl/workflow_impl/session.go diff --git a/internal/pkg/v2/dacctl/workflow_impl/session_test.go b/internal/pkg/dacctl/workflow_impl/session_test.go similarity index 100% rename from internal/pkg/v2/dacctl/workflow_impl/session_test.go rename to internal/pkg/dacctl/workflow_impl/session_test.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/dacd/brick_manager_impl/brick_manager.go similarity index 94% rename from internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go rename to internal/pkg/dacd/brick_manager_impl/brick_manager.go index 5d905f5b..de39d087 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager.go @@ -2,12 +2,12 @@ package brick_manager_impl import ( "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "log" ) diff --git a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go similarity index 95% rename from internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go rename to internal/pkg/dacd/brick_manager_impl/brick_manager_test.go index 43602dd8..6f111998 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go @@ -2,10 +2,10 @@ package brick_manager_impl import ( "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "os" diff --git a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go b/internal/pkg/dacd/brick_manager_impl/host_bricks.go similarity index 93% rename from internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go rename to internal/pkg/dacd/brick_manager_impl/host_bricks.go index 63176408..3fa974fa 100644 --- a/internal/pkg/v2/dacd/brick_manager_impl/host_bricks.go +++ b/internal/pkg/dacd/brick_manager_impl/host_bricks.go @@ -2,8 +2,8 @@ package brick_manager_impl import ( "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacd/config" ) func getDevices(brickManagerConfig config.BrickManagerConfig) []string { diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go similarity index 100% rename from internal/pkg/v2/dacd/brick_manager_impl/session_action_handler.go rename to internal/pkg/dacd/brick_manager_impl/session_action_handler.go diff --git a/internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler_test.go similarity index 100% rename from internal/pkg/v2/dacd/brick_manager_impl/session_action_handler_test.go rename to internal/pkg/dacd/brick_manager_impl/session_action_handler_test.go diff --git a/internal/pkg/v2/dacd/config/config.go b/internal/pkg/dacd/config/config.go similarity index 100% rename from internal/pkg/v2/dacd/config/config.go rename to internal/pkg/dacd/config/config.go diff --git a/internal/pkg/v2/dacd/config/config_test.go b/internal/pkg/dacd/config/config_test.go similarity index 100% rename from internal/pkg/v2/dacd/config/config_test.go rename to internal/pkg/dacd/config/config_test.go diff --git a/internal/pkg/v2/dacd/interface.go b/internal/pkg/dacd/interface.go similarity index 100% rename from internal/pkg/v2/dacd/interface.go rename to internal/pkg/dacd/interface.go diff --git a/internal/pkg/registry_impl/brick_allocation.go b/internal/pkg/registry_impl/brick_allocation.go index f0a15001..45bdbfe2 100644 --- a/internal/pkg/registry_impl/brick_allocation.go +++ b/internal/pkg/registry_impl/brick_allocation.go @@ -3,10 +3,10 @@ package registry_impl import ( "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "log" ) diff --git a/internal/pkg/registry_impl/brick_host.go b/internal/pkg/registry_impl/brick_host.go index 7903b27a..baf44ec7 100644 --- a/internal/pkg/registry_impl/brick_host.go +++ b/internal/pkg/registry_impl/brick_host.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "log" ) diff --git a/internal/pkg/registry_impl/session.go b/internal/pkg/registry_impl/session.go index 15f786ff..d7e7cc63 100644 --- a/internal/pkg/registry_impl/session.go +++ b/internal/pkg/registry_impl/session.go @@ -3,10 +3,10 @@ package registry_impl import ( "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "log" ) diff --git a/internal/pkg/registry_impl/session_actions.go b/internal/pkg/registry_impl/session_actions.go index 07ee77be..2c69f579 100644 --- a/internal/pkg/registry_impl/session_actions.go +++ b/internal/pkg/registry_impl/session_actions.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/dacctl/actions_impl/parsers" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" - "github.com/RSE-Cambridge/data-acc/internal/pkg/v2/dacctl/actions_impl/parsers" "github.com/google/uuid" "log" ) From be1255e4eb2f6abfc81d4c577f3fb06c273bf987 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 12:13:56 +0100 Subject: [PATCH 147/191] Improve error reporting for ansible failures Ensure we always tidy up the temp dir for teardown failures. --- internal/pkg/filesystem_impl/ansible.go | 48 +++++-------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index cf6f9a76..33f7b95b 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -207,11 +207,15 @@ func executeAnsibleSetup(internalName string, bricks []datamodel.Brick) error { formatArgs := "dac.yml -i inventory --tag format" err = executeAnsiblePlaybook(dir, formatArgs) if err != nil { - return err + return fmt.Errorf("error during server format: %s", err.Error()) } startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" - return executeAnsiblePlaybook(dir, startupArgs) + err = executeAnsiblePlaybook(dir, startupArgs) + if err != nil { + return fmt.Errorf("error during create fs: %s", err.Error()) + } + return nil } func executeAnsibleTeardown(internalName string, bricks []datamodel.Brick) error { @@ -219,53 +223,19 @@ func executeAnsibleTeardown(internalName string, bricks []datamodel.Brick) error if err != nil { return err } + defer os.RemoveAll(dir) stopArgs := "dac.yml -i inventory --tag stop_all,unmount,client_unmount" err = executeAnsiblePlaybook(dir, stopArgs) if err != nil { - return err + return fmt.Errorf("error during server umount: %s", err.Error()) } formatArgs := "dac.yml -i inventory --tag clean" err = executeAnsiblePlaybook(dir, formatArgs) if err != nil { - return err - } - - // only delete if everything worked, to aid debugging - os.RemoveAll(dir) - return nil -} - -func executeAnsibleMount(internalName string, bricks []datamodel.Brick) error { - dir, err := setupAnsible(Lustre, internalName, bricks) - if err != nil { - return err + return fmt.Errorf("error during server clean: %s", err.Error()) } - - startupArgs := "dac.yml -i inventory --tag client_mount" - err = executeAnsiblePlaybook(dir, startupArgs) - if err != nil { - return err - } - - os.RemoveAll(dir) - return nil -} - -func executeAnsibleUnmount(internalName string, bricks []datamodel.Brick) error { - dir, err := setupAnsible(Lustre, internalName, bricks) - if err != nil { - return err - } - - stopArgs := "dac.yml -i inventory --tag client_unmount" - err = executeAnsiblePlaybook(dir, stopArgs) - if err != nil { - return err - } - - os.RemoveAll(dir) return nil } From 268d84b8339219b4bd340fcacfc2faccccc93dd6 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 13:51:57 +0100 Subject: [PATCH 148/191] Fix missing logging and reporting in unmount --- .../session_action_handler.go | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 8f4712d2..2063cb9a 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -266,20 +266,56 @@ func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, s return s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus) } + +func (s *sessionActionHandler) doMutliJobUnmount(action datamodel.SessionAction, sessionName datamodel.SessionName) error { + sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) + if err != nil { + log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) + return err + } + if err = sessionMutex.Lock(context.TODO()); err != nil { + log.Printf("unable to lock session mutex: %s due to: %s\n", sessionName, err) + return err + } + defer func() { + if err := sessionMutex.Unlock(context.TODO()); err != nil { + log.Println("failed to drop mutex for:", sessionName) + } + }() + + multiJobSession, err := s.sessionRegistry.GetSession(sessionName) + if err != nil { + return err + } + if !multiJobSession.VolumeRequest.MultiJob { + log.Panicf("trying multi-job attach to non-multi job session %s", multiJobSession.Name) + } + + attachments, ok := multiJobSession.CurrentAttachments[action.Session.Name] + if !ok { + log.Println("skip detach, already seems to be detached") + return nil + } + if err := s.fsProvider.Unmount(multiJobSession, attachments); err != nil { + return err + } + + // update multi job session to note our attachments have now gone + delete(multiJobSession.CurrentAttachments, action.Session.Name) + _, err = s.sessionRegistry.UpdateSession(multiJobSession) + return err +} + func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) error { if action.Session.ActualSizeBytes > 0 { if err := s.fsProvider.Unmount(action.Session, action.Session.CurrentAttachments[action.Session.Name]); err != nil { return err } - // TODO: delete attachments? } for _, sessionName := range action.Session.MultiJobAttachments { - multiJobSession, _ := s.sessionRegistry.GetSession(sessionName) - attachments := multiJobSession.CurrentAttachments[action.Session.Name] - if err := s.fsProvider.Unmount(multiJobSession, attachments); err != nil { + if err := s.doMutliJobUnmount(action, sessionName); err != nil { return err } - // TODO: delete attachments to prevent double unmount on teardown? } return nil } From 6d0f678e21ef7e013766f2896a80bf400ad89f1e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 14:17:18 +0100 Subject: [PATCH 149/191] Panic when we can't kill a process at timeout --- internal/pkg/filesystem_impl/ansible.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 33f7b95b..7e74b353 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -258,7 +258,9 @@ func executeAnsiblePlaybook(dir string, args string) error { timer := time.AfterFunc(time.Minute*5, func() { log.Println("Time up, waited more than 5 mins to complete.") - cmd.Process.Kill() + if err := cmd.Process.Kill(); err != nil { + log.Panicf("error trying to kill process: %s", err.Error()) + } }) output, currentErr := cmd.CombinedOutput() timer.Stop() From 9b27145282e34bfe4ce8f6fbfaa1c72122af487e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 14:52:41 +0100 Subject: [PATCH 150/191] Revert "Stop formatting disks when deleting a filesystem" This reverts commit 5567c092d4c9e1211a98246314afe9acafcea120. Seem to have issues with fs-ansible not cleaning up correctly, so lets revert back to a known OK ish state to start with. --- fs-ansible/roles/lustre/tasks/format.yaml | 4 ++-- internal/pkg/filesystem_impl/ansible.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index aa85e78d..d951153e 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -5,7 +5,7 @@ mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" osts: "{{ vars[fs_name + '_osts'] | default({}) }}" mdt_size: "{{ vars[fs_name + '_mdt_size'] | default('10%') }}" - tags: [ 'never', 'format_mgs', 'reformat_mgs', 'format', 'clean'] + tags: [ 'never', 'format_mgs', 'reformat_mgs', 'format'] - name: Ensure MGS has been formatted command: /usr/sbin/mkfs.lustre --mgs /dev/{{ mgs }} @@ -66,7 +66,7 @@ when: - osts is defined - tags: [ 'never', 'reformat_mdts', 'reformat_osts', 'format', 'clean'] + tags: [ 'never', 'reformat_mdts', 'reformat_osts', 'format'] - name: Ensure MDTs setup diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 7e74b353..c7f5bdd1 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -231,7 +231,7 @@ func executeAnsibleTeardown(internalName string, bricks []datamodel.Brick) error return fmt.Errorf("error during server umount: %s", err.Error()) } - formatArgs := "dac.yml -i inventory --tag clean" + formatArgs := "dac.yml -i inventory --tag format" err = executeAnsiblePlaybook(dir, formatArgs) if err != nil { return fmt.Errorf("error during server clean: %s", err.Error()) From 9aff651a579f2c44cea33879ff49df69836e5ee9 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 14:53:25 +0100 Subject: [PATCH 151/191] Revert "Refactor LVM formatting" This reverts commit 8c02d2f75151e6493b4fec8d13c4e8dfb19bad29. --- fs-ansible/roles/lustre/tasks/format.yaml | 141 ++++++++++++---------- 1 file changed, 78 insertions(+), 63 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index d951153e..3ea3ff3f 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -22,101 +22,116 @@ - mgs is defined tags: [ 'never', 'reformat_mgs'] - -- name: Clean all previous LV and VG +- name: Clean LVM block: - - - name: Remove LV for MDTs + - name: Remove LV for MDT lvol: lv: "mdt" vg: "dac-{{ item }}" state: absent force: yes loop: "{{ mdts.keys() | list }}" - when: - - mdts is defined - - - name: Remove LV for OST - lvol: - lv: "ost" - vg: "dac-{{ item }}" - state: absent - force: yes - loop: "{{ osts.keys() | list }}" - when: - - osts is defined - - name: Remove VG for MTDs + - name: Remove VG lvg: vg: "dac-{{ item }}" pvs: "/dev/{{ item }}" force: yes state: absent loop: "{{ mdts.keys() | list }}" - when: - - mdts is defined - - - name: Remove VG for OSTs - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - force: yes - state: absent - loop: "{{ osts.keys() | list }}" - when: - - osts is defined - - tags: [ 'never', 'reformat_mdts', 'reformat_osts', 'format'] - -- name: Ensure MDTs setup - block: - - - name: Add VG for MDT - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - state: present + - name: dmsetup remove lvm + command: "dmsetup remove dac--{{ item }}-mdt" + register: command_result + failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" loop: "{{ mdts.keys() | list }}" - - name: Add LV for MDT - lvol: - lv: "mdt" - vg: "dac-{{ item }}" - size: "{{ mdt_size }}" - state: present - loop: "{{ mdts.keys() | list }}" + when: + - mdts is defined + tags: [ 'never', 'reformat_mdts', 'format'] - - name: Reformat MDTs - command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} --mkfsoptions=\"-O large_dir\" /dev/mapper/dac--{{ item.key }}-mdt" - loop: "{{ mdts|dict2items }}" +- name: Add VG + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + state: present + loop: "{{ mdts.keys() | list }}" + when: + - mdts is defined + tags: [ 'never', 'reformat_mdts', 'format'] + +- name: Add LV for MDT + lvol: + lv: "mdt" + vg: "dac-{{ item }}" + size: "{{ mdt_size }}" + state: present + loop: "{{ mdts.keys() | list }}" + when: + - mdts is defined + tags: [ 'never', 'reformat_mdts', 'format'] +- name: Reformat MDTs + command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} --mkfsoptions=\"-O large_dir\" /dev/mapper/dac--{{ item.key }}-mdt" + loop: "{{ mdts|dict2items }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] -- name: Ensure OSTs setup +- name: Clean LVM block: - - name: Add VG for OSTs - lvg: + - name: Remove LV for OST + lvol: + lv: "ost" vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - state: present + state: absent + force: yes loop: "{{ osts.keys() | list }}" + when: + - mdts is defined - - name: Add LV for OSTs - lvol: - lv: "ost" + - name: Remove VG OST + lvg: vg: "dac-{{ item }}" - size: "+100%FREE" - state: present + pvs: "/dev/{{ item }}" + state: absent loop: "{{ osts.keys() | list }}" + when: + - osts is defined + - mdts is not defined + + - name: dmsetup remove lvm + command: "dmsetup remove dac--{{ item }}-ost" + register: command_result + failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" + loop: "{{ mdts.keys() | list }}" + tags: [ 'never', 'reformat_osts', 'format'] - - name: Reformat OSTs - command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/mapper/dac--{{ item.key }}-ost" - loop: "{{ osts|dict2items }}" +- name: Add VG + lvg: + vg: "dac-{{ item }}" + pvs: "/dev/{{ item }}" + state: present + loop: "{{ osts.keys() | list }}" + when: + - osts is defined + tags: [ 'never', 'reformat_osts', 'format'] + +- name: Add LV for OST + lvol: + lv: "ost" + vg: "dac-{{ item }}" + size: +100%FREE + state: present + loop: "{{ osts.keys() | list }}" + when: + - osts is defined + tags: [ 'never', 'reformat_osts', 'format'] +- name: Reformat OSTs + command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/mapper/dac--{{ item.key }}-ost" + loop: "{{ osts|dict2items }}" when: - osts is defined tags: [ 'never', 'reformat_osts', 'format'] From 2ca4a2e3883d2d61ea8d32f93f93a6202ba7dfbe Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 14:56:04 +0100 Subject: [PATCH 152/191] Fix formatting --- internal/pkg/dacd/brick_manager_impl/session_action_handler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 2063cb9a..9f417706 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -266,7 +266,6 @@ func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, s return s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus) } - func (s *sessionActionHandler) doMutliJobUnmount(action datamodel.SessionAction, sessionName datamodel.SessionName) error { sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { From 54ab3fe7175cf36dd98afd35167a479fd3381137 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 14:59:30 +0100 Subject: [PATCH 153/191] Use latest build --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 4aebc9ae..b39264bc 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.3' -data_acc_checksum: 'sha256:797cdde7f85628c75fb86a7c2af359df57184d5162bafe6a62f2765965ce726f' +data_acc_version: 'v2.0-alpha.4' +data_acc_checksum: 'sha256:71aaa54eb3a4fa688f8588774787b87b3bd2e3169a7ce9d0c65530561a827f61' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From 3521b0bbf576bacb6ab09f15132e2f943d96093d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 15:36:00 +0100 Subject: [PATCH 154/191] Move config to top level --- internal/pkg/{dacd => }/config/config.go | 0 internal/pkg/{dacd => }/config/config_test.go | 0 internal/pkg/dacd/brick_manager_impl/brick_manager.go | 2 +- internal/pkg/dacd/brick_manager_impl/brick_manager_test.go | 2 +- internal/pkg/dacd/brick_manager_impl/host_bricks.go | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) rename internal/pkg/{dacd => }/config/config.go (100%) rename internal/pkg/{dacd => }/config/config_test.go (100%) diff --git a/internal/pkg/dacd/config/config.go b/internal/pkg/config/config.go similarity index 100% rename from internal/pkg/dacd/config/config.go rename to internal/pkg/config/config.go diff --git a/internal/pkg/dacd/config/config_test.go b/internal/pkg/config/config_test.go similarity index 100% rename from internal/pkg/dacd/config/config_test.go rename to internal/pkg/config/config_test.go diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/dacd/brick_manager_impl/brick_manager.go index de39d087..b60e27ad 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager.go @@ -2,8 +2,8 @@ package brick_manager_impl import ( "context" + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go index 6f111998..e67abca7 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go @@ -2,7 +2,7 @@ package brick_manager_impl import ( "context" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/mock_registry" diff --git a/internal/pkg/dacd/brick_manager_impl/host_bricks.go b/internal/pkg/dacd/brick_manager_impl/host_bricks.go index 3fa974fa..9d149b7e 100644 --- a/internal/pkg/dacd/brick_manager_impl/host_bricks.go +++ b/internal/pkg/dacd/brick_manager_impl/host_bricks.go @@ -2,7 +2,7 @@ package brick_manager_impl import ( "fmt" - "github.com/RSE-Cambridge/data-acc/internal/pkg/dacd/config" + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" ) From 8a491f3eecc6553ab8e8efcf9df79529844dc95a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 15:48:55 +0100 Subject: [PATCH 155/191] Add keystore config --- internal/pkg/config/brick_manager.go | 29 ++++++++++++++++++++++++++ internal/pkg/config/config.go | 24 --------------------- internal/pkg/config/keystore.go | 31 ++++++++++++++++++++++++++++ internal/pkg/store_impl/keystore.go | 27 +++++++----------------- 4 files changed, 68 insertions(+), 43 deletions(-) create mode 100644 internal/pkg/config/brick_manager.go create mode 100644 internal/pkg/config/keystore.go diff --git a/internal/pkg/config/brick_manager.go b/internal/pkg/config/brick_manager.go new file mode 100644 index 00000000..6c54c475 --- /dev/null +++ b/internal/pkg/config/brick_manager.go @@ -0,0 +1,29 @@ +package config + +import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "log" +) + +type BrickManagerConfig struct { + BrickHostName datamodel.BrickHostName + PoolName datamodel.PoolName + DeviceCapacityGiB uint + DeviceCount uint + DeviceAddressPattern string + HostEnabled bool +} + +// TODO: need additional validation here +func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { + config := BrickManagerConfig{ + datamodel.BrickHostName(getHostname(env)), + datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), + getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB + getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT + getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), // TODO: DEVICE_TYPE + getBool(env, "DAC_HOST_ENABLED", true), + } + log.Println("Got brick manager config:", config) + return config +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index ca5180ce..cf02598f 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -1,40 +1,16 @@ package config import ( - "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "log" "os" "strconv" ) -type BrickManagerConfig struct { - BrickHostName datamodel.BrickHostName - PoolName datamodel.PoolName - DeviceCapacityGiB uint - DeviceCount uint - DeviceAddressPattern string - HostEnabled bool -} - type ReadEnvironemnt interface { LookupEnv(key string) (string, bool) Hostname() (string, error) } -// TODO: need additional validation here -func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { - config := BrickManagerConfig{ - datamodel.BrickHostName(getHostname(env)), - datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), - getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB - getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT - getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), // TODO: DEVICE_TYPE - getBool(env, "DAC_HOST_ENABLED", true), - } - log.Println("Got brick manager config:", config) - return config -} - func getHostname(env ReadEnvironemnt) string { hostname, err := env.Hostname() if err != nil { diff --git a/internal/pkg/config/keystore.go b/internal/pkg/config/keystore.go new file mode 100644 index 00000000..ab27e2d8 --- /dev/null +++ b/internal/pkg/config/keystore.go @@ -0,0 +1,31 @@ +package config + +import ( + "log" + "strings" +) + +type KeystoreConfig struct { + Endpoints []string + CertFile string + KeyFile string + CAFile string +} + +func GetKeystoreConfig(env ReadEnvironemnt) KeystoreConfig { + config := KeystoreConfig{ + CertFile: getString(env, "ETCDCTL_CERT_FILE", ""), + KeyFile: getString(env, "ETCDCTL_KEY_FILE", ""), + CAFile: getString(env, "ETCDCTL_CA_FILE", ""), + } + endpointsStr := getString(env, "ETCDCTL_ENDPOINTS", "") + if endpointsStr == "" { + log.Println("ETCD_ENDPOINTS is deprecated please use ETCDCTL_ENDPOINTS") + endpointsStr = getString(env, "ETCD_ENDPOINTS", "") + } + if endpointsStr == "" { + log.Fatalf("Must set ETCDCTL_ENDPOINTS environemnt variable, e.g. export ETCDCTL_ENDPOINTS=127.0.0.1:2379") + } + config.Endpoints = strings.Split(endpointsStr, ",") + return config +} diff --git a/internal/pkg/store_impl/keystore.go b/internal/pkg/store_impl/keystore.go index 7904b75d..ac13b281 100644 --- a/internal/pkg/store_impl/keystore.go +++ b/internal/pkg/store_impl/keystore.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "errors" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/clientv3util" @@ -13,15 +14,13 @@ import ( "github.com/coreos/etcd/mvcc/mvccpb" "github.com/coreos/etcd/pkg/transport" "log" - "os" - "strings" "time" ) -func getTLSConfig() *tls.Config { - certFile := os.Getenv("ETCDCTL_CERT_FILE") - keyFile := os.Getenv("ETCDCTL_KEY_FILE") - caFile := os.Getenv("ETCDCTL_CA_FILE") +func getTLSConfig(keystoreConfig config.KeystoreConfig) *tls.Config { + certFile := keystoreConfig.CertFile + keyFile := keystoreConfig.KeyFile + caFile := keystoreConfig.CAFile if certFile == "" || keyFile == "" || caFile == "" { return nil @@ -39,22 +38,12 @@ func getTLSConfig() *tls.Config { return tlsConfig } -func getEndpoints() []string { - endpoints := os.Getenv("ETCDCTL_ENDPOINTS") - if endpoints == "" { - endpoints = os.Getenv("ETCD_ENDPOINTS") - } - if endpoints == "" { - log.Fatalf("Must set ETCDCTL_ENDPOINTS environemnt variable, e.g. export ETCDCTL_ENDPOINTS=127.0.0.1:2379") - } - return strings.Split(endpoints, ",") -} - func newEtcdClient() *clientv3.Client { + conf := config.GetKeystoreConfig(config.DefaultEnv) cli, err := clientv3.New(clientv3.Config{ - Endpoints: getEndpoints(), + Endpoints: conf.Endpoints, DialTimeout: 10 * time.Second, - TLS: getTLSConfig(), + TLS: getTLSConfig(conf), }) if err != nil { fmt.Println("failed to create client") From deeb7851e3645a090ca182d94927ae312ebd26b0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 15:56:05 +0100 Subject: [PATCH 156/191] Move log config to central location --- cmd/dacctl/main.go | 6 ++---- internal/pkg/config/log.go | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 internal/pkg/config/log.go diff --git a/cmd/dacctl/main.go b/cmd/dacctl/main.go index 3bfc0e17..0d9e7e33 100644 --- a/cmd/dacctl/main.go +++ b/cmd/dacctl/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/pkg/version" "github.com/urfave/cli" "log" @@ -166,10 +167,7 @@ func runCli(args []string) error { } func main() { - logFilename := os.Getenv("DACCTL_LOG") - if logFilename == "" { - logFilename = "/var/log/dacctl.log" - } + logFilename := config.GetDacctlLog() f, err := os.OpenFile(logFilename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { log.Fatalf("please use DACCTL_LOG to configure an alternative, as error opening file: %v ", err) diff --git a/internal/pkg/config/log.go b/internal/pkg/config/log.go new file mode 100644 index 00000000..b8f31381 --- /dev/null +++ b/internal/pkg/config/log.go @@ -0,0 +1,5 @@ +package config + +func GetDacctlLog() string { + return getString(DefaultEnv, "DACCTL_LOG", "/var/log/dacctl.log") +} From 5b95db2c94db8b8cf3b9271e71cb380d123166c6 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 16:40:10 +0100 Subject: [PATCH 157/191] Add config for filesystem settings --- internal/pkg/config/filesystem.go | 29 ++++++++++++++++++ internal/pkg/filesystem_impl/ansible.go | 39 ++++++------------------- internal/pkg/filesystem_impl/mount.go | 30 ++++--------------- 3 files changed, 44 insertions(+), 54 deletions(-) create mode 100644 internal/pkg/config/filesystem.go diff --git a/internal/pkg/config/filesystem.go b/internal/pkg/config/filesystem.go new file mode 100644 index 00000000..39874e42 --- /dev/null +++ b/internal/pkg/config/filesystem.go @@ -0,0 +1,29 @@ +package config + +type FilesystemConfig struct { + MGSDevice string + MaxMDTs uint + HostGroup string + AnsibleDir string + SkipAnsible bool + LnetSuffix string + MDTSizeMB uint +} + +func GetFilesystemConfig() FilesystemConfig { + env := DefaultEnv + conf := FilesystemConfig{ + MGSDevice: getString(env, "DAC_MGS_DEV", "sdb"), + MaxMDTs: getUint(env, "DAC_MAX_MDT_COUNT", 24), + HostGroup: getString(env, "DAC_HOST_GROUP", "dac-prod"), + AnsibleDir: getString(env, "DAC_ANSIBLE_DIR", "/var/lib/data-acc/fs-ansible/"), + SkipAnsible: getBool(env, "DAC_SKIP_ANSIBLE", false), + LnetSuffix: getString(env, "DAC_LNET_SUFFIX", ""), + } + mdtSizeMB := getUint(env, "DAC_MDT_SIZE_GB", 0) * 1024 + if mdtSizeMB == 0 { + mdtSizeMB = getUint(env, "DAC_MDT_SIZE_MB", uint(20*1024)) + } + conf.MDTSizeMB = mdtSizeMB + return conf +} diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index c7f5bdd1..fb908d06 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -3,6 +3,7 @@ package filesystem_impl import ( "bytes" "fmt" + "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "gopkg.in/yaml.v2" "io/ioutil" @@ -11,7 +12,6 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "strings" "time" ) @@ -35,21 +35,9 @@ type Wrapper struct { All FileSystems } -var DefaultHostGroup = "dac-prod" -var DefaultMaxMDTs uint = 24 +var conf = config.GetFilesystemConfig() func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) string { - // NOTE: only used by lustre - mgsDevice := os.Getenv("DAC_MGS_DEV") - if mgsDevice == "" { - mgsDevice = "sdb" - } - maxMDTs := DefaultMaxMDTs - maxMDTsConf, err := strconv.ParseUint(os.Getenv("DAC_MAX_MDT_COUNT"), 10, 32) - if err == nil && maxMDTsConf > 0 { - maxMDTs = uint(maxMDTsConf) - } - allocationByHost := make(map[datamodel.BrickHostName][]datamodel.BrickAllocation) for i, brick := range allBricks { allocationByHost[brick.BrickHostName] = append(allocationByHost[brick.BrickHostName], datamodel.BrickAllocation{ @@ -62,7 +50,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str // assign at most one mdt per host. // While this may give us less MDTs than max MDTs, // but it helps spread MDTs across network connections - oneMdtPerHost := len(allBricks) > int(maxMDTs) + oneMdtPerHost := len(allBricks) > int(conf.MaxMDTs) hosts := make(map[string]HostInfo) mgsnode := "" @@ -86,7 +74,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str if allocations[0].AllocatedIndex == 0 { if fsType == Lustre { - hostInfo.MGS = mgsDevice + hostInfo.MGS = conf.MGSDevice } else { hostInfo.MGS = allocations[0].Brick.Device } @@ -101,8 +89,8 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str Vars: map[string]string{ "mgsnode": mgsnode, //"client_port": fmt.Sprintf("%d", volume.ClientPort), - "lnet_suffix": getLnetSuffix(), - "mdt_size": fmt.Sprintf("%dm", getMdtSizeMB()), + "lnet_suffix": conf.LnetSuffix, + "mdt_size": fmt.Sprintf("%dm", conf.MDTSizeMB), }, Hosts: hosts, } @@ -121,11 +109,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str strOut = strings.Replace(strOut, " client_port:", fmt.Sprintf(" %s_client_port:", fsname), -1) strOut = strings.Replace(strOut, " mdt_size:", fmt.Sprintf(" %s_mdt_size:", fsname), -1) - hostGroup := os.Getenv("DAC_HOST_GROUP") - if hostGroup == "" { - hostGroup = DefaultHostGroup - } - strOut = strings.Replace(strOut, "all:", hostGroup+":", -1) + strOut = strings.Replace(strOut, "all:", conf.HostGroup+":", -1) return strOut } @@ -146,11 +130,7 @@ func getPlaybook(fsType FSType, fsUuid string) string { } func getAnsibleDir(suffix string) string { - ansibleDir := os.Getenv("DAC_ANSIBLE_DIR") - if ansibleDir == "" { - ansibleDir = "/var/lib/data-acc/fs-ansible/" - } - return path.Join(ansibleDir, suffix) + return path.Join(conf.AnsibleDir, suffix) } func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) (string, error) { @@ -244,8 +224,7 @@ func executeAnsiblePlaybook(dir string, args string) error { cmdStr := fmt.Sprintf(`cd %s; . .venv/bin/activate; ansible-playbook %s;`, dir, args) log.Println("Requested ansible:", cmdStr) - skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") - if skipAnsible == "True" { + if conf.SkipAnsible { log.Println("Skip as DAC_SKIP_ANSIBLE=True") time.Sleep(time.Millisecond * 200) return nil diff --git a/internal/pkg/filesystem_impl/mount.go b/internal/pkg/filesystem_impl/mount.go index 3d3b154d..e7368c7b 100644 --- a/internal/pkg/filesystem_impl/mount.go +++ b/internal/pkg/filesystem_impl/mount.go @@ -4,10 +4,8 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "log" - "os" "os/exec" "path" - "strconv" "time" ) @@ -19,22 +17,6 @@ func getMountDir(sourceName datamodel.SessionName, isMultiJob bool, attachingFor return fmt.Sprintf("/dac/%s_job", sourceName) } -func getLnetSuffix() string { - return os.Getenv("DAC_LNET_SUFFIX") -} - -func getMdtSizeMB() uint { - mdtSizeGB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_GB"), 10, 32) - if err == nil && mdtSizeGB > 0 { - return uint(mdtSizeGB * 1024) - } - mdtSizeMB, err := strconv.ParseUint(os.Getenv("DAC_MDT_SIZE_MB"), 10, 32) - if err == nil && mdtSizeMB > 0 { - return uint(mdtSizeMB) - } - return uint(20 * 1024) -} - func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, internalName string, primaryBrickHost datamodel.BrickHostName, attachment datamodel.AttachmentSessionStatus, owner uint, group uint) error { @@ -44,8 +26,6 @@ func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, in log.Panicf("failed to find primary brick for volume: %s", sessionName) } - lnetSuffix := getLnetSuffix() - if fsType == BeegFS { // Write out the config needed, and do the mount using ansible // TODO: Move Lustre mount here that is done below @@ -60,7 +40,7 @@ func mount(fsType FSType, sessionName datamodel.SessionName, isMultiJob bool, in if err := mkdir(attachHost, mountDir); err != nil { return err } - if err := mountRemoteFilesystem(fsType, attachHost, lnetSuffix, + if err := mountRemoteFilesystem(fsType, attachHost, conf.LnetSuffix, string(primaryBrickHost), internalName, mountDir); err != nil { return err } @@ -260,11 +240,11 @@ type Run interface { type run struct { } +// TODO: need some code sharing here!!! func (*run) Execute(hostname string, cmdStr string) error { log.Println("SSH to:", hostname, "with command:", cmdStr) - skipAnsible := os.Getenv("DAC_SKIP_ANSIBLE") - if skipAnsible == "True" { + if conf.SkipAnsible { log.Println("Skip as DAC_SKIP_ANSIBLE=True") time.Sleep(time.Millisecond * 200) return nil @@ -275,7 +255,9 @@ func (*run) Execute(hostname string, cmdStr string) error { timer := time.AfterFunc(time.Minute, func() { log.Println("Time up, waited more than 5 mins to complete.") - cmd.Process.Kill() + if err := cmd.Process.Kill(); err != nil { + log.Panicf("error trying to kill process: %s", err.Error()) + } }) output, err := cmd.CombinedOutput() From 2afde66ccff428501cc68e6cb19d5aa2d40f506c Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 16:42:56 +0100 Subject: [PATCH 158/191] Add backwards compatibility for config changes --- internal/pkg/config/brick_manager.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/pkg/config/brick_manager.go b/internal/pkg/config/brick_manager.go index 6c54c475..e2e6ae3d 100644 --- a/internal/pkg/config/brick_manager.go +++ b/internal/pkg/config/brick_manager.go @@ -19,9 +19,12 @@ func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { config := BrickManagerConfig{ datamodel.BrickHostName(getHostname(env)), datamodel.PoolName(getString(env, "DAC_POOL_NAME", "default")), - getUint(env, "DAC_BRICK_CAPACITY_GB", 1400), // TODO: DAC_DEVICE_CAPACITY_GB - getUint(env, "DAC_BRICK_COUNT", 12), // TODO: DEVICE_COUNT - getString(env, "DAC_BRICK_ADDRESS_PATTERN", "nvme%dn1"), // TODO: DEVICE_TYPE + getUint(env, "DAC_BRICK_CAPACITY_GB", + getUint(env, "DAC_DEVICE_CAPACITY_GB", 1400)), + getUint(env, "DAC_BRICK_COUNT", + getUint(env, "DEVICE_COUNT", 12)), + getString(env, "DAC_BRICK_ADDRESS_PATTERN", + getString(env, "DEVICE_TYPE", "nvme%dn1")), getBool(env, "DAC_HOST_ENABLED", true), } log.Println("Got brick manager config:", config) From 01fe35173589f2b73fdab83c5607d3a4d3cd47c5 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 17:15:39 +0100 Subject: [PATCH 159/191] Add 30 min timeout for all dacctl operations --- internal/pkg/dacctl/workflow_impl/session.go | 22 ++++++++++++++----- .../pkg/dacctl/workflow_impl/session_test.go | 12 +++++----- internal/pkg/registry_impl/session_actions.go | 3 ++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/internal/pkg/dacctl/workflow_impl/session.go b/internal/pkg/dacctl/workflow_impl/session.go index 00fd049a..5758de97 100644 --- a/internal/pkg/dacctl/workflow_impl/session.go +++ b/internal/pkg/dacctl/workflow_impl/session.go @@ -31,30 +31,42 @@ type sessionFacade struct { func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType datamodel.SessionActionType, getSession func() (datamodel.Session, error)) error { + // 30 min timeout to aquire lock and send action + ctxt, cancelFunc := context.WithTimeout(context.Background(), time.Minute*30) + defer func() { + cancelFunc() + }() sessionMutex, err := s.session.GetSessionMutex(sessionName) if err != nil { return fmt.Errorf("unable to get session mutex: %s due to: %s", sessionName, err) } - err = sessionMutex.Lock(context.TODO()) + err = sessionMutex.Lock(ctxt) if err != nil { return fmt.Errorf("unable to lock session mutex: %s due to: %s", sessionName, err) } session, err := getSession() if err != nil { - sessionMutex.Unlock(context.TODO()) + unlockErr := sessionMutex.Unlock(context.TODO()) + if unlockErr != nil { + log.Println("failed to drop mutex", unlockErr) + } return err } - if session.Name == "" && err == nil { + if session.Name == "" { // skip processing for this session // e.g. its a delete and we have already been deleted - sessionMutex.Unlock(context.TODO()) + unlockErr := sessionMutex.Unlock(context.TODO()) + if unlockErr != nil { + log.Println("failed to drop mutex", unlockErr) + } return nil } // This will error out if the host is not currently up - sessionActions, err := s.actions.SendSessionAction(context.TODO(), actionType, session) + sessionActions, err := s.actions.SendSessionAction(ctxt, actionType, session) + // Drop mutex regardless of if we had an error or not mutexErr := sessionMutex.Unlock(context.TODO()) if err != nil { return err diff --git a/internal/pkg/dacctl/workflow_impl/session_test.go b/internal/pkg/dacctl/workflow_impl/session_test.go index 7f77f2ec..fccc9cd9 100644 --- a/internal/pkg/dacctl/workflow_impl/session_test.go +++ b/internal/pkg/dacctl/workflow_impl/session_test.go @@ -31,7 +31,7 @@ func TestSessionFacade_CreateSession_NoBricks(t *testing.T) { allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) - sessionMutex.EXPECT().Lock(context.TODO()) + sessionMutex.EXPECT().Lock(gomock.Any()) brickList := []datamodel.Brick{{Device: "sda", BrickHostName: datamodel.BrickHostName("host1")}} allocations.EXPECT().GetAllPoolInfos().Return([]datamodel.PoolInfo{{AvailableBricks: brickList}}, nil) initialSession.PrimaryBrickHost = "host1" @@ -63,7 +63,7 @@ func TestSessionFacade_CreateSession_WithBricks_AllocationError(t *testing.T) { allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) - sessionMutex.EXPECT().Lock(context.TODO()) + sessionMutex.EXPECT().Lock(gomock.Any()) allocationMutex := mock_store.NewMockMutex(mockCtrl) allocations.EXPECT().GetAllocationMutex().Return(allocationMutex, nil) allocationMutex.EXPECT().Lock(context.TODO()) @@ -100,7 +100,7 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) allocations.EXPECT().GetPool(datamodel.PoolName("pool1")).Return(datamodel.Pool{Name: "pool1"}, nil) sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(initialSession.Name).Return(sessionMutex, nil) - sessionMutex.EXPECT().Lock(context.TODO()) + sessionMutex.EXPECT().Lock(gomock.Any()) allocationMutex := mock_store.NewMockMutex(mockCtrl) allocations.EXPECT().GetAllocationMutex().Return(allocationMutex, nil) allocationMutex.EXPECT().Lock(context.TODO()) @@ -129,7 +129,7 @@ func TestSessionFacade_CreateSession_WithBricks_CreateSessionError(t *testing.T) allocationMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") actionChan := make(chan datamodel.SessionAction) - actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionCreateFilesystem, returnedSession).Return(actionChan, nil) + actions.EXPECT().SendSessionAction(gomock.Any(), datamodel.SessionCreateFilesystem, returnedSession).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) go func() { actionChan <- datamodel.SessionAction{Error: fakeErr.Error()} @@ -150,7 +150,7 @@ func TestSessionFacade_DeleteSession(t *testing.T) { facade := sessionFacade{session: sessionRegistry, actions: actions} sessionMutex := mock_store.NewMockMutex(mockCtrl) sessionRegistry.EXPECT().GetSessionMutex(sessionName).Return(sessionMutex, nil) - sessionMutex.EXPECT().Lock(context.TODO()) + sessionMutex.EXPECT().Lock(gomock.Any()) initialSession := datamodel.Session{Name: "foo"} sessionRegistry.EXPECT().GetSession(sessionName).Return(initialSession, nil) updatedSession := datamodel.Session{ @@ -162,7 +162,7 @@ func TestSessionFacade_DeleteSession(t *testing.T) { } sessionRegistry.EXPECT().UpdateSession(updatedSession).Return(initialSession, nil) actionChan := make(chan datamodel.SessionAction) - actions.EXPECT().SendSessionAction(context.TODO(), datamodel.SessionDelete, initialSession).Return(actionChan, nil) + actions.EXPECT().SendSessionAction(gomock.Any(), datamodel.SessionDelete, initialSession).Return(actionChan, nil) sessionMutex.EXPECT().Unlock(context.TODO()) fakeErr := errors.New("fake") go func() { diff --git a/internal/pkg/registry_impl/session_actions.go b/internal/pkg/registry_impl/session_actions.go index 2c69f579..95ac2389 100644 --- a/internal/pkg/registry_impl/session_actions.go +++ b/internal/pkg/registry_impl/session_actions.go @@ -121,7 +121,8 @@ func (s *sessionActions) SendSessionAction( close(responseChan) return } - // TODO: don't we need to stop the watch above? will we see the delete event? + log.Println("stopped waiting for action response, likely the context timed out") + // TODO: double check watch gets stopped somehow? assume context has been cancelled externally? }() return responseChan, nil } From f8816fb99f6ad996dfb0c36e5363f289cd199738 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 30 Aug 2019 17:24:02 +0100 Subject: [PATCH 160/191] Don't show bricks for dead hosts as available --- internal/pkg/registry_impl/brick_allocation.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/pkg/registry_impl/brick_allocation.go b/internal/pkg/registry_impl/brick_allocation.go index 45bdbfe2..7f501c2a 100644 --- a/internal/pkg/registry_impl/brick_allocation.go +++ b/internal/pkg/registry_impl/brick_allocation.go @@ -134,6 +134,17 @@ func (a *allocationRegistry) GetAllPoolInfos() ([]datamodel.PoolInfo, error) { } for _, brickHost := range brickHosts { + // skip disabled hosts + if !brickHost.Enabled { + continue + } + // skip dead hosts + hostAlive, _ := a.brickHostRegistry.IsBrickHostAlive(brickHost.Name) + if !hostAlive { + continue + } + + // look for any unallocated bricks for _, brick := range brickHost.Bricks { allocated := false for _, allocatedDevice := range allocatedDevicesByBrickHost[brick.BrickHostName] { From edbfde73bf048f4a3b523607f1adb2ff3d6683f1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 14:58:37 +0100 Subject: [PATCH 161/191] Harden the state machine handing in session_action_handler --- internal/pkg/config/keystore.go | 1 - internal/pkg/dacctl/workflow_impl/session.go | 9 +- .../session_action_handler.go | 159 ++++++++++-------- 3 files changed, 100 insertions(+), 69 deletions(-) diff --git a/internal/pkg/config/keystore.go b/internal/pkg/config/keystore.go index ab27e2d8..fbf1a93e 100644 --- a/internal/pkg/config/keystore.go +++ b/internal/pkg/config/keystore.go @@ -20,7 +20,6 @@ func GetKeystoreConfig(env ReadEnvironemnt) KeystoreConfig { } endpointsStr := getString(env, "ETCDCTL_ENDPOINTS", "") if endpointsStr == "" { - log.Println("ETCD_ENDPOINTS is deprecated please use ETCDCTL_ENDPOINTS") endpointsStr = getString(env, "ETCD_ENDPOINTS", "") } if endpointsStr == "" { diff --git a/internal/pkg/dacctl/workflow_impl/session.go b/internal/pkg/dacctl/workflow_impl/session.go index 5758de97..1dc9633a 100644 --- a/internal/pkg/dacctl/workflow_impl/session.go +++ b/internal/pkg/dacctl/workflow_impl/session.go @@ -222,6 +222,11 @@ func (s sessionFacade) DeleteSession(sessionName datamodel.SessionName, hurry bo return session, nil } + if session.Status.DeleteRequested { + // TODO: is there anything we can do about this? + log.Println("Warning, delete already called") + } + // Record we want this deleted, in case host is not alive // can be deleted when it is next stated session.Status.DeleteRequested = true @@ -242,8 +247,8 @@ func (s sessionFacade) Mount(sessionName datamodel.SessionName, computeNodes []s func() (datamodel.Session, error) { session, err := s.session.GetSession(sessionName) if err != nil { - log.Println("Unable to find session, skipping delete:", sessionName) - return session, nil + log.Println("Unable to find session we want to mount:", sessionName) + return session, err } // TODO: what about the login nodes? what do we want to do there? diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 9f417706..288f178d 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -2,6 +2,7 @@ package brick_manager_impl import ( "context" + "errors" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" @@ -100,6 +101,9 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { if err != nil { return action.Session, fmt.Errorf("error getting session: %s", err) } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") + } fsStatus, err := s.fsProvider.Create(action.Session) session.FilesystemStatus = fsStatus @@ -121,39 +125,49 @@ func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - if !action.Session.Status.UnmountComplete { - if err := s.doAllUnmounts(action); err != nil { - log.Println("failed unmount during delete", action.Session.Name) + session, err := s.sessionRegistry.GetSession(action.Session.Name) + if err != nil { + // TODO: deal with already being deleted? add check if exists call? + return action.Session, fmt.Errorf("error getting session: %s", err) + } + + if !session.Status.UnmountComplete { + if err := s.doAllUnmounts(session); err != nil { + log.Println("failed unmount during delete", session.Name) } } - if !action.Session.Status.CopyDataOutComplete && !action.Session.Status.DeleteSkipCopyDataOut { + if !session.Status.CopyDataOutComplete && !session.Status.DeleteSkipCopyDataOut { if err := s.fsProvider.DataCopyOut(action.Session); err != nil { log.Println("failed DataCopyOut during delete", action.Session.Name) } } // Only try delete if we have bricks to delete - if action.Session.ActualSizeBytes > 0 { - if err := s.fsProvider.Delete(action.Session); err != nil { - return action.Session, err + if session.ActualSizeBytes > 0 { + if err := s.fsProvider.Delete(session); err != nil { + return session, err } } - return action.Session, s.sessionRegistry.DeleteSession(action.Session) + return session, s.sessionRegistry.DeleteSession(session) }) } func (s *sessionActionHandler) handleCopyIn(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.fsProvider.DataCopyIn(action.Session) + // Get latest session now we have the mutex + session, err := s.sessionRegistry.GetSession(action.Session.Name) if err != nil { - return action.Session, err + return action.Session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") } - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - session = action.Session + if err := s.fsProvider.DataCopyIn(session); err != nil { + return session, err } + session.Status.CopyDataInComplete = true return s.sessionRegistry.UpdateSession(session) }) @@ -161,61 +175,62 @@ func (s *sessionActionHandler) handleCopyIn(action datamodel.SessionAction) { func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.fsProvider.DataCopyOut(action.Session) + // Get latest session now we have the mutex + session, err := s.sessionRegistry.GetSession(action.Session.Name) if err != nil { - return action.Session, err + return action.Session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") } - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - session = action.Session + if err := s.fsProvider.DataCopyOut(session); err != nil { + return session, err } + session.Status.CopyDataOutComplete = true return s.sessionRegistry.UpdateSession(session) }) } -func (s *sessionActionHandler) doAllMounts(action datamodel.SessionAction) error { +func (s *sessionActionHandler) doAllMounts(actionSession datamodel.Session) error { attachmentSession := datamodel.AttachmentSession{ - Hosts: action.Session.RequestedAttachHosts, - SessionName: action.Session.Name, + Hosts: actionSession.RequestedAttachHosts, + SessionName: actionSession.Name, } - if action.Session.ActualSizeBytes > 0 { - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - return err - } + if actionSession.ActualSizeBytes > 0 { jobAttachmentStatus := datamodel.AttachmentSessionStatus{ AttachmentSession: attachmentSession, - GlobalMount: session.VolumeRequest.Access == datamodel.Striped || session.VolumeRequest.Access == datamodel.PrivateAndStriped, - PrivateMount: session.VolumeRequest.Access == datamodel.Private || session.VolumeRequest.Access == datamodel.PrivateAndStriped, - SwapBytes: session.VolumeRequest.SwapBytes, - } - if session.CurrentAttachments == nil { - session.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ - session.Name: jobAttachmentStatus, - } - } else { - session.CurrentAttachments[session.Name] = jobAttachmentStatus + GlobalMount: actionSession.VolumeRequest.Access == datamodel.Striped || actionSession.VolumeRequest.Access == datamodel.PrivateAndStriped, + PrivateMount: actionSession.VolumeRequest.Access == datamodel.Private || actionSession.VolumeRequest.Access == datamodel.PrivateAndStriped, + SwapBytes: actionSession.VolumeRequest.SwapBytes, } - session, err = s.sessionRegistry.UpdateSession(session) - if err != nil { - return err - } - - if err := s.fsProvider.Mount(action.Session, jobAttachmentStatus); err != nil { + //if actionSession.CurrentAttachments == nil { + // session.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ + // session.Name: jobAttachmentStatus, + // } + //} else { + // session.CurrentAttachments[session.Name] = jobAttachmentStatus + //} + //session, err = s.sessionRegistry.UpdateSession(session) + //if err != nil { + // return err + //} + + if err := s.fsProvider.Mount(actionSession, jobAttachmentStatus); err != nil { return err } + // TODO: should we update the session? and delete attachments later? } - for _, sessionName := range action.Session.MultiJobAttachments { - if err := s.doMutliJobMount(action, sessionName); err != nil { + for _, sessionName := range actionSession.MultiJobAttachments { + if err := s.doMultiJobMount(actionSession, sessionName); err != nil { return nil } } return nil } -func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, sessionName datamodel.SessionName) error { +func (s *sessionActionHandler) doMultiJobMount(actionSession datamodel.Session, sessionName datamodel.SessionName) error { sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) @@ -240,8 +255,8 @@ func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, s } attachmentSession := datamodel.AttachmentSession{ - Hosts: action.Session.RequestedAttachHosts, - SessionName: action.Session.Name, + Hosts: actionSession.RequestedAttachHosts, + SessionName: actionSession.Name, } multiJobAttachmentStatus := datamodel.AttachmentSessionStatus{ AttachmentSession: attachmentSession, @@ -266,7 +281,7 @@ func (s *sessionActionHandler) doMutliJobMount(action datamodel.SessionAction, s return s.fsProvider.Mount(multiJobSession, multiJobAttachmentStatus) } -func (s *sessionActionHandler) doMutliJobUnmount(action datamodel.SessionAction, sessionName datamodel.SessionName) error { +func (s *sessionActionHandler) doMultiJobUnmount(actionSession datamodel.Session, sessionName datamodel.SessionName) error { sessionMutex, err := s.sessionRegistry.GetSessionMutex(sessionName) if err != nil { log.Printf("unable to get session mutex: %s due to: %s\n", sessionName, err) @@ -290,7 +305,7 @@ func (s *sessionActionHandler) doMutliJobUnmount(action datamodel.SessionAction, log.Panicf("trying multi-job attach to non-multi job session %s", multiJobSession.Name) } - attachments, ok := multiJobSession.CurrentAttachments[action.Session.Name] + attachments, ok := multiJobSession.CurrentAttachments[actionSession.Name] if !ok { log.Println("skip detach, already seems to be detached") return nil @@ -300,19 +315,19 @@ func (s *sessionActionHandler) doMutliJobUnmount(action datamodel.SessionAction, } // update multi job session to note our attachments have now gone - delete(multiJobSession.CurrentAttachments, action.Session.Name) + delete(multiJobSession.CurrentAttachments, actionSession.Name) _, err = s.sessionRegistry.UpdateSession(multiJobSession) return err } -func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) error { - if action.Session.ActualSizeBytes > 0 { - if err := s.fsProvider.Unmount(action.Session, action.Session.CurrentAttachments[action.Session.Name]); err != nil { +func (s *sessionActionHandler) doAllUnmounts(actionSession datamodel.Session) error { + if actionSession.ActualSizeBytes > 0 { + if err := s.fsProvider.Unmount(actionSession, actionSession.CurrentAttachments[actionSession.Name]); err != nil { return err } } - for _, sessionName := range action.Session.MultiJobAttachments { - if err := s.doMutliJobUnmount(action, sessionName); err != nil { + for _, sessionName := range actionSession.MultiJobAttachments { + if err := s.doMultiJobUnmount(actionSession, sessionName); err != nil { return err } } @@ -321,18 +336,24 @@ func (s *sessionActionHandler) doAllUnmounts(action datamodel.SessionAction) err func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.doAllMounts(action) + session, err := s.sessionRegistry.GetSession(action.Session.Name) if err != nil { - if err := s.doAllUnmounts(action); err != nil { + return action.Session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") + } + if session.Status.MountComplete { + return session, errors.New("already mounted, can't mount again") + } + + if err := s.doAllMounts(session); err != nil { + if err := s.doAllUnmounts(session); err != nil { log.Println("error while rolling back possible partial mount", action.Session.Name, err) } return action.Session, err } - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - session = action.Session - } session.Status.MountComplete = true return s.sessionRegistry.UpdateSession(session) }) @@ -340,15 +361,21 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - err := s.doAllUnmounts(action) + session, err := s.sessionRegistry.GetSession(action.Session.Name) if err != nil { - return action.Session, err + return action.Session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") + } + if session.Status.UnmountComplete { + return session, errors.New("already unmounted, can't umount again") } - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - session = action.Session + if err := s.doAllUnmounts(session); err != nil { + return action.Session, err } + session.Status.UnmountComplete = true return s.sessionRegistry.UpdateSession(session) }) From 6403d6f67f4800ba998a75a5e6a683e382f6164a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 15:07:09 +0100 Subject: [PATCH 162/191] Move where the goroutine is started for session actions --- .../pkg/dacd/brick_manager_impl/brick_manager.go | 3 ++- .../brick_manager_impl/session_action_handler.go | 12 ++++++------ internal/pkg/registry/session_actions.go | 3 +++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/dacd/brick_manager_impl/brick_manager.go index b60e27ad..01060665 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager.go @@ -45,7 +45,8 @@ func (bm *brickManager) Startup() { go func() { for event := range events { - bm.sessionActionHandler.ProcessSessionAction(event) + // TODO: we could limit the number of workers + go bm.sessionActionHandler.ProcessSessionAction(event) } log.Println("ERROR: stopped waiting for new Session Actions") }() diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 288f178d..334a3d2e 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -34,17 +34,17 @@ type sessionActionHandler struct { func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionAction) { switch action.ActionType { case datamodel.SessionDelete: - go s.handleDelete(action) + s.handleDelete(action) case datamodel.SessionCreateFilesystem: - go s.handleCreate(action) + s.handleCreate(action) case datamodel.SessionCopyDataIn: - go s.handleCopyIn(action) + s.handleCopyIn(action) case datamodel.SessionMount: - go s.handleMount(action) + s.handleMount(action) case datamodel.SessionUnmount: - go s.handleUnmount(action) + s.handleUnmount(action) case datamodel.SessionCopyDataOut: - go s.handleCopyOut(action) + s.handleCopyOut(action) default: log.Panicf("not yet implemented action for %+v", action) } diff --git a/internal/pkg/registry/session_actions.go b/internal/pkg/registry/session_actions.go index c1aed9ae..670176fe 100644 --- a/internal/pkg/registry/session_actions.go +++ b/internal/pkg/registry/session_actions.go @@ -17,6 +17,9 @@ type SessionActions interface { // Gets all actions for the given host GetSessionActionRequests(ctxt context.Context, brickHostName datamodel.BrickHostName) (<-chan datamodel.SessionAction, error) + // Get any actions that have not been completed + GetOutstandingSessionActionRequests(brickHostName datamodel.BrickHostName) ([]datamodel.SessionAction, error) + // Server reports given action is complete // Includes callbacks for Create Session Volume // From f0c92e60da4a8f82755a4a89264fe3635e068d9b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 16:17:12 +0100 Subject: [PATCH 163/191] Try to restore existing sessions on restart of dacd --- internal/pkg/config/brick_manager.go | 1 + .../dacd/brick_manager_impl/brick_manager.go | 62 +++++++++++-- .../brick_manager_impl/brick_manager_test.go | 10 ++- .../session_action_handler.go | 90 +++++++++++++------ internal/pkg/facade/session_action_handler.go | 1 + .../pkg/mock_facade/session_action_handler.go | 12 +++ internal/pkg/mock_registry/session_actions.go | 15 ++++ internal/pkg/registry_impl/session_actions.go | 17 ++++ 8 files changed, 174 insertions(+), 34 deletions(-) diff --git a/internal/pkg/config/brick_manager.go b/internal/pkg/config/brick_manager.go index e2e6ae3d..fecbf578 100644 --- a/internal/pkg/config/brick_manager.go +++ b/internal/pkg/config/brick_manager.go @@ -25,6 +25,7 @@ func GetBrickManagerConfig(env ReadEnvironemnt) BrickManagerConfig { getUint(env, "DEVICE_COUNT", 12)), getString(env, "DAC_BRICK_ADDRESS_PATTERN", getString(env, "DEVICE_TYPE", "nvme%dn1")), + // Disabled means don't accept new Sessions, but allow Actions on existing Sessions getBool(env, "DAC_HOST_ENABLED", true), } log.Println("Got brick manager config:", config) diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/dacd/brick_manager_impl/brick_manager.go index 01060665..e8ca97b8 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager.go @@ -15,6 +15,7 @@ func NewBrickManager(keystore store.Keystore) dacd.BrickManager { return &brickManager{ config: config.GetBrickManagerConfig(config.DefaultEnv), brickRegistry: registry_impl.NewBrickHostRegistry(keystore), + sessionRegistry: registry_impl.NewSessionRegistry(keystore), sessionActions: registry_impl.NewSessionActionsRegistry(keystore), sessionActionHandler: NewSessionActionHandler(keystore), } @@ -23,6 +24,7 @@ func NewBrickManager(keystore store.Keystore) dacd.BrickManager { type brickManager struct { config config.BrickManagerConfig brickRegistry registry.BrickHostRegistry + sessionRegistry registry.SessionRegistry sessionActions registry.SessionActions sessionActionHandler facade.SessionActionHandler } @@ -43,6 +45,20 @@ func (bm *brickManager) Startup() { // If we are are enabled, this includes new create session requests events, err := bm.sessionActions.GetSessionActionRequests(context.TODO(), bm.config.BrickHostName) + // Assume we got restarted, first try to finish all pending actions + bm.completePendingActions() + + // If we were restarted, likely no one is listening for pending actions any more + // so don'y worry the above pending actions may have failed due to not restoring sessions first + bm.restoreSessions() + + // Tell everyone we are listening + err = bm.brickRegistry.KeepAliveHost(context.TODO(), bm.config.BrickHostName) + if err != nil { + log.Panicf("failed to start keep alive host: %s", err) + } + + // Process any events, given others know we are alive go func() { for event := range events { // TODO: we could limit the number of workers @@ -50,14 +66,50 @@ func (bm *brickManager) Startup() { } log.Println("ERROR: stopped waiting for new Session Actions") }() +} + +func (bm *brickManager) completePendingActions() { + // Assume the service has been restarted, lets + // retry any actions that haven't been completed + // making the assumption that actions are idempotent + actions, err := bm.sessionActions.GetOutstandingSessionActionRequests(bm.config.BrickHostName) + if err != nil { + log.Fatalf("unable to get outstanding session action requests due to: %s", err.Error()) + } - // TODO: try to recover all existing filesystems on restart - // including a check to make sure all related brick hosts are alive + for _, action := range actions { + // TODO: what about the extra response if no one is listening any more? + bm.sessionActionHandler.ProcessSessionAction(action) + } +} - // Tell everyone we are listening - err = bm.brickRegistry.KeepAliveHost(context.TODO(), bm.config.BrickHostName) +func (bm *brickManager) restoreSessions() { + // In case the server was restarted, double check everything is up + // If marked deleted, and not already deleted, delete it + sessions, err := bm.sessionRegistry.GetAllSessions() if err != nil { - log.Panicf("failed to start keep alive host: %s", err) + log.Panicf("unable to fetch all sessions due to: %s", err) + } + for _, session := range sessions { + hasLocalBrick := false + for _, brick := range session.AllocatedBricks { + if brick.BrickHostName == bm.config.BrickHostName { + hasLocalBrick = true + } + } + if !hasLocalBrick { + continue + } + + if session.Status.FileSystemCreated && !session.Status.DeleteRequested { + // If we have previously finished creating the session, + // and we don't have a pending delete, try to restore the session + log.Println("Restoring session with local brick", session.Name) + go bm.sessionActionHandler.RestoreSession(session) + } else { + // TODO: should we just do the delete here? + log.Printf("WARNING session in strange state: %+v\n", session) + } } } diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go index e67abca7..a6e10cff 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager_test.go @@ -22,15 +22,21 @@ func TestBrickManager_Startup(t *testing.T) { defer mockCtrl.Finish() brickRegistry := mock_registry.NewMockBrickHostRegistry(mockCtrl) sessionActions := mock_registry.NewMockSessionActions(mockCtrl) + sessionRegistry := mock_registry.NewMockSessionRegistry(mockCtrl) handler := mock_facade.NewMockSessionActionHandler(mockCtrl) brickManager := brickManager{ - config: config.GetBrickManagerConfig(config.DefaultEnv), - brickRegistry: brickRegistry, sessionActions: sessionActions, sessionActionHandler: handler, + config: config.GetBrickManagerConfig(config.DefaultEnv), + brickRegistry: brickRegistry, + sessionActions: sessionActions, + sessionActionHandler: handler, + sessionRegistry: sessionRegistry, } // TODO... brickRegistry.EXPECT().UpdateBrickHost(gomock.Any()) sessionActions.EXPECT().GetSessionActionRequests(context.TODO(), gomock.Any()) + sessionActions.EXPECT().GetOutstandingSessionActionRequests(brickManager.config.BrickHostName) + sessionRegistry.EXPECT().GetAllSessions() hostname, _ := os.Hostname() brickRegistry.EXPECT().KeepAliveHost(context.TODO(), datamodel.BrickHostName(hostname)) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 334a3d2e..99f05ea2 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -50,6 +50,38 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi } } +func (s *sessionActionHandler) RestoreSession(session datamodel.Session) { + // Get session lock before attempting the restore + sessionMutex, err := s.sessionRegistry.GetSessionMutex(session.Name) + if err != nil { + log.Printf("unable to get session mutex: %s due to: %s\n", session.Name, err) + return + } + err = sessionMutex.Lock(context.TODO()) + if err != nil { + log.Printf("unable to lock session mutex: %s due to: %s\n", session.Name, err) + return + } + // Always drop mutex on function exit + defer func() { + if err := sessionMutex.Unlock(context.TODO()); err != nil { + log.Printf("failed to drop mutex for: %s due to: %s\n", session.Name, err.Error()) + } + }() + + // TODO: need a way that doesn't try to do format!! + _, err = s.doCreate(session) + if err != nil { + log.Printf("unable to restore session: %+v\n", session) + session.Status.Error = err.Error() + if _, err := s.sessionRegistry.UpdateSession(session); err != nil { + log.Panicf("unable to report that session restore failed for session: %s", session.Name) + } + } + + // TODO: do we just assume any pending mounts will resume in their own time? or should we retry mounts too? +} + func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, process func() (datamodel.Session, error)) { sessionName := action.Session.Name @@ -90,37 +122,41 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - // Nothing to create, just complete the action - // TODO: why do we send the action? - if action.Session.ActualSizeBytes == 0 { - return action.Session, nil - } + return s.doCreate(action.Session) + }) +} - // Get latest session now we have the mutex - session, err := s.sessionRegistry.GetSession(action.Session.Name) - if err != nil { - return action.Session, fmt.Errorf("error getting session: %s", err) - } - if session.Status.DeleteRequested { - return session, fmt.Errorf("can't do action once delete has been requested for") - } +func (s *sessionActionHandler) doCreate(session datamodel.Session) (datamodel.Session, error) { + // Nothing to create, just complete the action + // TODO: why do we send the action? + if session.ActualSizeBytes == 0 { + return session, nil + } - fsStatus, err := s.fsProvider.Create(action.Session) - session.FilesystemStatus = fsStatus - session.Status.FileSystemCreated = err == nil - if err != nil { - session.Status.Error = err.Error() - } + // Get latest session now we have the mutex + session, err := s.sessionRegistry.GetSession(session.Name) + if err != nil { + return session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") + } - session, updateErr := s.sessionRegistry.UpdateSession(session) - if updateErr != nil { - log.Println("Failed to update session:", updateErr) - if err == nil { - err = updateErr - } + fsStatus, err := s.fsProvider.Create(session) + session.FilesystemStatus = fsStatus + session.Status.FileSystemCreated = err == nil + if err != nil { + session.Status.Error = err.Error() + } + + session, updateErr := s.sessionRegistry.UpdateSession(session) + if updateErr != nil { + log.Println("Failed to update session:", updateErr) + if err == nil { + err = updateErr } - return session, err - }) + } + return session, err } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { diff --git a/internal/pkg/facade/session_action_handler.go b/internal/pkg/facade/session_action_handler.go index 3e765bef..52dd44d0 100644 --- a/internal/pkg/facade/session_action_handler.go +++ b/internal/pkg/facade/session_action_handler.go @@ -4,4 +4,5 @@ import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type SessionActionHandler interface { ProcessSessionAction(action datamodel.SessionAction) + RestoreSession(session datamodel.Session) } diff --git a/internal/pkg/mock_facade/session_action_handler.go b/internal/pkg/mock_facade/session_action_handler.go index e5e4b6db..61626363 100644 --- a/internal/pkg/mock_facade/session_action_handler.go +++ b/internal/pkg/mock_facade/session_action_handler.go @@ -44,3 +44,15 @@ func (mr *MockSessionActionHandlerMockRecorder) ProcessSessionAction(action inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSessionAction", reflect.TypeOf((*MockSessionActionHandler)(nil).ProcessSessionAction), action) } + +// RestoreSession mocks base method +func (m *MockSessionActionHandler) RestoreSession(session datamodel.Session) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "RestoreSession", session) +} + +// RestoreSession indicates an expected call of RestoreSession +func (mr *MockSessionActionHandlerMockRecorder) RestoreSession(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RestoreSession", reflect.TypeOf((*MockSessionActionHandler)(nil).RestoreSession), session) +} diff --git a/internal/pkg/mock_registry/session_actions.go b/internal/pkg/mock_registry/session_actions.go index 5bdc2e47..40d73269 100644 --- a/internal/pkg/mock_registry/session_actions.go +++ b/internal/pkg/mock_registry/session_actions.go @@ -64,6 +64,21 @@ func (mr *MockSessionActionsMockRecorder) GetSessionActionRequests(ctxt, brickHo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionActionRequests", reflect.TypeOf((*MockSessionActions)(nil).GetSessionActionRequests), ctxt, brickHostName) } +// GetOutstandingSessionActionRequests mocks base method +func (m *MockSessionActions) GetOutstandingSessionActionRequests(brickHostName datamodel.BrickHostName) ([]datamodel.SessionAction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOutstandingSessionActionRequests", brickHostName) + ret0, _ := ret[0].([]datamodel.SessionAction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetOutstandingSessionActionRequests indicates an expected call of GetOutstandingSessionActionRequests +func (mr *MockSessionActionsMockRecorder) GetOutstandingSessionActionRequests(brickHostName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOutstandingSessionActionRequests", reflect.TypeOf((*MockSessionActions)(nil).GetOutstandingSessionActionRequests), brickHostName) +} + // CompleteSessionAction mocks base method func (m *MockSessionActions) CompleteSessionAction(action datamodel.SessionAction) error { m.ctrl.T.Helper() diff --git a/internal/pkg/registry_impl/session_actions.go b/internal/pkg/registry_impl/session_actions.go index 95ac2389..87bf354d 100644 --- a/internal/pkg/registry_impl/session_actions.go +++ b/internal/pkg/registry_impl/session_actions.go @@ -10,6 +10,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/store" "github.com/google/uuid" "log" + "sort" ) func NewSessionActionsRegistry(store store.Keystore) registry.SessionActions { @@ -157,6 +158,22 @@ func (s *sessionActions) GetSessionActionRequests(ctxt context.Context, return sessionActionChan, nil } +func (s *sessionActions) GetOutstandingSessionActionRequests(brickHostName datamodel.BrickHostName) ([]datamodel.SessionAction, error) { + rawRequests, err := s.store.GetAll(getSessionActionRequestHostPrefix(brickHostName)) + if err != nil { + return nil, err + } + // Return actions in order they were sent, i.e. create revision order + sort.Slice(rawRequests, func(i, j int) bool { + return rawRequests[i].CreateRevision < rawRequests[j].CreateRevision + }) + var actions []datamodel.SessionAction + for _, request := range rawRequests { + actions = append(actions, sessionActionFromRaw(request.Value)) + } + return actions, nil +} + func (s *sessionActions) CompleteSessionAction(sessionAction datamodel.SessionAction) error { // TODO: when you delete a session, you should delete all completion records? From d4720378bc1861308947a839e6f63cab3a1bb0c5 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 16:38:23 +0100 Subject: [PATCH 164/191] Skip format when doing a restore --- .../dacd/brick_manager_impl/brick_manager.go | 1 + .../session_action_handler.go | 128 +++++++++--------- internal/pkg/filesystem/provider.go | 1 + internal/pkg/filesystem_impl/ansible.go | 13 +- internal/pkg/filesystem_impl/provider.go | 6 +- internal/pkg/mock_filesystem/provider.go | 14 ++ 6 files changed, 94 insertions(+), 69 deletions(-) diff --git a/internal/pkg/dacd/brick_manager_impl/brick_manager.go b/internal/pkg/dacd/brick_manager_impl/brick_manager.go index e8ca97b8..1cd4e68b 100644 --- a/internal/pkg/dacd/brick_manager_impl/brick_manager.go +++ b/internal/pkg/dacd/brick_manager_impl/brick_manager.go @@ -77,6 +77,7 @@ func (bm *brickManager) completePendingActions() { log.Fatalf("unable to get outstanding session action requests due to: %s", err.Error()) } + // We wait for these to finish before starting keepalive for _, action := range actions { // TODO: what about the extra response if no one is listening any more? bm.sessionActionHandler.ProcessSessionAction(action) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 99f05ea2..5aba16ae 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -50,38 +50,6 @@ func (s *sessionActionHandler) ProcessSessionAction(action datamodel.SessionActi } } -func (s *sessionActionHandler) RestoreSession(session datamodel.Session) { - // Get session lock before attempting the restore - sessionMutex, err := s.sessionRegistry.GetSessionMutex(session.Name) - if err != nil { - log.Printf("unable to get session mutex: %s due to: %s\n", session.Name, err) - return - } - err = sessionMutex.Lock(context.TODO()) - if err != nil { - log.Printf("unable to lock session mutex: %s due to: %s\n", session.Name, err) - return - } - // Always drop mutex on function exit - defer func() { - if err := sessionMutex.Unlock(context.TODO()); err != nil { - log.Printf("failed to drop mutex for: %s due to: %s\n", session.Name, err.Error()) - } - }() - - // TODO: need a way that doesn't try to do format!! - _, err = s.doCreate(session) - if err != nil { - log.Printf("unable to restore session: %+v\n", session) - session.Status.Error = err.Error() - if _, err := s.sessionRegistry.UpdateSession(session); err != nil { - log.Panicf("unable to report that session restore failed for session: %s", session.Name) - } - } - - // TODO: do we just assume any pending mounts will resume in their own time? or should we retry mounts too? -} - func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, process func() (datamodel.Session, error)) { sessionName := action.Session.Name @@ -122,41 +90,38 @@ func (s *sessionActionHandler) processWithMutex(action datamodel.SessionAction, func (s *sessionActionHandler) handleCreate(action datamodel.SessionAction) { s.processWithMutex(action, func() (datamodel.Session, error) { - return s.doCreate(action.Session) - }) -} - -func (s *sessionActionHandler) doCreate(session datamodel.Session) (datamodel.Session, error) { - // Nothing to create, just complete the action - // TODO: why do we send the action? - if session.ActualSizeBytes == 0 { - return session, nil - } + session := action.Session + // Nothing to create, just complete the action + // TODO: why do we send the action? + if session.ActualSizeBytes == 0 { + return session, nil + } - // Get latest session now we have the mutex - session, err := s.sessionRegistry.GetSession(session.Name) - if err != nil { - return session, fmt.Errorf("error getting session: %s", err) - } - if session.Status.DeleteRequested { - return session, fmt.Errorf("can't do action once delete has been requested for") - } + // Get latest session now we have the mutex + session, err := s.sessionRegistry.GetSession(session.Name) + if err != nil { + return session, fmt.Errorf("error getting session: %s", err) + } + if session.Status.DeleteRequested { + return session, fmt.Errorf("can't do action once delete has been requested for") + } - fsStatus, err := s.fsProvider.Create(session) - session.FilesystemStatus = fsStatus - session.Status.FileSystemCreated = err == nil - if err != nil { - session.Status.Error = err.Error() - } + fsStatus, err := s.fsProvider.Create(session) + session.FilesystemStatus = fsStatus + session.Status.FileSystemCreated = err == nil + if err != nil { + session.Status.Error = err.Error() + } - session, updateErr := s.sessionRegistry.UpdateSession(session) - if updateErr != nil { - log.Println("Failed to update session:", updateErr) - if err == nil { - err = updateErr + session, updateErr := s.sessionRegistry.UpdateSession(session) + if updateErr != nil { + log.Println("Failed to update session:", updateErr) + if err == nil { + err = updateErr + } } - } - return session, err + return session, err + }) } func (s *sessionActionHandler) handleDelete(action datamodel.SessionAction) { @@ -416,3 +381,40 @@ func (s *sessionActionHandler) handleUnmount(action datamodel.SessionAction) { return s.sessionRegistry.UpdateSession(session) }) } + +func (s *sessionActionHandler) RestoreSession(session datamodel.Session) { + if session.ActualSizeBytes == 0 { + // Nothing to do + return + } + + // Get session lock before attempting the restore + sessionMutex, err := s.sessionRegistry.GetSessionMutex(session.Name) + if err != nil { + log.Printf("unable to get session mutex: %s due to: %s\n", session.Name, err) + return + } + err = sessionMutex.Lock(context.TODO()) + if err != nil { + log.Printf("unable to lock session mutex: %s due to: %s\n", session.Name, err) + return + } + // Always drop mutex on function exit + defer func() { + if err := sessionMutex.Unlock(context.TODO()); err != nil { + log.Printf("failed to drop mutex for: %s due to: %s\n", session.Name, err.Error()) + } + }() + + err = s.fsProvider.Restore(session) + + if err != nil { + log.Printf("unable to restore session: %+v\n", session) + session.Status.Error = err.Error() + if _, err := s.sessionRegistry.UpdateSession(session); err != nil { + log.Panicf("unable to report that session restore failed for session: %s", session.Name) + } + } + + // TODO: do we just assume any pending mounts will resume in their own time? or should we retry mounts too? +} diff --git a/internal/pkg/filesystem/provider.go b/internal/pkg/filesystem/provider.go index 4d502382..5e4ef62c 100644 --- a/internal/pkg/filesystem/provider.go +++ b/internal/pkg/filesystem/provider.go @@ -4,6 +4,7 @@ import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" type Provider interface { Create(session datamodel.Session) (datamodel.FilesystemStatus, error) + Restore(session datamodel.Session) error Delete(session datamodel.Session) error DataCopyIn(session datamodel.Session) error diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index fb908d06..2f91e1c9 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -176,7 +176,7 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) return dir, err } -func executeAnsibleSetup(internalName string, bricks []datamodel.Brick) error { +func executeAnsibleSetup(internalName string, bricks []datamodel.Brick, doFormat bool) error { // TODO: restore beegfs support dir, err := setupAnsible(Lustre, internalName, bricks) if err != nil { @@ -184,10 +184,13 @@ func executeAnsibleSetup(internalName string, bricks []datamodel.Brick) error { } defer os.RemoveAll(dir) - formatArgs := "dac.yml -i inventory --tag format" - err = executeAnsiblePlaybook(dir, formatArgs) - if err != nil { - return fmt.Errorf("error during server format: %s", err.Error()) + // allow skip format when trying to rebuild + if doFormat { + formatArgs := "dac.yml -i inventory --tag format" + err = executeAnsiblePlaybook(dir, formatArgs) + if err != nil { + return fmt.Errorf("error during server format: %s", err.Error()) + } } startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" diff --git a/internal/pkg/filesystem_impl/provider.go b/internal/pkg/filesystem_impl/provider.go index 1a38aca9..6624a92d 100644 --- a/internal/pkg/filesystem_impl/provider.go +++ b/internal/pkg/filesystem_impl/provider.go @@ -30,10 +30,14 @@ func (f *fileSystemProvider) Create(session datamodel.Session) (datamodel.Filesy InternalName: GetNewUUID(), InternalData: "", } - err := executeAnsibleSetup(session.FilesystemStatus.InternalName, session.AllocatedBricks) + err := executeAnsibleSetup(session.FilesystemStatus.InternalName, session.AllocatedBricks, true) return session.FilesystemStatus, err } +func (f *fileSystemProvider) Restore(session datamodel.Session) error { + return executeAnsibleSetup(session.FilesystemStatus.InternalName, session.AllocatedBricks, false) +} + func (f *fileSystemProvider) Delete(session datamodel.Session) error { return executeAnsibleTeardown(session.FilesystemStatus.InternalName, session.AllocatedBricks) } diff --git a/internal/pkg/mock_filesystem/provider.go b/internal/pkg/mock_filesystem/provider.go index 062773f0..ec65fa8d 100644 --- a/internal/pkg/mock_filesystem/provider.go +++ b/internal/pkg/mock_filesystem/provider.go @@ -48,6 +48,20 @@ func (mr *MockProviderMockRecorder) Create(session interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockProvider)(nil).Create), session) } +// Restore mocks base method +func (m *MockProvider) Restore(session datamodel.Session) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Restore", session) + ret0, _ := ret[0].(error) + return ret0 +} + +// Restore indicates an expected call of Restore +func (mr *MockProviderMockRecorder) Restore(session interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Restore", reflect.TypeOf((*MockProvider)(nil).Restore), session) +} + // Delete mocks base method func (m *MockProvider) Delete(session datamodel.Session) error { m.ctrl.T.Helper() From 2e8d50600005478332f4d65990b9215079ee1c01 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 16:43:42 +0100 Subject: [PATCH 165/191] Pick up new build --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index b39264bc..6b7f4117 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.4' -data_acc_checksum: 'sha256:71aaa54eb3a4fa688f8588774787b87b3bd2e3169a7ce9d0c65530561a827f61' +data_acc_version: 'v2.0-alpha.5' +data_acc_checksum: 'sha256:1aaa54eb3a4fa688f8588774787b87b3bd2e3169a7ce9d0c65530561a827f61' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From f88d43958a01d1927a119dc4e4f724b19325b576 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Sat, 31 Aug 2019 17:29:07 +0100 Subject: [PATCH 166/191] Add new CLI option to generate ansible --- cmd/dacctl/actions.go | 6 ++++ cmd/dacctl/main.go | 6 +++- cmd/dacctl/main_test.go | 4 +++ internal/pkg/dacctl/actions_impl/actions.go | 8 +++++ internal/pkg/dacctl/interface.go | 1 + internal/pkg/dacctl/workflow_impl/session.go | 17 +++++++++ internal/pkg/facade/session.go | 3 ++ internal/pkg/filesystem/ansible.go | 21 ++--------- internal/pkg/filesystem_impl/ansible.go | 12 +++++++ internal/pkg/mock_facade/session.go | 14 ++++++++ internal/pkg/mock_filesystem/ansible.go | 37 +++----------------- 11 files changed, 77 insertions(+), 52 deletions(-) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index 01fe276d..8df44dbf 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -129,3 +129,9 @@ func dataOut(c *cli.Context) error { defer keystore.Close() return getActions(keystore).DataOut(c) } + +func generateAnsible(c *cli.Context) error { + keystore := getKeystore() + defer keystore.Close() + return getActions(keystore).GenerateAnsible(c) +} diff --git a/cmd/dacctl/main.go b/cmd/dacctl/main.go index 0d9e7e33..ca3e84f1 100644 --- a/cmd/dacctl/main.go +++ b/cmd/dacctl/main.go @@ -161,8 +161,12 @@ func runCli(args []string) error { Usage: "Returns fake data to keep burst buffer plugin happy.", Action: showConfigurations, }, + { + Name: "generate_ansible", + Usage: "Creates debug ansible in debug ansible.", + Action: generateAnsible, + }, } - return app.Run(stripFunctionArg(args)) } diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 0ef12733..48b0ba2c 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -245,3 +245,7 @@ func (*stubDacctlActions) PostRun(c dacctl.CliContext) error { func (*stubDacctlActions) DataOut(c dacctl.CliContext) error { return errors.New("DataOut") } + +func (*stubDacctlActions) GenerateAnsible(c dacctl.CliContext) error { + return errors.New("DataOut") +} diff --git a/internal/pkg/dacctl/actions_impl/actions.go b/internal/pkg/dacctl/actions_impl/actions.go index d5d7b283..5cd36984 100644 --- a/internal/pkg/dacctl/actions_impl/actions.go +++ b/internal/pkg/dacctl/actions_impl/actions.go @@ -117,3 +117,11 @@ func (d *dacctlActions) DataOut(c dacctl.CliContext) error { } return d.session.CopyDataOut(sessionName) } + +func (d *dacctlActions) GenerateAnsible(c dacctl.CliContext) error { + sessionName, err := d.getSessionName(c) + if err != nil { + return err + } + return d.session.GenerateAnsible(sessionName) +} diff --git a/internal/pkg/dacctl/interface.go b/internal/pkg/dacctl/interface.go index 3ab1ad5e..93f72ce9 100644 --- a/internal/pkg/dacctl/interface.go +++ b/internal/pkg/dacctl/interface.go @@ -21,4 +21,5 @@ type DacctlActions interface { PreRun(c CliContext) error PostRun(c CliContext) error DataOut(c CliContext) error + GenerateAnsible(c CliContext) error } diff --git a/internal/pkg/dacctl/workflow_impl/session.go b/internal/pkg/dacctl/workflow_impl/session.go index 1dc9633a..8d0fef38 100644 --- a/internal/pkg/dacctl/workflow_impl/session.go +++ b/internal/pkg/dacctl/workflow_impl/session.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" @@ -27,6 +28,7 @@ type sessionFacade struct { session registry.SessionRegistry actions registry.SessionActions allocations registry.AllocationRegistry + ansible filesystem.Ansible } func (s sessionFacade) submitJob(sessionName datamodel.SessionName, actionType datamodel.SessionActionType, @@ -282,3 +284,18 @@ func (s sessionFacade) GetSession(sessionName datamodel.SessionName) (datamodel. func (s sessionFacade) GetAllSessions() ([]datamodel.Session, error) { return s.session.GetAllSessions() } + +func (s sessionFacade) GenerateAnsible(sessionName datamodel.SessionName) error { + session, err := s.session.GetSession(sessionName) + if err != nil { + log.Println("Unable to find session we want to mount:", sessionName) + return err + } + directory, err := s.ansible.CreateEnvironment(session) + if err != nil { + return err + } + log.Println("Created ansible in directory:", directory) + log.Println("Please delete directory when your work is done.") + return nil +} diff --git a/internal/pkg/facade/session.go b/internal/pkg/facade/session.go index 09f83dcf..7ca022c8 100644 --- a/internal/pkg/facade/session.go +++ b/internal/pkg/facade/session.go @@ -36,4 +36,7 @@ type Session interface { // Get all sessions GetAllSessions() ([]datamodel.Session, error) + + // Generate ansible test dir + GenerateAnsible(sessionName datamodel.SessionName) error } diff --git a/internal/pkg/filesystem/ansible.go b/internal/pkg/filesystem/ansible.go index d4cc9431..7d455f01 100644 --- a/internal/pkg/filesystem/ansible.go +++ b/internal/pkg/filesystem/ansible.go @@ -2,24 +2,7 @@ package filesystem import "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" -type AnsibleEnv struct { - directory string -} - type Ansible interface { - CreateEnvironment(session datamodel.Session, createAllPlaybooks bool, groupAllVars map[string]string) (AnsibleEnv, error) - DestroyEnvironment(env AnsibleEnv) error - RunPlaybook(env AnsibleEnv, playbook AnsiblePlaybook, extraVars map[string]string) + // returns temp dir environment was created in + CreateEnvironment(session datamodel.Session) (string, error) } - -type AnsiblePlaybook int - -const ( - Create AnsiblePlaybook = iota - Delete - DataCopyIn - DataCopyOut - // Mount and unmount require extraVars with the hosts to be mounted - Mount - Unmount -) diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 2f91e1c9..5946a366 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" "gopkg.in/yaml.v2" "io/ioutil" "log" @@ -35,6 +36,17 @@ type Wrapper struct { All FileSystems } +func NewAnsible() filesystem.Ansible { + return &ansibleImpl{} +} + +type ansibleImpl struct { +} + +func (*ansibleImpl) CreateEnvironment(session datamodel.Session) (string, error) { + return setupAnsible(Lustre, session.FilesystemStatus.InternalName, session.AllocatedBricks) +} + var conf = config.GetFilesystemConfig() func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) string { diff --git a/internal/pkg/mock_facade/session.go b/internal/pkg/mock_facade/session.go index 0009481b..2fd50e17 100644 --- a/internal/pkg/mock_facade/session.go +++ b/internal/pkg/mock_facade/session.go @@ -161,3 +161,17 @@ func (mr *MockSessionMockRecorder) GetAllSessions() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllSessions", reflect.TypeOf((*MockSession)(nil).GetAllSessions)) } + +// GenerateAnsible mocks base method +func (m *MockSession) GenerateAnsible(sessionName datamodel.SessionName) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateAnsible", sessionName) + ret0, _ := ret[0].(error) + return ret0 +} + +// GenerateAnsible indicates an expected call of GenerateAnsible +func (mr *MockSessionMockRecorder) GenerateAnsible(sessionName interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateAnsible", reflect.TypeOf((*MockSession)(nil).GenerateAnsible), sessionName) +} diff --git a/internal/pkg/mock_filesystem/ansible.go b/internal/pkg/mock_filesystem/ansible.go index febe4f80..15dd8ade 100644 --- a/internal/pkg/mock_filesystem/ansible.go +++ b/internal/pkg/mock_filesystem/ansible.go @@ -6,7 +6,6 @@ package mock_filesystem import ( datamodel "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" - filesystem "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" gomock "github.com/golang/mock/gomock" reflect "reflect" ) @@ -35,42 +34,16 @@ func (m *MockAnsible) EXPECT() *MockAnsibleMockRecorder { } // CreateEnvironment mocks base method -func (m *MockAnsible) CreateEnvironment(session datamodel.Session, createAllPlaybooks bool, groupAllVars map[string]string) (filesystem.AnsibleEnv, error) { +func (m *MockAnsible) CreateEnvironment(session datamodel.Session) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateEnvironment", session, createAllPlaybooks, groupAllVars) - ret0, _ := ret[0].(filesystem.AnsibleEnv) + ret := m.ctrl.Call(m, "CreateEnvironment", session) + ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateEnvironment indicates an expected call of CreateEnvironment -func (mr *MockAnsibleMockRecorder) CreateEnvironment(session, createAllPlaybooks, groupAllVars interface{}) *gomock.Call { +func (mr *MockAnsibleMockRecorder) CreateEnvironment(session interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEnvironment", reflect.TypeOf((*MockAnsible)(nil).CreateEnvironment), session, createAllPlaybooks, groupAllVars) -} - -// DestroyEnvironment mocks base method -func (m *MockAnsible) DestroyEnvironment(env filesystem.AnsibleEnv) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DestroyEnvironment", env) - ret0, _ := ret[0].(error) - return ret0 -} - -// DestroyEnvironment indicates an expected call of DestroyEnvironment -func (mr *MockAnsibleMockRecorder) DestroyEnvironment(env interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DestroyEnvironment", reflect.TypeOf((*MockAnsible)(nil).DestroyEnvironment), env) -} - -// RunPlaybook mocks base method -func (m *MockAnsible) RunPlaybook(env filesystem.AnsibleEnv, playbook filesystem.AnsiblePlaybook, extraVars map[string]string) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "RunPlaybook", env, playbook, extraVars) -} - -// RunPlaybook indicates an expected call of RunPlaybook -func (mr *MockAnsibleMockRecorder) RunPlaybook(env, playbook, extraVars interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunPlaybook", reflect.TypeOf((*MockAnsible)(nil).RunPlaybook), env, playbook, extraVars) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEnvironment", reflect.TypeOf((*MockAnsible)(nil).CreateEnvironment), session) } From d3ab3f23e281de2fcc91aa7e216e87edc55b8acd Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 09:22:05 +0100 Subject: [PATCH 167/191] Remove unused lustre-multirail role --- .../roles/lustre-multirail/tasks/main.yaml | 33 ------------------- .../roles/lustre-multirail/tasks/main.yaml~ | 0 2 files changed, 33 deletions(-) delete mode 100644 fs-ansible/roles/lustre-multirail/tasks/main.yaml delete mode 100644 fs-ansible/roles/lustre-multirail/tasks/main.yaml~ diff --git a/fs-ansible/roles/lustre-multirail/tasks/main.yaml b/fs-ansible/roles/lustre-multirail/tasks/main.yaml deleted file mode 100644 index cc971e39..00000000 --- a/fs-ansible/roles/lustre-multirail/tasks/main.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: flush arp ib - command: ip neigh flush dev {{ item }} - with_items: - - ib0 - - ib1 - -- name: update routes2 - blockinfile: - path: /etc/iproute2/rt_tables - block: | - 200 ib0 - 201 ib1 - -- name: static routes {{ item.name }} - command: ip route add 10.47.0.0/16 dev {{ item.name }} proto kernel scope link src {{ item.addr }} table {{ item.name }} - with_items: "{{ ip }}" - when: - - ip is defined - - item.name is match(target|default(".*")) - tags: [ 'never' , 'init_routes' ] - -- name: ib static routes - command: ip rule add from {{ item.addr }} table {{ item.name }} - with_items: "{{ ip }}" - when: - - ip is defined - - item.name is match(target|default(".*")) - tags: [ 'never' , 'init_routes' ] - -- name: ib route flush - command: ip route flush cache diff --git a/fs-ansible/roles/lustre-multirail/tasks/main.yaml~ b/fs-ansible/roles/lustre-multirail/tasks/main.yaml~ deleted file mode 100644 index e69de29b..00000000 From 317a922c01340d99189906e5d0eaace4e4901d2a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 09:23:13 +0100 Subject: [PATCH 168/191] Remove unused test inventory and playbooks --- fs-ansible/dac-beegfs.yml | 8 ----- fs-ansible/demo.sh | 56 -------------------------------- fs-ansible/inv_gen.py | 47 --------------------------- fs-ansible/inventory | 13 -------- fs-ansible/test-dac-beegfs.yml | 16 --------- fs-ansible/test-dac-lustre.yml | 16 --------- fs-ansible/test-dac.yml | 7 ---- fs-ansible/test-inventory-beegfs | 24 -------------- fs-ansible/test-inventory-lustre | 26 --------------- 9 files changed, 213 deletions(-) delete mode 100644 fs-ansible/dac-beegfs.yml delete mode 100755 fs-ansible/demo.sh delete mode 100755 fs-ansible/inv_gen.py delete mode 100644 fs-ansible/inventory delete mode 100644 fs-ansible/test-dac-beegfs.yml delete mode 100644 fs-ansible/test-dac-lustre.yml delete mode 100644 fs-ansible/test-dac.yml delete mode 100644 fs-ansible/test-inventory-beegfs delete mode 100644 fs-ansible/test-inventory-lustre diff --git a/fs-ansible/dac-beegfs.yml b/fs-ansible/dac-beegfs.yml deleted file mode 100644 index 6709cf44..00000000 --- a/fs-ansible/dac-beegfs.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Setup beegfs for io500 - hosts: fs1001 - become: yes - roles: - - role: beegfs - vars: - fs_name: fs1001 diff --git a/fs-ansible/demo.sh b/fs-ansible/demo.sh deleted file mode 100755 index 8755eef0..00000000 --- a/fs-ansible/demo.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -. .venv/bin/activate - -set -eux - -echo Get FS1 up -echo - -ansible-playbook test-dac.yml -i test-inventory --tag reformat_mgs - -ansible-playbook test-dac.yml -i test-inventory --tag format_mgs --tag reformat_mdts --tag reformat_osts -ansible-playbook test-dac.yml -i test-inventory --tag start_mgs --tag start_mdts --tag start_osts --tag mount_fs - -echo Show FS works -echo - -ls /mnt/lustre/fs1 -ssh dac2 'sudo bash -c "hostname > /mnt/lustre/fs1/demo"' -cat /mnt/lustre/fs1/demo - -echo Get FS1 down -echo - -ansible-playbook test-dac.yml -i test-inventory --tag umount_fs --tag stop_osts --tag stop_mdts -ansible-playbook test-dac.yml -i test-inventory --tag reformat_mdts --tag reformat_osts - -cat /mnt/lustre/fs1/demo || true - -echo Clean MGS -echo - -ansible-playbook test-dac.yml -i test-inventory --tag stop_mgs -ansible-playbook test-dac.yml -i test-inventory --tag reformat_mgs - -echo Bring up an down FS2 - -ansible-playbook test-dac2.yml -i test-inventory2 --tag format_mgs --tag reformat_mdts --tag reformat_osts -ansible-playbook test-dac2.yml -i test-inventory2 --tag start_mgs --tag start_mdts --tag start_osts --tag mount_fs -ansible-playbook test-dac2.yml -i test-inventory2 --tag start_mgs --tag start_mdts --tag start_osts --tag mount_fs - -ls /mnt/lustre/fs2 -ssh dac2 'sudo bash -c "hostname > /mnt/lustre/fs2/demo"' -cat /mnt/lustre/fs2/demo - -ansible-playbook test-dac2.yml -i test-inventory2 --tag umount_fs --tag stop_osts --tag stop_mdts -ansible-playbook test-dac2.yml -i test-inventory2 --tag umount_fs --tag stop_osts --tag stop_mdts -ansible-playbook test-dac2.yml -i test-inventory2 --tag reformat_mdts --tag reformat_osts - -cat /mnt/lustre/fs2/demo || true - -echo Clean MGS -echo - -ansible-playbook test-dac.yml -i test-inventory --tag stop_mgs -ansible-playbook test-dac.yml -i test-inventory --tag reformat_mgs diff --git a/fs-ansible/inv_gen.py b/fs-ansible/inv_gen.py deleted file mode 100755 index ed4bd87c..00000000 --- a/fs-ansible/inv_gen.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python2 -import yaml -from collections import OrderedDict - -def main(): - - dac_no = 2 - #init yaml structure - dac = {'dac-prod':{'children': {'fs1001':{'hosts':{}}}}} - - #create nvme,index as orderd dict to preserve lustre global index order in relation to nvme - for i in range(1,dac_no+1): - dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)] = {'fs1001_osts': OrderedDict()} - - #set dac 1 as mdt and mgs - dac['dac-prod']['children']['fs1001']['hosts']['dac-e-1']['fs1001_mgs'] = "nvme0n1" - dac['dac-prod']['children']['fs1001']['hosts']['dac-e-1']['fs1001_mdt'] = "nvme0n1" - - #create keys for nvmes and index init to 0 - for i in range(1,12): - dac['dac-prod']['children']['fs1001']['hosts']['dac-e-1']['fs1001_osts']["nvme{}n1".format(i)] = 0 - - for i in range(2,dac_no+1): - for j in range(0,12): - dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)]['fs1001_osts']["nvme{}n1".format(j)] = 0 - - #globaly index all nvmes - n = 1 - for i in range(1,dac_no+1): - for key in dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)]['fs1001_osts']: - dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)]['fs1001_osts'][key] = n - n+=1 - - #cast orderdict back to dict for yaml - for i in range(1,dac_no+1): - dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)]['fs1001_osts'] = dict(dac['dac-prod']['children']['fs1001']['hosts']["dac-e-{}".format(i)]['fs1001_osts']) - - dac['dac-prod']['children']['fs1001']['vars'] = { - 'fs1001_mgsnode': 'dac-e-1', - 'fs1001_client_port': '10001' - } - - print(yaml.dump(dac)) - - -if __name__ == "__main__": - main() diff --git a/fs-ansible/inventory b/fs-ansible/inventory deleted file mode 100644 index bf1447b6..00000000 --- a/fs-ansible/inventory +++ /dev/null @@ -1,13 +0,0 @@ -dac-prod: - children: - fs1001: - hosts: - dac-e-1: - fs1001_mgs: nvme0n1 - fs1001_mdt: nvme1n1 - fs1001_osts: {nvme0n1: 2} - dac2: - fs1001_osts: {nvme3n1: 1} - vars: - fs1001_mgsnode: dac-e-1 - fs1001_client_port: 1001 diff --git a/fs-ansible/test-dac-beegfs.yml b/fs-ansible/test-dac-beegfs.yml deleted file mode 100644 index a21c2138..00000000 --- a/fs-ansible/test-dac-beegfs.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Setup beegfs for fs1 - hosts: fs1 - become: yes - roles: - - role: beegfs - vars: - fs_name: fs1 - -- name: Setup beegfs for fs2 - hosts: fs2 - become: yes - roles: - - role: beegfs - vars: - fs_name: fs2 diff --git a/fs-ansible/test-dac-lustre.yml b/fs-ansible/test-dac-lustre.yml deleted file mode 100644 index 02531021..00000000 --- a/fs-ansible/test-dac-lustre.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Setup beegfs for fs1 - hosts: fs1 - become: yes - roles: - - role: lustre - vars: - fs_name: fs1 - -- name: Setup beegfs for fs2 - hosts: fs2 - become: yes - roles: - - role: lustre - vars: - fs_name: fs2 diff --git a/fs-ansible/test-dac.yml b/fs-ansible/test-dac.yml deleted file mode 100644 index ccef00c9..00000000 --- a/fs-ansible/test-dac.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Install Lustre for fs1 - hosts: fs1 - become: yes - gather_facts: no - roles: - - role: lustre diff --git a/fs-ansible/test-inventory-beegfs b/fs-ansible/test-inventory-beegfs deleted file mode 100644 index 98756e62..00000000 --- a/fs-ansible/test-inventory-beegfs +++ /dev/null @@ -1,24 +0,0 @@ -dac-fake: - children: - fs1: - hosts: - dac1: - fs1_mgs: nvme0n1 - fs1_mdt: nvme0n1 - fs1_osts: {nvme0n1: 2} - dac2: - fs1_osts: {nvme3n1: 1} - vars: - fs1_mgsnode: dac1 - fs1_client_port: 1001 - fs2: - hosts: - dac1: - fs2_mgs: nvme3n1 - fs2_mdt: nvme3n1 - fs2_osts: {nvme4n1: 2} - dac2: - fs2_osts: {nvme2n1: 1} - vars: - fs2_mgsnode: dac1 - fs2_client_port: 1002 diff --git a/fs-ansible/test-inventory-lustre b/fs-ansible/test-inventory-lustre deleted file mode 100644 index 5429ef8a..00000000 --- a/fs-ansible/test-inventory-lustre +++ /dev/null @@ -1,26 +0,0 @@ -dac-fake: - children: - fs1: - hosts: - dac1: - fs1_mgs: nvme0n1 - fs1_mdt: nvme1n1 - fs1_osts: {nvme2n1: 2} - dac2: - fs1_osts: {nvme3n1: 1} - vars: - fs1_mgsnode: dac1 - fs1_client_port: 1001 - lnet_suffix: "" - fs2: - hosts: - dac1: - fs2_mgs: nvme0n1 - fs2_mdt: nvme3n1 - fs2_osts: {nvme4n1: 2} - dac2: - fs2_osts: {nvme2n1: 1} - vars: - fs2_mgsnode: dac1 - fs2_client_port: 1002 - lnet_suffix: "" From e7efb51bc5d0e9822105fc33418c2ded70e68bba Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 10:23:10 +0100 Subject: [PATCH 169/191] New playbook based fs-ansible interface --- fs-ansible/create.yml | 16 ++++ fs-ansible/delete.yml | 16 ++++ fs-ansible/group_vars/dac-fake.yaml | 32 ------- fs-ansible/group_vars/dac-prod.yaml | 74 --------------- fs-ansible/restore.yml | 8 ++ internal/pkg/filesystem_impl/ansible.go | 66 +++++--------- internal/pkg/filesystem_impl/ansible_test.go | 95 ++++++++++---------- 7 files changed, 112 insertions(+), 195 deletions(-) create mode 100644 fs-ansible/create.yml create mode 100644 fs-ansible/delete.yml delete mode 100644 fs-ansible/group_vars/dac-fake.yaml delete mode 100644 fs-ansible/group_vars/dac-prod.yaml create mode 100644 fs-ansible/restore.yml diff --git a/fs-ansible/create.yml b/fs-ansible/create.yml new file mode 100644 index 00000000..0c79c267 --- /dev/null +++ b/fs-ansible/create.yml @@ -0,0 +1,16 @@ +--- +- name: Create Lustre filesystem (format) + hosts: all + any_errors_fatal: true + become: yes + roles: + - role: lustre + tags: [ format ] + +- name: Create Lustre filesystem (format) + hosts: all + any_errors_fatal: true + become: yes + roles: + - role: lustre + tags: [ mount, create_mdt, create_mgs, create_osts, client_mount ] \ No newline at end of file diff --git a/fs-ansible/delete.yml b/fs-ansible/delete.yml new file mode 100644 index 00000000..8bbabcc9 --- /dev/null +++ b/fs-ansible/delete.yml @@ -0,0 +1,16 @@ +--- +- name: Delete Lustre filesystem (unmount) + hosts: all + any_errors_fatal: true + become: yes + roles: + - role: lustre + tags: [ stop_all, unmount, client_unmount ] + +- name: Delete Lustre filesystem (format) + hosts: all + any_errors_fatal: true + become: yes + roles: + - role: lustre + tags: [ format ] diff --git a/fs-ansible/group_vars/dac-fake.yaml b/fs-ansible/group_vars/dac-fake.yaml deleted file mode 100644 index 783bdfa1..00000000 --- a/fs-ansible/group_vars/dac-fake.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -beegfs_host_info: - nvme0n1: - if: eth0 - str_port: 8100 - mdt_port: 8200 - mgs_port: 8300 - numa: 0 - nvme1n1: - if: eth0 - str_port: 8101 - mdt_port: 8201 - mgs_port: 8301 - numa: 0 - nvme2n1: - if: eth0 - str_port: 8102 - mdt_port: 8202 - mgs_port: 8302 - numa: 0 - nvme3n1: - if: eth0 - str_port: 8103 - mdt_port: 8203 - mgs_port: 8303 - numa: 0 - nvme4n1: - if: eth0 - str_port: 8104 - mdt_port: 8204 - mgs_port: 8304 - numa: 0 diff --git a/fs-ansible/group_vars/dac-prod.yaml b/fs-ansible/group_vars/dac-prod.yaml deleted file mode 100644 index 224f3ad4..00000000 --- a/fs-ansible/group_vars/dac-prod.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -beegfs_host_info: - nvme0n1: - if: ib0 - str_port: 8100 - mdt_port: 8200 - mgs_port: 8300 - numa: 0 - nvme1n1: - if: ib0 - str_port: 8101 - mdt_port: 8201 - mgs_port: 8301 - numa: 0 - nvme2n1: - if: ib0 - str_port: 8102 - mdt_port: 8202 - mgs_port: 8302 - numa: 0 - nvme3n1: - if: ib0 - str_port: 8103 - mdt_port: 8203 - mgs_port: 8303 - numa: 0 - nvme4n1: - if: ib0 - str_port: 8104 - mdt_port: 8204 - mgs_port: 8304 - numa: 0 - nvme5n1: - if: ib0 - str_port: 8105 - mdt_port: 8205 - mgs_port: 8305 - numa: 0 - nvme6n1: - if: ib1 - str_port: 8106 - mdt_port: 8206 - mgs_port: 8306 - numa: 1 - nvme7n1: - if: ib1 - str_port: 8107 - mdt_port: 8207 - mgs_port: 8307 - numa: 1 - nvme8n1: - if: ib1 - str_port: 8108 - mdt_port: 8208 - mgs_port: 8308 - numa: 1 - nvme9n1: - if: ib1 - str_port: 8109 - mdt_port: 8209 - mgs_port: 8309 - numa: 1 - nvme10n1: - if: ib1 - str_port: 8110 - mdt_port: 8210 - mgs_port: 8310 - numa: 1 - nvme11n1: - if: ib1 - str_port: 8111 - mdt_port: 8211 - mgs_port: 8311 - numa: 1 diff --git a/fs-ansible/restore.yml b/fs-ansible/restore.yml new file mode 100644 index 00000000..eb693e0b --- /dev/null +++ b/fs-ansible/restore.yml @@ -0,0 +1,8 @@ +--- +- name: Restore Lustre filesystem + hosts: all + any_errors_fatal: true + become: yes + roles: + - role: lustre + tags: [ mount, create_mdt, create_mgs, create_osts, client_mount ] \ No newline at end of file diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 5946a366..76693c74 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -1,7 +1,6 @@ package filesystem_impl import ( - "bytes" "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" @@ -13,7 +12,6 @@ import ( "os/exec" "path" "path/filepath" - "strings" "time" ) @@ -33,7 +31,7 @@ type FileSystems struct { } type Wrapper struct { - All FileSystems + Dacs FileSystems } func NewAnsible() filesystem.Ansible { @@ -103,25 +101,18 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str //"client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": conf.LnetSuffix, "mdt_size": fmt.Sprintf("%dm", conf.MDTSizeMB), + "fs_name": fsUuid, }, Hosts: hosts, } fsname := fmt.Sprintf("%s", fsUuid) - data := Wrapper{All: FileSystems{Children: map[string]FSInfo{fsname: fsinfo}}} + data := Wrapper{Dacs: FileSystems{Children: map[string]FSInfo{fsname: fsinfo}}} output, err := yaml.Marshal(data) if err != nil { log.Fatalln(err) } strOut := string(output) - strOut = strings.Replace(strOut, " mgs:", fmt.Sprintf(" %s_mgs:", fsname), -1) - strOut = strings.Replace(strOut, " mdts:", fmt.Sprintf(" %s_mdts:", fsname), -1) - strOut = strings.Replace(strOut, " osts:", fmt.Sprintf(" %s_osts:", fsname), -1) - strOut = strings.Replace(strOut, " mgsnode:", fmt.Sprintf(" %s_mgsnode:", fsname), -1) - strOut = strings.Replace(strOut, " client_port:", fmt.Sprintf(" %s_client_port:", fsname), -1) - strOut = strings.Replace(strOut, " mdt_size:", fmt.Sprintf(" %s_mdt_size:", fsname), -1) - - strOut = strings.Replace(strOut, "all:", conf.HostGroup+":", -1) return strOut } @@ -156,16 +147,9 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) } log.Println("Using ansible tempdir:", dir) - playbook := getPlaybook(fsType, internalName) - tmpPlaybook := filepath.Join(dir, "dac.yml") - if err := ioutil.WriteFile(tmpPlaybook, bytes.NewBufferString(playbook).Bytes(), 0666); err != nil { - return dir, err - } - log.Println(playbook) - inventory := getInventory(fsType, internalName, bricks) tmpInventory := filepath.Join(dir, "inventory") - if err := ioutil.WriteFile(tmpInventory, bytes.NewBufferString(inventory).Bytes(), 0666); err != nil { + if err := ioutil.WriteFile(tmpInventory, []byte(tmpInventory), 0666); err != nil { return dir, err } log.Println(inventory) @@ -179,12 +163,6 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) cmd = exec.Command("cp", "-r", getAnsibleDir(".venv"), dir) output, err = cmd.CombinedOutput() log.Println("copy venv", string(output)) - if err != nil { - return dir, err - } - cmd = exec.Command("cp", "-r", getAnsibleDir("group_vars"), dir) - output, err = cmd.CombinedOutput() - log.Println("copy group vars", string(output)) return dir, err } @@ -194,21 +172,25 @@ func executeAnsibleSetup(internalName string, bricks []datamodel.Brick, doFormat if err != nil { return err } - defer os.RemoveAll(dir) + defer func() { + if err := os.RemoveAll(dir); err != nil { + log.Printf("error removing %s due to %s\n", dir, err) + } + }() // allow skip format when trying to rebuild if doFormat { - formatArgs := "dac.yml -i inventory --tag format" + formatArgs := "create.yml -i inventory" err = executeAnsiblePlaybook(dir, formatArgs) if err != nil { - return fmt.Errorf("error during server format: %s", err.Error()) + return fmt.Errorf("error during ansible create: %s", err.Error()) + } + } else { + formatArgs := "restore.yml -i inventory" + err = executeAnsiblePlaybook(dir, formatArgs) + if err != nil { + return fmt.Errorf("error during ansible create: %s", err.Error()) } - } - - startupArgs := "dac.yml -i inventory --tag mount,create_mdt,create_mgs,create_osts,client_mount" - err = executeAnsiblePlaybook(dir, startupArgs) - if err != nil { - return fmt.Errorf("error during create fs: %s", err.Error()) } return nil } @@ -218,15 +200,13 @@ func executeAnsibleTeardown(internalName string, bricks []datamodel.Brick) error if err != nil { return err } - defer os.RemoveAll(dir) - - stopArgs := "dac.yml -i inventory --tag stop_all,unmount,client_unmount" - err = executeAnsiblePlaybook(dir, stopArgs) - if err != nil { - return fmt.Errorf("error during server umount: %s", err.Error()) - } + defer func() { + if err := os.RemoveAll(dir); err != nil { + log.Printf("error removing %s due to %s\n", dir, err) + } + }() - formatArgs := "dac.yml -i inventory --tag format" + formatArgs := "delete.yml -i inventory" err = executeAnsiblePlaybook(dir, formatArgs) if err != nil { return fmt.Errorf("error during server clean: %s", err.Error()) diff --git a/internal/pkg/filesystem_impl/ansible_test.go b/internal/pkg/filesystem_impl/ansible_test.go index a5c106fe..026950f2 100644 --- a/internal/pkg/filesystem_impl/ansible_test.go +++ b/internal/pkg/filesystem_impl/ansible_test.go @@ -17,21 +17,22 @@ func TestPlugin_GetInventory(t *testing.T) { } fsUuid := "abcdefgh" result := getInventory(BeegFS, fsUuid, brickAllocations) - expected := `dac-prod: + expected := `dacs: children: abcdefgh: hosts: dac1: - abcdefgh_mgs: nvme1n1 - abcdefgh_mdts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} - abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} + mgs: nvme1n1 + mdts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} + osts: {nvme1n1: 0, nvme2n1: 1, nvme3n1: 2} dac2: - abcdefgh_mdts: {nvme2n1: 3, nvme3n1: 4} - abcdefgh_osts: {nvme2n1: 3, nvme3n1: 4} + mdts: {nvme2n1: 3, nvme3n1: 4} + osts: {nvme2n1: 3, nvme3n1: 4} vars: + fs_name: abcdefgh lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 + mdt_size: 20480m + mgsnode: dac1 ` assert.Equal(t, expected, result) } @@ -44,21 +45,22 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { } fsUuid := "abcdefgh" result := getInventory(Lustre, fsUuid, brickAllocations) - expected := `dac-prod: + expected := `dacs: children: abcdefgh: hosts: dac1: - abcdefgh_mgs: sdb - abcdefgh_mdts: {nvme1n1: 0} - abcdefgh_osts: {nvme1n1: 0} + mgs: sdb + mdts: {nvme1n1: 0} + osts: {nvme1n1: 0} dac2: - abcdefgh_mdts: {nvme2n1: 1, nvme3n1: 2} - abcdefgh_osts: {nvme2n1: 1, nvme3n1: 2} + mdts: {nvme2n1: 1, nvme3n1: 2} + osts: {nvme2n1: 1, nvme3n1: 2} vars: + fs_name: abcdefgh lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 + mdt_size: 20480m + mgsnode: dac1 ` assert.Equal(t, expected, result) } @@ -106,54 +108,55 @@ func TestPlugin_GetInventory_MaxMDT(t *testing.T) { fsUuid := "abcdefgh" result := getInventory(Lustre, fsUuid, brickAllocations) - expected := `dac-prod: + expected := `dacs: children: abcdefgh: hosts: dac1: - abcdefgh_mgs: sdb - abcdefgh_mdts: {nvme1n1: 0} - abcdefgh_osts: {nvme1n1: 0, nvme2n1: 1} + mgs: sdb + mdts: {nvme1n1: 0} + osts: {nvme1n1: 0, nvme2n1: 1} dac3: - abcdefgh_mdts: {nvme1n1: 2} - abcdefgh_osts: {nvme1n1: 2, nvme2n1: 3} + mdts: {nvme1n1: 2} + osts: {nvme1n1: 2, nvme2n1: 3} dac5: - abcdefgh_mdts: {nvme1n1: 4} - abcdefgh_osts: {nvme1n1: 4, nvme2n1: 5} + mdts: {nvme1n1: 4} + osts: {nvme1n1: 4, nvme2n1: 5} dac7: - abcdefgh_mdts: {nvme1n1: 6} - abcdefgh_osts: {nvme1n1: 6, nvme2n1: 7} + mdts: {nvme1n1: 6} + osts: {nvme1n1: 6, nvme2n1: 7} dac9: - abcdefgh_mdts: {nvme1n1: 8} - abcdefgh_osts: {nvme1n1: 8, nvme2n1: 9} + mdts: {nvme1n1: 8} + osts: {nvme1n1: 8, nvme2n1: 9} dac11: - abcdefgh_mdts: {nvme1n1: 10} - abcdefgh_osts: {nvme1n1: 10, nvme2n1: 11} + mdts: {nvme1n1: 10} + osts: {nvme1n1: 10, nvme2n1: 11} dac13: - abcdefgh_mdts: {nvme1n1: 12} - abcdefgh_osts: {nvme1n1: 12, nvme2n1: 13} + mdts: {nvme1n1: 12} + osts: {nvme1n1: 12, nvme2n1: 13} dac15: - abcdefgh_mdts: {nvme1n1: 14} - abcdefgh_osts: {nvme1n1: 14, nvme2n1: 15} + mdts: {nvme1n1: 14} + osts: {nvme1n1: 14, nvme2n1: 15} dac17: - abcdefgh_mdts: {nvme1n1: 16} - abcdefgh_osts: {nvme1n1: 16, nvme2n1: 17} + mdts: {nvme1n1: 16} + osts: {nvme1n1: 16, nvme2n1: 17} dac19: - abcdefgh_mdts: {nvme1n1: 18} - abcdefgh_osts: {nvme1n1: 18, nvme2n1: 19} + mdts: {nvme1n1: 18} + osts: {nvme1n1: 18, nvme2n1: 19} dac21: - abcdefgh_mdts: {nvme1n1: 20} - abcdefgh_osts: {nvme1n1: 20, nvme2n1: 21} + mdts: {nvme1n1: 20} + osts: {nvme1n1: 20, nvme2n1: 21} dac23: - abcdefgh_mdts: {nvme1n1: 22} - abcdefgh_osts: {nvme1n1: 22, nvme2n1: 23} + mdts: {nvme1n1: 22} + osts: {nvme1n1: 22, nvme2n1: 23} dac25: - abcdefgh_mdts: {nvme1n1: 24} - abcdefgh_osts: {nvme1n1: 24, nvme2n1: 25} + mdts: {nvme1n1: 24} + osts: {nvme1n1: 24, nvme2n1: 25} vars: + fs_name: abcdefgh lnet_suffix: "" - abcdefgh_mdt_size: 20480m - abcdefgh_mgsnode: dac1 + mdt_size: 20480m + mgsnode: dac1 ` assert.Equal(t, expected, result) } From 722d02090081819b2c23ae227896c5933add11d3 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 11:17:19 +0100 Subject: [PATCH 170/191] Revert "add lvm changes for prod dac" This reverts commit 5f0fdf224a1912a02b3ff5f570a4911ce116c26e. --- fs-ansible/roles/lustre/tasks/format.yaml | 122 +++++++--------------- fs-ansible/roles/lustre/tasks/mount.yaml | 6 +- 2 files changed, 37 insertions(+), 91 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 3ea3ff3f..085de338 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -1,7 +1,7 @@ --- - set_fact: mgs: "{{ vars[fs_name + '_mgs'] | default(omit) }}" - mgsnode: "{{ vars[fs_name + '_mgsnode']}}" + mgsnode: "{{ vars[fs_name + '_mgsnode'] }}" mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" osts: "{{ vars[fs_name + '_osts'] | default({}) }}" mdt_size: "{{ vars[fs_name + '_mdt_size'] | default('10%') }}" @@ -22,115 +22,63 @@ - mgs is defined tags: [ 'never', 'reformat_mgs'] -- name: Clean LVM - block: - - name: Remove LV for MDT - lvol: - lv: "mdt" - vg: "dac-{{ item }}" - state: absent - force: yes - loop: "{{ mdts.keys() | list }}" - - - name: Remove VG - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - force: yes - state: absent - loop: "{{ mdts.keys() | list }}" - - - name: dmsetup remove lvm - command: "dmsetup remove dac--{{ item }}-mdt" - register: command_result - failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" - loop: "{{ mdts.keys() | list }}" - +- name: Remove old MDT Partition + parted: + device: "/dev/{{ item }}" + number: 1 + state: absent + loop: "{{ mdts.keys() }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] -- name: Add VG - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - state: present - loop: "{{ mdts.keys() | list }}" - when: - - mdts is defined - tags: [ 'never', 'reformat_mdts', 'format'] -- name: Add LV for MDT - lvol: - lv: "mdt" - vg: "dac-{{ item }}" - size: "{{ mdt_size }}" - state: present - loop: "{{ mdts.keys() | list }}" +- name: Add MDT Partition + parted: + device: "/dev/{{ item }}" + number: 1 + part_start: "0%" + part_end: "{{ mdt_size }}" + label: gpt + state: present + loop: "{{ mdts.keys() }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] - name: Reformat MDTs - command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} --mkfsoptions=\"-O large_dir\" /dev/mapper/dac--{{ item.key }}-mdt" + command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" loop: "{{ mdts|dict2items }}" when: - mdts is defined tags: [ 'never', 'reformat_mdts', 'format'] -- name: Clean LVM - block: - - name: Remove LV for OST - lvol: - lv: "ost" - vg: "dac-{{ item }}" - state: absent - force: yes - loop: "{{ osts.keys() | list }}" - when: - - mdts is defined - - - name: Remove VG OST - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - state: absent - loop: "{{ osts.keys() | list }}" - when: - - osts is defined - - mdts is not defined - - - name: dmsetup remove lvm - command: "dmsetup remove dac--{{ item }}-ost" - register: command_result - failed_when: "command_result.rc != 0 and ('device' not in command_result.stderr)" - loop: "{{ mdts.keys() | list }}" - tags: [ 'never', 'reformat_osts', 'format'] - -- name: Add VG - lvg: - vg: "dac-{{ item }}" - pvs: "/dev/{{ item }}" - state: present - loop: "{{ osts.keys() | list }}" +- name: Remove old OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + state: absent + loop: "{{ osts.keys() }}" when: - osts is defined - tags: [ 'never', 'reformat_osts', 'format'] + tags: [ 'never', 'reformat_mdts', 'format'] -- name: Add LV for OST - lvol: - lv: "ost" - vg: "dac-{{ item }}" - size: +100%FREE - state: present - loop: "{{ osts.keys() | list }}" +- name: Add OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + part_start: "{{ mdt_size }}" + part_end: "100%" + label: gpt + state: present + loop: "{{ osts.keys() }}" when: - osts is defined - tags: [ 'never', 'reformat_osts', 'format'] + tags: [ 'never', 'reformat_mdts', 'format'] - name: Reformat OSTs - command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/mapper/dac--{{ item.key }}-ost" + command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2" loop: "{{ osts|dict2items }}" when: - osts is defined diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index 0c7bb639..74bca2c6 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -10,11 +10,9 @@ command: modprobe -v lustre tags: [ 'never', 'start_lustre', 'mount'] - # Using ordering specified in here: # http://wiki.lustre.org/Starting_and_Stopping_Lustre_Services - - name: Start MGS block: - name: Create MGS mount dir @@ -42,7 +40,7 @@ with_items: "{{ mdts.keys() | list }}" - name: mount MDTs - command: mount -t lustre /dev/mapper/dac--{{ item }}-mdt /lustre/{{ fs_name }}/MDT/{{ item }} + command: mount -t lustre /dev/{{ item }}p1 /lustre/{{ fs_name }}/MDT/{{ item }} register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" @@ -74,7 +72,7 @@ with_items: "{{ osts.keys() | list }}" - name: mount OSTs - command: mount -t lustre /dev/mapper/dac--{{ item }}-ost /lustre/{{ fs_name }}/OST/{{ item }} + command: mount -t lustre /dev/{{ item }}p2 /lustre/{{ fs_name }}/OST/{{ item }} register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" From 114a39e228961994993886f4740b627221a05984 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 11:24:07 +0100 Subject: [PATCH 171/191] Change the role to match the new var format Dropped the fs_name extra bits. --- fs-ansible/roles/lustre/tasks/format.yaml | 8 +++----- fs-ansible/roles/lustre/tasks/mount.yaml | 6 ++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 085de338..b6ad3521 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -1,10 +1,8 @@ --- - set_fact: - mgs: "{{ vars[fs_name + '_mgs'] | default(omit) }}" - mgsnode: "{{ vars[fs_name + '_mgsnode'] }}" - mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" - osts: "{{ vars[fs_name + '_osts'] | default({}) }}" - mdt_size: "{{ vars[fs_name + '_mdt_size'] | default('10%') }}" + mdts: "{{ mdts | default({}) }}" + osts: "{{ osts | default({}) }}" + mdt_size: "{{ mdt_size | default('10%') }}" tags: [ 'never', 'format_mgs', 'reformat_mgs', 'format'] - name: Ensure MGS has been formatted diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index 74bca2c6..aab6f5d9 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -1,9 +1,7 @@ --- - set_fact: - mgs: "{{ vars[fs_name + '_mgs'] | default(omit) }}" - mgsnode: "{{ vars[fs_name + '_mgsnode'] }}" - mdts: "{{ vars[fs_name + '_mdts'] | default({}) }}" - osts: "{{ vars[fs_name + '_osts'] | default({}) }}" + mdts: "{{ mdts | default({}) }}" + osts: "{{ osts | default({}) }}" tags: [ 'never', 'start_mgs', 'start_mdts', 'start_osts', 'create_mgs', 'create_mdt', 'create_osts', 'stop_all', 'client_mount','client_unmount', 'stop_mgs'] - name: load lustre module From 78b36cd12cad3c44d37dfa00e2f62d17b424e9c1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 13:03:19 +0100 Subject: [PATCH 172/191] Move to latest data-acc release --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 6b7f4117..673b4a4a 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.5' -data_acc_checksum: 'sha256:1aaa54eb3a4fa688f8588774787b87b3bd2e3169a7ce9d0c65530561a827f61' +data_acc_version: 'v2.0-alpha.6' +data_acc_checksum: 'sha256:ee41fb09dbc7e8b0a0fdaeed22133ca74a82fd391eea7abdaa4fb8bb8a086a12' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From b1b80b72214b054c58d9874d38e7ac0f61b8aa4c Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 13:48:11 +0100 Subject: [PATCH 173/191] Copy the playbooks into the tmp dir --- internal/pkg/filesystem_impl/ansible.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 76693c74..3e8a5365 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -160,6 +160,14 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) if err != nil { return dir, err } + + cmd = exec.Command("cp", getAnsibleDir("*.yml"), dir) + output, err = cmd.CombinedOutput() + log.Println("copy playbooks", string(output)) + if err != nil { + return dir, err + } + cmd = exec.Command("cp", "-r", getAnsibleDir(".venv"), dir) output, err = cmd.CombinedOutput() log.Println("copy venv", string(output)) From 63c5cc1eb36370cceaead5828100134a79e24027 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:16:10 +0100 Subject: [PATCH 174/191] Refactor fs-ansible --- fs-ansible/create.yml | 14 +- fs-ansible/delete.yml | 15 +-- fs-ansible/restore.yml | 8 +- fs-ansible/roles/lustre/defaults/main.yaml | 5 + fs-ansible/roles/lustre/tasks/format.yaml | 108 +++++++-------- fs-ansible/roles/lustre/tasks/lnet.yaml | 7 - fs-ansible/roles/lustre/tasks/main.yaml | 15 ++- fs-ansible/roles/lustre/tasks/mount.yaml | 123 ++---------------- fs-ansible/roles/lustre/tasks/repo.yaml | 24 ---- fs-ansible/roles/lustre/tasks/unmount.yaml | 51 ++++++++ fs-ansible/roles/lustre/tasks/wipe.yml | 20 +++ fs-ansible/roles/lustre/vars/main.yaml | 4 - .../lustre_client_mount/defaults/main.yaml | 2 + .../roles/lustre_client_mount/tasks/main.yaml | 28 ++++ 14 files changed, 194 insertions(+), 230 deletions(-) create mode 100644 fs-ansible/roles/lustre/defaults/main.yaml delete mode 100644 fs-ansible/roles/lustre/tasks/lnet.yaml delete mode 100644 fs-ansible/roles/lustre/tasks/repo.yaml create mode 100644 fs-ansible/roles/lustre/tasks/unmount.yaml create mode 100644 fs-ansible/roles/lustre/tasks/wipe.yml delete mode 100644 fs-ansible/roles/lustre/vars/main.yaml create mode 100644 fs-ansible/roles/lustre_client_mount/defaults/main.yaml create mode 100644 fs-ansible/roles/lustre_client_mount/tasks/main.yaml diff --git a/fs-ansible/create.yml b/fs-ansible/create.yml index 0c79c267..18c2e538 100644 --- a/fs-ansible/create.yml +++ b/fs-ansible/create.yml @@ -5,12 +5,10 @@ become: yes roles: - role: lustre - tags: [ format ] + vars: + lustre_state: "present" + lustre_format_disks: true -- name: Create Lustre filesystem (format) - hosts: all - any_errors_fatal: true - become: yes - roles: - - role: lustre - tags: [ mount, create_mdt, create_mgs, create_osts, client_mount ] \ No newline at end of file + - role: lustre_client_mount + vars: + lustre_client_mount_present: true \ No newline at end of file diff --git a/fs-ansible/delete.yml b/fs-ansible/delete.yml index 8bbabcc9..3a6233c4 100644 --- a/fs-ansible/delete.yml +++ b/fs-ansible/delete.yml @@ -1,16 +1,13 @@ --- -- name: Delete Lustre filesystem (unmount) +- name: Create Lustre filesystem (format) hosts: all any_errors_fatal: true become: yes roles: - - role: lustre - tags: [ stop_all, unmount, client_unmount ] + - role: lustre_client_mount + vars: + lustre_client_mount_present: false -- name: Delete Lustre filesystem (format) - hosts: all - any_errors_fatal: true - become: yes - roles: - role: lustre - tags: [ format ] + vars: + lustre_state: "absent" \ No newline at end of file diff --git a/fs-ansible/restore.yml b/fs-ansible/restore.yml index eb693e0b..49b3dd3e 100644 --- a/fs-ansible/restore.yml +++ b/fs-ansible/restore.yml @@ -5,4 +5,10 @@ become: yes roles: - role: lustre - tags: [ mount, create_mdt, create_mgs, create_osts, client_mount ] \ No newline at end of file + vars: + lustre_state: "present" + lustre_format_disks: false + + - role: lustre_client_mount + vars: + lustre_client_mount_present: true \ No newline at end of file diff --git a/fs-ansible/roles/lustre/defaults/main.yaml b/fs-ansible/roles/lustre/defaults/main.yaml new file mode 100644 index 00000000..257d3af8 --- /dev/null +++ b/fs-ansible/roles/lustre/defaults/main.yaml @@ -0,0 +1,5 @@ +--- +# state is either present or absent +lustre_state: "present" +lustre_format_disks: true +lustre_stop_mgs: false \ No newline at end of file diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index b6ad3521..6bc10a79 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -3,7 +3,6 @@ mdts: "{{ mdts | default({}) }}" osts: "{{ osts | default({}) }}" mdt_size: "{{ mdt_size | default('10%') }}" - tags: [ 'never', 'format_mgs', 'reformat_mgs', 'format'] - name: Ensure MGS has been formatted command: /usr/sbin/mkfs.lustre --mgs /dev/{{ mgs }} @@ -12,72 +11,53 @@ changed_when: "command_result.rc == 0" when: - mgs is defined - tags: [ 'never', 'format_mgs', 'format'] -- name: Reformat MGS - command: /usr/sbin/mkfs.lustre --mgs --reformat /dev/{{ mgs }} - when: - - mgs is defined - tags: [ 'never', 'reformat_mgs'] - -- name: Remove old MDT Partition - parted: - device: "/dev/{{ item }}" - number: 1 - state: absent - loop: "{{ mdts.keys() }}" +- name: Format MDTs + block: + - name: Remove old MDT Partition + parted: + device: "/dev/{{ item }}" + number: 1 + state: absent + loop: "{{ mdts.keys() }}" + + - name: Add MDT Partition + parted: + device: "/dev/{{ item }}" + number: 1 + part_start: "0%" + part_end: "{{ mdt_size }}" + label: gpt + state: present + loop: "{{ mdts.keys() }}" + + - name: Reformat MDTs + command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" + loop: "{{ mdts|dict2items }}" when: - mdts is defined - tags: [ 'never', 'reformat_mdts', 'format'] - - -- name: Add MDT Partition - parted: - device: "/dev/{{ item }}" - number: 1 - part_start: "0%" - part_end: "{{ mdt_size }}" - label: gpt - state: present - loop: "{{ mdts.keys() }}" - when: - - mdts is defined - tags: [ 'never', 'reformat_mdts', 'format'] - -- name: Reformat MDTs - command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" - loop: "{{ mdts|dict2items }}" - when: - - mdts is defined - tags: [ 'never', 'reformat_mdts', 'format'] - - -- name: Remove old OST Partition - parted: - device: "/dev/{{ item }}" - number: 2 - state: absent - loop: "{{ osts.keys() }}" - when: - - osts is defined - tags: [ 'never', 'reformat_mdts', 'format'] - -- name: Add OST Partition - parted: - device: "/dev/{{ item }}" - number: 2 - part_start: "{{ mdt_size }}" - part_end: "100%" - label: gpt - state: present - loop: "{{ osts.keys() }}" - when: - - osts is defined - tags: [ 'never', 'reformat_mdts', 'format'] -- name: Reformat OSTs - command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2" - loop: "{{ osts|dict2items }}" +- name: Format OST + block: + - name: Remove old OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + state: absent + loop: "{{ osts.keys() }}" + + - name: Add OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + part_start: "{{ mdt_size }}" + part_end: "100%" + label: gpt + state: present + loop: "{{ osts.keys() }}" + + - name: Reformat OSTs + command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2" + loop: "{{ osts|dict2items }}" when: - osts is defined - tags: [ 'never', 'reformat_osts', 'format'] diff --git a/fs-ansible/roles/lustre/tasks/lnet.yaml b/fs-ansible/roles/lustre/tasks/lnet.yaml deleted file mode 100644 index 67efb688..00000000 --- a/fs-ansible/roles/lustre/tasks/lnet.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: set lnet module options - lineinfile: - path: /etc/modprobe.d/lnet.conf - regexp: '^options' - line: 'options lnet networks=o2ib1(ib0)' - tags: ['never', 'start_lnet'] diff --git a/fs-ansible/roles/lustre/tasks/main.yaml b/fs-ansible/roles/lustre/tasks/main.yaml index 7f451c85..545fe187 100644 --- a/fs-ansible/roles/lustre/tasks/main.yaml +++ b/fs-ansible/roles/lustre/tasks/main.yaml @@ -1,5 +1,12 @@ --- -- include: repo.yaml -- include: lnet.yaml -- include: format.yaml -- include: mount.yaml +- import_tasks: format.yaml + when: lustre_format_disks|bool and lustre_state == "present" + +- import_tasks: mount.yaml + when: lustre_state == "present" + +- import_tasks: unmount.yaml + when: lustre_state == "absent" + +- import_tasks: wipe.yaml + when: lustre_state == "absent" diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index aab6f5d9..bbfd22dc 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -20,13 +20,24 @@ recurse: yes - name: mount MGSs - command: mount -t lustre /dev/{{ mgs }} /lustre/MGS + command: mount -t lustre /dev/{{ mgs | default("mgs") }} /lustre/MGS register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" when: - mgs is defined - tags: [ 'never', 'start_mgs', 'create_mgs'] + +- name: Enable multiple mdt target support (DNE2) + block: + - name: Enable DNE2 remote dir + command: "lctl set_param mdt.*.enable_remote_dir=1" + register: command_result + failed_when: "command_result.rc != 0 and ('lctl' not in command_result.stderr)" + + - name: Enable DNE2 all users + command: "lctl set_param mdt.*.enable_remote_dir_gid=-1" + when: + - mgs is defined - name: Start MDTs block: @@ -45,20 +56,6 @@ with_items: "{{ mdts.keys() | list }}" when: - mdts is defined - tags: [ 'never', 'start_mdts', 'create_mdt'] - -- name: DNE2 - block: - - name: Enable DNE2 remote dir - command: "lctl set_param mdt.*.enable_remote_dir=1" - register: command_result - failed_when: "command_result.rc != 0 and ('lctl' not in command_result.stderr)" - - - name: Enable DNE2 all users - command: "lctl set_param mdt.*.enable_remote_dir_gid=-1" - when: - - mgs is defined - tags: [ 'never', 'start_mgs', 'create_mgs'] - name: Start OSTs block: @@ -75,97 +72,5 @@ failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" with_items: "{{ osts.keys() | list }}" - - when: - - osts is defined - tags: [ 'never', 'start_osts', 'create_osts'] - - -- name: mount lustre FS - block: - - name: ensure mount dir exists - file: - path: "/mnt/lustre/{{ fs_name }}" - state: directory - recurse: yes - - name: mount lustre fs - command: "mount -t lustre {{ mgsnode }}{{ lnet_suffix }}:/{{ fs_name }} /mnt/lustre/{{ fs_name }}" - register: command_result - failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" - changed_when: "command_result.rc == 0" - tags: [ 'never', 'mount_fs', 'client_mount'] - - -- name: umount lustre FS - block: - - name: umount lustre fs - command: "umount -l /mnt/lustre/{{ fs_name }}" - register: command_result - failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" - changed_when: "command_result.rc == 0" - - - name: ensure mount dir deleted - file: - path: "/mnt/lustre/{{ fs_name }}" - state: absent - tags: [ 'never', 'umount_fs', 'client_unmount'] - - -- name: Stop MDTs - block: - - name: umount mdts - command: umount /lustre/{{ fs_name }}/MDT/{{ item }} - register: command_result - failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" - changed_when: "command_result.rc == 0" - with_items: "{{ mdts.keys() | list }}" - - - name: remove mdt mount dir - file: - path: /lustre/{{ fs_name }}/MDT - state: absent - when: - - mdts is defined - tags: [ 'never', 'stop_mdts', 'stop_all'] - - -- name: Stop OST - block: - - name: umount OSTs - command: umount /lustre/{{ fs_name }}/OST/{{ item }} - register: command_result - failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" - changed_when: "command_result.rc == 0" - with_items: "{{ osts.keys() | list }}" - - - name: Remove OST mount dir - file: - path: /lustre/{{ fs_name }}/OST - state: absent - when: - - osts is defined - tags: [ 'never', 'stop_osts', 'stop_all'] - - -- name: Stop MGS - block: - - name: umount MGS - command: umount /lustre/MGS - register: command_result - failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" - changed_when: "command_result.rc == 0" - - - name: Remove MGSDT mount dir - file: - path: /lustre/MGS - state: absent - when: - - mgs is defined - tags: [ 'never', 'stop_mgs'] - - -- name: umount all lustre - command: umount -a -l -t lustre when: - - osts is defined - tags: [ 'never', 'stop_alllustre'] + - osts is defined \ No newline at end of file diff --git a/fs-ansible/roles/lustre/tasks/repo.yaml b/fs-ansible/roles/lustre/tasks/repo.yaml deleted file mode 100644 index b781aa8c..00000000 --- a/fs-ansible/roles/lustre/tasks/repo.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: enable lustre server repo - yum_repository: - name: lustre-server - description: lustre-server - file: lustre-repo - baseurl: https://downloads.whamcloud.com/public/lustre/{{ lustre.release }}/el7/{{ lustre.servertype }} - gpgcheck: no - -- name: enable lustre client repo - yum_repository: - name: lustre-client - description: lustre-client - file: lustre-repo - baseurl: https://downloads.whamcloud.com/public/lustre/{{ lustre.release }}/el7/client - gpgcheck: no - -- name: enable lustre e2fs repo - yum_repository: - name: e2fsprogs-wc - description: e2fsprogs-wc - file: lustre-repo - baseurl: https://downloads.whamcloud.com/public/e2fsprogs/latest/el7 - gpgcheck: no diff --git a/fs-ansible/roles/lustre/tasks/unmount.yaml b/fs-ansible/roles/lustre/tasks/unmount.yaml new file mode 100644 index 00000000..d7df9095 --- /dev/null +++ b/fs-ansible/roles/lustre/tasks/unmount.yaml @@ -0,0 +1,51 @@ +--- +# Using ordering specified in here: +# http://wiki.lustre.org/Starting_and_Stopping_Lustre_Services + +- name: Stop MDTs + block: + - name: umount mdts + command: umount /lustre/{{ fs_name }}/MDT/{{ item }} + register: command_result + failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" + changed_when: "command_result.rc == 0" + with_items: "{{ mdts.keys() | list }}" + + - name: remove mdt mount dir + file: + path: /lustre/{{ fs_name }}/MDT + state: absent + when: + - mdts is defined + +- name: Stop OST + block: + - name: umount OSTs + command: umount /lustre/{{ fs_name }}/OST/{{ item }} + register: command_result + failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" + changed_when: "command_result.rc == 0" + with_items: "{{ osts.keys() | list }}" + + - name: Remove OST mount dir + file: + path: /lustre/{{ fs_name }}/OST + state: absent + when: + - osts is defined + + +- name: Stop MGS + block: + - name: umount MGS + command: umount /lustre/MGS + register: command_result + failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" + changed_when: "command_result.rc == 0" + + - name: Remove MGSDT mount dir + file: + path: /lustre/MGS + state: absent + when: + - mgs is defined and lustre_stop_mgs \ No newline at end of file diff --git a/fs-ansible/roles/lustre/tasks/wipe.yml b/fs-ansible/roles/lustre/tasks/wipe.yml new file mode 100644 index 00000000..caffb014 --- /dev/null +++ b/fs-ansible/roles/lustre/tasks/wipe.yml @@ -0,0 +1,20 @@ +--- +- set_fact: + mdts: "{{ mdts | default({}) }}" + osts: "{{ osts | default({}) }}" + +# TODO: maybe call wipefs or ss out the block device headers + +- name: Remove old MDT Partition + parted: + device: "/dev/{{ item }}" + number: 1 + state: absent + loop: "{{ mdts.keys() }}" + +- name: Remove old OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + state: absent + loop: "{{ osts.keys() }}" \ No newline at end of file diff --git a/fs-ansible/roles/lustre/vars/main.yaml b/fs-ansible/roles/lustre/vars/main.yaml deleted file mode 100644 index ea3846a1..00000000 --- a/fs-ansible/roles/lustre/vars/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- - lustre: - release: lustre-2.10.4 - servertype: patchless-ldiskfs-server diff --git a/fs-ansible/roles/lustre_client_mount/defaults/main.yaml b/fs-ansible/roles/lustre_client_mount/defaults/main.yaml new file mode 100644 index 00000000..5638ea37 --- /dev/null +++ b/fs-ansible/roles/lustre_client_mount/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +lustre_client_mount_present: true \ No newline at end of file diff --git a/fs-ansible/roles/lustre_client_mount/tasks/main.yaml b/fs-ansible/roles/lustre_client_mount/tasks/main.yaml new file mode 100644 index 00000000..25eda07b --- /dev/null +++ b/fs-ansible/roles/lustre_client_mount/tasks/main.yaml @@ -0,0 +1,28 @@ +--- +- name: mount lustre FS + block: + - name: ensure mount dir exists + file: + path: "/mnt/lustre/{{ fs_name }}" + state: directory + recurse: yes + - name: mount lustre fs + command: "mount -t lustre {{ mgsnode }}{{ lnet_suffix }}:/{{ fs_name }} /mnt/lustre/{{ fs_name }}" + register: command_result + failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" + changed_when: "command_result.rc == 0" + when: "lustre_client_mount_present|bool" + +- name: umount lustre FS + block: + - name: umount lustre fs + command: "umount -l /mnt/lustre/{{ fs_name }}" + register: command_result + failed_when: "command_result.rc != 0 and ('not mounted' not in command_result.stderr) and ('mountpoint not found' not in command_result.stderr)" + changed_when: "command_result.rc == 0" + + - name: ensure mount dir deleted + file: + path: "/mnt/lustre/{{ fs_name }}" + state: absent + when: "not (lustre_client_mount_present|bool)" \ No newline at end of file From 75180659f703b056b258fd44c7e9748266049699 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:20:06 +0100 Subject: [PATCH 175/191] Fix wrong name in wipe.yml --- fs-ansible/roles/lustre/tasks/{wipe.yml => wipe.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fs-ansible/roles/lustre/tasks/{wipe.yml => wipe.yaml} (100%) diff --git a/fs-ansible/roles/lustre/tasks/wipe.yml b/fs-ansible/roles/lustre/tasks/wipe.yaml similarity index 100% rename from fs-ansible/roles/lustre/tasks/wipe.yml rename to fs-ansible/roles/lustre/tasks/wipe.yaml From 7754f2dd4b5b40301bf27e50cbefa8199fd6da51 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:36:10 +0100 Subject: [PATCH 176/191] Fix issues with mdt_size --- fs-ansible/roles/lustre/defaults/main.yaml | 8 +++++++- fs-ansible/roles/lustre/tasks/format.yaml | 13 ++----------- fs-ansible/roles/lustre/tasks/mount.yaml | 6 +----- fs-ansible/roles/lustre/tasks/unmount.yaml | 8 +------- internal/pkg/filesystem_impl/ansible.go | 2 +- internal/pkg/filesystem_impl/ansible_test.go | 6 +++--- 6 files changed, 15 insertions(+), 28 deletions(-) diff --git a/fs-ansible/roles/lustre/defaults/main.yaml b/fs-ansible/roles/lustre/defaults/main.yaml index 257d3af8..14a5581e 100644 --- a/fs-ansible/roles/lustre/defaults/main.yaml +++ b/fs-ansible/roles/lustre/defaults/main.yaml @@ -2,4 +2,10 @@ # state is either present or absent lustre_state: "present" lustre_format_disks: true -lustre_stop_mgs: false \ No newline at end of file +lustre_stop_mgs: false + +mdt_size_mb: "50" + +# following are usually overwritten by host vars +mdts: {} +osts: {} \ No newline at end of file diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 6bc10a79..3c8457a4 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -1,9 +1,4 @@ --- -- set_fact: - mdts: "{{ mdts | default({}) }}" - osts: "{{ osts | default({}) }}" - mdt_size: "{{ mdt_size | default('10%') }}" - - name: Ensure MGS has been formatted command: /usr/sbin/mkfs.lustre --mgs /dev/{{ mgs }} register: command_result @@ -26,7 +21,7 @@ device: "/dev/{{ item }}" number: 1 part_start: "0%" - part_end: "{{ mdt_size }}" + part_end: "{{ mdt_size_mb }}mb" label: gpt state: present loop: "{{ mdts.keys() }}" @@ -34,8 +29,6 @@ - name: Reformat MDTs command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" loop: "{{ mdts|dict2items }}" - when: - - mdts is defined - name: Format OST block: @@ -50,7 +43,7 @@ parted: device: "/dev/{{ item }}" number: 2 - part_start: "{{ mdt_size }}" + part_start: "{{ mdt_size_mb }}mb" part_end: "100%" label: gpt state: present @@ -59,5 +52,3 @@ - name: Reformat OSTs command: "/usr/sbin/mkfs.lustre --ost --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p2" loop: "{{ osts|dict2items }}" - when: - - osts is defined diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index bbfd22dc..15e2565b 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -54,8 +54,6 @@ failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" with_items: "{{ mdts.keys() | list }}" - when: - - mdts is defined - name: Start OSTs block: @@ -71,6 +69,4 @@ register: command_result failed_when: "command_result.rc != 0 and ('is already mounted' not in command_result.stderr)" changed_when: "command_result.rc == 0" - with_items: "{{ osts.keys() | list }}" - when: - - osts is defined \ No newline at end of file + with_items: "{{ osts.keys() | list }}" \ No newline at end of file diff --git a/fs-ansible/roles/lustre/tasks/unmount.yaml b/fs-ansible/roles/lustre/tasks/unmount.yaml index d7df9095..e91f6fdc 100644 --- a/fs-ansible/roles/lustre/tasks/unmount.yaml +++ b/fs-ansible/roles/lustre/tasks/unmount.yaml @@ -15,8 +15,6 @@ file: path: /lustre/{{ fs_name }}/MDT state: absent - when: - - mdts is defined - name: Stop OST block: @@ -31,9 +29,6 @@ file: path: /lustre/{{ fs_name }}/OST state: absent - when: - - osts is defined - - name: Stop MGS block: @@ -47,5 +42,4 @@ file: path: /lustre/MGS state: absent - when: - - mgs is defined and lustre_stop_mgs \ No newline at end of file + when: mgs is defined and lustre_stop_mgs \ No newline at end of file diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 3e8a5365..2ef94157 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -100,7 +100,7 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str "mgsnode": mgsnode, //"client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": conf.LnetSuffix, - "mdt_size": fmt.Sprintf("%dm", conf.MDTSizeMB), + "mdt_size_mb": fmt.Sprintf("%d", conf.MDTSizeMB), "fs_name": fsUuid, }, Hosts: hosts, diff --git a/internal/pkg/filesystem_impl/ansible_test.go b/internal/pkg/filesystem_impl/ansible_test.go index 026950f2..30f82796 100644 --- a/internal/pkg/filesystem_impl/ansible_test.go +++ b/internal/pkg/filesystem_impl/ansible_test.go @@ -31,7 +31,7 @@ func TestPlugin_GetInventory(t *testing.T) { vars: fs_name: abcdefgh lnet_suffix: "" - mdt_size: 20480m + mdt_size_mb: "20480" mgsnode: dac1 ` assert.Equal(t, expected, result) @@ -59,7 +59,7 @@ func TestPlugin_GetInventory_withNoOstOnOneHost(t *testing.T) { vars: fs_name: abcdefgh lnet_suffix: "" - mdt_size: 20480m + mdt_size_mb: "20480" mgsnode: dac1 ` assert.Equal(t, expected, result) @@ -155,7 +155,7 @@ func TestPlugin_GetInventory_MaxMDT(t *testing.T) { vars: fs_name: abcdefgh lnet_suffix: "" - mdt_size: 20480m + mdt_size_mb: "20480" mgsnode: dac1 ` assert.Equal(t, expected, result) From 3c5355012914263c4b10910be0aaba26457dc161 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:37:27 +0100 Subject: [PATCH 177/191] Fix units for parted --- fs-ansible/roles/lustre/tasks/format.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index 3c8457a4..c987349b 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -21,7 +21,7 @@ device: "/dev/{{ item }}" number: 1 part_start: "0%" - part_end: "{{ mdt_size_mb }}mb" + part_end: "{{ mdt_size_mb }}MB" label: gpt state: present loop: "{{ mdts.keys() }}" @@ -43,7 +43,7 @@ parted: device: "/dev/{{ item }}" number: 2 - part_start: "{{ mdt_size_mb }}mb" + part_start: "{{ mdt_size_mb }}MB" part_end: "100%" label: gpt state: present From 6ddafdf84d48088722a5b1df32cbb599d0b2236d Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:40:46 +0100 Subject: [PATCH 178/191] Remove broken dne2 commands for now --- fs-ansible/roles/lustre/tasks/mount.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/mount.yaml b/fs-ansible/roles/lustre/tasks/mount.yaml index 15e2565b..5030e221 100644 --- a/fs-ansible/roles/lustre/tasks/mount.yaml +++ b/fs-ansible/roles/lustre/tasks/mount.yaml @@ -27,18 +27,6 @@ when: - mgs is defined -- name: Enable multiple mdt target support (DNE2) - block: - - name: Enable DNE2 remote dir - command: "lctl set_param mdt.*.enable_remote_dir=1" - register: command_result - failed_when: "command_result.rc != 0 and ('lctl' not in command_result.stderr)" - - - name: Enable DNE2 all users - command: "lctl set_param mdt.*.enable_remote_dir_gid=-1" - when: - - mgs is defined - - name: Start MDTs block: - name: Create MDT mount dir From cd60c4c04dc742fe0c31de1ee3a95677322e300f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 15:45:20 +0100 Subject: [PATCH 179/191] Remove left over directory --- fs-ansible/roles/lustre/tasks/unmount.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs-ansible/roles/lustre/tasks/unmount.yaml b/fs-ansible/roles/lustre/tasks/unmount.yaml index e91f6fdc..9ccdf1f6 100644 --- a/fs-ansible/roles/lustre/tasks/unmount.yaml +++ b/fs-ansible/roles/lustre/tasks/unmount.yaml @@ -30,6 +30,11 @@ path: /lustre/{{ fs_name }}/OST state: absent +- name: Remove OST mount dir + file: + path: /lustre/{{ fs_name }} + state: absent + - name: Stop MGS block: - name: umount MGS From 611d4799e6dd5f3e67a41b22dd9d58e947bf493e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 16:47:17 +0100 Subject: [PATCH 180/191] Fix generate ansible cli --- cmd/dacctl/main.go | 1 + cmd/dacctl/main_test.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/dacctl/main.go b/cmd/dacctl/main.go index ca3e84f1..0cbd9d34 100644 --- a/cmd/dacctl/main.go +++ b/cmd/dacctl/main.go @@ -165,6 +165,7 @@ func runCli(args []string) error { Name: "generate_ansible", Usage: "Creates debug ansible in debug ansible.", Action: generateAnsible, + Flags: []cli.Flag{token}, }, } return app.Run(stripFunctionArg(args)) diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 48b0ba2c..82d144cd 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -109,6 +109,9 @@ func TestShow(t *testing.T) { err = runCli([]string{"--function", "show_configurations"}) assert.Equal(t, "ShowConfigurations", err.Error()) + + err = runCli([]string{"--function", "generate_ansible", "--token", "foo"}) + assert.Equal(t, "GenerateAnsible", err.Error()) } func TestFlow(t *testing.T) { @@ -247,5 +250,5 @@ func (*stubDacctlActions) DataOut(c dacctl.CliContext) error { } func (*stubDacctlActions) GenerateAnsible(c dacctl.CliContext) error { - return errors.New("DataOut") + return errors.New("GenerateAnsible") } From 342ee2ff3905b23566817f6ad975c6204601d648 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 17:08:10 +0100 Subject: [PATCH 181/191] Pick up new release --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 673b4a4a..0bc306ea 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.6' -data_acc_checksum: 'sha256:ee41fb09dbc7e8b0a0fdaeed22133ca74a82fd391eea7abdaa4fb8bb8a086a12' +data_acc_version: 'v2.0-alpha.7' +data_acc_checksum: 'sha256:667ff9e5f3aa215ee346f4245dac88f8f79749599096ec4227563bee749d42ae' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From 33fefe756dd5ce4e3ea5e22d25e7be8fe99f43f5 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 18:58:39 +0100 Subject: [PATCH 182/191] Fix the copying of the playbooks --- internal/pkg/filesystem_impl/ansible.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index 2ef94157..c6a0d3c0 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -161,11 +161,13 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) return dir, err } - cmd = exec.Command("cp", getAnsibleDir("*.yml"), dir) - output, err = cmd.CombinedOutput() - log.Println("copy playbooks", string(output)) - if err != nil { - return dir, err + for _, playbook := range []string{"create.yml", "delete.yml", "restore.yml"} { + cmd = exec.Command("cp", getAnsibleDir(playbook), dir) + output, err = cmd.CombinedOutput() + log.Println("copy playbooks", playbook, string(output)) + if err != nil { + return dir, err + } } cmd = exec.Command("cp", "-r", getAnsibleDir(".venv"), dir) From 63be08d89a8df32316d183f8292e1458baab324f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 19:13:56 +0100 Subject: [PATCH 183/191] Pick up new version v2.0-alpha.8 --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 0bc306ea..60da6dd5 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.7' -data_acc_checksum: 'sha256:667ff9e5f3aa215ee346f4245dac88f8f79749599096ec4227563bee749d42ae' +data_acc_version: 'v2.0-alpha.8' +data_acc_checksum: 'sha256:3f61ad455abad836e09da6c0cbd77deb3413fbec4d831c1e5af3e6dd83598c35' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From a9b6905417c884fd28c2c18dba773f7f0fe47d96 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 23:23:00 +0100 Subject: [PATCH 184/191] Fix up inventory generation --- cmd/dacctl/actions.go | 4 +++- cmd/dacctl/main.go | 2 ++ cmd/dacctl/main_test.go | 4 ++-- internal/pkg/dacctl/actions_impl/actions.go | 4 ++-- internal/pkg/dacctl/interface.go | 2 +- internal/pkg/dacctl/workflow_impl/session.go | 14 +++++--------- internal/pkg/facade/session.go | 2 +- internal/pkg/filesystem_impl/ansible.go | 6 +++--- internal/pkg/mock_facade/session.go | 7 ++++--- 9 files changed, 23 insertions(+), 22 deletions(-) diff --git a/cmd/dacctl/actions.go b/cmd/dacctl/actions.go index 8df44dbf..7c7c4034 100644 --- a/cmd/dacctl/actions.go +++ b/cmd/dacctl/actions.go @@ -133,5 +133,7 @@ func dataOut(c *cli.Context) error { func generateAnsible(c *cli.Context) error { keystore := getKeystore() defer keystore.Close() - return getActions(keystore).GenerateAnsible(c) + return printOutput(func() (s string, e error) { + return getActions(keystore).GenerateAnsible(c) + }) } diff --git a/cmd/dacctl/main.go b/cmd/dacctl/main.go index 0cbd9d34..5d0bf86b 100644 --- a/cmd/dacctl/main.go +++ b/cmd/dacctl/main.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "github.com/RSE-Cambridge/data-acc/internal/pkg/config" "github.com/RSE-Cambridge/data-acc/pkg/version" "github.com/urfave/cli" @@ -191,6 +192,7 @@ func main() { log.Println("dacctl start, called with:", strings.Join(os.Args, " ")) if err := runCli(os.Args); err != nil { + fmt.Println(err) log.Println("dacctl error, called with:", strings.Join(os.Args, " ")) log.Fatal(err) } else { diff --git a/cmd/dacctl/main_test.go b/cmd/dacctl/main_test.go index 82d144cd..ae73ada7 100644 --- a/cmd/dacctl/main_test.go +++ b/cmd/dacctl/main_test.go @@ -249,6 +249,6 @@ func (*stubDacctlActions) DataOut(c dacctl.CliContext) error { return errors.New("DataOut") } -func (*stubDacctlActions) GenerateAnsible(c dacctl.CliContext) error { - return errors.New("GenerateAnsible") +func (*stubDacctlActions) GenerateAnsible(c dacctl.CliContext) (string, error) { + return "", errors.New("GenerateAnsible") } diff --git a/internal/pkg/dacctl/actions_impl/actions.go b/internal/pkg/dacctl/actions_impl/actions.go index 5cd36984..89f15668 100644 --- a/internal/pkg/dacctl/actions_impl/actions.go +++ b/internal/pkg/dacctl/actions_impl/actions.go @@ -118,10 +118,10 @@ func (d *dacctlActions) DataOut(c dacctl.CliContext) error { return d.session.CopyDataOut(sessionName) } -func (d *dacctlActions) GenerateAnsible(c dacctl.CliContext) error { +func (d *dacctlActions) GenerateAnsible(c dacctl.CliContext) (string, error) { sessionName, err := d.getSessionName(c) if err != nil { - return err + return "", err } return d.session.GenerateAnsible(sessionName) } diff --git a/internal/pkg/dacctl/interface.go b/internal/pkg/dacctl/interface.go index 93f72ce9..d4b58d7b 100644 --- a/internal/pkg/dacctl/interface.go +++ b/internal/pkg/dacctl/interface.go @@ -21,5 +21,5 @@ type DacctlActions interface { PreRun(c CliContext) error PostRun(c CliContext) error DataOut(c CliContext) error - GenerateAnsible(c CliContext) error + GenerateAnsible(c CliContext) (string, error) } diff --git a/internal/pkg/dacctl/workflow_impl/session.go b/internal/pkg/dacctl/workflow_impl/session.go index 8d0fef38..c43f4e04 100644 --- a/internal/pkg/dacctl/workflow_impl/session.go +++ b/internal/pkg/dacctl/workflow_impl/session.go @@ -7,6 +7,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/facade" "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" + "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry" "github.com/RSE-Cambridge/data-acc/internal/pkg/registry_impl" "github.com/RSE-Cambridge/data-acc/internal/pkg/store" @@ -21,6 +22,7 @@ func NewSessionFacade(keystore store.Keystore) facade.Session { session: registry_impl.NewSessionRegistry(keystore), actions: registry_impl.NewSessionActionsRegistry(keystore), allocations: registry_impl.NewAllocationRegistry(keystore), + ansible: filesystem_impl.NewAnsible(), } } @@ -285,17 +287,11 @@ func (s sessionFacade) GetAllSessions() ([]datamodel.Session, error) { return s.session.GetAllSessions() } -func (s sessionFacade) GenerateAnsible(sessionName datamodel.SessionName) error { +func (s sessionFacade) GenerateAnsible(sessionName datamodel.SessionName) (string, error) { session, err := s.session.GetSession(sessionName) if err != nil { log.Println("Unable to find session we want to mount:", sessionName) - return err + return "", err } - directory, err := s.ansible.CreateEnvironment(session) - if err != nil { - return err - } - log.Println("Created ansible in directory:", directory) - log.Println("Please delete directory when your work is done.") - return nil + return s.ansible.CreateEnvironment(session) } diff --git a/internal/pkg/facade/session.go b/internal/pkg/facade/session.go index 7ca022c8..6b4281b0 100644 --- a/internal/pkg/facade/session.go +++ b/internal/pkg/facade/session.go @@ -38,5 +38,5 @@ type Session interface { GetAllSessions() ([]datamodel.Session, error) // Generate ansible test dir - GenerateAnsible(sessionName datamodel.SessionName) error + GenerateAnsible(sessionName datamodel.SessionName) (string, error) } diff --git a/internal/pkg/filesystem_impl/ansible.go b/internal/pkg/filesystem_impl/ansible.go index c6a0d3c0..33fc040b 100644 --- a/internal/pkg/filesystem_impl/ansible.go +++ b/internal/pkg/filesystem_impl/ansible.go @@ -100,8 +100,8 @@ func getInventory(fsType FSType, fsUuid string, allBricks []datamodel.Brick) str "mgsnode": mgsnode, //"client_port": fmt.Sprintf("%d", volume.ClientPort), "lnet_suffix": conf.LnetSuffix, - "mdt_size_mb": fmt.Sprintf("%d", conf.MDTSizeMB), - "fs_name": fsUuid, + "mdt_size_mb": fmt.Sprintf("%d", conf.MDTSizeMB), + "fs_name": fsUuid, }, Hosts: hosts, } @@ -149,7 +149,7 @@ func setupAnsible(fsType FSType, internalName string, bricks []datamodel.Brick) inventory := getInventory(fsType, internalName, bricks) tmpInventory := filepath.Join(dir, "inventory") - if err := ioutil.WriteFile(tmpInventory, []byte(tmpInventory), 0666); err != nil { + if err := ioutil.WriteFile(tmpInventory, []byte(inventory), 0666); err != nil { return dir, err } log.Println(inventory) diff --git a/internal/pkg/mock_facade/session.go b/internal/pkg/mock_facade/session.go index 2fd50e17..0899939a 100644 --- a/internal/pkg/mock_facade/session.go +++ b/internal/pkg/mock_facade/session.go @@ -163,11 +163,12 @@ func (mr *MockSessionMockRecorder) GetAllSessions() *gomock.Call { } // GenerateAnsible mocks base method -func (m *MockSession) GenerateAnsible(sessionName datamodel.SessionName) error { +func (m *MockSession) GenerateAnsible(sessionName datamodel.SessionName) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GenerateAnsible", sessionName) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 } // GenerateAnsible indicates an expected call of GenerateAnsible From f4b08e50d9145dfb2c00470133dc3e609e4dbb9b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 2 Sep 2019 23:28:41 +0100 Subject: [PATCH 185/191] Move to latest release --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 60da6dd5..6c6716d8 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.8' -data_acc_checksum: 'sha256:3f61ad455abad836e09da6c0cbd77deb3413fbec4d831c1e5af3e6dd83598c35' +data_acc_version: 'v2.0-alpha.10' +data_acc_checksum: 'sha256:5db079614a80f6607e0514d8b8f84bf1f7a5486ab8a63778ea22fa4d0c504f3d' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From f6693e08d94751564db9437902236461f04fd1a0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 3 Sep 2019 00:01:58 +0100 Subject: [PATCH 186/191] Fix fs-ansible when nvme only gets an OST no MDT --- fs-ansible/roles/lustre/tasks/format.yaml | 35 ++++++++++++++--------- fs-ansible/roles/lustre/tasks/main.yaml | 3 ++ fs-ansible/roles/lustre/tasks/wipe.yaml | 4 +-- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/fs-ansible/roles/lustre/tasks/format.yaml b/fs-ansible/roles/lustre/tasks/format.yaml index c987349b..483f7c08 100644 --- a/fs-ansible/roles/lustre/tasks/format.yaml +++ b/fs-ansible/roles/lustre/tasks/format.yaml @@ -7,15 +7,29 @@ when: - mgs is defined -- name: Format MDTs +- name: Partition disks block: - - name: Remove old MDT Partition + - name: Add MDT Partition parted: - device: "/dev/{{ item }}" - number: 1 - state: absent - loop: "{{ mdts.keys() }}" + device: "/dev/{{ item }}" + number: 1 + part_start: "0%" + part_end: "{{ mdt_size_mb }}MB" + label: gpt + state: present + loop: "{{ (mdts.keys() + osts.keys()) | unique }}" + - name: Add OST Partition + parted: + device: "/dev/{{ item }}" + number: 2 + part_start: "{{ mdt_size_mb }}MB" + part_end: "100%" + label: gpt + state: present + loop: "{{ (mdts.keys() + osts.keys()) | unique }}" +- name: Format MDTs + block: - name: Add MDT Partition parted: device: "/dev/{{ item }}" @@ -30,15 +44,8 @@ command: "/usr/sbin/mkfs.lustre --mdt --reformat --fsname={{ fs_name }} --index={{ item.value }} --mgsnode={{ mgsnode }}{{ lnet_suffix }} /dev/{{ item.key }}p1" loop: "{{ mdts|dict2items }}" -- name: Format OST +- name: Format OSTs block: - - name: Remove old OST Partition - parted: - device: "/dev/{{ item }}" - number: 2 - state: absent - loop: "{{ osts.keys() }}" - - name: Add OST Partition parted: device: "/dev/{{ item }}" diff --git a/fs-ansible/roles/lustre/tasks/main.yaml b/fs-ansible/roles/lustre/tasks/main.yaml index 545fe187..193bee27 100644 --- a/fs-ansible/roles/lustre/tasks/main.yaml +++ b/fs-ansible/roles/lustre/tasks/main.yaml @@ -1,4 +1,7 @@ --- +- import_tasks: wipe.yaml + when: lustre_format_disks|bool and lustre_state == "present" + - import_tasks: format.yaml when: lustre_format_disks|bool and lustre_state == "present" diff --git a/fs-ansible/roles/lustre/tasks/wipe.yaml b/fs-ansible/roles/lustre/tasks/wipe.yaml index caffb014..e36327f7 100644 --- a/fs-ansible/roles/lustre/tasks/wipe.yaml +++ b/fs-ansible/roles/lustre/tasks/wipe.yaml @@ -10,11 +10,11 @@ device: "/dev/{{ item }}" number: 1 state: absent - loop: "{{ mdts.keys() }}" + loop: "{{ (mdts.keys() + osts.keys()) | unique }}" - name: Remove old OST Partition parted: device: "/dev/{{ item }}" number: 2 state: absent - loop: "{{ osts.keys() }}" \ No newline at end of file + loop: "{{ (mdts.keys() + osts.keys()) | unique }}" \ No newline at end of file From 6c77f1aae66c2951faf666455f5db894d667c54f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 3 Sep 2019 00:09:20 +0100 Subject: [PATCH 187/191] Start using v2.0-alpha.11 --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index 6c6716d8..fe3f372e 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.10' -data_acc_checksum: 'sha256:5db079614a80f6607e0514d8b8f84bf1f7a5486ab8a63778ea22fa4d0c504f3d' +data_acc_version: 'v2.0-alpha.11' +data_acc_checksum: 'sha256:ee2e7025666b96d726499b72ff591c930c73c29bfde69b9f5d8b5ab6015b455e' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From 84bd2490f00360ba83f47bc27d80e2cda36b773a Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 3 Sep 2019 00:35:41 +0100 Subject: [PATCH 188/191] Make sure we use a random fs-name We were getting fs-name clashes due to the deterministic random number generator. As such, we had fs-ansible failures due to the same fs_name being used in two places. --- internal/pkg/filesystem_impl/provider.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/filesystem_impl/provider.go b/internal/pkg/filesystem_impl/provider.go index 6624a92d..9b95c851 100644 --- a/internal/pkg/filesystem_impl/provider.go +++ b/internal/pkg/filesystem_impl/provider.go @@ -4,6 +4,7 @@ import ( "github.com/RSE-Cambridge/data-acc/internal/pkg/datamodel" "github.com/RSE-Cambridge/data-acc/internal/pkg/filesystem" "math/rand" + "time" ) func NewFileSystemProvider(ansible filesystem.Ansible) filesystem.Provider { @@ -17,10 +18,13 @@ type fileSystemProvider struct { const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +var source = rand.NewSource(time.Now().UnixNano()) +var randGenerator = rand.New(source) + func GetNewUUID() string { b := make([]byte, 8) for i := range b { - b[i] = letters[rand.Int63()%int64(len(letters))] + b[i] = letters[randGenerator.Int63()%int64(len(letters))] } return string(b) } From 6419b80d4f6e22650a8f216f533cec261f74a1fe Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 3 Sep 2019 00:40:54 +0100 Subject: [PATCH 189/191] Use latest release v2.0-alpha.12 --- dac-ansible/roles/data-acc/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dac-ansible/roles/data-acc/defaults/main.yml b/dac-ansible/roles/data-acc/defaults/main.yml index fe3f372e..c5591795 100644 --- a/dac-ansible/roles/data-acc/defaults/main.yml +++ b/dac-ansible/roles/data-acc/defaults/main.yml @@ -1,6 +1,6 @@ --- -data_acc_version: 'v2.0-alpha.11' -data_acc_checksum: 'sha256:ee2e7025666b96d726499b72ff591c930c73c29bfde69b9f5d8b5ab6015b455e' +data_acc_version: 'v2.0-alpha.12' +data_acc_checksum: 'sha256:3e2d42757c5a0a77d967fabac6b722e597077aeb8edf1fe6225c5203827829c5' data_acc_platform: linux-amd64 data_acc_mirror: https://github.com/RSE-Cambridge/data-acc/releases/download data_acc_install_dir: /usr/local/bin From d4512b23cc3cc65c6d47fe2d22e63d3fc41baf00 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 4 Sep 2019 21:07:43 +0100 Subject: [PATCH 190/191] Fix unmount of per job buffer Previously we only unmounted persistent buffers as we didn't record in the session that the mount had happened. --- .../session_action_handler.go | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 5aba16ae..8c0613c4 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -194,7 +194,7 @@ func (s *sessionActionHandler) handleCopyOut(action datamodel.SessionAction) { }) } -func (s *sessionActionHandler) doAllMounts(actionSession datamodel.Session) error { +func (s *sessionActionHandler) doAllMounts(actionSession datamodel.Session) (datamodel.Session, error) { attachmentSession := datamodel.AttachmentSession{ Hosts: actionSession.RequestedAttachHosts, SessionName: actionSession.Name, @@ -206,29 +206,30 @@ func (s *sessionActionHandler) doAllMounts(actionSession datamodel.Session) erro PrivateMount: actionSession.VolumeRequest.Access == datamodel.Private || actionSession.VolumeRequest.Access == datamodel.PrivateAndStriped, SwapBytes: actionSession.VolumeRequest.SwapBytes, } - //if actionSession.CurrentAttachments == nil { - // session.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ - // session.Name: jobAttachmentStatus, - // } - //} else { - // session.CurrentAttachments[session.Name] = jobAttachmentStatus - //} - //session, err = s.sessionRegistry.UpdateSession(session) - //if err != nil { - // return err - //} + if actionSession.CurrentAttachments == nil { + actionSession.CurrentAttachments = map[datamodel.SessionName]datamodel.AttachmentSessionStatus{ + actionSession.Name: jobAttachmentStatus, + } + } else { + actionSession.CurrentAttachments[actionSession.Name] = jobAttachmentStatus + } + session, err := s.sessionRegistry.UpdateSession(actionSession) + if err != nil { + return actionSession, err + } + actionSession = session if err := s.fsProvider.Mount(actionSession, jobAttachmentStatus); err != nil { - return err + return actionSession, err } // TODO: should we update the session? and delete attachments later? } for _, sessionName := range actionSession.MultiJobAttachments { if err := s.doMultiJobMount(actionSession, sessionName); err != nil { - return nil + return actionSession, nil } } - return nil + return actionSession, nil } func (s *sessionActionHandler) doMultiJobMount(actionSession datamodel.Session, sessionName datamodel.SessionName) error { @@ -348,7 +349,8 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { return session, errors.New("already mounted, can't mount again") } - if err := s.doAllMounts(session); err != nil { + session, err = s.doAllMounts(session); + if err != nil { if err := s.doAllUnmounts(session); err != nil { log.Println("error while rolling back possible partial mount", action.Session.Name, err) } From 9b592df21ec099e331aa3e0a0bd4673a9920c46f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Wed, 4 Sep 2019 21:55:33 +0100 Subject: [PATCH 191/191] Fix formatting --- internal/pkg/dacd/brick_manager_impl/session_action_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go index 8c0613c4..85d9880f 100644 --- a/internal/pkg/dacd/brick_manager_impl/session_action_handler.go +++ b/internal/pkg/dacd/brick_manager_impl/session_action_handler.go @@ -349,7 +349,7 @@ func (s *sessionActionHandler) handleMount(action datamodel.SessionAction) { return session, errors.New("already mounted, can't mount again") } - session, err = s.doAllMounts(session); + session, err = s.doAllMounts(session) if err != nil { if err := s.doAllUnmounts(session); err != nil { log.Println("error while rolling back possible partial mount", action.Session.Name, err)