From e5497d35ae507a3d20b1537fc2deb4ea4e0f1561 Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Wed, 27 Sep 2023 23:07:13 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=A7=B9=20pin=20copyright=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pr-test-lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-test-lint.yml b/.github/workflows/pr-test-lint.yml index bcd17e6943..b1403be6c5 100644 --- a/.github/workflows/pr-test-lint.yml +++ b/.github/workflows/pr-test-lint.yml @@ -87,6 +87,8 @@ jobs: - name: Setup Copywrite uses: hashicorp/setup-copywrite@v1.1.2 + with: + version: v0.16.4 - name: Check Header Compliance run: copywrite headers --plan From c5c9bf4fd934172e132f993d1f6bad46e2605844 Mon Sep 17 00:00:00 2001 From: Victoria Jeffrey Date: Wed, 27 Sep 2023 12:21:12 -0600 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=90=9B=20ensure=20aws=20ebs=20scan=20?= =?UTF-8?q?cleans=20up=20after=20itself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aws/connection/awsec2ebsconn/destroy.go | 6 ++- .../aws/connection/awsec2ebsconn/provider.go | 38 +++++++++++++++++-- .../aws/connection/awsec2ebsconn/setup.go | 37 +++++++++--------- providers/aws/provider/provider.go | 7 +++- 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/providers/aws/connection/awsec2ebsconn/destroy.go b/providers/aws/connection/awsec2ebsconn/destroy.go index 0a84387426..de5a4be784 100644 --- a/providers/aws/connection/awsec2ebsconn/destroy.go +++ b/providers/aws/connection/awsec2ebsconn/destroy.go @@ -16,8 +16,12 @@ import ( func (c *AwsEbsConnection) DetachVolumeFromInstance(ctx context.Context, volume *awsec2ebstypes.VolumeInfo) error { log.Info().Msg("detach volume") + var deviceName string + if c.volumeMounter != nil { + deviceName = c.volumeMounter.VolumeAttachmentLoc + } res, err := c.scannerRegionEc2svc.DetachVolume(ctx, &ec2.DetachVolumeInput{ - Device: aws.String(c.volumeMounter.VolumeAttachmentLoc), VolumeId: &volume.Id, + Device: aws.String(deviceName), VolumeId: &volume.Id, InstanceId: &c.scannerInstance.Id, }) if err != nil { diff --git a/providers/aws/connection/awsec2ebsconn/provider.go b/providers/aws/connection/awsec2ebsconn/provider.go index 475aa2ce4f..605a69420a 100644 --- a/providers/aws/connection/awsec2ebsconn/provider.go +++ b/providers/aws/connection/awsec2ebsconn/provider.go @@ -106,7 +106,7 @@ func NewAwsEbsConnection(id uint32, conf *inventory.Config, asset *inventory.Ass // check if we got the no setup override option. this implies the target volume is already attached to the instance // this is used in cases where we need to test a snapshot created from a public marketplace image. the volume gets attached to a brand // new instance, and then that instance is started and we scan the attached fs - var volLocation string + var volLocation, volId string if conf.Options[snapshot.NoSetup] == "true" || conf.Options[snapshot.IsSetup] == "true" { log.Info().Msg("skipping setup step") } else { @@ -114,13 +114,13 @@ func NewAwsEbsConnection(id uint32, conf *inventory.Config, asset *inventory.Ass var err error switch c.targetType { case awsec2ebstypes.EBSTargetInstance: - ok, volLocation, err = c.SetupForTargetInstance(ctx, instanceinfo) + ok, volLocation, volId, err = c.SetupForTargetInstance(ctx, instanceinfo) conf.PlatformId = awsec2.MondooInstanceID(i.AccountID, conf.Options["region"], convert.ToString(instanceinfo.InstanceId)) case awsec2ebstypes.EBSTargetVolume: - ok, volLocation, err = c.SetupForTargetVolume(ctx, *volumeid) + ok, volLocation, volId, err = c.SetupForTargetVolume(ctx, *volumeid) conf.PlatformId = awsec2.MondooVolumeID(volumeid.Account, volumeid.Region, volumeid.Id) case awsec2ebstypes.EBSTargetSnapshot: - ok, volLocation, err = c.SetupForTargetSnapshot(ctx, *snapshotid) + ok, volLocation, volId, err = c.SetupForTargetSnapshot(ctx, *snapshotid) conf.PlatformId = awsec2.MondooSnapshotID(snapshotid.Account, snapshotid.Region, snapshotid.Id) default: return c, errors.New("invalid target type") @@ -136,6 +136,12 @@ func NewAwsEbsConnection(id uint32, conf *inventory.Config, asset *inventory.Ass } // set is setup to true asset.Connections[0].Options[snapshot.IsSetup] = "true" + // save the other information to asset connection options too + asset.Connections[0].Options["volume-id"] = volId + asset.Connections[0].Options["volume-loc"] = volLocation + if c.scanVolumeInfo.Tags["createdBy"] == "Mondoo" { + asset.Connections[0].Options["createdBy"] = "Mondoo" + } } asset.PlatformIds = []string{conf.PlatformId} @@ -187,6 +193,8 @@ func NewAwsEbsConnection(id uint32, conf *inventory.Config, asset *inventory.Ass } asset.Id = conf.Type asset.Platform.Runtime = c.Runtime() + asset.Connections[0].Options["scanner-id"] = c.scannerInstance.Id + asset.Connections[0].Options["scanner-region"] = c.scannerInstance.Region return c, nil } @@ -205,7 +213,29 @@ func (c *AwsEbsConnection) Close() { return } } + // we seem to be losing all the connection info we + // had when we started by the time we get here. + // we should figure out what is happening + // for now, reassemble the info needed from the asset + // connection options ctx := context.Background() + opts := c.asset.Connections[0].Options + c.volumeMounter = &snapshot.VolumeMounter{ + ScanDir: opts["mounted"], + VolumeAttachmentLoc: opts["volume-loc"], + } + c.scanVolumeInfo = &awsec2ebstypes.VolumeInfo{ + Id: opts["volume-id"], + Tags: map[string]string{"createdBy": opts["createdBy"]}, + } + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + log.Error().Err(err).Msg("cfg") + return + } + cfg.Region = opts["scanner-region"] + c.scannerRegionEc2svc = ec2.NewFromConfig(cfg) + c.scannerInstance.Id = opts["scanner-id"] if c.volumeMounter != nil { err := c.volumeMounter.UnmountVolumeFromInstance() if err != nil { diff --git a/providers/aws/connection/awsec2ebsconn/setup.go b/providers/aws/connection/awsec2ebsconn/setup.go index c023c527e9..751eb73321 100644 --- a/providers/aws/connection/awsec2ebsconn/setup.go +++ b/providers/aws/connection/awsec2ebsconn/setup.go @@ -61,7 +61,7 @@ func (c *AwsEbsConnection) Validate(ctx context.Context) (*types.Instance, *awse return nil, nil, nil, errors.New("cannot validate; unrecognized ebs target") } -func (c *AwsEbsConnection) SetupForTargetVolume(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, error) { +func (c *AwsEbsConnection) SetupForTargetVolume(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, string, error) { log.Debug().Interface("volume", volume).Msg("setup for target volume") if !volume.IsAvailable { return c.SetupForTargetVolumeUnavailable(ctx, volume) @@ -70,7 +70,7 @@ func (c *AwsEbsConnection) SetupForTargetVolume(ctx context.Context, volume awse return c.AttachVolumeToInstance(ctx, volume) } -func (c *AwsEbsConnection) SetupForTargetVolumeUnavailable(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, error) { +func (c *AwsEbsConnection) SetupForTargetVolumeUnavailable(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, string, error) { found, snapId, err := c.FindRecentSnapshotForVolume(ctx, volume) if err != nil { // only log the error here, this is not a blocker @@ -79,41 +79,41 @@ func (c *AwsEbsConnection) SetupForTargetVolumeUnavailable(ctx context.Context, if !found { snapId, err = c.CreateSnapshotFromVolume(ctx, volume) if err != nil { - return false, "", err + return false, "", "", err } } snapId, err = c.CopySnapshotToRegion(ctx, snapId) if err != nil { - return false, "", err + return false, "", "", err } volId, err := c.CreateVolumeFromSnapshot(ctx, snapId) if err != nil { - return false, "", err + return false, "", "", err } c.scanVolumeInfo = &volId return c.AttachVolumeToInstance(ctx, volId) } -func (c *AwsEbsConnection) SetupForTargetSnapshot(ctx context.Context, snapshot awsec2ebstypes.SnapshotId) (bool, string, error) { +func (c *AwsEbsConnection) SetupForTargetSnapshot(ctx context.Context, snapshot awsec2ebstypes.SnapshotId) (bool, string, string, error) { log.Debug().Interface("snapshot", snapshot).Msg("setup for target snapshot") snapId, err := c.CopySnapshotToRegion(ctx, snapshot) if err != nil { - return false, "", err + return false, "", "", err } volId, err := c.CreateVolumeFromSnapshot(ctx, snapId) if err != nil { - return false, "", err + return false, "", "", err } c.scanVolumeInfo = &volId return c.AttachVolumeToInstance(ctx, volId) } -func (c *AwsEbsConnection) SetupForTargetInstance(ctx context.Context, instanceinfo *types.Instance) (bool, string, error) { +func (c *AwsEbsConnection) SetupForTargetInstance(ctx context.Context, instanceinfo *types.Instance) (bool, string, string, error) { log.Debug().Str("instance id", *instanceinfo.InstanceId).Msg("setup for target instance") var err error v, err := c.GetVolumeInfoForInstance(ctx, instanceinfo) if err != nil { - return false, "", err + return false, "", "", err } found, snapId, err := c.FindRecentSnapshotForVolume(ctx, v) if err != nil { @@ -123,16 +123,16 @@ func (c *AwsEbsConnection) SetupForTargetInstance(ctx context.Context, instancei if !found { snapId, err = c.CreateSnapshotFromVolume(ctx, v) if err != nil { - return false, "", err + return false, "", "", err } } snapId, err = c.CopySnapshotToRegion(ctx, snapId) if err != nil { - return false, "", err + return false, "", "", err } volId, err := c.CreateVolumeFromSnapshot(ctx, snapId) if err != nil { - return false, "", err + return false, "", "", err } c.scanVolumeInfo = &volId return c.AttachVolumeToInstance(ctx, volId) @@ -391,15 +391,14 @@ func AttachVolume(ctx context.Context, ec2svc *ec2.Client, location string, volI return location, res.State, nil } -func (c *AwsEbsConnection) AttachVolumeToInstance(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, error) { +func (c *AwsEbsConnection) AttachVolumeToInstance(ctx context.Context, volume awsec2ebstypes.VolumeInfo) (bool, string, string, error) { log.Info().Str("volume id", volume.Id).Msg("attach volume") - location := newVolumeAttachmentLoc() ready := false loc, state, err := AttachVolume(ctx, c.scannerRegionEc2svc, newVolumeAttachmentLoc(), volume.Id, c.scannerInstance.Id) if err != nil { - return ready, "", err + return ready, "", "", err } - location = loc // warning: there is no guarantee from AWS that the device will be placed there + location := loc // warning: there is no guarantee from AWS that the device will be placed there log.Debug().Str("location", location).Msg("target volume") /* @@ -415,7 +414,7 @@ func (c *AwsEbsConnection) AttachVolumeToInstance(ctx context.Context, volume aw time.Sleep(10 * time.Second) resp, err := c.scannerRegionEc2svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []string{volume.Id}}) if err != nil { - return ready, location, err + return ready, location, "", err } if len(resp.Volumes) == 1 { volState = resp.Volumes[0].State @@ -423,7 +422,7 @@ func (c *AwsEbsConnection) AttachVolumeToInstance(ctx context.Context, volume aw log.Info().Interface("state", volState).Msg("waiting for volume attachment completion") } } - return true, location, nil + return true, location, volume.Id, nil } func awsTagsToMap(tags []types.Tag) map[string]string { diff --git a/providers/aws/provider/provider.go b/providers/aws/provider/provider.go index 6aa466ab4c..ff21b3989d 100644 --- a/providers/aws/provider/provider.go +++ b/providers/aws/provider/provider.go @@ -97,8 +97,11 @@ func parseFlagsToOptions(m map[string]*llx.Primitive) map[string]string { func (s *Service) Shutdown(req *plugin.ShutdownReq) (*plugin.ShutdownRes, error) { for i := range s.runtimes { runtime := s.runtimes[i] - if conn, ok := runtime.Connection.(awsec2ebsconn.AwsEbsConnection); ok { - conn.Close() + if conn, ok := runtime.Connection.(shared.Connection); ok { + if conn.Type() == awsec2ebsconn.EBSConnectionType { + conn := runtime.Connection.(*awsec2ebsconn.AwsEbsConnection) + conn.Close() + } } } return &plugin.ShutdownRes{}, nil