Skip to content

Commit

Permalink
resource scanning added to obtain creator code (#2519)
Browse files Browse the repository at this point in the history
* resource scanning added to obtain creator code

* add get resource creator account info code

* add resource create action

* update query rule

* update create time parse code
  • Loading branch information
bruceybian authored Feb 5, 2024
1 parent b9a8fa4 commit bea7f20
Show file tree
Hide file tree
Showing 41 changed files with 8,028 additions and 66 deletions.
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ require (
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/clb v1.0.847
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cloudaudit v1.0.544
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cls v1.0.711
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cwp v1.0.762
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.692
Expand Down Expand Up @@ -110,6 +110,7 @@ require (
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cdwpg v1.0.772
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dasb v1.0.798
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/oceanus v1.0.831
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856
github.com/wI2L/jsondiff v0.3.0
k8s.io/apimachinery v0.22.4
)
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.770 h1:5FWt9F+a
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.770/go.mod h1:zuWcAJVIXbHbOR7i6SKK0wNlvPLt5Su4ijOZGMJbUi8=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.824 h1:DVKvZ6h+qd7tadUrCjVAkCCmE3TsbK2ZmwGd3AJcpWc=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/bi v1.0.824/go.mod h1:DvBpDX/qdJG4KKLeULmRvhAjPYiw8za0HeTSu2y/lFw=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856 h1:zJBb69FBBNZko4PbDScqj9wogPatzNYEXAJ862IBoMQ=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing v1.0.856/go.mod h1:P+VGibeVh8pW4N85x9ee0xzXsZoT9jzjHYD8GaAiyG4=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam v1.0.760 h1:Ky9dRsTL2HXKWUrTFpQFZWQ1TrM+o+P35kczR7thalo=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam v1.0.760/go.mod h1:AAfdrxknvUedvigxbbzKQLxN+1EG5NPbytpiqmfuFvU=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cat v1.0.760 h1:oGY4IigfIw0iQKh3/cOY29KBeEeFbvJft69e0beyfdI=
Expand Down Expand Up @@ -954,6 +956,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.845 h1:fiiV
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.845/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847 h1:ITZmxAWfbr5yikJ4T30yVYMW3jpa/oTmNbPnw/h1Vq0=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.847/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856 h1:4PaaKxPjh0dmRzzz0CRZSZlbvzQIPcg4TE1Ibz0Cdlk=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.856/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624 h1:nEZqsoqt1pEoaP9JjkHQy3/H00suCfzlHW1qOm2nYD8=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.624/go.mod h1:+TXSVyeKwt1IhZRqKPbTREteBcP+K07Q846/ilNzLWA=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cwp v1.0.762 h1:2egy69SP/wPsmnfozcQVZ6tUY6F6N/TpEe/7xtXrc/8=
Expand Down
184 changes: 184 additions & 0 deletions tencentcloud/common/cloud_common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
package common

import (
"encoding/json"
cls "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cls/v20201016"
"log"
"strconv"
"strings"
"time"

cam "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cam/v20190116"
"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/connectivity"
"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/internal/helper"
)

const (
// DefaultSearchLogStartTimestamp sync logs start time 2023-11-07 16:41:00
DefaultSearchLogStartTimestamp = 1699346460000

DefaultTopicId = "aef50d54-b17d-4782-8618-a7873203ec29"

QueryGrammarRule = " AND "
)

// ResourceAccountInfo 资源账户信息
type ResourceAccountInfo struct {
ResourceType string // 资源类型
ResourceName string // 资源名称
AccountId string // 主账号ID
PrincipalId string // 用户ID
UserName string // 用户名
}

// GetResourceCreatorAccountInfo get resource creator user info
func GetResourceCreatorAccountInfo(client *connectivity.TencentCloudClient, resourceCreateAction string, resources []*ResourceInstance) map[string]*ResourceAccountInfo {
resourceIdToSubAccountInfoMap := make(map[string]*ResourceAccountInfo)
if resourceCreateAction == "" {
return resourceIdToSubAccountInfoMap
}

request := cls.NewSearchLogRequest()
request.From = helper.IntInt64(DefaultSearchLogStartTimestamp)
request.To = helper.Int64(CurrentTimeMillisecond())
request.TopicId = helper.String(DefaultTopicId)

for _, r := range resources {
query := resourceCreateAction + QueryGrammarRule
if r.Id != "" {
query = query + r.Id
} else if r.Name != "" {
query = query + r.Name
} else {
continue
}
request.Query = helper.String(query)

response, err := client.UseClsClient().SearchLog(request)
if err != nil {
log.Printf("[CRITAL] search resource[%v] log data error: %v", r.Id, err.Error())
return resourceIdToSubAccountInfoMap
}
if response == nil || response.Response == nil {
log.Printf("[CRITAL] search resource[%v] log data response is nil", r.Id)
return resourceIdToSubAccountInfoMap
}
if len(response.Response.Results) == 0 {
log.Printf("[CRITAL] search resource[%v] log data response results is empty", r.Id)
return resourceIdToSubAccountInfoMap
}

result := response.Response.Results[0]
if result != nil {
var jsonData string
if len(*result.LogJson) > 2 {
jsonData = *result.LogJson
} else if len(*result.RawLog) > 2 {
jsonData = *result.RawLog
} else {
continue
}

resourceAccountInfo := ParseLogJsonData(jsonData)
if resourceAccountInfo.PrincipalId == resourceAccountInfo.UserName &&
resourceAccountInfo.PrincipalId != resourceAccountInfo.AccountId {
userName := GetSubAccountUserName(client, resourceAccountInfo.PrincipalId)
resourceAccountInfo.UserName = userName
}
resourceIdToSubAccountInfoMap[r.Id] = resourceAccountInfo
}
}

return resourceIdToSubAccountInfoMap
}

// GetSubAccountUserName get sub account user name
func GetSubAccountUserName(client *connectivity.TencentCloudClient, uin string) string {
uinNum, err := strconv.ParseUint(uin, 10, 64)
if err != nil {
log.Printf("[CRITAL] parse uin[%v] to uint64 type error: %v", uin, err.Error())
return ""
}

request := cam.NewDescribeSubAccountsRequest()

uinArray := []*uint64{helper.Uint64(uinNum)}
request.FilterSubAccountUin = uinArray

response, err := client.UseCamClient().DescribeSubAccounts(request)
if err != nil {
log.Printf("[CRITAL] get sub account[%v] data error: %v", uin, err.Error())
return ""
}
if response == nil || response.Response == nil {
log.Printf("[CRITAL] get sub account[%v] data response is nil", uin)
return ""
}

name := response.Response.SubAccounts[0].Name
return *name
}

// CurrentTimeMillisecond get the current millisecond timestamp
func CurrentTimeMillisecond() int64 {
return time.Now().UnixNano() / int64(time.Millisecond)
}

func ParseLogJsonData(jsonData string) *ResourceAccountInfo {
if jsonData == "" {
return nil
}

var data map[string]interface{}
err := json.Unmarshal([]byte(jsonData), &data)
if err != nil {
log.Printf("[CRITAL] parse log json data[%v] error: %v", jsonData, err.Error())
return nil
}

resourceType := ""
if v, ok := data["resourceType"]; ok {
resourceType = v.(string)
}
resourceName := ""
if v, ok := data["resourceName"]; ok {
resourceName = v.(string)
if resourceName != "" {
resourceName = strings.Split(resourceName, "/")[0]
}
}
accountId, principalId, userName := parseUserIdentityFields(data)

return &ResourceAccountInfo{
ResourceType: resourceType,
ResourceName: resourceName,
AccountId: accountId,
PrincipalId: principalId,
UserName: userName,
}
}

func parseUserIdentityFields(data map[string]interface{}) (accountId, principalId, userName string) {
if v, ok := data["userIdentity.accountId"]; ok {
accountId = v.(string)
}
if v, ok := data["userIdentity.principalId"]; ok {
principalId = v.(string)
}
if v, ok := data["userIdentity.userName"]; ok {
userName = v.(string)
}
if v, ok := data["userIdentity"]; ok {
switch v := v.(type) {
case string:
var userIdentity map[string]string
err := json.Unmarshal([]byte(v), &userIdentity)
if err == nil {
accountId = userIdentity["accountId"]
principalId = userIdentity["principalId"]
userName = userIdentity["userName"]
}
}
}
return
}
4 changes: 2 additions & 2 deletions tencentcloud/common/file_common.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ const (
SweeperNonKeepResourceScanDir = "../../../tmp/non_keep_resource_scan/"
)

var ResourceScanHeader = []string{"资源类型", "资源名称", "实例ID", "实例名称", "分类", "创建时长(天)"}
var NonKeepResourceScanHeader = []string{"ResourceType", "ResourceName", "InstanceId", "InstanceName"}
var ResourceScanHeader = []string{"资源类型", "资源名称", "实例ID", "实例名称", "分类", "创建时长(天)", "创建者用户ID", "创建者用户名"}
var NonKeepResourceScanHeader = []string{"ResourceType", "ResourceName", "InstanceId", "InstanceName", "PrincipalId", "UserName"}

// WriteCsvFileData write data to csv file
func WriteCsvFileData(dirPath string, header []string, data [][]string) error {
Expand Down
40 changes: 34 additions & 6 deletions tencentcloud/common/resource_scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"regexp"
"strconv"
"time"

"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/connectivity"
)

const (
Expand All @@ -27,14 +29,16 @@ type ResourceInstance struct {
DefaultKeep bool
}

func ProcessScanCloudResources(resources, nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
ProcessResources(resources, resourceType, resourceName)
func ProcessScanCloudResources(client *connectivity.TencentCloudClient, resources, nonKeepResources []*ResourceInstance, resourceCreateAction string) {
ProcessResources(client, resources, resourceCreateAction)

ProcessNonKeepResources(nonKeepResources, resourceType, resourceName)
ProcessNonKeepResources(client, nonKeepResources, resourceCreateAction)
}

// ProcessResources Process all scanned cloud resources
func ProcessResources(resources []*ResourceInstance, resourceType, resourceName string) {
func ProcessResources(client *connectivity.TencentCloudClient, resources []*ResourceInstance, resourceCreateAction string) {
resourceIdToSubAccountInfoMap := GetResourceCreatorAccountInfo(client, resourceCreateAction, resources)

data := make([][]string, len(resources))
for i, r := range resources {
isResourceKeep := CheckResourceNameKeep(r.Name)
Expand All @@ -48,13 +52,24 @@ func ProcessResources(resources []*ResourceInstance, resourceType, resourceName
log.Printf("[CRITAL] compute resource creation duration error: %v", err.Error())
}

var resourceType, resourceName, principalId, userName string
creatorAccountInfo := resourceIdToSubAccountInfoMap[r.Id]
if creatorAccountInfo != nil {
resourceType = creatorAccountInfo.ResourceType
resourceName = creatorAccountInfo.ResourceName
principalId = creatorAccountInfo.PrincipalId
userName = creatorAccountInfo.UserName
}

data[i] = []string{
resourceType,
resourceName,
r.Id,
r.Name,
isResourceKeep,
creationDuration,
principalId,
userName,
}
}
err := WriteCsvFileData(SweeperResourceScanDir, ResourceScanHeader, data)
Expand All @@ -64,14 +79,27 @@ func ProcessResources(resources []*ResourceInstance, resourceType, resourceName
}

// ProcessNonKeepResources Processing scanned non-keep cloud resources
func ProcessNonKeepResources(nonKeepResources []*ResourceInstance, resourceType, resourceName string) {
func ProcessNonKeepResources(client *connectivity.TencentCloudClient, nonKeepResources []*ResourceInstance, resourceCreateAction string) {
resourceIdToSubAccountInfoMap := GetResourceCreatorAccountInfo(client, resourceCreateAction, nonKeepResources)

data := make([][]string, len(nonKeepResources))
for i, r := range nonKeepResources {
var resourceType, resourceName, principalId, userName string
creatorAccountInfo := resourceIdToSubAccountInfoMap[r.Id]
if creatorAccountInfo != nil {
resourceType = creatorAccountInfo.ResourceType
resourceName = creatorAccountInfo.ResourceName
principalId = creatorAccountInfo.PrincipalId
userName = creatorAccountInfo.UserName
}

data[i] = []string{
resourceType,
resourceName,
r.Id,
r.Name,
principalId,
userName,
}
}
err := WriteCsvFileData(SweeperNonKeepResourceScanDir, NonKeepResourceScanHeader, data)
Expand Down Expand Up @@ -133,7 +161,7 @@ func ParsedTime(createTime string) (*time.Time, error) {
} else {
// try parsing input strings using different time formats
for _, format := range TimeFormats {
parsedTime, err = time.Parse(format, createTime)
parsedTime, err = time.ParseInLocation(format, createTime, time.Local)
if err == nil {
break
}
Expand Down
16 changes: 16 additions & 0 deletions tencentcloud/connectivity/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strconv"
"time"

billing "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/billing/v20180709"
dasb "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dasb/v20191018"

oceanus "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/oceanus/v20190422"
Expand Down Expand Up @@ -199,6 +200,7 @@ type TencentCloudClient struct {
trocketConn *trocket.Client
biConn *bi.Client
cdwpgConn *cdwpg.Client
billingConn *billing.Client
//internal version: replace client begin, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
//internal version: replace client end, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
}
Expand Down Expand Up @@ -1376,6 +1378,20 @@ func (me *TencentCloudClient) UseCdwpgClient() *cdwpg.Client {
return me.cdwpgConn
}

// UseBillingClient returns billing client for service
func (me *TencentCloudClient) UseBillingClient() *billing.Client {
if me.billingConn != nil {
return me.billingConn
}

cpf := me.NewClientProfile(300)
cpf.Language = "zh-CN"
me.billingConn, _ = billing.NewClient(me.Credential, me.Region, cpf)
me.billingConn.WithHttpTransport(&LogRoundTripper{})

return me.billingConn
}

//internal version: replace useClient begin, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.
//internal version: replace useClient end, please do not modify this annotation and refrain from inserting any code between the beginning and end lines of the annotation.

Expand Down
5 changes: 3 additions & 2 deletions tencentcloud/services/as/resource_tc_as_attachment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ func testSweepAsAttachment(r string) error {
logId := tccommon.GetLogId(tccommon.ContextNil)
ctx := context.WithValue(context.TODO(), tccommon.LogIdKey, logId)
cli, _ := tcacctest.SharedClientForRegion(r)
asService := svcas.NewAsService(cli.(tccommon.ProviderMeta).GetAPIV3Conn())
client := cli.(tccommon.ProviderMeta).GetAPIV3Conn()
asService := svcas.NewAsService(client)

scalingGroups, err := asService.DescribeAutoScalingGroupByFilter(ctx, "", "", "", nil)
if err != nil {
Expand All @@ -50,7 +51,7 @@ func testSweepAsAttachment(r string) error {
CreatTime: *v.CreatedTime,
})
}
tccommon.ProcessScanCloudResources(resources, nonKeepResources, "as", "attachment")
tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateAutoScalingGroup")

for _, v := range scalingGroups {
scalingGroupId := *v.AutoScalingGroupId
Expand Down
2 changes: 1 addition & 1 deletion tencentcloud/services/cam/resource_tc_cam_group_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func init() {
CreatTime: *v.CreateTime,
})
}
tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cam", "group")
tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateGroup")

for _, v := range groups {
name := *v.GroupName
Expand Down
2 changes: 1 addition & 1 deletion tencentcloud/services/cbs/resource_tc_cbs_storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func init() {
CreatTime: *v.CreateTime,
})
}
tccommon.ProcessScanCloudResources(resources, nonKeepResources, "cbs", "storage")
tccommon.ProcessScanCloudResources(client, resources, nonKeepResources, "CreateDisks")

for i := range disks {
disk := disks[i]
Expand Down
Loading

0 comments on commit bea7f20

Please sign in to comment.