Skip to content

Commit

Permalink
pb-4872: Added change to support ignore file option for kdmp snapshot.
Browse files Browse the repository at this point in the history
	- Added KDMP_EXCLUDE_FILE_LIST configmap parameter to
	  kdmp-config configmap for used to specify the ignore file list for kdmp snapshot.
	- Added set policy command execution in kopiabackup executor for setting ignore file list to kopia.
  • Loading branch information
sivakumar subraani committed Nov 20, 2023
1 parent fdf611b commit 705ac31
Show file tree
Hide file tree
Showing 7 changed files with 247 additions and 6 deletions.
52 changes: 50 additions & 2 deletions pkg/controllers/dataexport/reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ const (
pxbackupAnnotationCreateByValue = "px-backup"
backupObjectUIDKey = kdmpAnnotationPrefix + "backupobject-uid"
pvcUIDKey = kdmpAnnotationPrefix + "pvc-uid"
kdmpStorageClassKey = kdmpAnnotationPrefix + "storage-class"
volumeSnapShotCRDirectory = "csi-generic"
snapDeleteAnnotation = "snapshotScheduledForDeletion"
snapRestoreAnnotation = "snapshotScheduledForRestore"
Expand All @@ -76,6 +77,7 @@ const (
defaultTimeout = 1 * time.Minute
progressCheckInterval = 5 * time.Second
compressionKey = "KDMP_COMPRESSION"
excludeFileListKey = "KDMP_EXCLUDE_FILE_LIST"
backupPath = "KDMP_BACKUP_PATH"
)

Expand Down Expand Up @@ -261,6 +263,8 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er

var compressionType string
var podDataPath string
var excludeFileList string
pvcStorageClass := dataExport.Labels[kdmpStorageClassKey]
var backupLocation *storkapi.BackupLocation
var data updateDataExportDetail
if driverName != drivers.Rsync {
Expand All @@ -273,11 +277,33 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
kdmpData, err := core.Instance().GetConfigMap(utils.KdmpConfigmapName, utils.KdmpConfigmapNamespace)
if err != nil {
logrus.Errorf("failed reading config map %v: %v", utils.KdmpConfigmapName, err)
logrus.Warnf("default to %s compression", utils.DefaultCompresion)
compressionType = utils.DefaultCompresion
if err != nil {
msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName)
logrus.Errorf(msg)
data := updateDataExportDetail{
status: kdmpapi.DataExportStatusFailed,
reason: msg,
}
return false, c.updateStatus(dataExport, data)
}
} else {
compressionType = kdmpData.Data[compressionKey]
if len(compressionType) == 0 {
compressionType = utils.DefaultCompresion
}
podDataPath = kdmpData.Data[backupPath]
if len(kdmpData.Data[excludeFileListKey]) != 0 {
excludeFileList, err = parseExcludeFileListKey(pvcStorageClass, kdmpData.Data[excludeFileListKey])
if err != nil {
msg := fmt.Sprintf("Failed in parsing the excludeFileList configmap parameter from configmap [%v/%v]", utils.KdmpConfigmapNamespace, utils.KdmpConfigmapName)
logrus.Errorf(msg)
data := updateDataExportDetail{
status: kdmpapi.DataExportStatusFailed,
reason: msg,
}
return false, c.updateStatus(dataExport, data)
}
}
}
blName := dataExport.Spec.Destination.Name
blNamespace := dataExport.Spec.Destination.Namespace
Expand Down Expand Up @@ -308,6 +334,7 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
driver,
srcPVCName,
compressionType,
excludeFileList,
dataExport,
podDataPath,
utils.KdmpConfigmapName,
Expand Down Expand Up @@ -503,6 +530,25 @@ func (c *Controller) sync(ctx context.Context, in *kdmpapi.DataExport) (bool, er
return false, nil
}

// If pvc storageclass and the configured storageclass matches, extract the configured ignore file list and return it.
func parseExcludeFileListKey(pvcStorageClass string, excludeFileListValue string) (string, error) {
storageClassList := strings.Split(excludeFileListValue, ",")
var excludeFileList string
for _, storageClass := range storageClassList {
colonSplit := strings.Split(storageClass, ":")
if len(colonSplit) != 2 {
return "", fmt.Errorf("invalid exclude file list in the configmap parameter. It should of format: \"storageClass1:dir1#dir2, storageClass2:dir1#file1\"")
}
// if the PVC storageclass and configure storageclass are same, extract the configured ignore file list
if pvcStorageClass == colonSplit[0] {
excludeFileList = colonSplit[1]
}

}
logrus.Infof("parseExcludeFileListKey: configured excludeFileList - %v", excludeFileList)
return excludeFileList, nil
}

func appendPodLogToStork(jobName string, namespace string) {
// Get job and check whether it has live pod attaced to it
job, err := batch.Instance().GetJob(jobName, namespace)
Expand Down Expand Up @@ -1752,6 +1798,7 @@ func startTransferJob(
drv drivers.Interface,
srcPVCName string,
compressionType string,
excludeFileList string,
dataExport *kdmpapi.DataExport,
podDataPath string,
jobConfigMap string,
Expand Down Expand Up @@ -1812,6 +1859,7 @@ func startTransferJob(
drivers.WithCertSecretName(utils.GetCertSecretName(dataExport.GetName())),
drivers.WithCertSecretNamespace(dataExport.Spec.Source.Namespace),
drivers.WithCompressionType(compressionType),
drivers.WithExcludeFileList(excludeFileList),
drivers.WithPodDatapathType(podDataPath),
drivers.WithJobConfigMap(jobConfigMap),
drivers.WithJobConfigMapNs(jobConfigMapNs),
Expand Down
7 changes: 7 additions & 0 deletions pkg/drivers/kopiabackup/kopiabackup.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,13 @@ func jobFor(
splitCmd = append(splitCmd, "--compression", jobOption.Compression)
cmd = strings.Join(splitCmd, " ")
}

if jobOption.ExcludeFileList != "" {
splitCmd := strings.Split(cmd, " ")
splitCmd = append(splitCmd, "--exclude-file-list", jobOption.ExcludeFileList)
cmd = strings.Join(splitCmd, " ")
}

kopiaExecutorImage, imageRegistrySecret, err := utils.GetExecutorImageAndSecret(drivers.KopiaExecutorImage,
jobOption.KopiaImageExecutorSource,
jobOption.KopiaImageExecutorSourceNs,
Expand Down
9 changes: 9 additions & 0 deletions pkg/drivers/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type JobOpts struct {
MaintenanceType string
RepoPVCName string
Compression string
ExcludeFileList string
PodDataPath string
// JobConfigMap holds any config needs to be provided to job
// from the caller. Eg: executor image name, secret, etc..
Expand Down Expand Up @@ -456,6 +457,14 @@ func WithCompressionType(compressionType string) JobOption {
}
}

// WithExcludeFileList is job parameter.
func WithExcludeFileList(excludeFileList string) JobOption {
return func(opts *JobOpts) error {
opts.ExcludeFileList = excludeFileList
return nil
}
}

// WithPodDatapathType is job parameter.
func WithPodDatapathType(podDataPath string) JobOption {
return func(opts *JobOpts) error {
Expand Down
4 changes: 2 additions & 2 deletions pkg/executor/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const (
secretAccessKeyPath = "/etc/cred-secret/secretAccessKey"
bucketPath = "/etc/cred-secret/path"
endpointPath = "/etc/cred-secret/endpoint"
sseTypePath = "/etc/cred-secret/sse"
sseTypePath = "/etc/cred-secret/sse"
passwordPath = "/etc/cred-secret/password"
regionPath = "/etc/cred-secret/region"
disableSslPath = "/etc/cred-secret/disablessl"
Expand Down Expand Up @@ -91,7 +91,7 @@ type S3Config struct {
// Region will be defaulted to us-east-1 if not provided
Region string
DisableSSL bool
SseType string
SseType string
}

// AzureConfig specifies the config required to connect to Azure Blob Storage
Expand Down
61 changes: 59 additions & 2 deletions pkg/executor/kopia/kopiabackup.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ const (
)

var (
bkpNamespace string
compression string
bkpNamespace string
compression string
excludeFileList string
)

var (
Expand Down Expand Up @@ -67,6 +68,7 @@ func newBackupCommand() *cobra.Command {
backupCommand.Flags().StringVar(&sourcePath, "source-path", "", "Source for kopia backup")
backupCommand.Flags().StringVar(&sourcePathGlob, "source-path-glob", "", "The regexp should match only one path that will be used for backup")
backupCommand.Flags().StringVar(&compression, "compression", "", "Compression type to be used")
backupCommand.Flags().StringVar(&excludeFileList, "exclude-file-list", "", " list of dir names that need to be exclude in the kopia snapshot")

return backupCommand
}
Expand Down Expand Up @@ -164,6 +166,15 @@ func runBackup(sourcePath string) error {
}
}

// if excludeFileList is not set in config map, it means no need to exclude any dir in the snapshot.
if excludeFileList != "" {
if err = runKopiaExcludeFileList(repo, sourcePath); err != nil {
errMsg := fmt.Sprintf("setting exclude file list failed for path %s: %v", sourcePath, err)
logrus.Errorf("%s: %v", fn, errMsg)
return fmt.Errorf(errMsg)
}
}

if err = runKopiaBackup(repo, sourcePath); err != nil {
errMsg := fmt.Sprintf("backup failed for repository %s: %v", repo.Name, err)
logrus.Errorf("%s: %v", fn, errMsg)
Expand Down Expand Up @@ -463,6 +474,52 @@ func setGlobalPolicy() error {
return nil
}

func runKopiaExcludeFileList(repository *executor.Repository, sourcePath string) error {
logrus.Infof("setting exclude file list for the snapshot")
excludeFileListCmd, err := kopia.GetExcludeFileListCommand(
sourcePath,
excludeFileList,
)
if err != nil {
return err
}
excludeFileListExecutor := kopia.NewExcludeFileListExecutor(excludeFileListCmd)
if err := excludeFileListExecutor.Run(); err != nil {
err = fmt.Errorf("failed to run exclude file list command: %v", err)
return err
}
t := func() (interface{}, bool, error) {
status, err := excludeFileListExecutor.Status()
if err != nil {
return "", true, err
}
if status.LastKnownError != nil {
if err = executor.WriteVolumeBackupStatus(
status,
volumeBackupName,
bkpNamespace,
); err != nil {
errMsg := fmt.Sprintf("failed to write a VolumeBackup status: %v", err)
logrus.Errorf("%v", errMsg)
return "", true, fmt.Errorf(errMsg)
}
return "", true, status.LastKnownError
}
if status.Done {
return "", false, nil
}

return "", true, fmt.Errorf("setting exclude file list for snapshot command status not available")
}
if _, err := task.DoRetryWithTimeout(t, executor.DefaultTimeout, progressCheckInterval); err != nil {
logrus.Errorf("failed setting snapshot exclude file list for path %v: %v", sourcePath, err)
return err
}

logrus.Infof("setting exclude file list is successfully")
return nil
}

func runKopiaCompression(repository *executor.Repository, sourcePath string) error {
logrus.Infof("Compression started")
compressionCmd, err := kopia.GetCompressionCommand(
Expand Down
32 changes: 32 additions & 0 deletions pkg/kopia/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package kopia
import (
"os"
"os/exec"
"strings"

cmdexec "github.com/portworx/kdmp/pkg/executor"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -46,6 +47,8 @@ type Command struct {
DisableSsl bool
// Compression to be used for backup
Compression string
// ExcludeFileList to be used for backup
ExcludeFileList string
// Region for S3 object
Region string
}
Expand Down Expand Up @@ -487,3 +490,32 @@ func (c *Command) CompressionCmd() *exec.Cmd {

return cmd
}

// ExcludeFileListCmd returns os/exec.Cmd object for the kopia policy set
func (c *Command) ExcludeFileListCmd() *exec.Cmd {
// Get all the flags
argsSlice := []string{
c.Name, // compression command
"set",
c.Path,
"--log-dir",
logDir,
"--config-file",
configFile,
}
hashSplit := strings.Split(c.ExcludeFileList, "#")
for _, file := range hashSplit {
argsSlice = append(argsSlice, "--add-ignore")
argsSlice = append(argsSlice, file)
}
argsSlice = append(argsSlice, c.Flags...)
// Get the cmd args
argsSlice = append(argsSlice, c.Args...)
cmd := exec.Command(baseCmd, argsSlice...)
if len(c.Env) > 0 {
cmd.Env = append(os.Environ(), c.Env...)
}
cmd.Dir = c.Dir
logrus.Infof("ExcludeFileListCmd: %+v", cmd)
return cmd
}
88 changes: 88 additions & 0 deletions pkg/kopia/excludefilelist.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package kopia

import (
"bytes"
"fmt"
"os"
"os/exec"

cmdexec "github.com/portworx/kdmp/pkg/executor"
"github.com/sirupsen/logrus"
)

type excludeFileListExecutor struct {
cmd *Command
execCmd *exec.Cmd
outBuf *bytes.Buffer
errBuf *bytes.Buffer
lastError error
isRunning bool
}

// GetExcludeFileListCommand returns a wrapper over the kopia policy set
func GetExcludeFileListCommand(path, excludeFileList string) (*Command, error) {
if path == "" {
return nil, fmt.Errorf("path name cannot be empty")
}
return &Command{
Name: "policy",
Path: path,
ExcludeFileList: excludeFileList,
}, nil
}

// NewExcludeFileListExecutor returns an instance of Executor that can be used for
// running a kopia policy set command for setting exclude dir list
func NewExcludeFileListExecutor(cmd *Command) Executor {
return &excludeFileListExecutor{
cmd: cmd,
outBuf: new(bytes.Buffer),
errBuf: new(bytes.Buffer),
}
}

func (c *excludeFileListExecutor) Run() error {
c.execCmd = c.cmd.ExcludeFileListCmd()
c.execCmd.Stdout = c.outBuf
c.execCmd.Stderr = c.errBuf

if err := c.execCmd.Start(); err != nil {
c.lastError = err
return err
}
c.isRunning = true
go func() {
err := c.execCmd.Wait()
if err != nil {
c.lastError = fmt.Errorf("failed to run the kopia exclude file list setting command: %v"+
" stdout: %v stderr: %v", err, c.outBuf.String(), c.errBuf.String())
logrus.Errorf("%v", c.lastError)
return
}
c.isRunning = false
}()

return nil
}

func (c *excludeFileListExecutor) Status() (*cmdexec.Status, error) {
if c.lastError != nil {
fmt.Fprintln(os.Stderr, c.errBuf.String())
return &cmdexec.Status{
LastKnownError: c.lastError,
Done: true,
}, nil
}

if c.isRunning {
return &cmdexec.Status{
Done: false,
LastKnownError: nil,
}, nil
}

return &cmdexec.Status{
Done: true,
}, nil

}

0 comments on commit 705ac31

Please sign in to comment.