fix: replace duplicate log files with symlinks

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/engine/launcher/args"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem"
 	"github.com/kurtosis-tech/stacktrace"
 	"github.com/sirupsen/logrus"
@@ -119,7 +120,7 @@ func CreateEngine(
 		// do a first removal
 		logRemover.Run()
 
-		logRemovalTicker := time.NewTicker(removeLogsWaitHours)
+		logRemovalTicker := time.NewTicker(volume_consts.RemoveLogsWaitHours)
 		for range logRemovalTicker.C {
 			logRemover.Run()
 		}

engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator/log_file_creator.go

@@ -0,0 +1,142 @@
+package log_file_creator
+
+import (
+	"context"
+	"fmt"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem"
+	"github.com/kurtosis-tech/stacktrace"
+	"github.com/sirupsen/logrus"
+	"os"
+	"strconv"
+)
+
+// LogFileCreator is responsible for creating the necessary file paths for service logs across all enclaves.
+// Context:
+// The LogsAggregator is configured to write logs to three different log file paths, one for uuid, service name, and shortened uuid.
+// This is so that the logs are retrievable by each identifier even when enclaves are stopped.
+// (More context on this here: https://github.com/kurtosis-tech/kurtosis/pull/1213)
+// To prevent storing duplicate logs, the LogFileCreator will ensure that the service name and short uuid log files are just
+// symlinks to the uuid log file path.
+type LogFileCreator struct {
+	kurtosisBackend backend_interface.KurtosisBackend
+
+	filesystem volume_filesystem.VolumeFilesystem
+
+	time logs_clock.LogsClock
+}
+
+func NewLogFileCreator(
+	kurtosisBackend backend_interface.KurtosisBackend,
+	filesystem volume_filesystem.VolumeFilesystem,
+	time logs_clock.LogsClock) *LogFileCreator {
+	return &LogFileCreator{
+		kurtosisBackend: kurtosisBackend,
+		filesystem:      filesystem,
+		time:            time,
+	}
+}
+
+// CreateLogFiles creates three log files for every service across all running enclaves.
+// The first is a file with the name ending in the uuid of the service.
+// The other two file paths are symlinks to the uuid file, ending with the shortened uuid and service name respectively.
+// If files exist for the shortened uuid and service name files, but they are not symlinks, they are removed and symlink files
+// are created to prevent duplicate log storage.
+func (creator *LogFileCreator) CreateLogFiles(ctx context.Context) error {
+	var err error
+
+	year, week := creator.time.Now().ISOWeek()
+
+	enclaveToServicesMap, err := creator.getEnclaveAndServiceInfo(ctx)
+	if err != nil {
+		// already wrapped with propagate
+		return err
+	}
+
+	for enclaveUuid, serviceRegistrations := range enclaveToServicesMap {
+		for _, serviceRegistration := range serviceRegistrations {
+			serviceUuidStr := string(serviceRegistration.GetUUID())
+			serviceNameStr := string(serviceRegistration.GetName())
+			serviceShortUuidStr := uuid_generator.ShortenedUUIDString(serviceUuidStr)
+
+			serviceUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceUuidStr)
+			if err = creator.createLogFileIdempotently(serviceUuidFilePathStr); err != nil {
+				return err
+			}
+
+			serviceNameFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceNameStr)
+			if err = creator.createSymlinkLogFile(serviceUuidFilePathStr, serviceNameFilePathStr); err != nil {
+				return err
+			}
+			logrus.Tracef("Created symlinked log file: '%v'", serviceNameFilePathStr)
+
+			serviceShortUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceShortUuidStr)
+			if err = creator.createSymlinkLogFile(serviceUuidFilePathStr, serviceShortUuidFilePathStr); err != nil {
+				return err
+			}
+			logrus.Tracef("Created symlinked log file: '%v'", serviceShortUuidFilePathStr)
+		}
+	}
+
+	return nil
+}
+
+func (creator *LogFileCreator) getEnclaveAndServiceInfo(ctx context.Context) (map[enclave.EnclaveUUID][]*service.ServiceRegistration, error) {
+	enclaveToServicesMap := map[enclave.EnclaveUUID][]*service.ServiceRegistration{}
+
+	enclaves, err := creator.kurtosisBackend.GetEnclaves(ctx, &enclave.EnclaveFilters{UUIDs: nil, Statuses: nil})
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred while trying to get all enclaves from kurtosis backend.")
+	}
+	for enclaveUuid := range enclaves {
+		var serviceRegistrations []*service.ServiceRegistration
+
+		enclaveServices, err := creator.kurtosisBackend.GetUserServices(ctx, enclaveUuid, &service.ServiceFilters{Names: nil, UUIDs: nil, Statuses: nil})
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred while trying to get user services for enclave '%v' from kurtosis backend.", enclaveUuid)
+		}
+		for _, serviceInfo := range enclaveServices {
+			serviceRegistrations = append(serviceRegistrations, serviceInfo.GetRegistration())
+		}
+
+		enclaveToServicesMap[enclaveUuid] = serviceRegistrations
+	}
+	return enclaveToServicesMap, nil
+}
+
+func (creator *LogFileCreator) createLogFileIdempotently(logFilePath string) error {
+	var err error
+	if _, err = creator.filesystem.Stat(logFilePath); os.IsNotExist(err) {
+		if _, err = creator.filesystem.Create(logFilePath); err != nil {
+			return stacktrace.Propagate(err, "An error occurred creating a log file path at '%v'", logFilePath)
+		}
+		logrus.Tracef("Created log file: '%v'", logFilePath)
+		return nil
+	}
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred checking if log file path at '%v' existed.", logFilePath)
+	}
+	return nil
+}
+
+func (creator *LogFileCreator) createSymlinkLogFile(targetLogFilePath, symlinkLogFilePath string) error {
+	// remove existing log files that could be storing logs at this path
+	if err := creator.filesystem.Remove(symlinkLogFilePath); err != nil {
+		return stacktrace.Propagate(err, "An error occurred attempting to remove an existing log file at the symlink file path '%v'.", symlinkLogFilePath)
+	}
+	// replace with symlink
+	if err := creator.filesystem.Symlink(targetLogFilePath, symlinkLogFilePath); err != nil {
+		return stacktrace.Propagate(err, "An error occurred creating a symlink file path '%v' for target file path '%v'.", targetLogFilePath, targetLogFilePath)
+	}
+	return nil
+}
+
+// creates a filepath of format /<filepath_base>/year/week/<enclave>/serviceIdentifier.<filetype>
+func getFilepathStr(year, week int, enclaveUuid, serviceIdentifier string) string {
+	return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceIdentifier, volume_consts.Filetype)
+}

engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover_test.go

@@ -6,9 +6,9 @@ import (
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem"
 	"github.com/stretchr/testify/require"
+	"os"
 	"strconv"
 	"testing"
-	"testing/fstest"
 )
 
 const (
@@ -19,35 +19,22 @@ const (
 )
 
 func TestLogRemover_Run(t *testing.T) {
+	mockFs := volume_filesystem.NewMockedVolumeFilesystem()
+
 	week49filepath := getWeekFilepathStr(2022, 49)
 	week50filepath := getWeekFilepathStr(2022, 50)
 	week51filepath := getWeekFilepathStr(2022, 51)
 	week52filepath := getWeekFilepathStr(2022, 52)
 	week1filepath := getWeekFilepathStr(2023, 1)
 	week2filepath := getWeekFilepathStr(2023, 2)
 
-	mapFs := &fstest.MapFS{
-		week49filepath: {
-			Data: []byte{},
-		},
-		week50filepath: {
-			Data: []byte{},
-		},
-		week51filepath: {
-			Data: []byte{},
-		},
-		week52filepath: {
-			Data: []byte{},
-		},
-		week1filepath: {
-			Data: []byte{},
-		},
-		week2filepath: {
-			Data: []byte{},
-		},
-	}
+	_, _ = mockFs.Create(week49filepath)
+	_, _ = mockFs.Create(week50filepath)
+	_, _ = mockFs.Create(week51filepath)
+	_, _ = mockFs.Create(week52filepath)
+	_, _ = mockFs.Create(week1filepath)
+	_, _ = mockFs.Create(week2filepath)
 
-	mockFs := volume_filesystem.NewMockedVolumeFilesystem(mapFs)
 	currentWeek := 2
 
 	mockTime := logs_clock.NewMockLogsClock(2023, currentWeek, defaultDay)
@@ -57,9 +44,10 @@ func TestLogRemover_Run(t *testing.T) {
 	logRemover.Run()
 
 	_, err := mockFs.Stat(week49filepath)
-	require.Error(t, err) // check the file doesn't exist
+	require.Error(t, err)
+	require.True(t, os.IsNotExist(err))
 }
 
 func getWeekFilepathStr(year, week int) string {
-	return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpathForTests, strconv.Itoa(year), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
+	return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
 }