Network: Return ACL logs from syslogs when the OVN controller is depl…

…oyed in MicroOVN (#14327)

closes #12836

We need to remember to enable the log tests for microovn on
canonical/lxd-ci#368 when this is merged

doc/.custom_wordlist.txt

@@ -143,6 +143,7 @@ MiB
 Mibit
 MicroCeph
 MicroCloud
+MicroOVN
 MII
 MinIO
 MITM

doc/howto/network_ovn_setup.md

@@ -162,6 +162,10 @@ See the linked YouTube video for the complete tutorial using four machines.
 
        lxc config set network.ovn.northbound_connection <ovn-northd-nb-db>
 
+   ```{note}
+   If you are using a MicroOVN deployment, pass the value of the MicroOVN node IP address you want to target. Prefix the IP address with `ssl:`, and suffix it with the `:6641` port number that corresponds to the OVN central service within MicroOVN.
+   ```
+
 1. Finally, create the actual OVN network (on the first machine):
 
        lxc network create my-ovn --type=ovn

lxd/network/acl/acl_interface.go

@@ -1,6 +1,8 @@
 package acl
 
 import (
+	"context"
+
 	"github.com/canonical/lxd/lxd/cluster/request"
 	"github.com/canonical/lxd/lxd/state"
 	"github.com/canonical/lxd/shared/api"
@@ -19,7 +21,7 @@ type NetworkACL interface {
 	UsedBy() ([]string, error)
 
 	// GetLog.
-	GetLog(clientType request.ClientType) (string, error)
+	GetLog(ctx context.Context, clientType request.ClientType) (string, error)
 
 	// Internal validation.
 	validateName(name string) error

lxd/network/acl/acl_ovn.go

@@ -1,10 +1,13 @@
 package acl
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net"
+	"strconv"
 	"strings"
 	"time"
 
@@ -1041,9 +1044,26 @@ type ovnLogEntry struct {
 	Action   string `json:"action"`
 }
 
-// ovnParseLogEntry takes a log line and expected ACL prefix and returns a re-formated log entry if matching.
-func ovnParseLogEntry(input string, prefix string) string {
-	fields := strings.Split(input, "|")
+// ovnParseLogEntry takes a log line (that comes from either an ovn controller log file or from the syslogs)
+// and expected ACL prefix and returns a re-formated log entry if matching.
+// The 'timestamp' string is in microseconds format. If empty, the timestamp is extracted from the log entry.
+func ovnParseLogEntry(logline string, syslogTimestamp string, prefix string) string {
+	parseLogTimeFromFields := func(fields []string) (time.Time, error) {
+		return time.Parse(time.RFC3339, fields[0])
+	}
+
+	parseLogTimeFromTimestamp := func(syslogTimestamp string) (time.Time, error) {
+		tsInt, err := strconv.ParseInt(syslogTimestamp, 10, 64)
+		if err != nil {
+			return time.Time{}, fmt.Errorf("Failed to parse timestamp: %w", err)
+		}
+
+		// The provided timestamp is in microseconds and need to be converted to nanoseconds.
+		tsNs := tsInt * 1000
+		return time.Unix(0, tsNs).UTC(), nil
+	}
+
+	fields := strings.Split(logline, "|")
 
 	// Skip unknown formatting.
 	if len(fields) != 5 {
@@ -1071,8 +1091,14 @@ func ovnParseLogEntry(input string, prefix string) string {
 		return ""
 	}
 
-	// Parse the timestamp.
-	logTime, err := time.Parse(time.RFC3339, fields[0])
+	var logTime time.Time
+	var err error
+	if syslogTimestamp == "" {
+		logTime, err = parseLogTimeFromFields(fields)
+	} else {
+		logTime, err = parseLogTimeFromTimestamp(syslogTimestamp)
+	}
+
 	if err != nil {
 		return ""
 	}
@@ -1131,3 +1157,57 @@ func ovnParseLogEntry(input string, prefix string) string {
 
 	return string(out)
 }
+
+// ovnParseLogEntriesFromJournald reads the OVN log entries from the systemd journal and returns them as a list of string entries.
+// Also, we chose to output the last 1000 entries to avoid overloading the system with too many log entries.
+func ovnParseLogEntriesFromJournald(ctx context.Context, systemdUnitName string, filter string) ([]string, error) {
+	var logEntries []string
+	cmd := []string{
+		"/usr/bin/journalctl",
+		"--unit", systemdUnitName,
+		"--directory", shared.HostPath("/var/log/journal"),
+		"--no-pager",
+		"--boot", "0",
+		"--case-sensitive",
+		"--grep", filter,
+		"--output-fields", "MESSAGE",
+		"-n", "1000",
+		"-o", "json",
+	}
+
+	stdout := bytes.Buffer{}
+	err := shared.RunCommandWithFds(ctx, nil, &stdout, cmd[0], cmd[1:]...)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to run journalctl to fetch OVN ACL logs: %w", err)
+	}
+
+	decoder := json.NewDecoder(&stdout)
+	for {
+		var sdLogEntry map[string]any
+		err = decoder.Decode(&sdLogEntry)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("Failed to parse log entry: %w", err)
+		}
+
+		message, ok := sdLogEntry["MESSAGE"].(string)
+		if !ok {
+			continue
+		}
+
+		timestamp, ok := sdLogEntry["__REALTIME_TIMESTAMP"].(string)
+		if !ok {
+			continue
+		}
+
+		logEntry := ovnParseLogEntry(message, timestamp, filter)
+		if logEntry == "" {
+			continue
+		}
+
+		logEntries = append(logEntries, logEntry)
+	}
+
+	return logEntries, nil
+}

lxd/network/acl/driver_common.go

@@ -749,35 +749,49 @@ func (d *common) Delete() error {
 }
 
 // GetLog gets the ACL log.
-func (d *common) GetLog(clientType request.ClientType) (string, error) {
+func (d *common) GetLog(ctx context.Context, clientType request.ClientType) (string, error) {
 	// ACLs aren't specific to a particular network type but the log only works with OVN.
-	logPath := shared.HostPath("/var/log/ovn/ovn-controller.log")
-	if !shared.PathExists(logPath) {
-		return "", fmt.Errorf("Only OVN log entries may be retrieved at this time")
-	}
+	var logEntries []string
+	var err error
 
-	// Open the log file.
-	logFile, err := os.Open(logPath)
-	if err != nil {
-		return "", fmt.Errorf("Couldn't open OVN log file: %w", err)
-	}
-
-	defer func() { _ = logFile.Close() }()
+	if shared.IsMicroOVNUsed() {
+		prefix := fmt.Sprintf("lxd_acl%d-", d.id)
+		logEntries, err = ovnParseLogEntriesFromJournald(ctx, "snap.microovn.chassis.service", prefix)
+		if err != nil {
+			return "", fmt.Errorf("Failed to get OVN log entries from syslog: %w", err)
+		}
+	} else {
+		// Else, if the current LXD deployment does not use MicroOVN,
+		// then try to read the OVN controller log file directly (a standalone OVN controller might be built-in with LXD).
+		logEntries = []string{}
+		prefix := fmt.Sprintf("lxd_acl%d-", d.id)
+		logPath := shared.HostPath("/var/log/ovn/ovn-controller.log")
+		if !shared.PathExists(logPath) {
+			return "", fmt.Errorf("Only OVN log entries may be retrieved at this time")
+		}
 
-	logEntries := []string{}
-	scanner := bufio.NewScanner(logFile)
-	for scanner.Scan() {
-		logEntry := ovnParseLogEntry(scanner.Text(), fmt.Sprintf("lxd_acl%d-", d.id))
-		if logEntry == "" {
-			continue
+		// Open the log file.
+		logFile, err := os.Open(logPath)
+		if err != nil {
+			return "", fmt.Errorf("Failed to open OVN log file: %w", err)
 		}
 
-		logEntries = append(logEntries, logEntry)
-	}
+		defer func() { _ = logFile.Close() }()
 
-	err = scanner.Err()
-	if err != nil {
-		return "", fmt.Errorf("Failed to read OVN log file: %w", err)
+		scanner := bufio.NewScanner(logFile)
+		for scanner.Scan() {
+			logEntry := ovnParseLogEntry(scanner.Text(), "", prefix)
+			if logEntry == "" {
+				continue
+			}
+
+			logEntries = append(logEntries, logEntry)
+		}
+
+		err = scanner.Err()
+		if err != nil {
+			return "", fmt.Errorf("Failed to read OVN log file: %w", err)
+		}
 	}
 
 	// Aggregates the entries from the rest of the cluster.

lxd/network_acls.go

@@ -637,7 +637,7 @@ func networkACLLogGet(d *Daemon, r *http.Request) response.Response {
 	}
 
 	clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent"))
-	log, err := netACL.GetLog(clientType)
+	log, err := netACL.GetLog(r.Context(), clientType)
 	if err != nil {
 		return response.SmartError(err)
 	}

shared/util.go

@@ -1514,3 +1514,15 @@ func ApplyDeviceOverrides(localDevices map[string]map[string]string, profileDevi
 
 	return localDevices, nil
 }
+
+// IsMicroOVNUsed returns whether the current LXD deployment is using a built-in openvswitch
+// or is connected to MicroOVN, which in this case, would make `/run/openvswitch` a symlink to
+// `/var/snap/lxd/common/microovn/chassis/switch`.
+func IsMicroOVNUsed() bool {
+	targetPath, err := os.Readlink("/run/openvswitch")
+	if err == nil && strings.HasSuffix(targetPath, "/microovn/chassis/switch") {
+		return true
+	}
+
+	return false
+}