Add support for adding metadata/attachments (#614)

Adds support for adding and getting metadata and attachments from an
existing mcap file. To add an attachment to a file,

    mcap add attachment demo.mcap -f Makefile

To get an attachment out of the file,

    mcap get attachment demo.mcap -n Makefile -o attachment.txt

To add metadata to a file,

    mcap add metadata demo.mcap -k foo=bar -k bar=baz -n "my metadata"

To get metadata out of the file,

    mcap get metadata demo.mcap -n "my metadata"
    {
      "bar": "baz",
      "foo": "bar"
    }

Also fixes a bug in the list attachments subcommand. Previously this was
listing chunks.

go/cli/mcap/cmd/add.go

@@ -0,0 +1,14 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var addCmd = &cobra.Command{
+	Use:   "add",
+	Short: "add records to an existing mcap file",
+	Run: func(cmd *cobra.Command, args []string) {
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(addCmd)
+}

go/cli/mcap/cmd/attachment.go

@@ -0,0 +1,190 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/foxglove/mcap/go/cli/mcap/utils"
+	"github.com/foxglove/mcap/go/mcap"
+	"github.com/spf13/cobra"
+)
+
+var (
+	addAttachmentLogTime      uint64
+	addAttachmentCreationTime uint64
+	addAttachmentFilename     string
+	addAttachmentMediaType    string
+)
+
+var (
+	getAttachmentName   string
+	getAttachmentOffset uint64
+	getAttachmentOutput string
+)
+
+func getAttachment(w io.Writer, rs io.ReadSeeker, idx *mcap.AttachmentIndex) error {
+	_, err := rs.Seek(int64(
+		idx.Offset+
+			1+ // opcode
+			8+ // record length
+			8+ // log time
+			8+ // creation time
+			4+ // name length
+			uint64(len(idx.Name))+
+			4+ // content type length
+			uint64(len(idx.MediaType))+
+			8), // data length
+		io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("failed to seek to offset %d: %w", idx.Offset, err)
+	}
+	_, err = io.CopyN(w, rs, int64(idx.DataSize))
+	if err != nil {
+		return fmt.Errorf("failed to copy attachment to output: %w", err)
+	}
+	return nil
+}
+
+var getAttachmentCmd = &cobra.Command{
+	Use:   "attachment",
+	Short: "Get an attachment by name or offset",
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.Background()
+		if len(args) != 1 {
+			die("Unexpected number of args")
+		}
+		filename := args[0]
+
+		var output io.Writer
+		var err error
+		if getAttachmentOutput == "" {
+			if !utils.StdoutRedirected() {
+				die(PleaseRedirect)
+			}
+			output = os.Stdout
+		} else {
+			output, err = os.Create(getAttachmentOutput)
+			if err != nil {
+				die("failed to create output file: %s", err)
+			}
+		}
+
+		err = utils.WithReader(ctx, filename, func(_ bool, rs io.ReadSeeker) error {
+			reader, err := mcap.NewReader(rs)
+			if err != nil {
+				return fmt.Errorf("failed to construct reader: %w", err)
+			}
+			info, err := reader.Info()
+			if err != nil {
+				return fmt.Errorf("failed to get mcap info: %w", err)
+			}
+			attachments := make(map[string][]*mcap.AttachmentIndex)
+			for _, attachmentIdx := range info.AttachmentIndexes {
+				attachments[attachmentIdx.Name] = append(
+					attachments[attachmentIdx.Name],
+					attachmentIdx,
+				)
+			}
+
+			switch {
+			case len(attachments[getAttachmentName]) == 0:
+				die("attachment %s not found", getAttachmentName)
+			case len(attachments[getAttachmentName]) == 1:
+				getAttachment(output, rs, attachments[getAttachmentName][0])
+			case len(attachments[getAttachmentName]) > 1:
+				if getAttachmentOffset == 0 {
+					return fmt.Errorf(
+						"multiple attachments named %s exist. Specify an offset.",
+						getAttachmentName,
+					)
+				}
+				for _, idx := range attachments[getAttachmentName] {
+					if idx.Offset == getAttachmentOffset {
+						return getAttachment(output, rs, idx)
+					}
+				}
+				return fmt.Errorf(
+					"failed to find attachment %s at offset %d",
+					getAttachmentName,
+					getAttachmentOffset,
+				)
+			}
+			return nil
+		})
+		if err != nil {
+			die("failed to extract attachment: %s", err)
+		}
+	},
+}
+
+var addAttachmentCmd = &cobra.Command{
+	Use:   "attachment",
+	Short: "Add an attachment to an mcap file",
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.Background()
+		if len(args) != 1 {
+			die("Unexpected number of args")
+		}
+		filename := args[0]
+		tempName := filename + ".new"
+		tmpfile, err := os.Create(tempName)
+		if err != nil {
+			die("failed to create temp file: %s", err)
+		}
+		attachment, err := os.ReadFile(addAttachmentFilename)
+		if err != nil {
+			die("failed to read attachment: %s", err)
+		}
+		err = utils.WithReader(ctx, filename, func(remote bool, rs io.ReadSeeker) error {
+			if remote {
+				die("not supported on remote mcap files")
+			}
+			fi, err := os.Stat(addAttachmentFilename)
+			if err != nil {
+				die("failed to stat file %s", addAttachmentFilename)
+			}
+			createTime := uint64(fi.ModTime().UTC().UnixNano())
+			if addAttachmentCreationTime > 0 {
+				createTime = addAttachmentCreationTime
+			}
+			logTime := uint64(time.Now().UTC().UnixNano())
+			if addAttachmentLogTime > 0 {
+				logTime = addAttachmentLogTime
+			}
+			return utils.RewriteMCAP(tmpfile, rs, func(w *mcap.Writer) error {
+				return w.WriteAttachment(&mcap.Attachment{
+					LogTime:    logTime,
+					CreateTime: createTime,
+					Name:       addAttachmentFilename,
+					MediaType:  addAttachmentMediaType,
+					Data:       attachment,
+				})
+			})
+		})
+		if err != nil {
+			die("failed to add attachment: %s", err)
+		}
+		err = os.Rename(tempName, filename)
+		if err != nil {
+			die("failed to rename temporary output: %s", err)
+		}
+	},
+}
+
+func init() {
+	addCmd.AddCommand(addAttachmentCmd)
+	addAttachmentCmd.PersistentFlags().StringVarP(&addAttachmentFilename, "file", "f", "", "filename of attachment to add")
+	addAttachmentCmd.PersistentFlags().StringVarP(&addAttachmentMediaType, "content-type", "", "application/octet-stream", "content type of attachment")
+	addAttachmentCmd.PersistentFlags().Uint64VarP(&addAttachmentLogTime, "log-time", "", 0, "attachment log time in nanoseconds (defaults to current timestamp)")
+	addAttachmentCmd.PersistentFlags().Uint64VarP(&addAttachmentLogTime, "creation-time", "", 0, "attachment creation time in nanoseconds (defaults to ctime)")
+	addAttachmentCmd.MarkPersistentFlagRequired("file")
+
+	getCmd.AddCommand(getAttachmentCmd)
+	getAttachmentCmd.PersistentFlags().StringVarP(&getAttachmentName, "name", "n", "", "name of attachment to extract")
+	getAttachmentCmd.PersistentFlags().Uint64VarP(&getAttachmentOffset, "offset", "", 0, "offset of attachment to extract")
+	getAttachmentCmd.PersistentFlags().StringVarP(&getAttachmentOutput, "output", "o", "", "location to write attachment to")
+	getAttachmentCmd.MarkPersistentFlagRequired("name")
+}

go/cli/mcap/cmd/attachments.go

@@ -14,17 +14,19 @@ import (
 func printAttachments(w io.Writer, attachmentIndexes []*mcap.AttachmentIndex) {
 	rows := make([][]string, 0, len(attachmentIndexes))
 	rows = append(rows, []string{
-		"log time",
 		"name",
 		"media type",
+		"log time",
+		"creation time",
 		"content length",
 		"offset",
 	})
 	for _, idx := range attachmentIndexes {
 		row := []string{
-			fmt.Sprintf("%d", idx.LogTime),
 			idx.Name,
 			idx.MediaType,
+			fmt.Sprintf("%d", idx.LogTime),
+			fmt.Sprintf("%d", idx.CreateTime),
 			fmt.Sprintf("%d", idx.DataSize),
 			fmt.Sprintf("%d", idx.Offset),
 		}
@@ -51,7 +53,7 @@ var attachmentsCmd = &cobra.Command{
 			if err != nil {
 				return fmt.Errorf("failed to get info: %w", err)
 			}
-			printChunks(os.Stdout, info.ChunkIndexes)
+			printAttachments(os.Stdout, info.AttachmentIndexes)
 			return nil
 		})
 		if err != nil {

go/cli/mcap/cmd/filter.go

@@ -122,7 +122,7 @@ usage:
 		var writer io.Writer
 		if filterOutput == "" {
 			if !utils.StdoutRedirected() {
-				die("Binary output can screw up your terminal. Supply -o or redirect to a file or pipe")
+				die(PleaseRedirect)
 			}
 			writer = os.Stdout
 		} else {

go/cli/mcap/cmd/get.go

@@ -0,0 +1,14 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var getCmd = &cobra.Command{
+	Use:   "get",
+	Short: "get a record from an mcap file",
+	Run: func(cmd *cobra.Command, args []string) {
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(getCmd)
+}

go/cli/mcap/cmd/merge.go

@@ -240,7 +240,7 @@ var mergeCmd = &cobra.Command{
 	Short: "Merge a selection of mcap files by record timestamp",
 	Run: func(cmd *cobra.Command, args []string) {
 		if mergeOutputFile == "" && !utils.StdoutRedirected() {
-			die("Binary output can screw up your terminal. Supply -o or redirect to a file or pipe.")
+			die(PleaseRedirect)
 		}
 		var readers []io.Reader
 		for _, arg := range args {