Skip to content

Commit

Permalink
Add ivfwriter support for VP9
Browse files Browse the repository at this point in the history
Adds the necessary wiring to get VP9 to work with `ivfwriter` as
well as a save-to-disk example.

It currently assumes 30 fps but it seems that the other codecs
also assume 30 fps so that is not a net-new assumption.
  • Loading branch information
kevmo314 committed Oct 24, 2024
1 parent 271ab55 commit fb8c6ac
Show file tree
Hide file tree
Showing 3 changed files with 296 additions and 9 deletions.
36 changes: 36 additions & 0 deletions examples/save-to-disk-vp9/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# save-to-disk-vp9
save-to-disk-vp9 is a simple application that shows how to save a video to disk using VP9.

If you wish to save VP8 and Opus instead of VP9 see [save-to-disk](https://github.com/pion/webrtc/tree/master/examples/save-to-disk)

If you wish to save VP8/Opus inside the same file see [save-to-webm](https://github.com/pion/example-webrtc-applications/tree/master/save-to-webm)

You can then send this video back to your browser using [play-from-disk](https://github.com/pion/example-webrtc-applications/tree/master/play-from-disk)

## Instructions
### Download save-to-disk-vp9
```
go install github.com/pion/webrtc/v4/examples/save-to-disk-vp9@latest
```

### Open save-to-disk-vp9 example page
[jsfiddle.net](https://jsfiddle.net/xjcve6d3/) you should see your Webcam, two text-areas and two buttons: `Copy browser SDP to clipboard`, `Start Session`.

### Run save-to-disk-vp9, with your browsers SessionDescription as stdin
In the jsfiddle the top textarea is your browser's Session Description. Press `Copy browser SDP to clipboard` or copy the base64 string manually.
We will use this value in the next step.

#### Linux/macOS
Run `echo $BROWSER_SDP | save-to-disk-vp9`
#### Windows
1. Paste the SessionDescription into a file.
1. Run `save-to-disk-vp9 < my_file`

### Input save-to-disk-vp9's SessionDescription into your browser
Copy the text that `save-to-disk-vp9` just emitted and copy into second text area

### Hit 'Start Session' in jsfiddle, wait, close jsfiddle, enjoy your video!
In the folder you ran `save-to-disk-vp9` you should now have a file `output.ivf` play with your video player of choice!
> Note: In order to correctly create the files, the remote client (JSFiddle) should be closed. The Go example will automatically close itself.
Congrats, you have used Pion WebRTC! Now start building something cool
215 changes: 215 additions & 0 deletions examples/save-to-disk-vp9/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
// SPDX-FileCopyrightText: 2023 The Pion community <https://pion.ly>
// SPDX-License-Identifier: MIT

//go:build !js
// +build !js

// save-to-disk-av1 is a simple application that shows how to save a video to disk using VP9.
package main

import (
"bufio"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"

"github.com/pion/interceptor"
"github.com/pion/interceptor/pkg/intervalpli"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
"github.com/pion/webrtc/v4/pkg/media/ivfwriter"
)

func saveToDisk(i media.Writer, track *webrtc.TrackRemote) {
defer func() {
if err := i.Close(); err != nil {
panic(err)
}
}()

for {
rtpPacket, _, err := track.ReadRTP()
if err != nil {
fmt.Println(err)
return
}
if err := i.WriteRTP(rtpPacket); err != nil {
fmt.Println(err)
return
}
}
}

func main() {
// Everything below is the Pion WebRTC API! Thanks for using it ❤️.

// Create a MediaEngine object to configure the supported codec
m := &webrtc.MediaEngine{}

// Setup the codecs you want to use.
// We'll use a VP9 and Opus but you can also define your own
if err := m.RegisterCodec(webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: nil},
PayloadType: 96,
}, webrtc.RTPCodecTypeVideo); err != nil {
panic(err)
}

// Create a InterceptorRegistry. This is the user configurable RTP/RTCP Pipeline.
// This provides NACKs, RTCP Reports and other features. If you use `webrtc.NewPeerConnection`
// this is enabled by default. If you are manually managing You MUST create a InterceptorRegistry
// for each PeerConnection.
i := &interceptor.Registry{}

// Register a intervalpli factory
// This interceptor sends a PLI every 3 seconds. A PLI causes a video keyframe to be generated by the sender.
// This makes our video seekable and more error resilent, but at a cost of lower picture quality and higher bitrates
// A real world application should process incoming RTCP packets from viewers and forward them to senders
intervalPliFactory, err := intervalpli.NewReceiverInterceptor()
if err != nil {
panic(err)
}
i.Add(intervalPliFactory)

// Use the default set of Interceptors
if err = webrtc.RegisterDefaultInterceptors(m, i); err != nil {
panic(err)
}

// Create the API object with the MediaEngine
api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithInterceptorRegistry(i))

// Prepare the configuration
config := webrtc.Configuration{}

// Create a new RTCPeerConnection
peerConnection, err := api.NewPeerConnection(config)
if err != nil {
panic(err)
}

// Allow us to receive 1 video track
if _, err = peerConnection.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo); err != nil {
panic(err)
}

ivfFile, err := ivfwriter.New("output.ivf", ivfwriter.WithCodec(webrtc.MimeTypeVP9))
if err != nil {
panic(err)
}

// Set a handler for when a new remote track starts, this handler saves buffers to disk as
// an ivf file, since we could have multiple video tracks we provide a counter.
// In your application this is where you would handle/process video
peerConnection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { //nolint: revive
if strings.EqualFold(track.Codec().MimeType, webrtc.MimeTypeVP9) {
fmt.Println("Got VP9 track, saving to disk as output.ivf")
saveToDisk(ivfFile, track)
}
})

// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
fmt.Printf("Connection State has changed %s \n", connectionState.String())

if connectionState == webrtc.ICEConnectionStateConnected {
fmt.Println("Ctrl+C the remote client to stop the demo")
} else if connectionState == webrtc.ICEConnectionStateFailed || connectionState == webrtc.ICEConnectionStateClosed {
if closeErr := ivfFile.Close(); closeErr != nil {
panic(closeErr)
}

fmt.Println("Done writing media files")

// Gracefully shutdown the peer connection
if closeErr := peerConnection.Close(); closeErr != nil {
panic(closeErr)
}

os.Exit(0)
}
})

// Wait for the offer to be pasted
offer := webrtc.SessionDescription{}
decode(readUntilNewline(), &offer)

// Set the remote SessionDescription
err = peerConnection.SetRemoteDescription(offer)
if err != nil {
panic(err)
}

// Create answer
answer, err := peerConnection.CreateAnswer(nil)
if err != nil {
panic(err)
}

// Create channel that is blocked until ICE Gathering is complete
gatherComplete := webrtc.GatheringCompletePromise(peerConnection)

// Sets the LocalDescription, and starts our UDP listeners
err = peerConnection.SetLocalDescription(answer)
if err != nil {
panic(err)
}

// Block until ICE Gathering is complete, disabling trickle ICE
// we do this because we only can exchange one signaling message
// in a production application you should exchange ICE Candidates via OnICECandidate
<-gatherComplete

// Output the answer in base64 so we can paste it in browser
fmt.Println(encode(peerConnection.LocalDescription()))

// Block forever
select {}
}

// Read from stdin until we get a newline
func readUntilNewline() (in string) {
var err error

r := bufio.NewReader(os.Stdin)
for {
in, err = r.ReadString('\n')
if err != nil && !errors.Is(err, io.EOF) {
panic(err)
}

if in = strings.TrimSpace(in); len(in) > 0 {
break
}
}

fmt.Println("")
return
}

// JSON encode + base64 a SessionDescription
func encode(obj *webrtc.SessionDescription) string {
b, err := json.Marshal(obj)
if err != nil {
panic(err)
}

return base64.StdEncoding.EncodeToString(b)
}

// Decode a base64 and unmarshal JSON into a SessionDescription
func decode(in string, obj *webrtc.SessionDescription) {
b, err := base64.StdEncoding.DecodeString(in)
if err != nil {
panic(err)
}

if err = json.Unmarshal(b, obj); err != nil {
panic(err)
}
}
54 changes: 45 additions & 9 deletions pkg/media/ivfwriter/ivfwriter.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ var (

const (
mimeTypeVP8 = "video/VP8"
mimeTypeVP9 = "video/VP9"
mimeTypeAV1 = "video/AV1"

ivfFileHeaderSignature = "DKIF"
Expand All @@ -35,9 +36,9 @@ type IVFWriter struct {
count uint64
seenKeyFrame bool

isVP8, isAV1 bool
isVP8, isVP9, isAV1 bool

// VP8
// VP8, VP9
currentFrame []byte

// AV1
Expand Down Expand Up @@ -75,7 +76,7 @@ func NewWith(out io.Writer, opts ...Option) (*IVFWriter, error) {
}
}

if !writer.isAV1 && !writer.isVP8 {
if !writer.isAV1 && !writer.isVP8 && !writer.isVP9 {
writer.isVP8 = true
}

Expand All @@ -92,9 +93,12 @@ func (i *IVFWriter) writeHeader() error {
binary.LittleEndian.PutUint16(header[6:], 32) // Header size

// FOURCC
if i.isVP8 {
switch {
case i.isVP8:
copy(header[8:], "VP80")
} else if i.isAV1 {
case i.isVP9:
copy(header[8:], "VP90")

Check warning on line 100 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L99-L100

Added lines #L99 - L100 were not covered by tests
case i.isAV1:
copy(header[8:], "AV01")
}

Expand Down Expand Up @@ -123,14 +127,15 @@ func (i *IVFWriter) writeFrame(frame []byte, timestamp uint64) error {
}

// WriteRTP adds a new packet and writes the appropriate headers for it
func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error {
func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error { //nolint:gocognit
if i.ioWriter == nil {
return errFileNotOpened
} else if len(packet.Payload) == 0 {
return nil
}

if i.isVP8 {
switch {
case i.isVP8:
vp8Packet := codecs.VP8Packet{}
if _, err := vp8Packet.Unmarshal(packet.Payload); err != nil {
return err
Expand All @@ -157,7 +162,36 @@ func (i *IVFWriter) WriteRTP(packet *rtp.Packet) error {
return err
}
i.currentFrame = nil
} else if i.isAV1 {
case i.isVP9:
vp9Packet := codecs.VP9Packet{}
if _, err := vp9Packet.Unmarshal(packet.Payload); err != nil {
return err
}

Check warning on line 169 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L165-L169

Added lines #L165 - L169 were not covered by tests

switch {
case !i.seenKeyFrame && vp9Packet.P:
return nil
case i.currentFrame == nil && !vp9Packet.B:
return nil

Check warning on line 175 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L171-L175

Added lines #L171 - L175 were not covered by tests
}

i.seenKeyFrame = true
i.currentFrame = append(i.currentFrame, vp9Packet.Payload[0:]...)

if !packet.Marker {
return nil
} else if len(i.currentFrame) == 0 {
return nil
}

Check warning on line 185 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L178-L185

Added lines #L178 - L185 were not covered by tests

// the timestamp must be sequential. webrtc mandates a clock rate of 90000
// and we've assumed 30fps in the header.
// TODO: can we not assume 30fps?
if err := i.writeFrame(i.currentFrame, uint64(packet.Header.Timestamp)/3000); err != nil {
return err
}
i.currentFrame = nil

Check warning on line 193 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L190-L193

Added lines #L190 - L193 were not covered by tests
case i.isAV1:
av1Packet := &codecs.AV1Packet{}
if _, err := av1Packet.Unmarshal(packet.Payload); err != nil {
return err
Expand Down Expand Up @@ -215,13 +249,15 @@ type Option func(i *IVFWriter) error
// WithCodec configures if IVFWriter is writing AV1 or VP8 packets to disk
func WithCodec(mimeType string) Option {
return func(i *IVFWriter) error {
if i.isVP8 || i.isAV1 {
if i.isVP8 || i.isVP9 || i.isAV1 {
return errCodecAlreadySet
}

switch mimeType {
case mimeTypeVP8:
i.isVP8 = true
case mimeTypeVP9:
i.isVP9 = true

Check warning on line 260 in pkg/media/ivfwriter/ivfwriter.go

View check run for this annotation

Codecov / codecov/patch

pkg/media/ivfwriter/ivfwriter.go#L259-L260

Added lines #L259 - L260 were not covered by tests
case mimeTypeAV1:
i.isAV1 = true
default:
Expand Down

0 comments on commit fb8c6ac

Please sign in to comment.