Support H265 (#695)

Author: cnderrauber

examples/filesaver/main.go

@@ -21,12 +21,14 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync"
 	"syscall"
 
 	"github.com/pion/rtp/codecs"
 	"github.com/pion/webrtc/v4"
 	"github.com/pion/webrtc/v4/pkg/media"
 	"github.com/pion/webrtc/v4/pkg/media/h264writer"
+	"github.com/pion/webrtc/v4/pkg/media/h265writer"
 	"github.com/pion/webrtc/v4/pkg/media/ivfwriter"
 	"github.com/pion/webrtc/v4/pkg/media/oggwriter"
 
@@ -37,6 +39,8 @@ import (
 
 var (
 	host, apiKey, apiSecret, roomName, identity string
+
+	writeWG sync.WaitGroup
 )
 
 func init() {
@@ -74,6 +78,7 @@ func main() {
 
 	<-sigChan
 	room.Disconnect()
+	writeWG.Wait()
 }
 
 func onTrackSubscribed(track *webrtc.TrackRemote, publication *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) {
@@ -113,6 +118,12 @@ func NewTrackWriter(track *webrtc.TrackRemote, pliWriter lksdk.PLIWriter, fileNa
 		}))
 		writer, err = h264writer.New(fileName + ".h264")
 
+	case strings.EqualFold(track.Codec().MimeType, "video/h265"):
+		sb = samplebuilder.New(maxVideoLate, &codecs.H265Packet{}, track.Codec().ClockRate, samplebuilder.WithPacketDroppedHandler(func() {
+			pliWriter(track.SSRC())
+		}))
+		writer, err = h265writer.New(fileName + ".h265")
+
 	case strings.EqualFold(track.Codec().MimeType, "audio/opus"):
 		sb = samplebuilder.New(maxAudioLate, &codecs.OpusPacket{}, track.Codec().ClockRate)
 		writer, err = oggwriter.New(fileName+".ogg", 48000, track.Codec().Channels)
@@ -130,21 +141,29 @@ func NewTrackWriter(track *webrtc.TrackRemote, pliWriter lksdk.PLIWriter, fileNa
 		writer: writer,
 		track:  track,
 	}
+	writeWG.Add(1)
 	go t.start()
 	return t, nil
 }
 
 func (t *TrackWriter) start() {
-	defer t.writer.Close()
+	defer func() {
+		t.writer.Close()
+		writeWG.Done()
+	}()
+
 	for {
 		pkt, _, err := t.track.ReadRTP()
 		if err != nil {
+			logger.Errorw("failed to read rtp packet", err)
 			break
 		}
 		t.sb.Push(pkt)
 
 		for _, p := range t.sb.PopPackets() {
-			t.writer.WriteRTP(p)
+			if err := t.writer.WriteRTP(p); err != nil {
+				logger.Errorw("failed to write rtp packet", err)
+			}
 		}
 	}
 }

go.mod

@@ -15,9 +15,9 @@ require (
 	github.com/pion/dtls/v3 v3.0.6
 	github.com/pion/interceptor v0.1.40
 	github.com/pion/rtcp v1.2.15
-	github.com/pion/rtp v1.8.19
-	github.com/pion/sdp/v3 v3.0.13
-	github.com/pion/webrtc/v4 v4.1.2
+	github.com/pion/rtp v1.8.20
+	github.com/pion/sdp/v3 v3.0.14
+	github.com/pion/webrtc/v4 v4.1.3
 	github.com/stretchr/testify v1.10.0
 	github.com/twitchtv/twirp v8.1.3+incompatible
 	go.uber.org/atomic v1.11.0
@@ -58,11 +58,11 @@ require (
 	github.com/opencontainers/runc v1.1.14 // indirect
 	github.com/pion/datachannel v1.5.10 // indirect
 	github.com/pion/ice/v4 v4.0.10 // indirect
-	github.com/pion/logging v0.2.3 // indirect
+	github.com/pion/logging v0.2.4 // indirect
 	github.com/pion/mdns/v2 v2.0.7 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/sctp v1.8.39 // indirect
-	github.com/pion/srtp/v3 v3.0.5 // indirect
+	github.com/pion/srtp/v3 v3.0.6 // indirect
 	github.com/pion/stun/v3 v3.0.0 // indirect
 	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pion/turn/v4 v4.0.2 // indirect

go.sum

@@ -133,30 +133,30 @@ github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
 github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
 github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
 github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic=
-github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI=
-github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90=
+github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
+github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
 github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
 github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
 github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
-github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c=
-github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
+github.com/pion/rtp v1.8.20 h1:8zcyqohadZE8FCBeGdyEvHiclPIezcwRQH9zfapFyYI=
+github.com/pion/rtp v1.8.20/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk=
 github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
 github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
-github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4=
-github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
-github.com/pion/srtp/v3 v3.0.5 h1:8XLB6Dt3QXkMkRFpoqC3314BemkpMQK2mZeJc4pUKqo=
-github.com/pion/srtp/v3 v3.0.5/go.mod h1:r1G7y5r1scZRLe2QJI/is+/O83W2d+JoEsuIexpw+uM=
+github.com/pion/sdp/v3 v3.0.14 h1:1h7gBr9FhOWH5GjWWY5lcw/U85MtdcibTyt/o6RxRUI=
+github.com/pion/sdp/v3 v3.0.14/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E=
+github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4=
+github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY=
 github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
 github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
 github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
 github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps=
 github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs=
-github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54=
-github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U=
+github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c=
+github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

readersampleprovider.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pion/webrtc/v4"
 	"github.com/pion/webrtc/v4/pkg/media"
 	"github.com/pion/webrtc/v4/pkg/media/h264reader"
+	"github.com/pion/webrtc/v4/pkg/media/h265reader"
 	"github.com/pion/webrtc/v4/pkg/media/ivfreader"
 
 	"github.com/livekit/server-sdk-go/v2/pkg/oggreader"
@@ -55,6 +56,9 @@ type ReaderSampleProvider struct {
 	// for h264
 	h264reader *h264reader.H264Reader
 
+	// for h265
+	h265reader *h265reader.H265Reader
+
 	// for ogg
 	oggReader *oggreader.OggReader
 }
@@ -126,6 +130,8 @@ func NewLocalFileTrack(file string, options ...ReaderSampleProviderOption) (*Loc
 			return nil, ErrCannotDetermineMime
 		}
 		_, _ = fp.Seek(0, 0)
+	case ".h265":
+		mime = webrtc.MimeTypeH265
 	case ".ogg":
 		mime = webrtc.MimeTypeOpus
 	default:
@@ -156,7 +162,7 @@ func NewLocalReaderTrack(in io.ReadCloser, mime string, options ...ReaderSampleP
 
 	// check if mime type is supported
 	switch provider.Mime {
-	case webrtc.MimeTypeH264, webrtc.MimeTypeOpus, webrtc.MimeTypeVP8, webrtc.MimeTypeVP9:
+	case webrtc.MimeTypeH264, webrtc.MimeTypeH265, webrtc.MimeTypeOpus, webrtc.MimeTypeVP8, webrtc.MimeTypeVP9:
 	// allow
 	default:
 		return nil, ErrUnsupportedFileType
@@ -178,14 +184,16 @@ func NewLocalReaderTrack(in io.ReadCloser, mime string, options ...ReaderSampleP
 
 func (p *ReaderSampleProvider) OnBind() error {
 	// If we are not closing on unbind, don't do anything on rebind
-	if p.ivfReader != nil || p.h264reader != nil || p.oggReader != nil {
+	if p.ivfReader != nil || p.h264reader != nil || p.oggReader != nil || p.h265reader != nil {
 		return nil
 	}
 
 	var err error
 	switch p.Mime {
 	case webrtc.MimeTypeH264:
 		p.h264reader, err = h264reader.NewReader(p.reader)
+	case webrtc.MimeTypeH265:
+		p.h265reader, err = h265reader.NewReader(p.reader)
 	case webrtc.MimeTypeVP8, webrtc.MimeTypeVP9:
 		var ivfHeader *ivfreader.IVFFileHeader
 		p.ivfReader, ivfHeader, err = ivfreader.NewWith(p.reader)
@@ -244,6 +252,46 @@ func (p *ReaderSampleProvider) NextSample(ctx context.Context) (media.Sample, er
 			return sample, nil
 		}
 		sample.Duration = defaultH264FrameDuration
+	case webrtc.MimeTypeH265:
+		var (
+			isFrame    bool
+			needPrefix bool
+		)
+
+		for {
+			nal, err := p.h265reader.NextNAL()
+			if err != nil {
+				return sample, err
+			}
+
+			// aggregate vps,sps,pps into a single AP packet (chrome requires this)
+			if nal.NalUnitType == 32 || nal.NalUnitType == 33 || nal.NalUnitType == 34 {
+				sample.Data = append(sample.Data, []byte{0, 0, 0, 1}...) // add NAL prefix
+				sample.Data = append(sample.Data, nal.Data...)
+				needPrefix = true
+				continue
+			}
+
+			if needPrefix {
+				sample.Data = append(sample.Data, []byte{0, 0, 0, 1}...) // add NAL prefix
+				sample.Data = append(sample.Data, nal.Data...)
+			} else {
+				sample.Data = nal.Data
+			}
+
+			if !isFrame {
+				isFrame = nal.NalUnitType < 32
+			}
+
+			if !isFrame {
+				// return it without duration
+				return sample, nil
+			}
+
+			sample.Duration = defaultH264FrameDuration
+			break
+		}
+
 	case webrtc.MimeTypeVP8, webrtc.MimeTypeVP9:
 		frame, header, err := p.ivfReader.ParseNextFrame()
 		if err != nil {