update to pion v3

client/src/neko/base.ts

@@ -2,7 +2,7 @@ import EventEmitter from 'eventemitter3'
 import { OPCODE } from './data'
 import { EVENT, WebSocketEvents } from './events'
 
-import { WebSocketMessages, WebSocketPayloads, SignalProvidePayload } from './messages'
+import { WebSocketMessages, WebSocketPayloads, SignalProvidePayload, SignalCandidatePayload } from './messages'
 
 export interface BaseEvents {
   info: (...message: any[]) => void
@@ -211,8 +211,8 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
     }
 
     this._peer.ontrack = this.onTrack.bind(this)
-    this._peer.addTransceiver('audio', { direction: 'recvonly' })
-    this._peer.addTransceiver('video', { direction: 'recvonly' })
+    this._peer.addTransceiver('audio', { direction: 'sendrecv' })
+    this._peer.addTransceiver('video', { direction: 'sendrecv' })
 
     this._channel = this._peer.createDataChannel('data')
     this._channel.onerror = this.onError.bind(this)
@@ -246,6 +246,15 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
       this.createPeer(sdp, lite, ice)
       return
     }
+    if (event === EVENT.SIGNAL.CANDIDATE) {
+      const { data } = payload as SignalCandidatePayload
+      let candidate: RTCIceCandidate = JSON.parse(data)
+      this._peer!.addIceCandidate(candidate)
+
+      return
+    }
+
+
 
     // @ts-ignore
     if (typeof this[event] === 'function') {

client/src/neko/events.ts

@@ -14,6 +14,7 @@ export const EVENT = {
   SIGNAL: {
     ANSWER: 'signal/answer',
     PROVIDE: 'signal/provide',
+    CANDIDATE: 'signal/candidate'
   },
   MEMBER: {
     LIST: 'member/list',
@@ -78,7 +79,7 @@ export type ControlEvents =
 
 export type SystemEvents = typeof EVENT.SYSTEM.DISCONNECT
 export type MemberEvents = typeof EVENT.MEMBER.LIST | typeof EVENT.MEMBER.CONNECTED | typeof EVENT.MEMBER.DISCONNECTED
-export type SignalEvents = typeof EVENT.SIGNAL.ANSWER | typeof EVENT.SIGNAL.PROVIDE
+export type SignalEvents = typeof EVENT.SIGNAL.ANSWER | typeof EVENT.SIGNAL.PROVIDE | typeof EVENT.SIGNAL.CANDIDATE
 export type ChatEvents = typeof EVENT.CHAT.MESSAGE | typeof EVENT.CHAT.EMOTE
 export type ScreenEvents = typeof EVENT.SCREEN.CONFIGURATIONS | typeof EVENT.SCREEN.RESOLUTION | typeof EVENT.SCREEN.SET
 

client/src/neko/messages.ts

@@ -15,6 +15,7 @@ export type WebSocketMessages =
   | WebSocketMessage
   | SignalProvideMessage
   | SignalAnswerMessage
+  | SignalCandidateMessage
   | MemberListMessage
   | MemberConnectMessage
   | MemberDisconnectMessage
@@ -26,6 +27,7 @@ export type WebSocketMessages =
 export type WebSocketPayloads =
   | SignalProvidePayload
   | SignalAnswerPayload
+  | SignalCandidatePayload
   | MemberListPayload
   | Member
   | ControlPayload
@@ -78,6 +80,14 @@ export interface SignalAnswerPayload {
   displayname: string
 }
 
+// signal/candidate
+export interface SignalCandidateMessage extends WebSocketMessage, SignalCandidatePayload {
+  event: typeof EVENT.SIGNAL.CANDIDATE
+}
+export interface SignalCandidatePayload {
+  data: string
+}
+
 /*
   MEMBER MESSAGES/PAYLOADS
 */

server/internal/gst/gst.go

@@ -10,10 +10,9 @@ import "C"
 import (
 	"fmt"
 	"sync"
+	"time"
 	"unsafe"
 
-	"github.com/pion/webrtc/v2"
-
 	"n.eko.moe/neko/internal/types"
 )
 
@@ -80,13 +79,13 @@ func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineS
 }
 
 // CreateAppPipeline creates a GStreamer Pipeline
-func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string) (*Pipeline, error) {
+func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, bitrate string) (*Pipeline, error) {
 	pipelineStr := " ! appsink name=appsink"
 
 	var clockRate float32
 
 	switch codecName {
-	case webrtc.VP8:
+	case "VP8":
 		// https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c
 		// gstreamer1.0-plugins-good
 		// vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1
@@ -99,9 +98,9 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		if pipelineSrc != "" {
 			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
 		} else {
-			pipelineStr = fmt.Sprintf(videoSrc+"vp8enc cpu-used=8 threads=2 deadline=1 error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true"+pipelineStr, pipelineDevice)
+			pipelineStr = fmt.Sprintf(videoSrc+"vp8enc cpu-used=-5 threads=4 deadline=1 error-resilient=partitions keyframe-max-dist=30 auto-alt-ref=true"+pipelineStr, pipelineDevice)
 		}
-	case webrtc.VP9:
+	case "VP9":
 		// https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c
 		// gstreamer1.0-plugins-good
 		// vp9enc
@@ -117,7 +116,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		} else {
 			pipelineStr = fmt.Sprintf(videoSrc+"vp9enc"+pipelineStr, pipelineDevice)
 		}
-	case webrtc.H264:
+	case "H264":
 		// https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc
 		// gstreamer1.0-plugins-bad
 		// openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000
@@ -130,22 +129,29 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		if pipelineSrc != "" {
 			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
 		} else {
-      var h264Str string
-			h264Str = "openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream"
+			var h264Str string
+			 h264Str = "openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream"
+			if bitrate != "" {
+				h264Str = "openh264enc multi-thread=4 complexity=high bitrate=" + bitrate + "000 max-bitrate=" + bitrate + "999 ! video/x-h264,stream-format=byte-stream"
+			}
 
 			// https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c
 			// gstreamer1.0-plugins-ugly
 			// video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream
 			if err := CheckPlugins([]string{"openh264"}); err != nil {
-				h264Str = "video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"
+
+				h264Str = "video/x-raw,format=I420 ! x264enc threads=4 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"
+				if bitrate != "" {
+					h264Str = "video/x-raw,format=I420 ! x264enc threads=4 bitrate=" + bitrate + " byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"
+				}
 
 				if err := CheckPlugins([]string{"x264"}); err != nil {
 					return nil, err
 				}
 			}
       pipelineStr = fmt.Sprintf(videoSrc+h264Str+pipelineStr, pipelineDevice)
 		}
-	case webrtc.Opus:
+	case "Opus":
 		// https://gstreamer.freedesktop.org/documentation/opus/opusenc.html
 		// gstreamer1.0-plugins-base
 		// opusenc
@@ -160,7 +166,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		} else {
 			pipelineStr = fmt.Sprintf(audioSrc+"opusenc"+pipelineStr, pipelineDevice)
 		}
-	case webrtc.G722:
+	case "G722":
 		// https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html?gi-language=c
 		// gstreamer1.0-libav
 		// avenc_g722
@@ -175,7 +181,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		} else {
 			pipelineStr = fmt.Sprintf(audioSrc+"avenc_g722"+pipelineStr, pipelineDevice)
 		}
-	case webrtc.PCMU:
+	case "PCMU":
 		// https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html?gi-language=c
 		// gstreamer1.0-plugins-good
 		// audio/x-raw, rate=8000 ! mulawenc
@@ -190,7 +196,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 		} else {
 			pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! mulawenc"+pipelineStr, pipelineDevice)
 		}
-	case webrtc.PCMA:
+	case "PCMA":
 		// https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html?gi-language=c
 		// gstreamer1.0-plugins-good
 		// audio/x-raw, rate=8000 ! alawenc
@@ -270,8 +276,7 @@ func goHandlePipelineBuffer(buffer unsafe.Pointer, bufferLen C.int, duration C.i
 	pipelinesLock.Unlock()
 
 	if ok {
-		samples := uint32(pipeline.ClockRate * (float32(duration) / 1000000000))
-		pipeline.Sample <- types.Sample{Data: C.GoBytes(buffer, bufferLen), Samples: samples}
+		pipeline.Sample <- types.Sample{Data: C.GoBytes(buffer, bufferLen), Timestamp: time.Now(), Duration: time.Duration(duration)}
 	} else {
 		fmt.Printf("discarding buffer, no pipeline with id %d", int(pipelineID))
 	}

server/internal/remote/manager.go

@@ -135,6 +135,7 @@ func (manager *RemoteManager) createPipelines() {
 		manager.config.VideoCodec,
 		manager.config.Display,
 		manager.config.VideoParams,
+		manager.config.Bitrate,
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create video pipeline")
@@ -144,6 +145,7 @@ func (manager *RemoteManager) createPipelines() {
 		manager.config.AudioCodec,
 		manager.config.Device,
 		manager.config.AudioParams,
+		"",
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create audio pipeline")
@@ -174,6 +176,7 @@ func (manager *RemoteManager) ChangeResolution(width int, height int, rate int)
 		manager.config.VideoCodec,
 		manager.config.Display,
 		manager.config.VideoParams,
+		manager.config.Bitrate,
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create new video pipeline")

server/internal/session/session.go

@@ -120,6 +120,16 @@ func (session *Session) SignalAnswer(sdp string) error {
 	return session.peer.SignalAnswer(sdp)
 }
 
+func (session *Session) SignalCandidate(data string) error {
+	if session.socket == nil {
+		return nil
+	}
+	return session.socket.Send(&message.SignalCandidate{
+		Event: event.SIGNAL_CANDIDATE,
+		Data: data,
+	});
+}
+
 func (session *Session) destroy() error {
 	if session.socket != nil {
 		if err := session.socket.Destroy(); err != nil {

server/internal/types/config/remote.go

@@ -4,7 +4,6 @@ import (
 	"regexp"
 	"strconv"
 
-	"github.com/pion/webrtc/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -19,6 +18,7 @@ type Remote struct {
 	ScreenWidth  int
 	ScreenHeight int
 	ScreenRate   int
+	Bitrate      string
 }
 
 func (Remote) Init(cmd *cobra.Command) error {
@@ -47,6 +47,12 @@ func (Remote) Init(cmd *cobra.Command) error {
 		return err
 	}
 
+	cmd.PersistentFlags().String("bitrate", "", "set this video bitrate when possible")
+	if err := viper.BindPFlag("bitrate", cmd.PersistentFlags().Lookup("bitrate")); err != nil {
+		return err
+	}
+
+
 	// video codecs
 	cmd.PersistentFlags().Bool("vp8", false, "use VP8 video codec")
 	if err := viper.BindPFlag("vp8", cmd.PersistentFlags().Lookup("vp8")); err != nil {
@@ -88,24 +94,24 @@ func (Remote) Init(cmd *cobra.Command) error {
 }
 
 func (s *Remote) Set() {
-	videoCodec := webrtc.VP8
+	videoCodec := "VP8"
 	if viper.GetBool("vp8") {
-		videoCodec = webrtc.VP8
+		videoCodec = "VP8"
 	} else if viper.GetBool("vp9") {
-		videoCodec = webrtc.VP9
+		videoCodec = "VP9"
 	} else if viper.GetBool("h264") {
-		videoCodec = webrtc.H264
+		videoCodec = "H264"
 	}
 
-	audioCodec := webrtc.Opus
+	audioCodec := "Opus"
 	if viper.GetBool("opus") {
-		audioCodec = webrtc.Opus
+		audioCodec = "Opus"
 	} else if viper.GetBool("g722") {
-		audioCodec = webrtc.G722
+		audioCodec = "G722"
 	} else if viper.GetBool("pcmu") {
-		audioCodec = webrtc.PCMU
+		audioCodec = "PCMU"
 	} else if viper.GetBool("pcma") {
-		audioCodec = webrtc.PCMA
+		audioCodec = "PCMA"
 	}
 
 	s.Device = viper.GetString("device")
@@ -114,6 +120,7 @@ func (s *Remote) Set() {
 	s.Display = viper.GetString("display")
 	s.VideoCodec = videoCodec
 	s.VideoParams = viper.GetString("video")
+	s.Bitrate = viper.GetString("bitrate")
 
 	s.ScreenWidth = 1280
 	s.ScreenHeight = 720

server/internal/types/event/events.go

@@ -5,8 +5,10 @@ const (
 )
 
 const (
-	SIGNAL_ANSWER  = "signal/answer"
-	SIGNAL_PROVIDE = "signal/provide"
+	SIGNAL_ANSWER    = "signal/answer"
+	SIGNAL_OFFER     = "signal/offer"
+	SIGNAL_PROVIDE   = "signal/provide"
+	SIGNAL_CANDIDATE = "signal/candidate"
 )
 
 const (

server/internal/types/message/messages.go

@@ -27,6 +27,11 @@ type SignalAnswer struct {
 	SDP         string `json:"sdp"`
 }
 
+type SignalCandidate struct {
+	Event   string `json:"event"`
+	Data    string `json:"data"`
+}
+
 type MembersList struct {
 	Event    string          `json:"event"`
 	Memebers []*types.Member `json:"members"`

server/internal/types/session.go

@@ -24,6 +24,7 @@ type Session interface {
 	Write(v interface{}) error
 	Send(v interface{}) error
 	SignalAnswer(sdp string) error
+	SignalCandidate(data string) error
 }
 
 type SessionManager interface {

server/internal/types/webrtc.go

@@ -1,8 +1,13 @@
 package types
 
+import (
+	"time"
+)
+
 type Sample struct {
-	Data    []byte
-	Samples uint32
+	Data       []byte
+	Timestamp  time.Time
+	Duration   time.Duration
 }
 
 type WebRTCManager interface {

server/internal/webrtc/handle.go

@@ -5,7 +5,7 @@ import (
 	"encoding/binary"
 	"strconv"
 
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 const OP_MOVE = 0x01

server/internal/webrtc/peer.go

@@ -3,7 +3,7 @@ package webrtc
 import (
 	"sync"
 
-	"github.com/pion/webrtc/v2"
+	"github.com/pion/webrtc/v3"
 )
 
 type Peer struct {