Skip to content

Commit

Permalink
support decoding M-JPEG streams with any quantization (#566) (#583)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aler9
aler9 authored Jun 22, 2024
1 parent 3ee5fbd commit 4750375
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 19 deletions.
76 changes: 68 additions & 8 deletions pkg/format/rtpmjpeg/decoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,65 @@ var chmAcSymbols = []byte{ //nolint:dupl
0xf9, 0xfa,
}

var lumaQuantizers = []int{
0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
}

var chromaQuantizers = []int{
0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
}

func makeQuantizationTables(q uint8) [][]byte {
var scale int
if q < 50 {
scale = 5000 / int(q)
} else {
scale = 200 - 2*int(q)
}

tables := make([][]byte, 2)

tables[0] = make([]byte, 64)

for i := 0; i < 64; i++ {
v := (lumaQuantizers[i]*scale + 50) / 100
if v > 255 {
v = 255
} else if v == 0 {
v = 1
}
tables[0][i] = byte(v)
}

tables[1] = make([]byte, 64)

for i := 0; i < 64; i++ {
v := (chromaQuantizers[i]*scale + 50) / 100
if v > 255 {
v = 255
} else if v == 0 {
v = 1
}
tables[1][i] = byte(v)
}

return tables
}

func joinFragments(fragments [][]byte, size int) []byte {
ret := make([]byte, size)
n := 0
Expand All @@ -107,7 +166,7 @@ type Decoder struct {
fragmentsSize int
fragments [][]byte
firstJpegHeader *headerJPEG
firstQTHeader *headerQuantizationTable
quantizationTables [][]byte
}

// Init initializes the decoder.
Expand All @@ -127,11 +186,11 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {
byts = byts[n:]

if jh.Width > maxDimension {
return nil, fmt.Errorf("Width of %d is not supported", jh.Width)
return nil, fmt.Errorf("width of %d is not supported", jh.Width)
}

if jh.Height > maxDimension {
return nil, fmt.Errorf("Height of %d is not supported", jh.Height)
return nil, fmt.Errorf("height of %d is not supported", jh.Height)
}

if jh.FragmentOffset == 0 {
Expand All @@ -140,14 +199,15 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {
d.firstPacketReceived = true

if jh.Quantization >= 128 {
d.firstQTHeader = &headerQuantizationTable{}
n, err := d.firstQTHeader.unmarshal(byts)
var hqt headerQuantizationTable
n, err := hqt.unmarshal(byts)
if err != nil {
return nil, err
}
d.quantizationTables = hqt.Tables
byts = byts[n:]
} else {
d.firstQTHeader = nil
d.quantizationTables = makeQuantizationTables(jh.Quantization)
}

d.fragments = append(d.fragments, byts)
Expand Down Expand Up @@ -186,10 +246,10 @@ func (d *Decoder) Decode(pkt *rtp.Packet) ([]byte, error) {

var dqt jpeg.DefineQuantizationTable
id := uint8(0)
for i := 0; i < len(d.firstQTHeader.Tables); i += 64 {
for _, table := range d.quantizationTables {
dqt.Tables = append(dqt.Tables, jpeg.QuantizationTable{
ID: id,
Data: d.firstQTHeader.Tables[i : i+64],
Data: table,
})
id++
}
Expand Down
102 changes: 102 additions & 0 deletions pkg/format/rtpmjpeg/decoder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,108 @@ func TestDecode(t *testing.T) {
}
}

func TestDecodeFixedQuantizationTable(t *testing.T) {
d := &Decoder{}
err := d.Init()
require.NoError(t, err)

image, err := d.Decode(&rtp.Packet{
Header: rtp.Header{
Version: 2,
Marker: true,
PayloadType: 26,
SequenceNumber: 17645,
SSRC: 0x9dbb7812,
},
Payload: []byte{
// JPEG header
0x00, 0x00, 0x00, 0x00, 0x01, 63, 0xf0, 0x87,
// JPEG data
1, 2,
},
})
require.NoError(t, err)

require.Equal(t, []byte{
0xff, 0xd8, 0xff, 0xdb, 0x00, 0x84, 0x00, 0x0c,
0x08, 0x09, 0x0a, 0x09, 0x07, 0x0c, 0x0a, 0x0a,
0x0a, 0x0d, 0x0d, 0x0c, 0x0e, 0x12, 0x1e, 0x13,
0x12, 0x10, 0x10, 0x12, 0x24, 0x1a, 0x1b, 0x15,
0x1e, 0x2b, 0x26, 0x2d, 0x2c, 0x2a, 0x26, 0x29,
0x29, 0x2f, 0x35, 0x44, 0x3a, 0x2f, 0x32, 0x40,
0x33, 0x29, 0x29, 0x3b, 0x51, 0x3c, 0x40, 0x46,
0x49, 0x4c, 0x4d, 0x4c, 0x2e, 0x39, 0x54, 0x5a,
0x53, 0x4a, 0x59, 0x44, 0x4b, 0x4c, 0x49, 0x01,
0x0d, 0x0d, 0x0d, 0x12, 0x10, 0x12, 0x23, 0x13,
0x13, 0x23, 0x49, 0x31, 0x29, 0x31, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,
0xff, 0xc0, 0x00, 0x11, 0x08, 0x04, 0x38, 0x07,
0x80, 0x03, 0x00, 0x22, 0x00, 0x01, 0x11, 0x01,
0x02, 0x11, 0x01, 0xff, 0xc4, 0x00, 0x1f, 0x00,
0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5,
0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04,
0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01,
0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05,
0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1,
0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1,
0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a,
0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27,
0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38,
0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6,
0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5,
0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4,
0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3,
0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1,
0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9,
0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
0xf8, 0xf9, 0xfa, 0xff, 0xc4, 0x00, 0x1f, 0x01,
0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5,
0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03,
0x04, 0x07, 0x05, 0x04, 0x04, 0x00, 0x01, 0x02,
0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05,
0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61,
0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42,
0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52,
0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24,
0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a,
0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37,
0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47,
0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57,
0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67,
0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77,
0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86,
0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95,
0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4,
0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3,
0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2,
0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca,
0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9,
0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8,
0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
0xf8, 0xf9, 0xfa, 0xff, 0xda, 0x00, 0x0c, 0x03,
0x00, 0x00, 0x01, 0x11, 0x02, 0x11, 0x00, 0x3f,
0x00, 0x01, 0x02, 0xff, 0xd9,
}, image)
}

func FuzzDecoder(f *testing.F) {
f.Fuzz(func(_ *testing.T, a []byte, am bool, b []byte, bm bool) {
d := &Decoder{}
Expand Down
5 changes: 4 additions & 1 deletion pkg/format/rtpmjpeg/encoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ outer:

qth := headerQuantizationTable{}

// gather and sort tables IDs
ids := make([]uint8, len(quantizationTables))
i := 0
for id := range quantizationTables {
Expand All @@ -238,8 +239,10 @@ outer:
sort.Slice(ids, func(i, j int) bool {
return ids[i] < ids[j]
})

// add tables sorted by ID
for _, id := range ids {
qth.Tables = append(qth.Tables, quantizationTables[id]...)
qth.Tables = append(qth.Tables, quantizationTables[id])
}

buf = qth.marshal(buf)
Expand Down
7 changes: 4 additions & 3 deletions pkg/format/rtpmjpeg/header_jpeg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ func (h *headerJPEG) unmarshal(byts []byte) (int, error) {

h.Type = byts[4]
if h.Type != 1 {
return 0, fmt.Errorf("Type %d is not supported", h.Type)
return 0, fmt.Errorf("type %d is not supported", h.Type)
}

h.Quantization = byts[5]
if h.Quantization != 255 {
return 0, fmt.Errorf("Q %d is not supported", h.Quantization)
if h.Quantization == 0 ||
(h.Quantization > 99 && h.Quantization < 127) {
return 0, fmt.Errorf("quantization %d is invalid", h.Quantization)
}

h.Width = int(byts[6]) * 8
Expand Down
25 changes: 19 additions & 6 deletions pkg/format/rtpmjpeg/header_quantization_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
type headerQuantizationTable struct {
MBZ uint8
Precision uint8
Tables []byte
Tables [][]byte
}

func (h *headerQuantizationTable) unmarshal(byts []byte) (int, error) {
Expand All @@ -18,29 +18,42 @@ func (h *headerQuantizationTable) unmarshal(byts []byte) (int, error) {
h.MBZ = byts[0]
h.Precision = byts[1]
if h.Precision != 0 {
return 0, fmt.Errorf("Precision %d is not supported", h.Precision)
return 0, fmt.Errorf("precision %d is not supported", h.Precision)
}

length := int(byts[2])<<8 | int(byts[3])
switch length {
case 64, 128:
default:
return 0, fmt.Errorf("Quantization table length %d is not supported", length)
return 0, fmt.Errorf("table length %d is not supported", length)
}

if (len(byts) - 4) < length {
return 0, fmt.Errorf("buffer is too short")
}

h.Tables = byts[4 : 4+length]
tableCount := length / 64
h.Tables = make([][]byte, tableCount)
n := 0

for i := 0; i < tableCount; i++ {
h.Tables[i] = byts[4+n : 4+64+n]
n += 64
}

return 4 + length, nil
}

func (h headerQuantizationTable) marshal(byts []byte) []byte {
byts = append(byts, h.MBZ)
byts = append(byts, h.Precision)
l := len(h.Tables)

l := len(h.Tables) * 64
byts = append(byts, []byte{byte(l >> 8), byte(l)}...)
byts = append(byts, h.Tables...)

for i := 0; i < len(h.Tables); i++ {
byts = append(byts, h.Tables[i]...)
}

return byts
}
2 changes: 1 addition & 1 deletion pkg/format/rtpmjpeg/header_quantization_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var casesQuantizationTable = []struct {
headerQuantizationTable{
MBZ: 1,
Precision: 0,
Tables: bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 64/4),
Tables: [][]byte{bytes.Repeat([]byte{0x01, 0x02, 0x03, 0x04}, 64/4)},
},
},
}
Expand Down

0 comments on commit 4750375

Please sign in to comment.