Skip to content

Commit

Permalink
Finished compression algorithm
Browse files Browse the repository at this point in the history
  • Loading branch information
maxinelasp committed Aug 29, 2024
1 parent ed0db75 commit 23e6ab4
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 21 deletions.
20 changes: 5 additions & 15 deletions imap_processing/mag/l1a/mag_l1a_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,6 @@ def append_vectors(
self.vectors = np.concatenate([self.vectors, additional_vectors])
self.packet_definitions[self.start_time] = packet_properties

print(self.vectors.shape)

# Every additional packet should be the next one in the sequence, if not, add
# the missing sequence(s) to the gap data
if not self.most_recent_sequence + 1 == vector_sequence:
Expand Down Expand Up @@ -359,7 +357,6 @@ def process_uncompressed_vectors(
vector sample.
"""

# TODO: error handling
def to_signed16(n: int) -> int:
"""
Convert an integer to a signed 16-bit integer.
Expand Down Expand Up @@ -722,8 +719,6 @@ def process_range_data_section(range_data: np.ndarray,
Updated array of vectors, identical to vectors with the range values
updated from range_data.
"""
print(
f"Lenght of range data: {len(range_data)}, length of vectors: {len(vectors)}")
if len(range_data) != len(vectors) * 2:
raise ValueError(
"Incorrect length for range_data, there should be two bits per vector."
Expand All @@ -733,7 +728,6 @@ def process_range_data_section(range_data: np.ndarray,
range_str = "".join([str(i) for i in range_data])
for i, vector in enumerate(vectors):
range_int = int(range_str[i * 2:i * 2 + 2], 2)
print(f"Calculating range {range} from {range_str[i * 2:i * 2 + 2]}")
updated_vectors[i][3] = range_int

return updated_vectors
Expand Down Expand Up @@ -822,20 +816,18 @@ def unpack_one_vector(
raise ValueError(
"unpack_one_vector method is expecting an array of bits as" "input."
)
if width % 8 != 0:
raise ValueError("Width of the vector data should be a multiple of 8.")

if len(vector_data) != width * 3 + 2 * has_range:
raise ValueError(
f"Invalid length {len(vector_data)} for vector data. Expected {width * 3}"
f" or {width * 3 + 2} if has_range.")
# TODO: could width ever be a non-multiple of 8? if yes pad with 0s at the
# beginning
padding = np.zeros(8 - (width % 8), dtype=np.uint8)

# take slices of the input data and pack from an array of bits to an array of
# uint8 bytes
x = np.packbits(vector_data[:width])
y = np.packbits(vector_data[width: 2 * width])
z = np.packbits(vector_data[2 * width: 3 * width])
x = np.packbits(np.concatenate((padding, vector_data[:width])))
y = np.packbits(np.concatenate((padding, vector_data[width: 2 * width])))
z = np.packbits(np.concatenate((padding, vector_data[2 * width: 3 * width])))

range_string = "".join([str(i) for i in vector_data[-2:]])

Expand Down Expand Up @@ -876,7 +868,6 @@ def twos_complement(value: np.ndarray, bits: int) -> np.int32:
two's complement of the input value, as a signed int.
"""
integer_value = int.from_bytes(value, "big")

if (integer_value & (1 << (bits - 1))) != 0:
output_value = integer_value - (1 << bits)
else:
Expand Down Expand Up @@ -905,7 +896,6 @@ def decode_fib_zig_zag(code: np.ndarray) -> int:
f"Error when decoding {code} - fibonacci encoded values "
f"should end in 2 sequential ones."
)
print(code)

# Fibonacci decoding
code = code[:-1]
Expand Down
65 changes: 59 additions & 6 deletions imap_processing/tests/mag/test_mag_l1a.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,14 +401,12 @@ def test_switch_to_uncompressed_vector_data(expected_vectors, uncompressed_vecto
"101110010011100101011010111001001110010101110001110010011000"
"000001100000000000000000000010111000000000000000000000010011000000000000000000000000100101011"
)
print(f"Primary compressed len: {len(primary_compressed)}")

# 4 uncompressed vectors from uncompressed_vector_bytearray
uncompressed_bits = ("00000010000001000000100000010000000100000010000011"
"00000010000001110000100000011101000100000011101011"
"00000010000010100000100000101010000100000101010011"
"00000010000011010000100000110111000100000110111111")
print(f"total primary len: {len(primary_compressed) + len(uncompressed_bits)}")

secondary_compressed = (
"0000001000000011000010000000111100010000000111111110001110"
Expand All @@ -422,10 +420,6 @@ def test_switch_to_uncompressed_vector_data(expected_vectors, uncompressed_vecto
)

uncompressed_expected_vectors = expected_vectors[0][:4]
print(len(primary_compressed))
print(len(secondary_compressed))
print(len(primary_compressed)+len(secondary_compressed)+len(uncompressed_bits)*2)
# expected index at 448

headers = "01000000"

Expand Down Expand Up @@ -460,6 +454,57 @@ def test_switch_to_uncompressed_vector_data(expected_vectors, uncompressed_vecto
assert np.array_equal(primary[0], expected_vectors[0][0])
assert np.array_equal(primary[2:], uncompressed_expected_vectors)

def test_different_compression_width():
# Compression headers - indicating a 12 bit width and no range section
headers = "00110000"

primary_compressed = (
"0101110010"
"011100101011010111001001110010101101011100100110000000011100"
"011100100111001010110101100001011000000001101011100100111001"
"010111000111001001110010101101011100100110000000011010111001"
"001110010101101011100100111001010111000110000101100000000110"
"101110010011100101011010111001001110010101110001110010011000"
"00000110101110010011100101011"
)

secondary_compressed = (
"10001110"
"0100111001010110101100001011000000001101011100100111001010"
"1110001110010011100101011010111001001100000000110101110010"
"0111001010110101110010011100101011100011000010110000000011"
"0101110010011100101011010111001001100000000111000111001001"
"1100101011010111001001110010101101011000010110000000011100"
"011100100111001010110101110010011100101011"
)

first_primary_vector = "00100000010010000001000000000010000011"
first_secondary_vector = "00000001011000000000000011111111111101"

expected_first_vector = [516, -2032, 32, 3]
expected_second_vector = [22, 0, -1, 1]

padding = "00000" # Pad to byte boundary

input_data = np.array(
[int(i) for i in headers + first_primary_vector + primary_compressed +
first_secondary_vector + secondary_compressed + padding],
dtype=np.uint8,
)

input_data = np.packbits(input_data)
(primary, secondary) = MagL1a.process_compressed_vectors(input_data, 16,
16)

assert np.array_equal(primary[0], expected_first_vector)
assert np.array_equal(secondary[0], expected_second_vector)

assert sum(primary[-1]) != 0
assert sum(secondary[-1]) != 0

assert len(primary) == 16
assert len(secondary) == 16


def test_real_uncompressed_vector_data(uncompressed_vector_bytearray, expected_vectors):
primary_expected = expected_vectors[0]
Expand Down Expand Up @@ -501,6 +546,14 @@ def test_unpack_one_vector(uncompressed_vector_bytearray, expected_vectors):

assert all(test_output == expected_vectors)

test_12bit_vector = np.array(
[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])

test_output = MagL1a.unpack_one_vector(test_12bit_vector, 12, 0)
expected_vectors = [22, 0, -1, 0]
assert all(test_output == expected_vectors)

def test_twos_complement():
# -19 in binary
Expand Down

0 comments on commit 23e6ab4

Please sign in to comment.