build: read full blocks from pipes in image headers

Many image recipes write header fields with printf or echo piped into
dd bs=N count=1 conv=sync. Bash writes the output of both line
buffered, so a field holding a 0x0a byte reaches the pipe in two writes.
If dd reads between them, it gets a short block, pads it with zeros due
to conv=sync and stops due to count=1, so the rest of the field is
lost. This depends on the scheduling of the builder, and the rebuilder
has caught it in a ZNET header of the Zyxel WSM20:

  -00000000: 5a4e 4554 80cd 7900 530a 4132 530a 0000  ZNET..y.S.A2S...
  +00000000: 5a4e 4554 80cd 7900 530a 4132 530a 4132  ZNET..y.S.A2S.A2

Fields holding sizes, CRCs or MD5 sums hit this at random, e.g. in
elx-header, iodata-mstc-header2, seil-header or nec-fw. Text fields
only hit it if they contain a newline, but they use the same pattern.
dongwon-header had the same problem and no longer uses dd since
commit f6ef30a ("ath79: dongwon-header: don't read the CRC from a
pipe with dd").

Pass iflag=fullblock to every dd that reads such a pipe with count= or
conv=sync, so dd reads until the block is full or the input ends.
dd with bs=1 is left alone, as a read always returns a full block there.

iflag=fullblock is supported by GNU and uutils dd, like
oflag=seek_bytes already used in ramips. Build dd in tools/coreutils as
well, so hosts other than Linux get GNU dd next to the GNU date and
stat the images already need.

Link: openwrt/openwrt#25897
Signed-off-by: Paul Spooren <mail@aparcar.org>

Author: aparcar

include/image-commands.mk

@@ -378,8 +378,8 @@ define Build/elecom-product-header
 	$(eval fw=$(if $(word 2,$(1)),$(word 2,$(1)),$@))
 
 	-( \
-		echo -n -e "ELECOM\x00\x00$(product)" | dd bs=40 count=1 conv=sync; \
-		echo -n "0.00" | dd bs=16 count=1 conv=sync; \
+		echo -n -e "ELECOM\x00\x00$(product)" | dd bs=40 count=1 conv=sync iflag=fullblock; \
+		echo -n "0.00" | dd bs=16 count=1 conv=sync iflag=fullblock; \
 		dd if=$(fw); \
 	) > $(fw).new \
 	&& mv $(fw).new $(fw) || rm -f $(fw)
@@ -392,7 +392,7 @@ define Build/elecom-wrc-gs-factory
 	$(MKHASH) md5 $(hash_opt) $@ >> $@
 	( \
 		echo -n "ELECOM $(product) v$(version)" | \
-			dd bs=32 count=1 conv=sync; \
+			dd bs=32 count=1 conv=sync iflag=fullblock; \
 		dd if=$@; \
 	) > $@.new
 	mv $@.new $@
@@ -403,14 +403,14 @@ define Build/elx-header
 	$(eval xor_pattern=$(word 2,$(1)))
 	( \
 		echo -ne "\x00\x00\x00\x00\x00\x00\x00\x03" | \
-			dd bs=42 count=1 conv=sync; \
+			dd bs=42 count=1 conv=sync iflag=fullblock; \
 		hw_id="$(hw_id)"; \
 		echo -ne "\x$${hw_id:0:2}\x$${hw_id:2:2}\x$${hw_id:4:2}\x$${hw_id:6:2}" | \
-			dd bs=20 count=1 conv=sync; \
+			dd bs=20 count=1 conv=sync iflag=fullblock; \
 		echo -ne "$$(printf '%08x' $$(stat -c%s $@) | fold -w2 | xargs -I {} echo \\x{} | tr -d '\n')" | \
-			dd bs=8 count=1 conv=sync; \
+			dd bs=8 count=1 conv=sync iflag=fullblock; \
 		echo -ne "$$($(MKHASH) md5 $@ | fold -w2 | xargs -I {} echo \\x{} | tr -d '\n')" | \
-			dd bs=58 count=1 conv=sync; \
+			dd bs=58 count=1 conv=sync iflag=fullblock; \
 	) > $(KDIR)/tmp/$(DEVICE_NAME).header
 	-$(call Build/xor-image,-p $(xor_pattern) -x) \
 	&& cat $(KDIR)/tmp/$(DEVICE_NAME).header $@ > $@.new \

target/linux/ath79/image/common-buffalo.mk

@@ -14,7 +14,7 @@ endef
 
 define Build/buffalo-tftp-header
 	( \
-		echo -n -e "# Airstation Public Fmt1" | dd bs=32 count=1 conv=sync; \
+		echo -n -e "# Airstation Public Fmt1" | dd bs=32 count=1 conv=sync iflag=fullblock; \
 		dd if=$@; \
 	) > $@.new
 	mv $@.new $@

target/linux/ath79/image/common-yuncore.mk

@@ -5,7 +5,7 @@ define Build/yuncore-tftp-header-16m
 			"&& saveenv" \
 			"&& erase 0x9f050000 +0xfa0000" \
 			"&& cp.b 0x800600c0 0x9f050000 0xfa0000" |\
-		dd bs=192 count=1 conv=sync; \
+		dd bs=192 count=1 conv=sync iflag=fullblock; \
 		dd if=$@; \
 	) > $@.new
 	mv $@.new $@

target/linux/ath79/image/generic.mk

@@ -113,10 +113,10 @@ define Build/nec-enc
 endef
 
 define Build/nec-fw
-  ( stat -c%s $@ | tr -d "\n" | dd bs=16 count=1 conv=sync; ) >> $@
+  ( stat -c%s $@ | tr -d "\n" | dd bs=16 count=1 conv=sync iflag=fullblock; ) >> $@
   ( \
-    echo -n -e "$(1)" | dd bs=16 count=1 conv=sync; \
-    echo -n "0.0.00" | dd bs=16 count=1 conv=sync; \
+    echo -n -e "$(1)" | dd bs=16 count=1 conv=sync iflag=fullblock; \
+    echo -n "0.0.00" | dd bs=16 count=1 conv=sync iflag=fullblock; \
     dd if=$@; \
   ) > $@.new
   mv $@.new $@

target/linux/mediatek/image/filogic.mk

@@ -109,7 +109,7 @@ define Build/mstc-header
     printf "$(magic)"; \
     tail -c+5 $@.crclen; head -c4 $@.crclen; \
     dd if=/dev/zero bs=4 count=2; \
-    printf "$(version)" | dd bs=56 count=1 conv=sync 2>/dev/null; \
+    printf "$(version)" | dd bs=56 count=1 conv=sync iflag=fullblock 2>/dev/null; \
     dd if=/dev/zero bs=$$((0x20000 - 0x84)) count=1 conv=sync 2>/dev/null | \
       tr "\0" "\377"; \
     cat $@; \
@@ -129,9 +129,9 @@ define Build/cetron-header
 	$(eval model=$(word 2,$(1)))
 	( \
 		dd if=/dev/zero bs=856 count=1 2>/dev/null; \
-		printf "$(model)," | dd bs=128 count=1 conv=sync 2>/dev/null; \
-		md5sum $@ | cut -f1 -d" " | dd bs=32 count=1 2>/dev/null; \
-		printf "$(magic)" | dd bs=4 count=1 conv=sync 2>/dev/null; \
+		printf "$(model)," | dd bs=128 count=1 conv=sync iflag=fullblock 2>/dev/null; \
+		md5sum $@ | cut -f1 -d" " | dd bs=32 count=1 iflag=fullblock 2>/dev/null; \
+		printf "$(magic)" | dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		cat $@; \
 	) > $@.tmp
 	fw_crc=$$(gzip -c $@.tmp | tail -c 8 | od -An -N4 -tx4 --endian little | tr -d ' \n'); \

target/linux/mvebu/image/cortexa9.mk

@@ -24,10 +24,10 @@ define Build/seil-header
         od -An -tx8 --endian little | tr -d ' \n')"; \
     printf "SEIL2015"; \
     printf "$(call toupper,$(LINUX_KARCH)) $(VERSION_DIST) Linux-$(LINUX_VERSION)" | \
-        dd bs=80 count=1 conv=sync 2>/dev/null; \
+        dd bs=80 count=1 conv=sync iflag=fullblock 2>/dev/null; \
     printf "$$(echo $${data_size_crc:8:8} | sed 's/../\\x&/g')"; \
     printf "\x00\x00\x00\x01\x00\x00\x00\x09\x00\x00\x00\x63"; \
-    printf "$(REVISION)" | dd bs=32 count=1 conv=sync 2>/dev/null; \
+    printf "$(REVISION)" | dd bs=32 count=1 conv=sync iflag=fullblock 2>/dev/null; \
     printf "\x00\x00\x00\x00"; \
     printf "$$(echo $${data_size_crc:0:8} | sed 's/../\\x&/g')"; \
     cat $@; \

target/linux/qualcommax/image/ipq50xx.mk

@@ -9,9 +9,9 @@ define Build/mstc-header
 		printf "CMOC"; \
 		tail -c+5 $@.crclen; head -c4 $@.crclen; \
 		printf '$(call toupper,$(LINUX_KARCH)) $(VERSION_DIST) Linux-$(LINUX_VERSION)' | \
-			dd bs=64 count=1 conv=sync 2>/dev/null; \
+			dd bs=64 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$(version)" | \
-			dd bs=64 count=1 conv=sync 2>/dev/null; \
+			dd bs=64 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		dd if=/dev/zero bs=$$(($(hdrlen) - 0x8c)) count=1 2>/dev/null; \
 		cat $@; \
 	) > $@.new

target/linux/ramips/image/mt7621.mk

@@ -136,14 +136,14 @@ define Build/iodata-mstc-header
 		data_size_crc="$$(dd if=$@ ibs=64 skip=1 2>/dev/null | gzip -c | \
 			tail -c 8 | od -An -tx8 --endian little | tr -d ' \n')"; \
 		echo -ne "$$(echo $$data_size_crc | sed 's/../\\x&/g')" | \
-			dd of=$@ bs=8 count=1 seek=7 conv=notrunc 2>/dev/null; \
+			dd of=$@ bs=8 count=1 seek=7 conv=notrunc iflag=fullblock 2>/dev/null; \
 	)
 	dd if=/dev/zero of=$@ bs=4 count=1 seek=1 conv=notrunc 2>/dev/null
 	( \
 		header_crc="$$(dd if=$@ bs=64 count=1 2>/dev/null | gzip -c | \
 			tail -c 8 | od -An -N4 -tx4 --endian little | tr -d ' \n')"; \
 		echo -ne "$$(echo $$header_crc | sed 's/../\\x&/g')" | \
-			dd of=$@ bs=4 count=1 seek=1 conv=notrunc 2>/dev/null; \
+			dd of=$@ bs=4 count=1 seek=1 conv=notrunc iflag=fullblock 2>/dev/null; \
 	)
 endef
 
@@ -155,18 +155,18 @@ define Build/iodata-mstc-header2
 		fw_len_crc=$$(gzip -c $@ | tail -c 8 | \
 			od -An -tx8 --endian little); \
 		printf "\x03\x1d\x61\x29\x07$(model)" | \
-			dd bs=21 count=1 conv=sync 2>/dev/null; \
-		printf "0.00.000" | dd bs=16 count=1 conv=sync 2>/dev/null; \
+			dd bs=21 count=1 conv=sync iflag=fullblock 2>/dev/null; \
+		printf "0.00.000" | dd bs=16 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$$(echo $(REVISION) | cut -d- -f1 | head -c8)" | \
-			dd bs=9 count=1 conv=sync 2>/dev/null; \
+			dd bs=9 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$(call toupper,$(LINUX_KARCH)) $(VERSION_DIST) Linux-$(LINUX_VERSION)" | \
-			dd bs=33 count=1 conv=sync 2>/dev/null; \
+			dd bs=33 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		date -d "@$(SOURCE_DATE_EPOCH)" "+%F" | tr -d "\n" | \
-			dd bs=15 count=1 conv=sync 2>/dev/null; \
+			dd bs=15 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$$(echo $(model_id) | sed 's/../\\x&/g')" | \
-			dd bs=8 count=1 conv=sync 2>/dev/null; \
+			dd bs=8 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$$(echo $$fw_len_crc | sed 's/../\\x&/g')" | \
-			dd bs=14 count=1 conv=sync 2>/dev/null; \
+			dd bs=14 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		cat $@; \
 	) > $@.new
 	( \
@@ -188,15 +188,15 @@ define Build/znet-header
 		payload_len="$$(dd if=$@ bs=4 count=1 skip=1 2>/dev/null | od -An -tdI --endian big | tr -d ' \n')"; \
 		payload_size_crc="$$(dd if=$@ ibs=1 count=$$payload_len 2>/dev/null | gzip -c | \
 			tail -c 8 | od -An -N4 -tx4 --endian big | tr -d ' \n')"; \
-		echo -ne "$(magic)" | dd bs=4 count=1 conv=sync 2>/dev/null; \
+		echo -ne "$(magic)" | dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		echo -ne "$$(printf '%08x' $$(stat -c%s $@) | fold -w2 | xargs -I {} echo \\x{} | tac | tr -d '\n')" | \
-			dd bs=4 count=1 conv=sync 2>/dev/null; \
+			dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		echo -ne "$$(echo $$data_size_crc | sed 's/../\\x&/g')" | \
-			dd bs=4 count=1 conv=sync 2>/dev/null; \
+			dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		echo -ne "$$(echo $$payload_size_crc | sed 's/../\\x&/g')" | \
-			dd bs=4 count=1 conv=sync 2>/dev/null; \
-		echo -ne "\x12\x34\x56\x78" | dd bs=4 count=1 conv=sync 2>/dev/null; \
-		echo -ne "$(version)" | dd bs=28 count=1 conv=sync 2>/dev/null; \
+			dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
+		echo -ne "\x12\x34\x56\x78" | dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
+		echo -ne "$(version)" | dd bs=28 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		dd if=/dev/zero bs=$$((0x40000 - $(hdrlen))) count=1 conv=sync 2>/dev/null | tr "\000" "\377"; \
 		cat $@; \
 	) > $@.new
@@ -216,7 +216,7 @@ define Build/belkin-header
 		fw_len_crc=$$(gzip -c $@ | tail -c 8 | od -An -tx8 | tr -d ' \n'); \
 		fw_crc_len="$${fw_len_crc:8:8}$${fw_len_crc:0:8}"; \
 		\
-		printf "$(magic)" | dd bs=4 count=1 conv=sync 2>/dev/null; \
+		printf "$(magic)" | dd bs=4 count=1 conv=sync iflag=fullblock 2>/dev/null; \
 		printf "$$(echo $${type_fw_date}$${hw_fw_ver} | \
 			sed 's/../\\x&/g')"; \
 		printf "$$(echo $${fw_crc_len}$${fw_crc_len} | \

target/linux/ramips/image/rt305x.mk

@@ -4,7 +4,7 @@
 define Build/buffalo-tftp-header
 	( \
 		echo -n -e "# Airstation FirmWare\nrun u_fw\nreset\n\n" | \
-			dd bs=512 count=1 conv=sync; \
+			dd bs=512 count=1 conv=sync iflag=fullblock; \
 		dd if=$@; \
 	) > $@.tmp && \
 	$(STAGING_DIR_HOST)/bin/buffalo-tftp -i $@.tmp -o $@.new

tools/coreutils/Makefile

@@ -16,7 +16,7 @@ PKG_HASH:=2bec616375002c92c1ed5ead32a092b174fe44c14bc736d32e5961053b821d84
 
 HOST_BUILD_PARALLEL := 1
 
-PKG_PROGRAMS:=chown cp date ginstall ln readlink realpath rmdir seq stat touch
+PKG_PROGRAMS:=chown cp date dd ginstall ln readlink realpath rmdir seq stat touch
 
 include $(INCLUDE_DIR)/host-build.mk