Update to latest libfdt and pylibfdt, with added size control

Update binman, dtoc, patman, buildman to Python 3
 Update move_config, rkmux, microcode_tool to Python 3
 -----BEGIN PGP SIGNATURE-----
 
 iQFFBAABCgAvFiEEslwAIq+Gp8wWVbYnfxc6PpAIreYFAl3BZicRHHNqZ0BjaHJv
 bWl1bS5vcmcACgkQfxc6PpAIrebBiwf9EJpAgEvWMBJmVRsnwWqyKr879OLh1/av
 EM/VFF0hjtFGEs1UsR30lk+4dCCSuhTzc4i8gpfFCmRcASFJ4IrRJeCQTLrRY5Bo
 YNjpQ4HT5wcF7oq58inqotrDZ7p6HLu2zt8oyz5HgzckqV+a+9ldD6k0rkuYR88f
 fSVAiji0QjPkvQECTzuG76iusQsYUxBxwKScFM3D0AD9m8aneotp7SGcLFPKPDd1
 NFJLqt2uJp7Zac7rQX/b6iEX9JCHOo1UXdurAmdf9ebXmlr4GWy3GP8yZpyRQa3q
 zOrkguCEiG4fIwAqesmM5RfL0ZYHjrkVaTDx1MVc3F3QwFi35JLTcw==
 =T7mD
 -----END PGP SIGNATURE-----

Merge tag 'fdt-pull-5nov19' of git://git.denx.de/u-boot-fdt

Update to latest libfdt and pylibfdt, with added size control
Update binman, dtoc, patman, buildman to Python 3
Update move_config, rkmux, microcode_tool to Python 3

configs/tbs2910_defconfig

@@ -80,4 +80,5 @@ CONFIG_USB_GADGET_DOWNLOAD=y
 CONFIG_I2C_EDID=y
 CONFIG_VIDEO_IPUV3=y
 CONFIG_VIDEO=y
+CONFIG_OF_LIBFDT_ASSUME_MASK=0xff
 # CONFIG_EFI_LOADER is not set

include/linux/kernel.h

@@ -37,6 +37,8 @@
 #define UINT32_MAX	U32_MAX
 #define UINT64_MAX	U64_MAX
 
+#define INT32_MAX	S32_MAX
+
 #define STACK_MAGIC	0xdeadbeef
 
 #define REPEAT_BYTE(x)	((~0ul / 0xff) * (x))

include/linux/libfdt_env.h

@@ -10,6 +10,7 @@
 #define LIBFDT_ENV_H
 
 #include <linux/string.h>
+#include <linux/kernel.h>
 
 #include <asm/byteorder.h>
 

lib/Kconfig

@@ -464,6 +464,17 @@ config OF_LIBFDT
 	  particular compatible nodes. The library operates on a flattened
 	  version of the device tree.
 
+config OF_LIBFDT_ASSUME_MASK
+	hex "Mask of conditions to assume for libfdt"
+	depends on OF_LIBFDT || FIT
+	default 0
+	help
+	  Use this to change the assumptions made by libfdt about the
+	  device tree it is working with. A value of 0 means that no assumptions
+	  are made, and libfdt is able to deal with malicious data. A value of
+	  0xff means all assumptions are made and any invalid data may cause
+	  unsafe execution. See FDT_ASSUME_PERFECT, etc. in libfdt_internal.h
+
 config OF_LIBFDT_OVERLAY
 	bool "Enable the FDT library overlay support"
 	depends on OF_LIBFDT
@@ -481,6 +492,17 @@ config SPL_OF_LIBFDT
 	  particular compatible nodes. The library operates on a flattened
 	  version of the device tree.
 
+config SPL_OF_LIBFDT_ASSUME_MASK
+	hex "Mask of conditions to assume for libfdt"
+	depends on SPL_OF_LIBFDT || FIT
+	default 0xff
+	help
+	  Use this to change the assumptions made by libfdt in SPL about the
+	  device tree it is working with. A value of 0 means that no assumptions
+	  are made, and libfdt is able to deal with malicious data. A value of
+	  0xff means all assumptions are made and any invalid data may cause
+	  unsafe execution. See FDT_ASSUME_PERFECT, etc. in libfdt_internal.h
+
 config TPL_OF_LIBFDT
 	bool "Enable the FDT library for TPL"
 	default y if TPL_OF_CONTROL
@@ -491,6 +513,17 @@ config TPL_OF_LIBFDT
 	  particular compatible nodes. The library operates on a flattened
 	  version of the device tree.
 
+config TPL_OF_LIBFDT_ASSUME_MASK
+	hex "Mask of conditions to assume for libfdt"
+	depends on TPL_OF_LIBFDT || FIT
+	default 0xff
+	help
+	  Use this to change the assumptions made by libfdt in TPL about the
+	  device tree it is working with. A value of 0 means that no assumptions
+	  are made, and libfdt is able to deal with malicious data. A value of
+	  0xff means all assumptions are made and any invalid data may cause
+	  unsafe execution. See FDT_ASSUME_PERFECT, etc. in libfdt_internal.h
+
 config FDT_FIXUP_PARTITIONS
 	bool "overwrite MTD partitions in DTS through defined in 'mtdparts'"
 	depends on OF_LIBFDT

lib/libfdt/Makefile

@@ -22,4 +22,5 @@ obj-y += fdt_ro.o
 # U-Boot own file
 obj-y += fdt_region.o
 
-ccflags-y := -I$(srctree)/scripts/dtc/libfdt
+ccflags-y := -I$(srctree)/scripts/dtc/libfdt \
+	-DFDT_ASSUME_MASK=$(CONFIG_$(SPL_TPL_)OF_LIBFDT_ASSUME_MASK)

lib/libfdt/fdt_ro.c

@@ -14,12 +14,13 @@
 
 #include "libfdt_internal.h"
 
-static int _fdt_nodename_eq(const void *fdt, int offset,
+static int fdt_nodename_eq_(const void *fdt, int offset,
 			    const char *s, int len)
 {
-	const char *p = fdt_offset_ptr(fdt, offset + FDT_TAGSIZE, len+1);
+	int olen;
+	const char *p = fdt_get_name(fdt, offset, &olen);
 
-	if (!p)
+	if (!p || (fdt_chk_extra() && olen < len))
 		/* short match */
 		return 0;
 
@@ -34,46 +35,85 @@ static int _fdt_nodename_eq(const void *fdt, int offset,
 		return 0;
 }
 
+const char *fdt_get_string(const void *fdt, int stroffset, int *lenp)
+{
+	int32_t totalsize;
+	uint32_t absoffset;
+	size_t len;
+	int err;
+	const char *s, *n;
+
+	if (!fdt_chk_extra()) {
+		s = (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset;
+
+		if (lenp)
+			*lenp = strlen(s);
+		return s;
+	}
+	totalsize = fdt_ro_probe_(fdt);
+	err = totalsize;
+	if (totalsize < 0)
+		goto fail;
+
+	err = -FDT_ERR_BADOFFSET;
+	absoffset = stroffset + fdt_off_dt_strings(fdt);
+	if (absoffset >= totalsize)
+		goto fail;
+	len = totalsize - absoffset;
+
+	if (fdt_magic(fdt) == FDT_MAGIC) {
+		if (stroffset < 0)
+			goto fail;
+		if (!fdt_chk_version() || fdt_version(fdt) >= 17) {
+			if (stroffset >= fdt_size_dt_strings(fdt))
+				goto fail;
+			if ((fdt_size_dt_strings(fdt) - stroffset) < len)
+				len = fdt_size_dt_strings(fdt) - stroffset;
+		}
+	} else if (fdt_magic(fdt) == FDT_SW_MAGIC) {
+		if ((stroffset >= 0)
+		    || (stroffset < -fdt_size_dt_strings(fdt)))
+			goto fail;
+		if ((-stroffset) < len)
+			len = -stroffset;
+	} else {
+		err = -FDT_ERR_INTERNAL;
+		goto fail;
+	}
+
+	s = (const char *)fdt + absoffset;
+	n = memchr(s, '\0', len);
+	if (!n) {
+		/* missing terminating NULL */
+		err = -FDT_ERR_TRUNCATED;
+		goto fail;
+	}
+
+	if (lenp)
+		*lenp = n - s;
+	return s;
+
+fail:
+	if (lenp)
+		*lenp = err;
+	return NULL;
+}
+
 const char *fdt_string(const void *fdt, int stroffset)
 {
-	return (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset;
+	return fdt_get_string(fdt, stroffset, NULL);
 }
 
-static int _fdt_string_eq(const void *fdt, int stroffset,
+static int fdt_string_eq_(const void *fdt, int stroffset,
 			  const char *s, int len)
 {
-	const char *p = fdt_string(fdt, stroffset);
+	int slen;
+	const char *p = fdt_get_string(fdt, stroffset, &slen);
 
-	return (strnlen(p, len + 1) == len) && (memcmp(p, s, len) == 0);
+	return p && (slen == len) && (memcmp(p, s, len) == 0);
 }
 
-uint32_t fdt_get_max_phandle(const void *fdt)
+int fdt_find_max_phandle(const void *fdt, uint32_t *phandle)
-{
-	uint32_t max_phandle = 0;
-	int offset;
-
-	for (offset = fdt_next_node(fdt, -1, NULL);;
-	     offset = fdt_next_node(fdt, offset, NULL)) {
-		uint32_t phandle;
-
-		if (offset == -FDT_ERR_NOTFOUND)
-			return max_phandle;
-
-		if (offset < 0)
-			return (uint32_t)-1;
-
-		phandle = fdt_get_phandle(fdt, offset);
-		if (phandle == (uint32_t)-1)
-			continue;
-
-		if (phandle > max_phandle)
-			max_phandle = phandle;
-	}
-
-	return 0;
-}
-
-int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 {
 	uint32_t max = 0;
 	int offset = -1;
@@ -95,6 +135,21 @@ int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 			max = value;
 	}
 
+	if (phandle)
+		*phandle = max;
+
+	return 0;
+}
+
+int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
+{
+	uint32_t max;
+	int err;
+
+	err = fdt_find_max_phandle(fdt, &max);
+	if (err < 0)
+		return err;
+
 	if (max == FDT_MAX_PHANDLE)
 		return -FDT_ERR_NOPHANDLES;
 
@@ -104,24 +159,48 @@ int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 	return 0;
 }
 
+static const struct fdt_reserve_entry *fdt_mem_rsv(const void *fdt, int n)
+{
+	int offset = n * sizeof(struct fdt_reserve_entry);
+	int absoffset = fdt_off_mem_rsvmap(fdt) + offset;
+
+	if (fdt_chk_extra()) {
+		if (absoffset < fdt_off_mem_rsvmap(fdt))
+			return NULL;
+		if (absoffset > fdt_totalsize(fdt) -
+		    sizeof(struct fdt_reserve_entry))
+			return NULL;
+	}
+	return fdt_mem_rsv_(fdt, n);
+}
+
 int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size)
 {
-	FDT_CHECK_HEADER(fdt);
+	const struct fdt_reserve_entry *re;
-	*address = fdt64_to_cpu(fdt_mem_rsv_(fdt, n)->address);
+
-	*size = fdt64_to_cpu(fdt_mem_rsv_(fdt, n)->size);
+	FDT_RO_PROBE(fdt);
+	re = fdt_mem_rsv(fdt, n);
+	if (fdt_chk_extra() && !re)
+		return -FDT_ERR_BADOFFSET;
+
+	*address = fdt64_ld(&re->address);
+	*size = fdt64_ld(&re->size);
 	return 0;
 }
 
 int fdt_num_mem_rsv(const void *fdt)
 {
-	int i = 0;
+	int i;
+	const struct fdt_reserve_entry *re;
 
-	while (fdt64_to_cpu(fdt_mem_rsv_(fdt, i)->size) != 0)
+	for (i = 0; (re = fdt_mem_rsv(fdt, i)) != NULL; i++) {
-		i++;
+		if (fdt64_ld(&re->size) == 0)
 			return i;
 	}
+	return -FDT_ERR_TRUNCATED;
+}
 
-static int _nextprop(const void *fdt, int offset)
+static int nextprop_(const void *fdt, int offset)
 {
 	uint32_t tag;
 	int nextoffset;
@@ -150,13 +229,13 @@ int fdt_subnode_offset_namelen(const void *fdt, int offset,
 {
 	int depth;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	for (depth = 0;
 	     (offset >= 0) && (depth >= 0);
 	     offset = fdt_next_node(fdt, offset, &depth))
 		if ((depth == 1)
-		    && _fdt_nodename_eq(fdt, offset, name, namelen))
+		    && fdt_nodename_eq_(fdt, offset, name, namelen))
 			return offset;
 
 	if (depth < 0)
@@ -170,36 +249,17 @@ int fdt_subnode_offset(const void *fdt, int parentoffset,
 	return fdt_subnode_offset_namelen(fdt, parentoffset, name, strlen(name));
 }
 
-/*
- * Find the next of path separator, note we need to search for both '/' and ':'
- * and then take the first one so that we do the right thing for e.g.
- * "foo/bar:option" and "bar:option/otheroption", both of which happen, so
- * first searching for either ':' or '/' does not work.
- */
-static const char *fdt_path_next_separator(const char *path, int len)
-{
-	const void *sep1 = memchr(path, '/', len);
-	const void *sep2 = memchr(path, ':', len);
-
-	if (sep1 && sep2)
-		return (sep1 < sep2) ? sep1 : sep2;
-	else if (sep1)
-		return sep1;
-	else
-		return sep2;
-}
-
 int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen)
 {
 	const char *end = path + namelen;
 	const char *p = path;
 	int offset = 0;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* see if we have an alias */
 	if (*path != '/') {
-		const char *q = fdt_path_next_separator(path, namelen);
+		const char *q = memchr(path, '/', end - p);
 
 		if (!q)
 			q = end;
@@ -212,16 +272,15 @@ int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen)
 		p = q;
 	}
 
-	while (*p && (p < end)) {
+	while (p < end) {
 		const char *q;
 
-		while (*p == '/')
+		while (*p == '/') {
 			p++;
-
+			if (p == end)
-		if (*p == '\0' || *p == ':')
 				return offset;
-
+		}
-		q = fdt_path_next_separator(p, end - p);
+		q = memchr(p, '/', end - p);
 		if (! q)
 			q = end;
 
@@ -243,16 +302,35 @@ int fdt_path_offset(const void *fdt, const char *path)
 const char *fdt_get_name(const void *fdt, int nodeoffset, int *len)
 {
 	const struct fdt_node_header *nh = fdt_offset_ptr_(fdt, nodeoffset);
+	const char *nameptr;
 	int err;
 
-	if (((err = fdt_check_header(fdt)) != 0)
+	if (fdt_chk_extra() &&
-	    || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0))
+	    (((err = fdt_ro_probe_(fdt)) < 0)
+	     || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0)))
 		goto fail;
 
-	if (len)
+	nameptr = nh->name;
-		*len = strlen(nh->name);
 
-	return nh->name;
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
+		/*
+		 * For old FDT versions, match the naming conventions of V16:
+		 * give only the leaf name (after all /). The actual tree
+		 * contents are loosely checked.
+		 */
+		const char *leaf;
+		leaf = strrchr(nameptr, '/');
+		if (leaf == NULL) {
+			err = -FDT_ERR_BADSTRUCTURE;
+			goto fail;
+		}
+		nameptr = leaf+1;
+	}
+
+	if (len)
+		*len = strlen(nameptr);
+
+	return nameptr;
 
  fail:
 	if (len)
@@ -267,7 +345,7 @@ int fdt_first_property_offset(const void *fdt, int nodeoffset)
 	if ((offset = fdt_check_node_offset_(fdt, nodeoffset)) < 0)
 		return offset;
 
-	return _nextprop(fdt, offset);
+	return nextprop_(fdt, offset);
 }
 
 int fdt_next_property_offset(const void *fdt, int offset)
@@ -275,17 +353,17 @@ int fdt_next_property_offset(const void *fdt, int offset)
 	if ((offset = fdt_check_prop_offset_(fdt, offset)) < 0)
 		return offset;
 
-	return _nextprop(fdt, offset);
+	return nextprop_(fdt, offset);
 }
 
-const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
+static const struct fdt_property *fdt_get_property_by_offset_(const void *fdt,
 						              int offset,
 						              int *lenp)
 {
 	int err;
 	const struct fdt_property *prop;
 
-	if ((err = fdt_check_prop_offset_(fdt, offset)) < 0) {
+	if (fdt_chk_basic() && (err = fdt_check_prop_offset_(fdt, offset)) < 0) {
 		if (lenp)
 			*lenp = err;
 		return NULL;
@@ -294,35 +372,76 @@ const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
 	prop = fdt_offset_ptr_(fdt, offset);
 
 	if (lenp)
-		*lenp = fdt32_to_cpu(prop->len);
+		*lenp = fdt32_ld(&prop->len);
 
 	return prop;
 }
 
-const struct fdt_property *fdt_get_property_namelen(const void *fdt,
+const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
+						      int offset,
+						      int *lenp)
+{
+	/* Prior to version 16, properties may need realignment
+	 * and this API does not work. fdt_getprop_*() will, however. */
+
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
+		if (lenp)
+			*lenp = -FDT_ERR_BADVERSION;
+		return NULL;
+	}
+
+	return fdt_get_property_by_offset_(fdt, offset, lenp);
+}
+
+static const struct fdt_property *fdt_get_property_namelen_(const void *fdt,
 						            int offset,
 						            const char *name,
-						    int namelen, int *lenp)
+						            int namelen,
+							    int *lenp,
+							    int *poffset)
 {
 	for (offset = fdt_first_property_offset(fdt, offset);
 	     (offset >= 0);
 	     (offset = fdt_next_property_offset(fdt, offset))) {
 		const struct fdt_property *prop;
 
-		if (!(prop = fdt_get_property_by_offset(fdt, offset, lenp))) {
+		prop = fdt_get_property_by_offset_(fdt, offset, lenp);
+		if (fdt_chk_extra() && !prop) {
 			offset = -FDT_ERR_INTERNAL;
 			break;
 		}
-		if (_fdt_string_eq(fdt, fdt32_to_cpu(prop->nameoff),
+		if (fdt_string_eq_(fdt, fdt32_ld(&prop->nameoff),
-				   name, namelen))
+				   name, namelen)) {
+			if (poffset)
+				*poffset = offset;
 			return prop;
 		}
+	}
 
 	if (lenp)
 		*lenp = offset;
 	return NULL;
 }
 
+
+const struct fdt_property *fdt_get_property_namelen(const void *fdt,
+						    int offset,
+						    const char *name,
+						    int namelen, int *lenp)
+{
+	/* Prior to version 16, properties may need realignment
+	 * and this API does not work. fdt_getprop_*() will, however. */
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
+		if (lenp)
+			*lenp = -FDT_ERR_BADVERSION;
+		return NULL;
+	}
+
+	return fdt_get_property_namelen_(fdt, offset, name, namelen, lenp,
+					 NULL);
+}
+
+
 const struct fdt_property *fdt_get_property(const void *fdt,
 					    int nodeoffset,
 					    const char *name, int *lenp)
@@ -334,12 +453,18 @@ const struct fdt_property *fdt_get_property(const void *fdt,
 const void *fdt_getprop_namelen(const void *fdt, int nodeoffset,
 				const char *name, int namelen, int *lenp)
 {
+	int poffset;
 	const struct fdt_property *prop;
 
-	prop = fdt_get_property_namelen(fdt, nodeoffset, name, namelen, lenp);
+	prop = fdt_get_property_namelen_(fdt, nodeoffset, name, namelen, lenp,
+					 &poffset);
 	if (!prop)
 		return NULL;
 
+	/* Handle realignment */
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10 &&
+	    (poffset + sizeof(*prop)) % 8 && fdt32_ld(&prop->len) >= 8)
+		return prop->data + 4;
 	return prop->data;
 }
 
@@ -348,11 +473,31 @@ const void *fdt_getprop_by_offset(const void *fdt, int offset,
 {
 	const struct fdt_property *prop;
 
-	prop = fdt_get_property_by_offset(fdt, offset, lenp);
+	prop = fdt_get_property_by_offset_(fdt, offset, lenp);
 	if (!prop)
 		return NULL;
-	if (namep)
+	if (namep) {
-		*namep = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
+		const char *name;
+		int namelen;
+
+		if (fdt_chk_extra()) {
+			name = fdt_get_string(fdt, fdt32_ld(&prop->nameoff),
+					      &namelen);
+			if (!name) {
+				if (lenp)
+					*lenp = namelen;
+				return NULL;
+			}
+			*namep = name;
+		} else {
+			*namep = fdt_string(fdt, fdt32_ld(&prop->nameoff));
+		}
+	}
+
+	/* Handle realignment */
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10 &&
+	    (offset + sizeof(*prop)) % 8 && fdt32_ld(&prop->len) >= 8)
+		return prop->data + 4;
 	return prop->data;
 }
 
@@ -376,7 +521,7 @@ uint32_t fdt_get_phandle(const void *fdt, int nodeoffset)
 			return 0;
 	}
 
-	return fdt32_to_cpu(*php);
+	return fdt32_ld(php);
 }
 
 const char *fdt_get_alias_namelen(const void *fdt,
@@ -402,7 +547,7 @@ int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 	int offset, depth, namelen;
 	const char *name;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	if (buflen < 2)
 		return -FDT_ERR_NOSPACE;
@@ -454,7 +599,7 @@ int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset,
 	int offset, depth;
 	int supernodeoffset = -FDT_ERR_INTERNAL;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	if (supernodedepth < 0)
 		return -FDT_ERR_NOTFOUND;
@@ -476,10 +621,12 @@ int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset,
 		}
 	}
 
+	if (fdt_chk_extra()) {
 		if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0))
 			return -FDT_ERR_BADOFFSET;
 		else if (offset == -FDT_ERR_BADOFFSET)
 			return -FDT_ERR_BADSTRUCTURE;
+	}
 
 	return offset; /* error from fdt_next_node() */
 }
@@ -491,7 +638,7 @@ int fdt_node_depth(const void *fdt, int nodeoffset)
 
 	err = fdt_supernode_atdepth_offset(fdt, nodeoffset, 0, &nodedepth);
 	if (err)
-		return (err < 0) ? err : -FDT_ERR_INTERNAL;
+		return (!fdt_chk_extra() || err < 0) ? err : -FDT_ERR_INTERNAL;
 	return nodedepth;
 }
 
@@ -513,7 +660,7 @@ int fdt_node_offset_by_prop_value(const void *fdt, int startoffset,
 	const void *val;
 	int len;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we scan each
 	 * property of a node in fdt_getprop(), then if that didn't
@@ -539,7 +686,7 @@ int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle)
 	if ((phandle == 0) || (phandle == -1))
 		return -FDT_ERR_BADPHANDLE;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we
 	 * potentially scan each property of a node in
@@ -692,7 +839,7 @@ int fdt_node_offset_by_compatible(const void *fdt, int startoffset,
 {
 	int offset, err;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we scan each
 	 * property of a node in fdt_node_check_compatible(), then if
@@ -711,3 +858,68 @@ int fdt_node_offset_by_compatible(const void *fdt, int startoffset,
 
 	return offset; /* error from fdt_next_node() */
 }
+
+#if !defined(CHECK_LEVEL) || CHECK_LEVEL > 0
+int fdt_check_full(const void *fdt, size_t bufsize)
+{
+	int err;
+	int num_memrsv;
+	int offset, nextoffset = 0;
+	uint32_t tag;
+	unsigned depth = 0;
+	const void *prop;
+	const char *propname;
+
+	if (bufsize < FDT_V1_SIZE)
+		return -FDT_ERR_TRUNCATED;
+	err = fdt_check_header(fdt);
+	if (err != 0)
+		return err;
+	if (bufsize < fdt_totalsize(fdt))
+		return -FDT_ERR_TRUNCATED;
+
+	num_memrsv = fdt_num_mem_rsv(fdt);
+	if (num_memrsv < 0)
+		return num_memrsv;
+
+	while (1) {
+		offset = nextoffset;
+		tag = fdt_next_tag(fdt, offset, &nextoffset);
+
+		if (nextoffset < 0)
+			return nextoffset;
+
+		switch (tag) {
+		case FDT_NOP:
+			break;
+
+		case FDT_END:
+			if (depth != 0)
+				return -FDT_ERR_BADSTRUCTURE;
+			return 0;
+
+		case FDT_BEGIN_NODE:
+			depth++;
+			if (depth > INT_MAX)
+				return -FDT_ERR_BADSTRUCTURE;
+			break;
+
+		case FDT_END_NODE:
+			if (depth == 0)
+				return -FDT_ERR_BADSTRUCTURE;
+			depth--;
+			break;
+
+		case FDT_PROP:
+			prop = fdt_getprop_by_offset(fdt, offset, &propname,
+						     &err);
+			if (!prop)
+				return err;
+			break;
+
+		default:
+			return -FDT_ERR_INTERNAL;
+		}
+	}
+}
+#endif

scripts/dtc/libfdt/Makefile.libfdt

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 # Makefile.libfdt
 #
 # This is not a complete Makefile of itself.  Instead, it is designed to
@@ -9,3 +10,9 @@ LIBFDT_VERSION = version.lds
 LIBFDT_SRCS = fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c fdt_empty_tree.c \
 	fdt_addresses.c fdt_overlay.c
 LIBFDT_OBJS = $(LIBFDT_SRCS:%.c=%.o)
+LIBFDT_LIB = libfdt-$(DTC_VERSION).$(SHAREDLIB_EXT)
+
+libfdt_clean:
+	@$(VECHO) CLEAN "(libfdt)"
+	rm -f $(STD_CLEANFILES:%=$(LIBFDT_dir)/%)
+	rm -f $(LIBFDT_dir)/$(LIBFDT_soname)

scripts/dtc/libfdt/fdt.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -55,14 +10,24 @@
 
 #include "libfdt_internal.h"
 
-int fdt_check_header(const void *fdt)
+/*
+ * Minimal sanity check for a read-only tree. fdt_ro_probe_() checks
+ * that the given buffer contains what appears to be a flattened
+ * device tree with sane information in its header.
+ */
+int32_t fdt_ro_probe_(const void *fdt)
 {
+	uint32_t totalsize = fdt_totalsize(fdt);
+
 	if (fdt_magic(fdt) == FDT_MAGIC) {
 		/* Complete tree */
+		if (fdt_chk_version()) {
 			if (fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
 				return -FDT_ERR_BADVERSION;
-		if (fdt_last_comp_version(fdt) > FDT_LAST_SUPPORTED_VERSION)
+			if (fdt_last_comp_version(fdt) >
+					FDT_LAST_SUPPORTED_VERSION)
 				return -FDT_ERR_BADVERSION;
+		}
 	} else if (fdt_magic(fdt) == FDT_SW_MAGIC) {
 		/* Unfinished sequential-write blob */
 		if (fdt_size_dt_struct(fdt) == 0)
@@ -71,6 +36,96 @@ int fdt_check_header(const void *fdt)
 		return -FDT_ERR_BADMAGIC;
 	}
 
+	if (totalsize < INT32_MAX)
+		return totalsize;
+	else
+		return -FDT_ERR_TRUNCATED;
+}
+
+static int check_off_(uint32_t hdrsize, uint32_t totalsize, uint32_t off)
+{
+	return (off >= hdrsize) && (off <= totalsize);
+}
+
+static int check_block_(uint32_t hdrsize, uint32_t totalsize,
+			uint32_t base, uint32_t size)
+{
+	if (!check_off_(hdrsize, totalsize, base))
+		return 0; /* block start out of bounds */
+	if ((base + size) < base)
+		return 0; /* overflow */
+	if (!check_off_(hdrsize, totalsize, base + size))
+		return 0; /* block end out of bounds */
+	return 1;
+}
+
+size_t fdt_header_size_(uint32_t version)
+{
+	if (version <= 1)
+		return FDT_V1_SIZE;
+	else if (version <= 2)
+		return FDT_V2_SIZE;
+	else if (version <= 3)
+		return FDT_V3_SIZE;
+	else if (version <= 16)
+		return FDT_V16_SIZE;
+	else
+		return FDT_V17_SIZE;
+}
+
+size_t fdt_header_size(const void *fdt)
+{
+	return fdt_chk_version() ? fdt_header_size_(fdt_version(fdt)) :
+		FDT_V17_SIZE;
+}
+
+int fdt_check_header(const void *fdt)
+{
+	size_t hdrsize;
+
+	if (fdt_magic(fdt) != FDT_MAGIC)
+		return -FDT_ERR_BADMAGIC;
+	if (fdt_chk_version()) {
+		if ((fdt_version(fdt) < FDT_FIRST_SUPPORTED_VERSION)
+		    || (fdt_last_comp_version(fdt) >
+			FDT_LAST_SUPPORTED_VERSION))
+			return -FDT_ERR_BADVERSION;
+		if (fdt_version(fdt) < fdt_last_comp_version(fdt))
+			return -FDT_ERR_BADVERSION;
+	}
+	hdrsize = fdt_header_size(fdt);
+	if (fdt_chk_basic()) {
+
+		if ((fdt_totalsize(fdt) < hdrsize)
+		    || (fdt_totalsize(fdt) > INT_MAX))
+			return -FDT_ERR_TRUNCATED;
+
+		/* Bounds check memrsv block */
+		if (!check_off_(hdrsize, fdt_totalsize(fdt),
+				fdt_off_mem_rsvmap(fdt)))
+			return -FDT_ERR_TRUNCATED;
+	}
+
+	if (fdt_chk_extra()) {
+		/* Bounds check structure block */
+		if (fdt_chk_version() && fdt_version(fdt) < 17) {
+			if (!check_off_(hdrsize, fdt_totalsize(fdt),
+					fdt_off_dt_struct(fdt)))
+				return -FDT_ERR_TRUNCATED;
+		} else {
+			if (!check_block_(hdrsize, fdt_totalsize(fdt),
+					  fdt_off_dt_struct(fdt),
+					  fdt_size_dt_struct(fdt)))
+				return -FDT_ERR_TRUNCATED;
+		}
+
+		/* Bounds check strings block */
+		if (!check_block_(hdrsize, fdt_totalsize(fdt),
+				  fdt_off_dt_strings(fdt),
+				  fdt_size_dt_strings(fdt)))
+			return -FDT_ERR_TRUNCATED;
+	}
+
 	return 0;
 }
 
@@ -78,12 +133,13 @@ const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int len)
 {
 	unsigned absoffset = offset + fdt_off_dt_struct(fdt);
 
+	if (fdt_chk_basic())
 		if ((absoffset < offset)
 		    || ((absoffset + len) < absoffset)
 		    || (absoffset + len) > fdt_totalsize(fdt))
 			return NULL;
 
-	if (fdt_version(fdt) >= 0x11)
+	if (!fdt_chk_version() || fdt_version(fdt) >= 0x11)
 		if (((offset + len) < offset)
 		    || ((offset + len) > fdt_size_dt_struct(fdt)))
 			return NULL;
@@ -100,7 +156,7 @@ uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset)
 
 	*nextoffset = -FDT_ERR_TRUNCATED;
 	tagp = fdt_offset_ptr(fdt, offset, FDT_TAGSIZE);
-	if (!tagp)
+	if (fdt_chk_basic() && !tagp)
 		return FDT_END; /* premature end */
 	tag = fdt32_to_cpu(*tagp);
 	offset += FDT_TAGSIZE;
@@ -112,18 +168,19 @@ uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset)
 		do {
 			p = fdt_offset_ptr(fdt, offset++, 1);
 		} while (p && (*p != '\0'));
-		if (!p)
+		if (fdt_chk_basic() && !p)
 			return FDT_END; /* premature end */
 		break;
 
 	case FDT_PROP:
 		lenp = fdt_offset_ptr(fdt, offset, sizeof(*lenp));
-		if (!lenp)
+		if (fdt_chk_basic() && !lenp)
 			return FDT_END; /* premature end */
 		/* skip-name offset, length and value */
 		offset += sizeof(struct fdt_property) - FDT_TAGSIZE
 			+ fdt32_to_cpu(*lenp);
-		if (fdt_version(fdt) < 0x10 && fdt32_to_cpu(*lenp) >= 8 &&
+		if (fdt_chk_version() &&
+		    fdt_version(fdt) < 0x10 && fdt32_to_cpu(*lenp) >= 8 &&
 		    ((offset - fdt32_to_cpu(*lenp)) % 8) != 0)
 			offset += 4;
 		break;
@@ -137,7 +194,8 @@ uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset)
 		return FDT_END;
 	}
 
-	if (!fdt_offset_ptr(fdt, startoffset, offset - startoffset))
+	if (fdt_chk_basic() &&
+	    !fdt_offset_ptr(fdt, startoffset, offset - startoffset))
 		return FDT_END; /* premature end */
 
 	*nextoffset = FDT_TAGALIGN(offset);
@@ -244,7 +302,7 @@ const char *fdt_find_string_(const char *strtab, int tabsize, const char *s)
 
 int fdt_move(const void *fdt, void *buf, int bufsize)
 {
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	if (fdt_totalsize(fdt) > bufsize)
 		return -FDT_ERR_NOSPACE;

scripts/dtc/libfdt/fdt.h

@@ -1,55 +1,10 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */
 #ifndef FDT_H
 #define FDT_H
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
  * Copyright 2012 Kim Phillips, Freescale Semiconductor.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __ASSEMBLY__

scripts/dtc/libfdt/fdt_addresses.c

@@ -1,53 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2014 David Gibson <david@gibson.dropbear.id.au>
  * Copyright (C) 2018 embedded brains GmbH
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -97,3 +52,50 @@ int fdt_size_cells(const void *fdt, int nodeoffset)
 		return 1;
 	return val;
 }
+
+/* This function assumes that [address|size]_cells is 1 or 2 */
+int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset,
+			     const char *name, uint64_t addr, uint64_t size)
+{
+	int addr_cells, size_cells, ret;
+	uint8_t data[sizeof(fdt64_t) * 2], *prop;
+
+	ret = fdt_address_cells(fdt, parent);
+	if (ret < 0)
+		return ret;
+	addr_cells = ret;
+
+	ret = fdt_size_cells(fdt, parent);
+	if (ret < 0)
+		return ret;
+	size_cells = ret;
+
+	/* check validity of address */
+	prop = data;
+	if (addr_cells == 1) {
+		if ((addr > UINT32_MAX) || ((UINT32_MAX + 1 - addr) < size))
+			return -FDT_ERR_BADVALUE;
+
+		fdt32_st(prop, (uint32_t)addr);
+	} else if (addr_cells == 2) {
+		fdt64_st(prop, addr);
+	} else {
+		return -FDT_ERR_BADNCELLS;
+	}
+
+	/* check validity of size */
+	prop += addr_cells * sizeof(fdt32_t);
+	if (size_cells == 1) {
+		if (size > UINT32_MAX)
+			return -FDT_ERR_BADVALUE;
+
+		fdt32_st(prop, (uint32_t)size);
+	} else if (size_cells == 2) {
+		fdt64_st(prop, size);
+	} else {
+		return -FDT_ERR_BADNCELLS;
+	}
+
+	return fdt_appendprop(fdt, nodeoffset, name, data,
+			      (addr_cells + size_cells) * sizeof(fdt32_t));
+}

scripts/dtc/libfdt/fdt_empty_tree.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2012 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 

scripts/dtc/libfdt/fdt_overlay.c

@@ -1,53 +1,8 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2016 Free Electrons
  * Copyright (C) 2016 NextThing Co.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -93,11 +48,11 @@ static uint32_t overlay_get_target_phandle(const void *fdto, int fragment)
  * @pathp: pointer which receives the path of the target (or NULL)
  *
  * overlay_get_target() retrieves the target offset in the base
- * device tree of a fragment, no matter how the actual targetting is
+ * device tree of a fragment, no matter how the actual targeting is
  * done (through a phandle or a path)
  *
  * returns:
- *      the targetted node offset in the base device tree
+ *      the targeted node offset in the base device tree
  *      Negative error code on error
  */
 static int overlay_get_target(const void *fdt, const void *fdto,
@@ -697,7 +652,7 @@ static int get_path_len(const void *fdt, int nodeoffset)
 	int len = 0, namelen;
 	const char *name;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	for (;;) {
 		name = fdt_get_name(fdt, nodeoffset, &namelen);
@@ -778,26 +733,36 @@ static int overlay_symbol_update(void *fdt, void *fdto)
 		/* keep end marker to avoid strlen() */
 		e = path + path_len;
 
-		/* format: /<fragment-name>/__overlay__/<relative-subnode-path> */
-
 		if (*path != '/')
 			return -FDT_ERR_BADVALUE;
 
 		/* get fragment name first */
 		s = strchr(path + 1, '/');
-		if (!s)
+		if (!s) {
-			return -FDT_ERR_BADOVERLAY;
+			/* Symbol refers to something that won't end
+			 * up in the target tree */
+			continue;
+		}
 
 		frag_name = path + 1;
 		frag_name_len = s - path - 1;
 
 		/* verify format; safe since "s" lies in \0 terminated prop */
 		len = sizeof("/__overlay__/") - 1;
-		if ((e - s) < len || memcmp(s, "/__overlay__/", len))
+		if ((e - s) > len && (memcmp(s, "/__overlay__/", len) == 0)) {
-			return -FDT_ERR_BADOVERLAY;
+			/* /<fragment-name>/__overlay__/<relative-subnode-path> */
-
 			rel_path = s + len;
 			rel_path_len = e - rel_path;
+		} else if ((e - s) == len
+			   && (memcmp(s, "/__overlay__", len - 1) == 0)) {
+			/* /<fragment-name>/__overlay__ */
+			rel_path = "";
+			rel_path_len = 0;
+		} else {
+			/* Symbol refers to something that won't end
+			 * up in the target tree */
+			continue;
+		}
 
 		/* find the fragment index in which the symbol lies */
 		ret = fdt_subnode_offset_namelen(fdto, 0, frag_name,
@@ -863,11 +828,15 @@ static int overlay_symbol_update(void *fdt, void *fdto)
 
 int fdt_overlay_apply(void *fdt, void *fdto)
 {
-	uint32_t delta = fdt_get_max_phandle(fdt);
+	uint32_t delta;
 	int ret;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
-	FDT_CHECK_HEADER(fdto);
+	FDT_RO_PROBE(fdto);
+
+	ret = fdt_find_max_phandle(fdt, &delta);
+	if (ret)
+		goto err;
 
 	ret = overlay_adjust_local_phandles(fdto, delta);
 	if (ret)

scripts/dtc/libfdt/fdt_ro.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -61,7 +16,7 @@ static int fdt_nodename_eq_(const void *fdt, int offset,
 	int olen;
 	const char *p = fdt_get_name(fdt, offset, &olen);
 
-	if (!p || olen < len)
+	if (!p || (fdt_chk_extra() && olen < len))
 		/* short match */
 		return 0;
 
@@ -76,46 +31,85 @@ static int fdt_nodename_eq_(const void *fdt, int offset,
 		return 0;
 }
 
+const char *fdt_get_string(const void *fdt, int stroffset, int *lenp)
+{
+	int32_t totalsize;
+	uint32_t absoffset;
+	size_t len;
+	int err;
+	const char *s, *n;
+
+	if (!fdt_chk_extra()) {
+		s = (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset;
+
+		if (lenp)
+			*lenp = strlen(s);
+		return s;
+	}
+	totalsize = fdt_ro_probe_(fdt);
+	err = totalsize;
+	if (totalsize < 0)
+		goto fail;
+
+	err = -FDT_ERR_BADOFFSET;
+	absoffset = stroffset + fdt_off_dt_strings(fdt);
+	if (absoffset >= totalsize)
+		goto fail;
+	len = totalsize - absoffset;
+
+	if (fdt_magic(fdt) == FDT_MAGIC) {
+		if (stroffset < 0)
+			goto fail;
+		if (!fdt_chk_version() || fdt_version(fdt) >= 17) {
+			if (stroffset >= fdt_size_dt_strings(fdt))
+				goto fail;
+			if ((fdt_size_dt_strings(fdt) - stroffset) < len)
+				len = fdt_size_dt_strings(fdt) - stroffset;
+		}
+	} else if (fdt_magic(fdt) == FDT_SW_MAGIC) {
+		if ((stroffset >= 0)
+		    || (stroffset < -fdt_size_dt_strings(fdt)))
+			goto fail;
+		if ((-stroffset) < len)
+			len = -stroffset;
+	} else {
+		err = -FDT_ERR_INTERNAL;
+		goto fail;
+	}
+
+	s = (const char *)fdt + absoffset;
+	n = memchr(s, '\0', len);
+	if (!n) {
+		/* missing terminating NULL */
+		err = -FDT_ERR_TRUNCATED;
+		goto fail;
+	}
+
+	if (lenp)
+		*lenp = n - s;
+	return s;
+
+fail:
+	if (lenp)
+		*lenp = err;
+	return NULL;
+}
+
 const char *fdt_string(const void *fdt, int stroffset)
 {
-	return (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset;
+	return fdt_get_string(fdt, stroffset, NULL);
 }
 
 static int fdt_string_eq_(const void *fdt, int stroffset,
 			  const char *s, int len)
 {
-	const char *p = fdt_string(fdt, stroffset);
+	int slen;
+	const char *p = fdt_get_string(fdt, stroffset, &slen);
 
-	return (strlen(p) == len) && (memcmp(p, s, len) == 0);
+	return p && (slen == len) && (memcmp(p, s, len) == 0);
 }
 
-uint32_t fdt_get_max_phandle(const void *fdt)
+int fdt_find_max_phandle(const void *fdt, uint32_t *phandle)
-{
-	uint32_t max_phandle = 0;
-	int offset;
-
-	for (offset = fdt_next_node(fdt, -1, NULL);;
-	     offset = fdt_next_node(fdt, offset, NULL)) {
-		uint32_t phandle;
-
-		if (offset == -FDT_ERR_NOTFOUND)
-			return max_phandle;
-
-		if (offset < 0)
-			return (uint32_t)-1;
-
-		phandle = fdt_get_phandle(fdt, offset);
-		if (phandle == (uint32_t)-1)
-			continue;
-
-		if (phandle > max_phandle)
-			max_phandle = phandle;
-	}
-
-	return 0;
-}
-
-int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 {
 	uint32_t max = 0;
 	int offset = -1;
@@ -137,6 +131,21 @@ int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 			max = value;
 	}
 
+	if (phandle)
+		*phandle = max;
+
+	return 0;
+}
+
+int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
+{
+	uint32_t max;
+	int err;
+
+	err = fdt_find_max_phandle(fdt, &max);
+	if (err < 0)
+		return err;
+
 	if (max == FDT_MAX_PHANDLE)
 		return -FDT_ERR_NOPHANDLES;
 
@@ -146,22 +155,46 @@ int fdt_generate_phandle(const void *fdt, uint32_t *phandle)
 	return 0;
 }
 
+static const struct fdt_reserve_entry *fdt_mem_rsv(const void *fdt, int n)
+{
+	int offset = n * sizeof(struct fdt_reserve_entry);
+	int absoffset = fdt_off_mem_rsvmap(fdt) + offset;
+
+	if (fdt_chk_extra()) {
+		if (absoffset < fdt_off_mem_rsvmap(fdt))
+			return NULL;
+		if (absoffset > fdt_totalsize(fdt) -
+		    sizeof(struct fdt_reserve_entry))
+			return NULL;
+	}
+	return fdt_mem_rsv_(fdt, n);
+}
+
 int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size)
 {
-	FDT_CHECK_HEADER(fdt);
+	const struct fdt_reserve_entry *re;
-	*address = fdt64_to_cpu(fdt_mem_rsv_(fdt, n)->address);
+
-	*size = fdt64_to_cpu(fdt_mem_rsv_(fdt, n)->size);
+	FDT_RO_PROBE(fdt);
+	re = fdt_mem_rsv(fdt, n);
+	if (fdt_chk_extra() && !re)
+		return -FDT_ERR_BADOFFSET;
+
+	*address = fdt64_ld(&re->address);
+	*size = fdt64_ld(&re->size);
 	return 0;
 }
 
 int fdt_num_mem_rsv(const void *fdt)
 {
-	int i = 0;
+	int i;
+	const struct fdt_reserve_entry *re;
 
-	while (fdt64_to_cpu(fdt_mem_rsv_(fdt, i)->size) != 0)
+	for (i = 0; (re = fdt_mem_rsv(fdt, i)) != NULL; i++) {
-		i++;
+		if (fdt64_ld(&re->size) == 0)
 			return i;
 	}
+	return -FDT_ERR_TRUNCATED;
+}
 
 static int nextprop_(const void *fdt, int offset)
 {
@@ -192,7 +225,7 @@ int fdt_subnode_offset_namelen(const void *fdt, int offset,
 {
 	int depth;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	for (depth = 0;
 	     (offset >= 0) && (depth >= 0);
@@ -218,7 +251,7 @@ int fdt_path_offset_namelen(const void *fdt, const char *path, int namelen)
 	const char *p = path;
 	int offset = 0;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* see if we have an alias */
 	if (*path != '/') {
@@ -268,13 +301,14 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *len)
 	const char *nameptr;
 	int err;
 
-	if (((err = fdt_check_header(fdt)) != 0)
+	if (fdt_chk_extra() &&
-	    || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0))
+	    (((err = fdt_ro_probe_(fdt)) < 0)
+	     || ((err = fdt_check_node_offset_(fdt, nodeoffset)) < 0)))
 		goto fail;
 
 	nameptr = nh->name;
 
-	if (fdt_version(fdt) < 0x10) {
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
 		/*
 		 * For old FDT versions, match the naming conventions of V16:
 		 * give only the leaf name (after all /). The actual tree
@@ -325,7 +359,7 @@ static const struct fdt_property *fdt_get_property_by_offset_(const void *fdt,
 	int err;
 	const struct fdt_property *prop;
 
-	if ((err = fdt_check_prop_offset_(fdt, offset)) < 0) {
+	if (fdt_chk_basic() && (err = fdt_check_prop_offset_(fdt, offset)) < 0) {
 		if (lenp)
 			*lenp = err;
 		return NULL;
@@ -334,7 +368,7 @@ static const struct fdt_property *fdt_get_property_by_offset_(const void *fdt,
 	prop = fdt_offset_ptr_(fdt, offset);
 
 	if (lenp)
-		*lenp = fdt32_to_cpu(prop->len);
+		*lenp = fdt32_ld(&prop->len);
 
 	return prop;
 }
@@ -346,7 +380,7 @@ const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
 	/* Prior to version 16, properties may need realignment
 	 * and this API does not work. fdt_getprop_*() will, however. */
 
-	if (fdt_version(fdt) < 0x10) {
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
 		if (lenp)
 			*lenp = -FDT_ERR_BADVERSION;
 		return NULL;
@@ -367,11 +401,12 @@ static const struct fdt_property *fdt_get_property_namelen_(const void *fdt,
 	     (offset = fdt_next_property_offset(fdt, offset))) {
 		const struct fdt_property *prop;
 
-		if (!(prop = fdt_get_property_by_offset_(fdt, offset, lenp))) {
+		prop = fdt_get_property_by_offset_(fdt, offset, lenp);
+		if (fdt_chk_extra() && !prop) {
 			offset = -FDT_ERR_INTERNAL;
 			break;
 		}
-		if (fdt_string_eq_(fdt, fdt32_to_cpu(prop->nameoff),
+		if (fdt_string_eq_(fdt, fdt32_ld(&prop->nameoff),
 				   name, namelen)) {
 			if (poffset)
 				*poffset = offset;
@@ -392,7 +427,7 @@ const struct fdt_property *fdt_get_property_namelen(const void *fdt,
 {
 	/* Prior to version 16, properties may need realignment
 	 * and this API does not work. fdt_getprop_*() will, however. */
-	if (fdt_version(fdt) < 0x10) {
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10) {
 		if (lenp)
 			*lenp = -FDT_ERR_BADVERSION;
 		return NULL;
@@ -423,8 +458,8 @@ const void *fdt_getprop_namelen(const void *fdt, int nodeoffset,
 		return NULL;
 
 	/* Handle realignment */
-	if (fdt_version(fdt) < 0x10 && (poffset + sizeof(*prop)) % 8 &&
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10 &&
-	    fdt32_to_cpu(prop->len) >= 8)
+	    (poffset + sizeof(*prop)) % 8 && fdt32_ld(&prop->len) >= 8)
 		return prop->data + 4;
 	return prop->data;
 }
@@ -437,12 +472,27 @@ const void *fdt_getprop_by_offset(const void *fdt, int offset,
 	prop = fdt_get_property_by_offset_(fdt, offset, lenp);
 	if (!prop)
 		return NULL;
-	if (namep)
+	if (namep) {
-		*namep = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
+		const char *name;
+		int namelen;
+
+		if (fdt_chk_extra()) {
+			name = fdt_get_string(fdt, fdt32_ld(&prop->nameoff),
+					      &namelen);
+			if (!name) {
+				if (lenp)
+					*lenp = namelen;
+				return NULL;
+			}
+			*namep = name;
+		} else {
+			*namep = fdt_string(fdt, fdt32_ld(&prop->nameoff));
+		}
+	}
 
 	/* Handle realignment */
-	if (fdt_version(fdt) < 0x10 && (offset + sizeof(*prop)) % 8 &&
+	if (fdt_chk_version() && fdt_version(fdt) < 0x10 &&
-	    fdt32_to_cpu(prop->len) >= 8)
+	    (offset + sizeof(*prop)) % 8 && fdt32_ld(&prop->len) >= 8)
 		return prop->data + 4;
 	return prop->data;
 }
@@ -467,7 +517,7 @@ uint32_t fdt_get_phandle(const void *fdt, int nodeoffset)
 			return 0;
 	}
 
-	return fdt32_to_cpu(*php);
+	return fdt32_ld(php);
 }
 
 const char *fdt_get_alias_namelen(const void *fdt,
@@ -493,7 +543,7 @@ int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 	int offset, depth, namelen;
 	const char *name;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	if (buflen < 2)
 		return -FDT_ERR_NOSPACE;
@@ -545,7 +595,7 @@ int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset,
 	int offset, depth;
 	int supernodeoffset = -FDT_ERR_INTERNAL;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	if (supernodedepth < 0)
 		return -FDT_ERR_NOTFOUND;
@@ -567,10 +617,12 @@ int fdt_supernode_atdepth_offset(const void *fdt, int nodeoffset,
 		}
 	}
 
+	if (fdt_chk_extra()) {
 		if ((offset == -FDT_ERR_NOTFOUND) || (offset >= 0))
 			return -FDT_ERR_BADOFFSET;
 		else if (offset == -FDT_ERR_BADOFFSET)
 			return -FDT_ERR_BADSTRUCTURE;
+	}
 
 	return offset; /* error from fdt_next_node() */
 }
@@ -582,7 +634,7 @@ int fdt_node_depth(const void *fdt, int nodeoffset)
 
 	err = fdt_supernode_atdepth_offset(fdt, nodeoffset, 0, &nodedepth);
 	if (err)
-		return (err < 0) ? err : -FDT_ERR_INTERNAL;
+		return (!fdt_chk_extra() || err < 0) ? err : -FDT_ERR_INTERNAL;
 	return nodedepth;
 }
 
@@ -604,7 +656,7 @@ int fdt_node_offset_by_prop_value(const void *fdt, int startoffset,
 	const void *val;
 	int len;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we scan each
 	 * property of a node in fdt_getprop(), then if that didn't
@@ -630,7 +682,7 @@ int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle)
 	if ((phandle == 0) || (phandle == -1))
 		return -FDT_ERR_BADPHANDLE;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we
 	 * potentially scan each property of a node in
@@ -783,7 +835,7 @@ int fdt_node_offset_by_compatible(const void *fdt, int startoffset,
 {
 	int offset, err;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	/* FIXME: The algorithm here is pretty horrible: we scan each
 	 * property of a node in fdt_node_check_compatible(), then if
@@ -802,3 +854,68 @@ int fdt_node_offset_by_compatible(const void *fdt, int startoffset,
 
 	return offset; /* error from fdt_next_node() */
 }
+
+#if !defined(FDT_ASSUME_MASK) || FDT_ASSUME_MASK != 0xff
+int fdt_check_full(const void *fdt, size_t bufsize)
+{
+	int err;
+	int num_memrsv;
+	int offset, nextoffset = 0;
+	uint32_t tag;
+	unsigned depth = 0;
+	const void *prop;
+	const char *propname;
+
+	if (bufsize < FDT_V1_SIZE)
+		return -FDT_ERR_TRUNCATED;
+	err = fdt_check_header(fdt);
+	if (err != 0)
+		return err;
+	if (bufsize < fdt_totalsize(fdt))
+		return -FDT_ERR_TRUNCATED;
+
+	num_memrsv = fdt_num_mem_rsv(fdt);
+	if (num_memrsv < 0)
+		return num_memrsv;
+
+	while (1) {
+		offset = nextoffset;
+		tag = fdt_next_tag(fdt, offset, &nextoffset);
+
+		if (nextoffset < 0)
+			return nextoffset;
+
+		switch (tag) {
+		case FDT_NOP:
+			break;
+
+		case FDT_END:
+			if (depth != 0)
+				return -FDT_ERR_BADSTRUCTURE;
+			return 0;
+
+		case FDT_BEGIN_NODE:
+			depth++;
+			if (depth > INT_MAX)
+				return -FDT_ERR_BADSTRUCTURE;
+			break;
+
+		case FDT_END_NODE:
+			if (depth == 0)
+				return -FDT_ERR_BADSTRUCTURE;
+			depth--;
+			break;
+
+		case FDT_PROP:
+			prop = fdt_getprop_by_offset(fdt, offset, &propname,
+						     &err);
+			if (!prop)
+				return err;
+			break;
+
+		default:
+			return -FDT_ERR_INTERNAL;
+		}
+	}
+}
+#endif

scripts/dtc/libfdt/fdt_rw.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -58,6 +13,8 @@
 static int fdt_blocks_misordered_(const void *fdt,
 				  int mem_rsv_size, int struct_size)
 {
+	if (!fdt_chk_basic())
+		return false;
 	return (fdt_off_mem_rsvmap(fdt) < FDT_ALIGN(sizeof(struct fdt_header), 8))
 		|| (fdt_off_dt_struct(fdt) <
 		    (fdt_off_mem_rsvmap(fdt) + mem_rsv_size))
@@ -67,25 +24,27 @@ static int fdt_blocks_misordered_(const void *fdt,
 		    (fdt_off_dt_strings(fdt) + fdt_size_dt_strings(fdt)));
 }
 
-static int fdt_rw_check_header_(void *fdt)
+static int fdt_rw_probe_(void *fdt)
 {
-	FDT_CHECK_HEADER(fdt);
+	if (!fdt_chk_basic())
+		return 0;
+	FDT_RO_PROBE(fdt);
 
-	if (fdt_version(fdt) < 17)
+	if (fdt_chk_version() && fdt_version(fdt) < 17)
 		return -FDT_ERR_BADVERSION;
 	if (fdt_blocks_misordered_(fdt, sizeof(struct fdt_reserve_entry),
 				   fdt_size_dt_struct(fdt)))
 		return -FDT_ERR_BADLAYOUT;
-	if (fdt_version(fdt) > 17)
+	if (fdt_chk_version() && fdt_version(fdt) > 17)
 		fdt_set_version(fdt, 17);
 
 	return 0;
 }
 
-#define FDT_RW_CHECK_HEADER(fdt) \
+#define FDT_RW_PROBE(fdt) \
 	{ \
 		int err_; \
-		if ((err_ = fdt_rw_check_header_(fdt)) != 0) \
+		if (fdt_chk_extra() && (err_ = fdt_rw_probe_(fdt)) != 0) \
 			return err_; \
 	}
 
@@ -136,6 +95,14 @@ static int fdt_splice_struct_(void *fdt, void *p,
 	return 0;
 }
 
+/* Must only be used to roll back in case of error */
+static void fdt_del_last_string_(void *fdt, const char *s)
+{
+	int newlen = strlen(s) + 1;
+
+	fdt_set_size_dt_strings(fdt, fdt_size_dt_strings(fdt) - newlen);
+}
+
 static int fdt_splice_string_(void *fdt, int newlen)
 {
 	void *p = (char *)fdt
@@ -149,7 +116,16 @@ static int fdt_splice_string_(void *fdt, int newlen)
 	return 0;
 }
 
-static int fdt_find_add_string_(void *fdt, const char *s)
+/**
+ * fdt_find_add_string_() - Find or allocate a string
+ *
+ * @fdt: pointer to the device tree to check/adjust
+ * @s: string to find/add
+ * @allocated: Set to 0 if the string was found, 1 if not found and so
+ *	allocated. Ignored if !fdt_chk_basic()
+ * @return offset of string in the string table (whether found or added)
+ */
+static int fdt_find_add_string_(void *fdt, const char *s, int *allocated)
 {
 	char *strtab = (char *)fdt + fdt_off_dt_strings(fdt);
 	const char *p;
@@ -157,6 +133,9 @@ static int fdt_find_add_string_(void *fdt, const char *s)
 	int len = strlen(s) + 1;
 	int err;
 
+	if (fdt_chk_basic())
+		*allocated = 0;
+
 	p = fdt_find_string_(strtab, fdt_size_dt_strings(fdt), s);
 	if (p)
 		/* found it */
@@ -167,6 +146,9 @@ static int fdt_find_add_string_(void *fdt, const char *s)
 	if (err)
 		return err;
 
+	if (fdt_chk_basic())
+		*allocated = 1;
+
 	memcpy(new, s, len);
 	return (new - strtab);
 }
@@ -176,7 +158,7 @@ int fdt_add_mem_rsv(void *fdt, uint64_t address, uint64_t size)
 	struct fdt_reserve_entry *re;
 	int err;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	re = fdt_mem_rsv_w_(fdt, fdt_num_mem_rsv(fdt));
 	err = fdt_splice_mem_rsv_(fdt, re, 0, 1);
@@ -192,7 +174,7 @@ int fdt_del_mem_rsv(void *fdt, int n)
 {
 	struct fdt_reserve_entry *re = fdt_mem_rsv_w_(fdt, n);
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	if (n >= fdt_num_mem_rsv(fdt))
 		return -FDT_ERR_NOTFOUND;
@@ -225,11 +207,12 @@ static int fdt_add_property_(void *fdt, int nodeoffset, const char *name,
 	int nextoffset;
 	int namestroff;
 	int err;
+	int allocated;
 
 	if ((nextoffset = fdt_check_node_offset_(fdt, nodeoffset)) < 0)
 		return nextoffset;
 
-	namestroff = fdt_find_add_string_(fdt, name);
+	namestroff = fdt_find_add_string_(fdt, name, &allocated);
 	if (namestroff < 0)
 		return namestroff;
 
@@ -237,8 +220,12 @@ static int fdt_add_property_(void *fdt, int nodeoffset, const char *name,
 	proplen = sizeof(**prop) + FDT_TAGALIGN(len);
 
 	err = fdt_splice_struct_(fdt, *prop, 0, proplen);
-	if (err)
+	if (err) {
+		/* Delete the string if we failed to add it */
+		if (fdt_chk_basic() && allocated)
+			fdt_del_last_string_(fdt, name);
 		return err;
+	}
 
 	(*prop)->tag = cpu_to_fdt32(FDT_PROP);
 	(*prop)->nameoff = cpu_to_fdt32(namestroff);
@@ -252,7 +239,7 @@ int fdt_set_name(void *fdt, int nodeoffset, const char *name)
 	int oldlen, newlen;
 	int err;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	namep = (char *)(uintptr_t)fdt_get_name(fdt, nodeoffset, &oldlen);
 	if (!namep)
@@ -275,7 +262,7 @@ int fdt_setprop_placeholder(void *fdt, int nodeoffset, const char *name,
 	struct fdt_property *prop;
 	int err;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	err = fdt_resize_property_(fdt, nodeoffset, name, len, &prop);
 	if (err == -FDT_ERR_NOTFOUND)
@@ -308,7 +295,7 @@ int fdt_appendprop(void *fdt, int nodeoffset, const char *name,
 	struct fdt_property *prop;
 	int err, oldlen, newlen;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen);
 	if (prop) {
@@ -334,7 +321,7 @@ int fdt_delprop(void *fdt, int nodeoffset, const char *name)
 	struct fdt_property *prop;
 	int len, proplen;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	prop = fdt_get_property_w(fdt, nodeoffset, name, &len);
 	if (!prop)
@@ -354,7 +341,7 @@ int fdt_add_subnode_namelen(void *fdt, int parentoffset,
 	uint32_t tag;
 	fdt32_t *endtag;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	offset = fdt_subnode_offset_namelen(fdt, parentoffset, name, namelen);
 	if (offset >= 0)
@@ -394,7 +381,7 @@ int fdt_del_node(void *fdt, int nodeoffset)
 {
 	int endoffset;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	endoffset = fdt_node_end_offset_(fdt, nodeoffset);
 	if (endoffset < 0)
@@ -435,12 +422,12 @@ int fdt_open_into(const void *fdt, void *buf, int bufsize)
 	const char *fdtend = fdtstart + fdt_totalsize(fdt);
 	char *tmp;
 
-	FDT_CHECK_HEADER(fdt);
+	FDT_RO_PROBE(fdt);
 
 	mem_rsv_size = (fdt_num_mem_rsv(fdt)+1)
 		* sizeof(struct fdt_reserve_entry);
 
-	if (fdt_version(fdt) >= 17) {
+	if (!fdt_chk_version() || fdt_version(fdt) >= 17) {
 		struct_size = fdt_size_dt_struct(fdt);
 	} else {
 		struct_size = 0;
@@ -494,7 +481,7 @@ int fdt_pack(void *fdt)
 {
 	int mem_rsv_size;
 
-	FDT_RW_CHECK_HEADER(fdt);
+	FDT_RW_PROBE(fdt);
 
 	mem_rsv_size = (fdt_num_mem_rsv(fdt)+1)
 		* sizeof(struct fdt_reserve_entry);

scripts/dtc/libfdt/fdt_strerror.c

@@ -1,51 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
@@ -82,6 +38,7 @@ static struct fdt_errtabent fdt_errtable[] = {
 	FDT_ERRTABENT(FDT_ERR_BADVALUE),
 	FDT_ERRTABENT(FDT_ERR_BADOVERLAY),
 	FDT_ERRTABENT(FDT_ERR_NOPHANDLES),
+	FDT_ERRTABENT(FDT_ERR_BADFLAGS),
 };
 #define FDT_ERRTABSIZE	(sizeof(fdt_errtable) / sizeof(fdt_errtable[0]))
 

scripts/dtc/libfdt/fdt_sw.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 
@@ -55,21 +10,90 @@
 
 #include "libfdt_internal.h"
 
-static int fdt_sw_check_header_(void *fdt)
+static int fdt_sw_probe_(void *fdt)
 {
-	if (fdt_magic(fdt) != FDT_SW_MAGIC)
+	if (fdt_chk_basic()) {
+		if (fdt_magic(fdt) == FDT_MAGIC)
+			return -FDT_ERR_BADSTATE;
+		else if (fdt_magic(fdt) != FDT_SW_MAGIC)
 			return -FDT_ERR_BADMAGIC;
-	/* FIXME: should check more details about the header state */
+	}
+
 	return 0;
 }
 
-#define FDT_SW_CHECK_HEADER(fdt) \
+#define FDT_SW_PROBE(fdt) \
 	{ \
 		int err; \
-		if ((err = fdt_sw_check_header_(fdt)) != 0) \
+		if (fdt_chk_basic() && (err = fdt_sw_probe_(fdt)) != 0) \
 			return err; \
 	}
 
+/* 'memrsv' state:	Initial state after fdt_create()
+ *
+ * Allowed functions:
+ *	fdt_add_reservmap_entry()
+ *	fdt_finish_reservemap()		[moves to 'struct' state]
+ */
+static int fdt_sw_probe_memrsv_(void *fdt)
+{
+	int err = fdt_sw_probe_(fdt);
+	if (err)
+		return err;
+
+	if (fdt_chk_extra() && fdt_off_dt_strings(fdt) != 0)
+		return -FDT_ERR_BADSTATE;
+	return 0;
+}
+
+#define FDT_SW_PROBE_MEMRSV(fdt) \
+	{ \
+		int err; \
+		if (fdt_chk_extra() && (err = fdt_sw_probe_memrsv_(fdt)) != 0) \
+			return err; \
+	}
+
+/* 'struct' state:	Enter this state after fdt_finish_reservemap()
+ *
+ * Allowed functions:
+ *	fdt_begin_node()
+ *	fdt_end_node()
+ *	fdt_property*()
+ *	fdt_finish()			[moves to 'complete' state]
+ */
+static int fdt_sw_probe_struct_(void *fdt)
+{
+	int err;
+
+	if (!fdt_chk_extra())
+		return 0;
+	err = fdt_sw_probe_(fdt);
+	if (err)
+		return err;
+
+	if (fdt_off_dt_strings(fdt) != fdt_totalsize(fdt))
+		return -FDT_ERR_BADSTATE;
+	return 0;
+}
+
+#define FDT_SW_PROBE_STRUCT(fdt) \
+	{ \
+		int err; \
+		if (fdt_chk_extra() && (err = fdt_sw_probe_struct_(fdt)) != 0) \
+			return err; \
+	}
+
+static inline uint32_t sw_flags(void *fdt)
+{
+	/* assert: (fdt_magic(fdt) == FDT_SW_MAGIC) */
+	return fdt_last_comp_version(fdt);
+}
+
+/* 'complete' state:	Enter this state after fdt_finish()
+ *
+ * Allowed functions: none
+ */
+
 static void *fdt_grab_space_(void *fdt, size_t len)
 {
 	int offset = fdt_size_dt_struct(fdt);
@@ -85,38 +109,58 @@ static void *fdt_grab_space_(void *fdt, size_t len)
 	return fdt_offset_ptr_w_(fdt, offset);
 }
 
-int fdt_create(void *buf, int bufsize)
+int fdt_create_with_flags(void *buf, int bufsize, uint32_t flags)
 {
+	const size_t hdrsize = FDT_ALIGN(sizeof(struct fdt_header),
+					 sizeof(struct fdt_reserve_entry));
 	void *fdt = buf;
 
-	if (bufsize < sizeof(struct fdt_header))
+	if (bufsize < hdrsize)
 		return -FDT_ERR_NOSPACE;
 
+	if (flags & ~FDT_CREATE_FLAGS_ALL)
+		return -FDT_ERR_BADFLAGS;
+
 	memset(buf, 0, bufsize);
 
+	/*
+	 * magic and last_comp_version keep intermediate state during the fdt
+	 * creation process, which is replaced with the proper FDT format by
+	 * fdt_finish().
+	 *
+	 * flags should be accessed with sw_flags().
+	 */
 	fdt_set_magic(fdt, FDT_SW_MAGIC);
 	fdt_set_version(fdt, FDT_LAST_SUPPORTED_VERSION);
-	fdt_set_last_comp_version(fdt, FDT_FIRST_SUPPORTED_VERSION);
+	fdt_set_last_comp_version(fdt, flags);
+
 	fdt_set_totalsize(fdt,  bufsize);
 
-	fdt_set_off_mem_rsvmap(fdt, FDT_ALIGN(sizeof(struct fdt_header),
+	fdt_set_off_mem_rsvmap(fdt, hdrsize);
-					      sizeof(struct fdt_reserve_entry)));
 	fdt_set_off_dt_struct(fdt, fdt_off_mem_rsvmap(fdt));
-	fdt_set_off_dt_strings(fdt, bufsize);
+	fdt_set_off_dt_strings(fdt, 0);
 
 	return 0;
 }
 
+int fdt_create(void *buf, int bufsize)
+{
+	return fdt_create_with_flags(buf, bufsize, 0);
+}
+
 int fdt_resize(void *fdt, void *buf, int bufsize)
 {
 	size_t headsize, tailsize;
 	char *oldtail, *newtail;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE(fdt);
 
 	headsize = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt);
 	tailsize = fdt_size_dt_strings(fdt);
 
+	if (fdt_chk_extra() && (headsize + tailsize) > fdt_totalsize(fdt))
+		return -FDT_ERR_INTERNAL;
+
 	if ((headsize + tailsize) > bufsize)
 		return -FDT_ERR_NOSPACE;
 
@@ -133,8 +177,9 @@ int fdt_resize(void *fdt, void *buf, int bufsize)
 		memmove(buf, fdt, headsize);
 	}
 
-	fdt_set_off_dt_strings(buf, bufsize);
 	fdt_set_totalsize(buf, bufsize);
+	if (fdt_off_dt_strings(buf))
+		fdt_set_off_dt_strings(buf, bufsize);
 
 	return 0;
 }
@@ -144,10 +189,7 @@ int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size)
 	struct fdt_reserve_entry *re;
 	int offset;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE_MEMRSV(fdt);
-
-	if (fdt_size_dt_struct(fdt))
-		return -FDT_ERR_BADSTATE;
 
 	offset = fdt_off_dt_struct(fdt);
 	if ((offset + sizeof(*re)) > fdt_totalsize(fdt))
@@ -164,16 +206,23 @@ int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size)
 
 int fdt_finish_reservemap(void *fdt)
 {
-	return fdt_add_reservemap_entry(fdt, 0, 0);
+	int err = fdt_add_reservemap_entry(fdt, 0, 0);
+
+	if (err)
+		return err;
+
+	fdt_set_off_dt_strings(fdt, fdt_totalsize(fdt));
+	return 0;
 }
 
 int fdt_begin_node(void *fdt, const char *name)
 {
 	struct fdt_node_header *nh;
-	int namelen = strlen(name) + 1;
+	int namelen;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE_STRUCT(fdt);
 
+	namelen = strlen(name) + 1;
 	nh = fdt_grab_space_(fdt, sizeof(*nh) + FDT_TAGALIGN(namelen));
 	if (! nh)
 		return -FDT_ERR_NOSPACE;
@@ -187,7 +236,7 @@ int fdt_end_node(void *fdt)
 {
 	fdt32_t *en;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE_STRUCT(fdt);
 
 	en = fdt_grab_space_(fdt, FDT_TAGSIZE);
 	if (! en)
@@ -197,19 +246,13 @@ int fdt_end_node(void *fdt)
 	return 0;
 }
 
-static int fdt_find_add_string_(void *fdt, const char *s)
+static int fdt_add_string_(void *fdt, const char *s)
 {
 	char *strtab = (char *)fdt + fdt_totalsize(fdt);
-	const char *p;
 	int strtabsize = fdt_size_dt_strings(fdt);
 	int len = strlen(s) + 1;
 	int struct_top, offset;
 
-	p = fdt_find_string_(strtab - strtabsize, strtabsize, s);
-	if (p)
-		return p - strtab;
-
-	/* Add it */
 	offset = -strtabsize - len;
 	struct_top = fdt_off_dt_struct(fdt) + fdt_size_dt_struct(fdt);
 	if (fdt_totalsize(fdt) + offset < struct_top)
@@ -220,20 +263,56 @@ static int fdt_find_add_string_(void *fdt, const char *s)
 	return offset;
 }
 
+/* Must only be used to roll back in case of error */
+static void fdt_del_last_string_(void *fdt, const char *s)
+{
+	int strtabsize = fdt_size_dt_strings(fdt);
+	int len = strlen(s) + 1;
+
+	fdt_set_size_dt_strings(fdt, strtabsize - len);
+}
+
+static int fdt_find_add_string_(void *fdt, const char *s, int *allocated)
+{
+	char *strtab = (char *)fdt + fdt_totalsize(fdt);
+	int strtabsize = fdt_size_dt_strings(fdt);
+	const char *p;
+
+	*allocated = 0;
+
+	p = fdt_find_string_(strtab - strtabsize, strtabsize, s);
+	if (p)
+		return p - strtab;
+
+	*allocated = 1;
+
+	return fdt_add_string_(fdt, s);
+}
+
 int fdt_property_placeholder(void *fdt, const char *name, int len, void **valp)
 {
 	struct fdt_property *prop;
 	int nameoff;
+	int allocated;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE_STRUCT(fdt);
 
-	nameoff = fdt_find_add_string_(fdt, name);
+	/* String de-duplication can be slow, _NO_NAME_DEDUP skips it */
+	if (sw_flags(fdt) & FDT_CREATE_FLAG_NO_NAME_DEDUP) {
+		allocated = 1;
+		nameoff = fdt_add_string_(fdt, name);
+	} else {
+		nameoff = fdt_find_add_string_(fdt, name, &allocated);
+	}
 	if (nameoff == 0)
 		return -FDT_ERR_NOSPACE;
 
 	prop = fdt_grab_space_(fdt, sizeof(*prop) + FDT_TAGALIGN(len));
-	if (! prop)
+	if (! prop) {
+		if (allocated)
+			fdt_del_last_string_(fdt, name);
 		return -FDT_ERR_NOSPACE;
+	}
 
 	prop->tag = cpu_to_fdt32(FDT_PROP);
 	prop->nameoff = cpu_to_fdt32(nameoff);
@@ -262,7 +341,7 @@ int fdt_finish(void *fdt)
 	uint32_t tag;
 	int offset, nextoffset;
 
-	FDT_SW_CHECK_HEADER(fdt);
+	FDT_SW_PROBE_STRUCT(fdt);
 
 	/* Add terminator */
 	end = fdt_grab_space_(fdt, sizeof(*end));
@@ -295,6 +374,10 @@ int fdt_finish(void *fdt)
 
 	/* Finally, adjust the header */
 	fdt_set_totalsize(fdt, newstroffset + fdt_size_dt_strings(fdt));
+
+	/* And fix up fields that were keeping intermediate state. */
+	fdt_set_last_comp_version(fdt, FDT_FIRST_SUPPORTED_VERSION);
 	fdt_set_magic(fdt, FDT_MAGIC);
+
 	return 0;
 }

scripts/dtc/libfdt/fdt_wip.c

@@ -1,52 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause)
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "libfdt_env.h"
 

scripts/dtc/libfdt/libfdt.h

@@ -1,54 +1,9 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */
 #ifndef LIBFDT_H
 #define LIBFDT_H
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "libfdt_env.h"
@@ -90,8 +45,9 @@
 
 /* Error codes: codes for bad device tree blobs */
 #define FDT_ERR_TRUNCATED	8
-	/* FDT_ERR_TRUNCATED: Structure block of the given device tree
+	/* FDT_ERR_TRUNCATED: FDT or a sub-block is improperly
-	 * ends without an FDT_END tag. */
+	 * terminated (overflows, goes outside allowed bounds, or
+	 * isn't properly terminated).  */
 #define FDT_ERR_BADMAGIC	9
 	/* FDT_ERR_BADMAGIC: Given "device tree" appears not to be a
 	 * device tree at all - it is missing the flattened device
@@ -137,7 +93,11 @@
 	/* FDT_ERR_NOPHANDLES: The device tree doesn't have any
 	 * phandle available anymore without causing an overflow */
 
-#define FDT_ERR_MAX		17
+#define FDT_ERR_BADFLAGS	18
+	/* FDT_ERR_BADFLAGS: The function was passed a flags field that
+	 * contains invalid flags or an invalid combination of flags. */
+
+#define FDT_ERR_MAX		18
 
 /* constants */
 #define FDT_MAX_PHANDLE 0xfffffffe
@@ -157,6 +117,61 @@ static inline void *fdt_offset_ptr_w(void *fdt, int offset, int checklen)
 
 uint32_t fdt_next_tag(const void *fdt, int offset, int *nextoffset);
 
+/*
+ * Alignment helpers:
+ *     These helpers access words from a device tree blob.  They're
+ *     built to work even with unaligned pointers on platforms (ike
+ *     ARM) that don't like unaligned loads and stores
+ */
+
+static inline uint32_t fdt32_ld(const fdt32_t *p)
+{
+	const uint8_t *bp = (const uint8_t *)p;
+
+	return ((uint32_t)bp[0] << 24)
+		| ((uint32_t)bp[1] << 16)
+		| ((uint32_t)bp[2] << 8)
+		| bp[3];
+}
+
+static inline void fdt32_st(void *property, uint32_t value)
+{
+	uint8_t *bp = (uint8_t *)property;
+
+	bp[0] = value >> 24;
+	bp[1] = (value >> 16) & 0xff;
+	bp[2] = (value >> 8) & 0xff;
+	bp[3] = value & 0xff;
+}
+
+static inline uint64_t fdt64_ld(const fdt64_t *p)
+{
+	const uint8_t *bp = (const uint8_t *)p;
+
+	return ((uint64_t)bp[0] << 56)
+		| ((uint64_t)bp[1] << 48)
+		| ((uint64_t)bp[2] << 40)
+		| ((uint64_t)bp[3] << 32)
+		| ((uint64_t)bp[4] << 24)
+		| ((uint64_t)bp[5] << 16)
+		| ((uint64_t)bp[6] << 8)
+		| bp[7];
+}
+
+static inline void fdt64_st(void *property, uint64_t value)
+{
+	uint8_t *bp = (uint8_t *)property;
+
+	bp[0] = value >> 56;
+	bp[1] = (value >> 48) & 0xff;
+	bp[2] = (value >> 40) & 0xff;
+	bp[3] = (value >> 32) & 0xff;
+	bp[4] = (value >> 24) & 0xff;
+	bp[5] = (value >> 16) & 0xff;
+	bp[6] = (value >> 8) & 0xff;
+	bp[7] = value & 0xff;
+}
+
 /**********************************************************************/
 /* Traversal functions                                                */
 /**********************************************************************/
@@ -199,7 +214,7 @@ int fdt_next_subnode(const void *fdt, int offset);
  *		...
  *	}
  *
- *	if ((node < 0) && (node != -FDT_ERR_NOT_FOUND)) {
+ *	if ((node < 0) && (node != -FDT_ERR_NOTFOUND)) {
  *		Error handling
  *	}
  *
@@ -217,7 +232,7 @@ int fdt_next_subnode(const void *fdt, int offset);
 /* General functions                                                  */
 /**********************************************************************/
 #define fdt_get_header(fdt, field) \
-	(fdt32_to_cpu(((const struct fdt_header *)(fdt))->field))
+	(fdt32_ld(&((const struct fdt_header *)(fdt))->field))
 #define fdt_magic(fdt)			(fdt_get_header(fdt, magic))
 #define fdt_totalsize(fdt)		(fdt_get_header(fdt, totalsize))
 #define fdt_off_dt_struct(fdt)		(fdt_get_header(fdt, off_dt_struct))
@@ -248,18 +263,32 @@ fdt_set_hdr_(size_dt_struct);
 #undef fdt_set_hdr_
 
 /**
- * fdt_check_header - sanity check a device tree or possible device tree
+ * fdt_header_size - return the size of the tree's header
+ * @fdt: pointer to a flattened device tree
+ */
+size_t fdt_header_size(const void *fdt);
+
+/**
+ * fdt_header_size_ - internal function which takes a version number
+ */
+size_t fdt_header_size_(uint32_t version);
+
+/**
+ * fdt_check_header - sanity check a device tree header
+
  * @fdt: pointer to data which might be a flattened device tree
  *
  * fdt_check_header() checks that the given buffer contains what
- * appears to be a flattened device tree with sane information in its
+ * appears to be a flattened device tree, and that the header contains
- * header.
+ * valid information (to the extent that can be determined from the
+ * header alone).
  *
  * returns:
  *     0, if the buffer appears to contain a valid device tree
  *     -FDT_ERR_BADMAGIC,
  *     -FDT_ERR_BADVERSION,
- *     -FDT_ERR_BADSTATE, standard meanings, as above
+ *     -FDT_ERR_BADSTATE,
+ *     -FDT_ERR_TRUNCATED, standard meanings, as above
  */
 int fdt_check_header(const void *fdt);
 
@@ -288,6 +317,24 @@ int fdt_move(const void *fdt, void *buf, int bufsize);
 /* Read-only functions                                                */
 /**********************************************************************/
 
+int fdt_check_full(const void *fdt, size_t bufsize);
+
+/**
+ * fdt_get_string - retrieve a string from the strings block of a device tree
+ * @fdt: pointer to the device tree blob
+ * @stroffset: offset of the string within the strings block (native endian)
+ * @lenp: optional pointer to return the string's length
+ *
+ * fdt_get_string() retrieves a pointer to a single string from the
+ * strings block of the device tree blob at fdt, and optionally also
+ * returns the string's length in *lenp.
+ *
+ * returns:
+ *     a pointer to the string, on success
+ *     NULL, if stroffset is out of bounds, or doesn't point to a valid string
+ */
+const char *fdt_get_string(const void *fdt, int stroffset, int *lenp);
+
 /**
  * fdt_string - retrieve a string from the strings block of a device tree
  * @fdt: pointer to the device tree blob
@@ -298,10 +345,24 @@ int fdt_move(const void *fdt, void *buf, int bufsize);
  *
  * returns:
  *     a pointer to the string, on success
- *     NULL, if stroffset is out of bounds
+ *     NULL, if stroffset is out of bounds, or doesn't point to a valid string
  */
 const char *fdt_string(const void *fdt, int stroffset);
 
+/**
+ * fdt_find_max_phandle - find and return the highest phandle in a tree
+ * @fdt: pointer to the device tree blob
+ * @phandle: return location for the highest phandle value found in the tree
+ *
+ * fdt_find_max_phandle() finds the highest phandle value in the given device
+ * tree. The value returned in @phandle is only valid if the function returns
+ * success.
+ *
+ * returns:
+ *     0 on success or a negative error code on failure
+ */
+int fdt_find_max_phandle(const void *fdt, uint32_t *phandle);
+
 /**
  * fdt_get_max_phandle - retrieves the highest phandle in a tree
  * @fdt: pointer to the device tree blob
@@ -310,12 +371,24 @@ const char *fdt_string(const void *fdt, int stroffset);
  * device tree. This will ignore badly formatted phandles, or phandles
  * with a value of 0 or -1.
  *
+ * This function is deprecated in favour of fdt_find_max_phandle().
+ *
  * returns:
  *      the highest phandle on success
  *      0, if no phandle was found in the device tree
  *      -1, if an error occurred
  */
-uint32_t fdt_get_max_phandle(const void *fdt);
+static inline uint32_t fdt_get_max_phandle(const void *fdt)
+{
+	uint32_t phandle;
+	int err;
+
+	err = fdt_find_max_phandle(fdt, &phandle);
+	if (err < 0)
+		return (uint32_t)-1;
+
+	return phandle;
+}
 
 /**
  * fdt_generate_phandle - return a new, unused phandle for a device tree blob
@@ -522,7 +595,7 @@ int fdt_next_property_offset(const void *fdt, int offset);
  *		...
  *	}
  *
- *	if ((property < 0) && (property != -FDT_ERR_NOT_FOUND)) {
+ *	if ((property < 0) && (property != -FDT_ERR_NOTFOUND)) {
  *		Error handling
  *	}
  *
@@ -625,7 +698,7 @@ static inline struct fdt_property *fdt_get_property_w(void *fdt, int nodeoffset,
 /**
  * fdt_getprop_by_offset - retrieve the value of a property at a given offset
  * @fdt: pointer to the device tree blob
- * @ffset: offset of the property to read
+ * @offset: offset of the property to read
  * @namep: pointer to a string variable (will be overwritten) or NULL
  * @lenp: pointer to an integer variable (will be overwritten) or NULL
  *
@@ -734,7 +807,7 @@ uint32_t fdt_get_phandle(const void *fdt, int nodeoffset);
 /**
  * fdt_get_alias_namelen - get alias based on substring
  * @fdt: pointer to the device tree blob
- * @name: name of the alias to look up
+ * @name: name of the alias th look up
  * @namelen: number of characters of name to consider
  *
  * Identical to fdt_get_alias(), but only examine the first namelen
@@ -1316,7 +1389,45 @@ int fdt_nop_node(void *fdt, int nodeoffset);
 /* Sequential write functions                                         */
 /**********************************************************************/
 
+/* fdt_create_with_flags flags */
+#define FDT_CREATE_FLAG_NO_NAME_DEDUP 0x1
+	/* FDT_CREATE_FLAG_NO_NAME_DEDUP: Do not try to de-duplicate property
+	 * names in the fdt. This can result in faster creation times, but
+	 * a larger fdt. */
+
+#define FDT_CREATE_FLAGS_ALL	(FDT_CREATE_FLAG_NO_NAME_DEDUP)
+
+/**
+ * fdt_create_with_flags - begin creation of a new fdt
+ * @fdt: pointer to memory allocated where fdt will be created
+ * @bufsize: size of the memory space at fdt
+ * @flags: a valid combination of FDT_CREATE_FLAG_ flags, or 0.
+ *
+ * fdt_create_with_flags() begins the process of creating a new fdt with
+ * the sequential write interface.
+ *
+ * fdt creation process must end with fdt_finished() to produce a valid fdt.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, bufsize is insufficient for a minimal fdt
+ *	-FDT_ERR_BADFLAGS, flags is not valid
+ */
+int fdt_create_with_flags(void *buf, int bufsize, uint32_t flags);
+
+/**
+ * fdt_create - begin creation of a new fdt
+ * @fdt: pointer to memory allocated where fdt will be created
+ * @bufsize: size of the memory space at fdt
+ *
+ * fdt_create() is equivalent to fdt_create_with_flags() with flags=0.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, bufsize is insufficient for a minimal fdt
+ */
 int fdt_create(void *buf, int bufsize);
+
 int fdt_resize(void *fdt, void *buf, int bufsize);
 int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size);
 int fdt_finish_reservemap(void *fdt);
@@ -1787,6 +1898,43 @@ static inline int fdt_appendprop_cell(void *fdt, int nodeoffset,
 #define fdt_appendprop_string(fdt, nodeoffset, name, str) \
 	fdt_appendprop((fdt), (nodeoffset), (name), (str), strlen(str)+1)
 
+/**
+ * fdt_appendprop_addrrange - append a address range property
+ * @fdt: pointer to the device tree blob
+ * @parent: offset of the parent node
+ * @nodeoffset: offset of the node to add a property at
+ * @name: name of property
+ * @addr: start address of a given range
+ * @size: size of a given range
+ *
+ * fdt_appendprop_addrrange() appends an address range value (start
+ * address and size) to the value of the named property in the given
+ * node, or creates a new property with that value if it does not
+ * already exist.
+ * If "name" is not specified, a default "reg" is used.
+ * Cell sizes are determined by parent's #address-cells and #size-cells.
+ *
+ * This function may insert data into the blob, and will therefore
+ * change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADNCELLS, if the node has a badly formatted or invalid
+ *		#address-cells property
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADVALUE, addr or size doesn't fit to respective cells size
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain a new property
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+int fdt_appendprop_addrrange(void *fdt, int parent, int nodeoffset,
+			     const char *name, uint64_t addr, uint64_t size);
+
 /**
  * fdt_delprop - delete a property
  * @fdt: pointer to the device tree blob

scripts/dtc/libfdt/libfdt_env.h

@@ -1,55 +1,10 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */
 #ifndef LIBFDT_ENV_H
 #define LIBFDT_ENV_H
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
  * Copyright 2012 Kim Phillips, Freescale Semiconductor.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <stdbool.h>
@@ -57,6 +12,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
+#include <limits.h>
 
 #ifdef __CHECKER__
 #define FDT_FORCE __attribute__((force))

scripts/dtc/libfdt/libfdt_internal.h

@@ -1,65 +1,24 @@
+/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */
 #ifndef LIBFDT_INTERNAL_H
 #define LIBFDT_INTERNAL_H
 /*
  * libfdt - Flat Device Tree manipulation
  * Copyright (C) 2006 David Gibson, IBM Corporation.
- *
- * libfdt is dual licensed: you can use it either under the terms of
- * the GPL, or the BSD license, at your option.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- *     You should have received a copy of the GNU General Public
- *     License along with this library; if not, write to the Free
- *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
- *     MA 02110-1301 USA
- *
- * Alternatively,
- *
- *  b) Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *     1. Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *     2. Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <fdt.h>
 
 #define FDT_ALIGN(x, a)		(((x) + (a) - 1) & ~((a) - 1))
 #define FDT_TAGALIGN(x)		(FDT_ALIGN((x), FDT_TAGSIZE))
 
-#define FDT_CHECK_HEADER(fdt) \
+int fdt_ro_probe_(const void *fdt);
+#define FDT_RO_PROBE(fdt)					\
 	{							\
-		int err_; \
+		int totalsize_;					\
-		if ((err_ = fdt_check_header(fdt)) != 0) \
+		if (fdt_chk_basic()) {				\
-			return err_; \
+			totalsize_ = fdt_ro_probe_(fdt);	\
+			if (totalsize_ < 0)			\
+				return totalsize_;		\
+		}						\
 	}
 
 int fdt_check_node_offset_(const void *fdt, int offset);
@@ -92,4 +51,87 @@ static inline struct fdt_reserve_entry *fdt_mem_rsv_w_(void *fdt, int n)
 
 #define FDT_SW_MAGIC		(~FDT_MAGIC)
 
+/**********************************************************************/
+/* Checking controls                                                  */
+/**********************************************************************/
+
+#ifndef FDT_ASSUME_MASK
+#define FDT_ASSUME_MASK 0
+#endif
+
+/*
+ * Defines assumptions which can be enabled. Each of these can be enabled
+ * individually. For maximum saftey, don't enable any assumptions!
+ *
+ * For minimal code size and no safety, use FDT_ASSUME_PERFECT at your own risk.
+ * You should have another method of validating the device tree, such as a
+ * signature or hash check before using libfdt.
+ *
+ * For situations where security is not a concern it may be safe to enable
+ * FDT_ASSUME_FRIENDLY.
+ */
+enum {
+	/*
+	 * This does essentially no checks. Only the latest device-tree
+	 * version is correctly handled. Incosistencies or errors in the device
+	 * tree may cause undefined behaviour or crashes.
+	 *
+	 * If an error occurs when modifying the tree it may leave the tree in
+	 * an intermediate (but valid) state. As an example, adding a property
+	 * where there is insufficient space may result in the property name
+	 * being added to the string table even though the property itself is
+	 * not added to the struct section.
+	 *
+	 * Only use this if you have a fully validated device tree with
+	 * the latest supported version and wish to minimise code size.
+	 */
+	FDT_ASSUME_PERFECT	= 0xff,
+
+	/*
+	 * This assumes that the device tree is sane. i.e. header metadata
+	 * and basic hierarchy are correct.
+	 *
+	 * These checks will be sufficient if you have a valid device tree with
+	 * no internal inconsistencies. With this assumption, libfdt will
+	 * generally not return -FDT_ERR_INTERNAL, -FDT_ERR_BADLAYOUT, etc.
+	 */
+	FDT_ASSUME_SANE		= 1 << 0,
+
+	/*
+	 * This disables checks for device-tree version and removes all code
+	 * which handles older versions.
+	 *
+	 * Only enable this if you know you have a device tree with the latest
+	 * version.
+	 */
+	FDT_ASSUME_LATEST	= 1 << 1,
+
+	/*
+	 * This disables any extensive checking of parameters and the device
+	 * tree, making various assumptions about correctness. Normal device
+	 * trees produced by libfdt and the compiler should be handled safely.
+	 * Malicious device trees and complete garbage may cause libfdt to
+	 * behave badly or crash.
+	 */
+	FDT_ASSUME_FRIENDLY	= 1 << 2,
+};
+
+/** fdt_chk_basic() - see if basic checking of params and DT data is enabled */
+static inline bool fdt_chk_basic(void)
+{
+	return !(FDT_ASSUME_MASK & FDT_ASSUME_SANE);
+}
+
+/** fdt_chk_version() - see if we need to handle old versions of the DT */
+static inline bool fdt_chk_version(void)
+{
+	return !(FDT_ASSUME_MASK & FDT_ASSUME_LATEST);
+}
+
+/** fdt_chk_extra() - see if extra checking is enabled */
+static inline bool fdt_chk_extra(void)
+{
+	return !(FDT_ASSUME_MASK & FDT_ASSUME_FRIENDLY);
+}
+
 #endif /* LIBFDT_INTERNAL_H */

scripts/dtc/pylibfdt/Makefile

@@ -21,7 +21,7 @@ quiet_cmd_pymod = PYMOD   $@
 		CPPFLAGS="$(HOSTCFLAGS) -I$(LIBFDT_srcdir)" OBJDIR=$(obj) \
 		SOURCES="$(PYLIBFDT_srcs)" \
 		SWIG_OPTS="-I$(LIBFDT_srcdir) -I$(LIBFDT_srcdir)/.." \
-		$(PYTHON2) $< --quiet build_ext --inplace
+		$(PYTHON3) $< --quiet build_ext --inplace
 
 $(obj)/_libfdt.so: $(src)/setup.py $(PYLIBFDT_srcs) FORCE
 	$(call if_changed,pymod)

scripts/dtc/pylibfdt/libfdt.i_shipped

@@ -18,7 +18,7 @@
  * a struct called fdt_property. That struct causes swig to create a class in
  * libfdt.py called fdt_property(), which confuses things.
  */
-static int fdt_property_stub(void *fdt, const char *name, const char *val,
+static int fdt_property_stub(void *fdt, const char *name, const void *val,
                              int len)
 {
     return fdt_property(fdt, name, val, len);
@@ -92,7 +92,7 @@ def check_err(val, quiet=()):
     Raises
         FdtException if val < 0
     """
-    if val < 0:
+    if isinstance(val, int) and val < 0:
         if -val not in quiet:
             raise FdtException(val)
     return val
@@ -417,7 +417,7 @@ class FdtRo(object):
                                quiet)
         if isinstance(pdata, (int)):
             return pdata
-        return Property(prop_name, bytearray(pdata[0]))
+        return Property(prop_name, bytes(pdata[0]))
 
     def get_phandle(self, nodeoffset):
         """Get the phandle of a node
@@ -431,6 +431,18 @@ class FdtRo(object):
         """
         return fdt_get_phandle(self._fdt, nodeoffset)
 
+    def get_alias(self, name):
+        """Get the full path referenced by a given alias
+
+        Args:
+            name: name of the alias to lookup
+
+        Returns:
+            Full path to the node for the alias named 'name', if it exists
+            None, if the given alias or the /aliases node does not exist
+        """
+        return fdt_get_alias(self._fdt, name)
+
     def parent_offset(self, nodeoffset, quiet=()):
         """Get the offset of a node's parent
 
@@ -624,7 +636,7 @@ class Fdt(FdtRo):
         Raises:
             FdtException if no parent found or other error occurs
         """
-        val = val.encode('utf-8') + '\0'
+        val = val.encode('utf-8') + b'\0'
         return check_err(fdt_setprop(self._fdt, nodeoffset, prop_name,
                                      val, len(val)), quiet)
 
@@ -727,6 +739,8 @@ class FdtSw(FdtRo):
 
     # First create the device tree with a node and property:
     sw = FdtSw()
+    sw.finish_reservemap()
+    with sw.add_node(''):
         with sw.add_node('node'):
             sw.property_u32('reg', 2)
     fdt = sw.as_fdt()
@@ -1029,17 +1043,24 @@ typedef uint32_t fdt32_t;
 	if (!$1)
 		$result = Py_None;
 	else
+        %#if PY_VERSION_HEX >= 0x03000000
+            $result = Py_BuildValue("y#", $1, *arg4);
+        %#else
             $result = Py_BuildValue("s#", $1, *arg4);
+        %#endif
 }
 
 /* typemap used for fdt_setprop() */
 %typemap(in) (const void *val) {
-    $1 = PyString_AsString($input);   /* char *str */
+    %#if PY_VERSION_HEX >= 0x03000000
+        if (!PyBytes_Check($input)) {
+            SWIG_exception_fail(SWIG_TypeError, "bytes expected in method '" "$symname"
+                "', argument " "$argnum"" of type '" "$type""'");
         }
-
+        $1 = PyBytes_AsString($input);
-/* typemap used for fdt_add_reservemap_entry() */
+    %#else
-%typemap(in) uint64_t {
+        $1 = PyString_AsString($input);   /* char *str */
-   $1 = PyLong_AsUnsignedLong($input);
+    %#endif
 }
 
 /* typemaps used for fdt_next_node() */
@@ -1061,7 +1082,7 @@ typedef uint32_t fdt32_t;
 }
 
 %typemap(argout) uint64_t * {
-        PyObject *val = PyLong_FromUnsignedLong(*arg$argnum);
+        PyObject *val = PyLong_FromUnsignedLongLong(*arg$argnum);
         if (!result) {
            if (PyTuple_GET_SIZE(resultobj) == 0)
               resultobj = val;
@@ -1092,6 +1113,6 @@ int fdt_property_cell(void *fdt, const char *name, uint32_t val);
  * This function has a stub since the name fdt_property is used for both a
   * function and a struct, which confuses SWIG.
  */
-int fdt_property_stub(void *fdt, const char *name, const char *val, int len);
+int fdt_property_stub(void *fdt, const char *name, const void *val, int len);
 
 %include <../libfdt/libfdt.h>

scripts/dtc/pylibfdt/setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 """
 setup.py file for SWIG libfdt

tools/binman/binman.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 
 # Copyright (c) 2016 Google, Inc

tools/binman/cbfs_util_test.py

@@ -56,7 +56,7 @@ class TestCbfs(unittest.TestCase):
         cls.have_lz4 = True
         try:
             tools.Run('lz4', '--no-frame-crc', '-c',
-                      tools.GetInputFilename('u-boot.bin'))
+                      tools.GetInputFilename('u-boot.bin'), binary=True)
         except:
             cls.have_lz4 = False
 

tools/binman/entry.py

@@ -7,16 +7,7 @@
 from __future__ import print_function
 
 from collections import namedtuple
-
-# importlib was introduced in Python 2.7 but there was a report of it not
-# working in 2.7.12, so we work around this:
-# http://lists.denx.de/pipermail/u-boot/2016-October/269729.html
-try:
 import importlib
-    have_importlib = True
-except:
-    have_importlib = False
-
 import os
 import sys
 
@@ -56,6 +47,8 @@ class Entry(object):
         offset: Offset of entry within the section, None if not known yet (in
             which case it will be calculated by Pack())
         size: Entry size in bytes, None if not known
+        pre_reset_size: size as it was before ResetForPack(). This allows us to
+            keep track of the size we started with and detect size changes
         uncomp_size: Size of uncompressed data in bytes, if the entry is
             compressed, else None
         contents_size: Size of contents in bytes, 0 by default
@@ -80,6 +73,7 @@ class Entry(object):
         self.name = node and (name_prefix + node.name) or 'none'
         self.offset = None
         self.size = None
+        self.pre_reset_size = None
         self.uncomp_size = None
         self.data = None
         self.contents_size = 0
@@ -119,10 +113,7 @@ class Entry(object):
             old_path = sys.path
             sys.path.insert(0, os.path.join(our_path, 'etype'))
             try:
-                if have_importlib:
                 module = importlib.import_module(module_name)
-                else:
-                    module = __import__(module_name)
             except ImportError as e:
                 raise ValueError("Unknown entry type '%s' in node '%s' (expected etype/%s.py, error '%s'" %
                                  (etype, node_path, module_name, e))
@@ -326,6 +317,7 @@ class Entry(object):
         self.Detail('ResetForPack: offset %s->%s, size %s->%s' %
                     (ToHex(self.offset), ToHex(self.orig_offset),
                      ToHex(self.size), ToHex(self.orig_size)))
+        self.pre_reset_size = self.size
         self.offset = self.orig_offset
         self.size = self.orig_size
 
@@ -769,7 +761,10 @@ features to produce new behaviours.
             True if the data did not result in a resize of this entry, False if
                  the entry must be resized
         """
+        if self.size is not None:
             self.contents_size = self.size
+        else:
+            self.contents_size = self.pre_reset_size
         ok = self.ProcessContentsUpdate(data)
         self.Detail('WriteData: size=%x, ok=%s' % (len(data), ok))
         section_ok = self.section.WriteChildData(self)

tools/binman/entry_test.py

@@ -39,21 +39,6 @@ class TestEntry(unittest.TestCase):
         else:
             import entry
 
-    def test1EntryNoImportLib(self):
-        """Test that we can import Entry subclassess successfully"""
-        sys.modules['importlib'] = None
-        global entry
-        self._ReloadEntry()
-        entry.Entry.Create(None, self.GetNode(), 'u-boot')
-        self.assertFalse(entry.have_importlib)
-
-    def test2EntryImportLib(self):
-        del sys.modules['importlib']
-        global entry
-        self._ReloadEntry()
-        entry.Entry.Create(None, self.GetNode(), 'u-boot-spl')
-        self.assertTrue(entry.have_importlib)
-
     def testEntryContents(self):
         """Test the Entry bass class"""
         import entry

tools/binman/etype/intel_fit.py

@@ -27,6 +27,6 @@ class Entry_intel_fit(Entry_blob):
         self.align = 16
 
     def ObtainContents(self):
-        data = struct.pack('<8sIHBB', '_FIT_   ', 1, 0x100, 0x80, 0x7d)
+        data = struct.pack('<8sIHBB', b'_FIT_   ', 1, 0x100, 0x80, 0x7d)
         self.SetContents(data)
         return True

tools/binman/ftest.py

@@ -174,7 +174,7 @@ class TestFunctional(unittest.TestCase):
         cls.have_lz4 = True
         try:
             tools.Run('lz4', '--no-frame-crc', '-c',
-                      os.path.join(cls._indir, 'u-boot.bin'))
+                      os.path.join(cls._indir, 'u-boot.bin'), binary=True)
         except:
             cls.have_lz4 = False
 
@@ -2113,7 +2113,7 @@ class TestFunctional(unittest.TestCase):
         data = self.data = self._DoReadFileRealDtb('115_fdtmap.dts')
         fdtmap_data = data[len(U_BOOT_DATA):]
         magic = fdtmap_data[:8]
-        self.assertEqual('_FDTMAP_', magic)
+        self.assertEqual(b'_FDTMAP_', magic)
         self.assertEqual(tools.GetBytes(0, 8), fdtmap_data[8:16])
 
         fdt_data = fdtmap_data[16:]
@@ -2156,7 +2156,7 @@ class TestFunctional(unittest.TestCase):
         dtb = fdt.Fdt.FromData(fdt_data)
         fdt_size = dtb.GetFdtObj().totalsize()
         hdr_data = data[-8:]
-        self.assertEqual('BinM', hdr_data[:4])
+        self.assertEqual(b'BinM', hdr_data[:4])
         offset = struct.unpack('<I', hdr_data[4:])[0] & 0xffffffff
         self.assertEqual(fdtmap_pos - 0x400, offset - (1 << 32))
 
@@ -2165,7 +2165,7 @@ class TestFunctional(unittest.TestCase):
         data = self.data = self._DoReadFileRealDtb('117_fdtmap_hdr_start.dts')
         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
         hdr_data = data[:8]
-        self.assertEqual('BinM', hdr_data[:4])
+        self.assertEqual(b'BinM', hdr_data[:4])
         offset = struct.unpack('<I', hdr_data[4:])[0]
         self.assertEqual(fdtmap_pos, offset)
 
@@ -2174,7 +2174,7 @@ class TestFunctional(unittest.TestCase):
         data = self.data = self._DoReadFileRealDtb('118_fdtmap_hdr_pos.dts')
         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
         hdr_data = data[0x80:0x88]
-        self.assertEqual('BinM', hdr_data[:4])
+        self.assertEqual(b'BinM', hdr_data[:4])
         offset = struct.unpack('<I', hdr_data[4:])[0]
         self.assertEqual(fdtmap_pos, offset)
 
@@ -2435,9 +2435,9 @@ class TestFunctional(unittest.TestCase):
 '  section               100   %x  section          100' % section_size,
 '    cbfs                100   400  cbfs               0',
 '      u-boot            138     4  u-boot            38',
-'      u-boot-dtb        180   10f  u-boot-dtb        80          3c9',
+'      u-boot-dtb        180   105  u-boot-dtb        80          3c9',
 '    u-boot-dtb          500   %x  u-boot-dtb       400          3c9' % fdt_size,
-'  fdtmap                %x   3b4  fdtmap           %x' %
+'  fdtmap                %x   3bd  fdtmap           %x' %
         (fdtmap_offset, fdtmap_offset),
 '  image-header          bf8     8  image-header     bf8',
             ]
@@ -2522,7 +2522,7 @@ class TestFunctional(unittest.TestCase):
         data = self._RunExtractCmd('section')
         cbfs_data = data[:0x400]
         cbfs = cbfs_util.CbfsReader(cbfs_data)
-        self.assertEqual(['u-boot', 'u-boot-dtb', ''], cbfs.files.keys())
+        self.assertEqual(['u-boot', 'u-boot-dtb', ''], list(cbfs.files.keys()))
         dtb_data = data[0x400:]
         dtb = self._decompress(dtb_data)
         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))

tools/buildman/board.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2012 The Chromium OS Authors.
 
+from collections import OrderedDict
 import re
 
 class Expr:
@@ -120,7 +121,7 @@ class Boards:
         Args:
             fname: Filename of boards.cfg file
         """
-        with open(fname, 'r') as fd:
+        with open(fname, 'r', encoding='utf-8') as fd:
             for line in fd:
                 if line[0] == '#':
                     continue
@@ -155,7 +156,7 @@ class Boards:
                 key is board.target
                 value is board
         """
-        board_dict = {}
+        board_dict = OrderedDict()
         for board in self._boards:
             board_dict[board.target] = board
         return board_dict
@@ -166,7 +167,7 @@ class Boards:
         Returns:
             List of Board objects that are marked selected
         """
-        board_dict = {}
+        board_dict = OrderedDict()
         for board in self._boards:
             if board.build_it:
                 board_dict[board.target] = board
@@ -259,7 +260,7 @@ class Boards:
                     due to each argument, arranged by argument.
                 List of errors found
         """
-        result = {}
+        result = OrderedDict()
         warnings = []
         terms = self._BuildTerms(args)
 

tools/buildman/bsettings.py

@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2012 The Chromium OS Authors.
 
-import ConfigParser
+import configparser
 import os
-import StringIO
+import io
 
 
 def Setup(fname=''):
@@ -15,20 +15,20 @@ def Setup(fname=''):
     global settings
     global config_fname
 
-    settings = ConfigParser.SafeConfigParser()
+    settings = configparser.SafeConfigParser()
     if fname is not None:
         config_fname = fname
         if config_fname == '':
             config_fname = '%s/.buildman' % os.getenv('HOME')
         if not os.path.exists(config_fname):
-            print 'No config file found ~/.buildman\nCreating one...\n'
+            print('No config file found ~/.buildman\nCreating one...\n')
             CreateBuildmanConfigFile(config_fname)
-            print 'To install tool chains, please use the --fetch-arch option'
+            print('To install tool chains, please use the --fetch-arch option')
         if config_fname:
             settings.read(config_fname)
 
 def AddFile(data):
-    settings.readfp(StringIO.StringIO(data))
+    settings.readfp(io.StringIO(data))
 
 def GetItems(section):
     """Get the items from a section of the config.
@@ -41,7 +41,7 @@ def GetItems(section):
     """
     try:
         return settings.items(section)
-    except ConfigParser.NoSectionError as e:
+    except configparser.NoSectionError as e:
         return []
     except:
         raise
@@ -68,10 +68,10 @@ def CreateBuildmanConfigFile(config_fname):
     try:
         f = open(config_fname, 'w')
     except IOError:
-        print "Couldn't create buildman config file '%s'\n" % config_fname
+        print("Couldn't create buildman config file '%s'\n" % config_fname)
         raise
 
-    print >>f, '''[toolchain]
+    print('''[toolchain]
 # name = path
 # e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux
 
@@ -93,5 +93,5 @@ openrisc = or1k
 # snapper-boards=ENABLE_AT91_TEST=1
 # snapper9260=${snapper-boards} BUILD_TAG=442
 # snapper9g45=${snapper-boards} BUILD_TAG=443
-'''
+''', file=f)
     f.close();

tools/buildman/builder.py

@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
 import glob
 import os
 import re
-import Queue
+import queue
 import shutil
 import signal
 import string
@@ -92,11 +92,10 @@ u-boot/             source directory
 """
 
 # Possible build outcomes
-OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
+OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 
 # Translate a commit subject into a valid filename (and handle unicode)
-trans_valid_chars = string.maketrans('/: ', '---')
+trans_valid_chars = str.maketrans('/: ', '---')
-trans_valid_chars = trans_valid_chars.decode('latin-1')
 
 BASE_CONFIG_FILENAMES = [
     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
@@ -122,8 +121,8 @@ class Config:
     def __hash__(self):
         val = 0
         for fname in self.config:
-            for key, value in self.config[fname].iteritems():
+            for key, value in self.config[fname].items():
-                print key, value
+                print(key, value)
                 val = val ^ hash(key) & hash(value)
         return val
 
@@ -293,8 +292,8 @@ class Builder:
         self._re_dtb_warning = re.compile('(.*): Warning .*')
         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 
-        self.queue = Queue.Queue()
+        self.queue = queue.Queue()
-        self.out_queue = Queue.Queue()
+        self.out_queue = queue.Queue()
         for i in range(self.num_threads):
             t = builderthread.BuilderThread(self, i, incremental,
                     per_board_out_dir)
@@ -781,7 +780,7 @@ class Builder:
         config = {}
         environment = {}
 
-        for board in boards_selected.itervalues():
+        for board in boards_selected.values():
             outcome = self.GetBuildOutcome(commit_upto, board.target,
                                            read_func_sizes, read_config,
                                            read_environment)
@@ -814,13 +813,13 @@ class Builder:
             tconfig = Config(self.config_filenames, board.target)
             for fname in self.config_filenames:
                 if outcome.config:
-                    for key, value in outcome.config[fname].iteritems():
+                    for key, value in outcome.config[fname].items():
                         tconfig.Add(fname, key, value)
             config[board.target] = tconfig
 
             tenvironment = Environment(board.target)
             if outcome.environment:
-                for key, value in outcome.environment.iteritems():
+                for key, value in outcome.environment.items():
                     tenvironment.Add(key, value)
             environment[board.target] = tenvironment
 
@@ -1040,12 +1039,12 @@ class Builder:
 
         # We now have a list of image size changes sorted by arch
         # Print out a summary of these
-        for arch, target_list in arch_list.iteritems():
+        for arch, target_list in arch_list.items():
             # Get total difference for each type
             totals = {}
             for result in target_list:
                 total = 0
-                for name, diff in result.iteritems():
+                for name, diff in result.items():
                     if name.startswith('_'):
                         continue
                     total += diff
@@ -1250,7 +1249,7 @@ class Builder:
             if self._show_unknown:
                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
                         self.col.MAGENTA)
-            for arch, target_list in arch_list.iteritems():
+            for arch, target_list in arch_list.items():
                 Print('%10s: %s' % (arch, target_list))
                 self._error_lines += 1
             if better_err:
@@ -1283,13 +1282,13 @@ class Builder:
                 environment_minus = {}
                 environment_change = {}
                 base = tbase.environment
-                for key, value in tenvironment.environment.iteritems():
+                for key, value in tenvironment.environment.items():
                     if key not in base:
                         environment_plus[key] = value
-                for key, value in base.iteritems():
+                for key, value in base.items():
                     if key not in tenvironment.environment:
                         environment_minus[key] = value
-                for key, value in base.iteritems():
+                for key, value in base.items():
                     new_value = tenvironment.environment.get(key)
                     if new_value and value != new_value:
                         desc = '%s -> %s' % (value, new_value)
@@ -1342,15 +1341,15 @@ class Builder:
                     config_minus = {}
                     config_change = {}
                     base = tbase.config[name]
-                    for key, value in tconfig.config[name].iteritems():
+                    for key, value in tconfig.config[name].items():
                         if key not in base:
                             config_plus[key] = value
                             all_config_plus[key] = value
-                    for key, value in base.iteritems():
+                    for key, value in base.items():
                         if key not in tconfig.config[name]:
                             config_minus[key] = value
                             all_config_minus[key] = value
-                    for key, value in base.iteritems():
+                    for key, value in base.items():
                         new_value = tconfig.config.get(key)
                         if new_value and value != new_value:
                             desc = '%s -> %s' % (value, new_value)
@@ -1368,7 +1367,7 @@ class Builder:
                 summary[target] = '\n'.join(lines)
 
             lines_by_target = {}
-            for target, lines in summary.iteritems():
+            for target, lines in summary.items():
                 if lines in lines_by_target:
                     lines_by_target[lines].append(target)
                 else:
@@ -1392,7 +1391,7 @@ class Builder:
                     Print('%s:' % arch)
                     _OutputConfigInfo(lines)
 
-            for lines, targets in lines_by_target.iteritems():
+            for lines, targets in lines_by_target.items():
                 if not lines:
                     continue
                 Print('%s :' % ' '.join(sorted(targets)))
@@ -1463,7 +1462,7 @@ class Builder:
             commits: Selected commits to build
         """
         # First work out how many commits we will build
-        count = (self.commit_count + self._step - 1) / self._step
+        count = (self.commit_count + self._step - 1) // self._step
         self.count = len(board_selected) * count
         self.upto = self.warned = self.fail = 0
         self._timestamps = collections.deque()
@@ -1566,7 +1565,7 @@ class Builder:
         self.ProcessResult(None)
 
         # Create jobs to build all commits for each board
-        for brd in board_selected.itervalues():
+        for brd in board_selected.values():
             job = builderthread.BuilderJob()
             job.board = brd
             job.commits = commits

tools/buildman/builderthread.py

@@ -28,7 +28,7 @@ def Mkdir(dirname, parents = False):
     except OSError as err:
         if err.errno == errno.EEXIST:
             if os.path.realpath('.') == os.path.realpath(dirname):
-                print "Cannot create the current working directory '%s'!" % dirname
+                print("Cannot create the current working directory '%s'!" % dirname)
                 sys.exit(1)
             pass
         else:
@@ -291,15 +291,13 @@ class BuilderThread(threading.Thread):
         outfile = os.path.join(build_dir, 'log')
         with open(outfile, 'w') as fd:
             if result.stdout:
-                # We don't want unicode characters in log files
+                fd.write(result.stdout)
-                fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
 
         errfile = self.builder.GetErrFile(result.commit_upto,
                 result.brd.target)
         if result.stderr:
             with open(errfile, 'w') as fd:
-                # We don't want unicode characters in log files
+                fd.write(result.stderr)
-                fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
         elif os.path.exists(errfile):
             os.remove(errfile)
 
@@ -314,17 +312,17 @@ class BuilderThread(threading.Thread):
                 else:
                     fd.write('%s' % result.return_code)
             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
-                print >>fd, 'gcc', result.toolchain.gcc
+                print('gcc', result.toolchain.gcc, file=fd)
-                print >>fd, 'path', result.toolchain.path
+                print('path', result.toolchain.path, file=fd)
-                print >>fd, 'cross', result.toolchain.cross
+                print('cross', result.toolchain.cross, file=fd)
-                print >>fd, 'arch', result.toolchain.arch
+                print('arch', result.toolchain.arch, file=fd)
                 fd.write('%s' % result.return_code)
 
             # Write out the image and function size information and an objdump
             env = result.toolchain.MakeEnvironment(self.builder.full_path)
             with open(os.path.join(build_dir, 'env'), 'w') as fd:
                 for var in sorted(env.keys()):
-                    print >>fd, '%s="%s"' % (var, env[var])
+                    print('%s="%s"' % (var, env[var]), file=fd)
             lines = []
             for fname in ['u-boot', 'spl/u-boot-spl']:
                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
@@ -335,7 +333,7 @@ class BuilderThread(threading.Thread):
                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
                                     result.brd.target, fname)
                     with open(nm, 'w') as fd:
-                        print >>fd, nm_result.stdout,
+                        print(nm_result.stdout, end=' ', file=fd)
 
                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
                 dump_result = command.RunPipe([cmd], capture=True,
@@ -346,7 +344,7 @@ class BuilderThread(threading.Thread):
                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
                                     result.brd.target, fname)
                     with open(objdump, 'w') as fd:
-                        print >>fd, dump_result.stdout,
+                        print(dump_result.stdout, end=' ', file=fd)
                     for line in dump_result.stdout.splitlines():
                         fields = line.split()
                         if len(fields) > 5 and fields[1] == '.rodata':
@@ -378,7 +376,7 @@ class BuilderThread(threading.Thread):
                 sizes = self.builder.GetSizesFile(result.commit_upto,
                                 result.brd.target)
                 with open(sizes, 'w') as fd:
-                    print >>fd, '\n'.join(lines)
+                    print('\n'.join(lines), file=fd)
 
         # Write out the configuration files, with a special case for SPL
         for dirname in ['', 'spl', 'tpl']:

tools/buildman/buildman.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Copyright (c) 2012 The Chromium OS Authors.
@@ -6,6 +6,8 @@
 
 """See README for more information"""
 
+from __future__ import print_function
+
 import multiprocessing
 import os
 import re
@@ -46,11 +48,11 @@ def RunTests(skip_net_tests):
         suite = unittest.TestLoader().loadTestsFromTestCase(module)
         suite.run(result)
 
-    print result
+    print(result)
     for test, err in result.errors:
-        print err
+        print(err)
     for test, err in result.failures:
-        print err
+        print(err)
 
 
 options, args = cmdline.ParseArgs()

tools/buildman/control.py

@@ -30,7 +30,7 @@ def GetActionSummary(is_summary, commits, selected, options):
     """
     if commits:
         count = len(commits)
-        count = (count + options.step - 1) / options.step
+        count = (count + options.step - 1) // options.step
         commit_str = '%d commit%s' % (count, GetPlural(count))
     else:
         commit_str = 'current source'
@@ -59,31 +59,31 @@ def ShowActions(series, why_selected, boards_selected, builder, options,
         board_warnings: List of warnings obtained from board selected
     """
     col = terminal.Color()
-    print 'Dry run, so not doing much. But I would do this:'
+    print('Dry run, so not doing much. But I would do this:')
-    print
+    print()
     if series:
         commits = series.commits
     else:
         commits = None
-    print GetActionSummary(False, commits, boards_selected,
+    print(GetActionSummary(False, commits, boards_selected,
-            options)
+            options))
-    print 'Build directory: %s' % builder.base_dir
+    print('Build directory: %s' % builder.base_dir)
     if commits:
         for upto in range(0, len(series.commits), options.step):
             commit = series.commits[upto]
-            print '   ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
+            print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
-            print commit.subject
+            print(commit.subject)
-    print
+    print()
     for arg in why_selected:
         if arg != 'all':
-            print arg, ': %d boards' % len(why_selected[arg])
+            print(arg, ': %d boards' % len(why_selected[arg]))
             if options.verbose:
-                print '   %s' % ' '.join(why_selected[arg])
+                print('   %s' % ' '.join(why_selected[arg]))
-    print ('Total boards to build for each commit: %d\n' %
+    print(('Total boards to build for each commit: %d\n' %
-            len(why_selected['all']))
+            len(why_selected['all'])))
     if board_warnings:
         for warning in board_warnings:
-            print col.Color(col.YELLOW, warning)
+            print(col.Color(col.YELLOW, warning))
 
 def CheckOutputDir(output_dir):
     """Make sure that the output directory is not within the current directory
@@ -146,17 +146,17 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
     if options.fetch_arch:
         if options.fetch_arch == 'list':
             sorted_list = toolchains.ListArchs()
-            print col.Color(col.BLUE, 'Available architectures: %s\n' %
+            print(col.Color(col.BLUE, 'Available architectures: %s\n' %
-                            ' '.join(sorted_list))
+                            ' '.join(sorted_list)))
             return 0
         else:
             fetch_arch = options.fetch_arch
             if fetch_arch == 'all':
                 fetch_arch = ','.join(toolchains.ListArchs())
-                print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
+                print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
-                                fetch_arch)
+                                fetch_arch))
             for arch in fetch_arch.split(','):
-                print
+                print()
                 ret = toolchains.FetchAndInstall(arch)
                 if ret:
                     return ret
@@ -167,7 +167,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
         toolchains.Scan(options.list_tool_chains and options.verbose)
     if options.list_tool_chains:
         toolchains.List()
-        print
+        print()
         return 0
 
     # Work out how many commits to build. We want to build everything on the
@@ -191,7 +191,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
                                    options.branch))
             if msg:
-                print col.Color(col.YELLOW, msg)
+                print(col.Color(col.YELLOW, msg))
             count += 1   # Build upstream commit also
 
     if not count:
@@ -268,7 +268,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
         options.threads = min(multiprocessing.cpu_count(), len(selected))
     if not options.jobs:
         options.jobs = max(1, (multiprocessing.cpu_count() +
-                len(selected) - 1) / len(selected))
+                len(selected) - 1) // len(selected))
 
     if not options.step:
         options.step = len(series.commits) - 1

tools/buildman/func_test.py

@@ -270,7 +270,7 @@ class TestFunctional(unittest.TestCase):
                                             stdout=''.join(commit_log[:count]))
 
         # Not handled, so abort
-        print 'git log', args
+        print('git log', args)
         sys.exit(1)
 
     def _HandleCommandGitConfig(self, args):
@@ -286,7 +286,7 @@ class TestFunctional(unittest.TestCase):
                                          stdout='refs/heads/master\n')
 
         # Not handled, so abort
-        print 'git config', args
+        print('git config', args)
         sys.exit(1)
 
     def _HandleCommandGit(self, in_args):
@@ -320,7 +320,7 @@ class TestFunctional(unittest.TestCase):
             return command.CommandResult(return_code=0)
 
         # Not handled, so abort
-        print 'git', git_args, sub_cmd, args
+        print('git', git_args, sub_cmd, args)
         sys.exit(1)
 
     def _HandleCommandNm(self, args):
@@ -351,7 +351,7 @@ class TestFunctional(unittest.TestCase):
             if pipe_list[1] == ['wc', '-l']:
                 wc = True
             else:
-                print 'invalid pipe', kwargs
+                print('invalid pipe', kwargs)
                 sys.exit(1)
         cmd = pipe_list[0][0]
         args = pipe_list[0][1:]
@@ -371,7 +371,7 @@ class TestFunctional(unittest.TestCase):
 
         if not result:
             # Not handled, so abort
-            print 'unknown command', kwargs
+            print('unknown command', kwargs)
             sys.exit(1)
 
         if wc:
@@ -404,14 +404,14 @@ class TestFunctional(unittest.TestCase):
             return command.CommandResult(return_code=0)
 
         # Not handled, so abort
-        print 'make', stage
+        print('make', stage)
         sys.exit(1)
 
     # Example function to print output lines
     def print_lines(self, lines):
-        print len(lines)
+        print(len(lines))
         for line in lines:
-            print line
+            print(line)
         #self.print_lines(terminal.GetPrintTestLines())
 
     def testNoBoards(self):

tools/buildman/test.py

@@ -212,11 +212,11 @@ class TestBuild(unittest.TestCase):
         self.assertEqual(lines[1].text, '02: %s' % commits[1][1])
 
         col = terminal.Color()
-        self.assertSummary(lines[2].text, 'sandbox', 'w+', ['board4'],
+        self.assertSummary(lines[2].text, 'arm', 'w+', ['board1'],
                            outcome=OUTCOME_WARN)
-        self.assertSummary(lines[3].text, 'arm', 'w+', ['board1'],
+        self.assertSummary(lines[3].text, 'powerpc', 'w+', ['board2', 'board3'],
                            outcome=OUTCOME_WARN)
-        self.assertSummary(lines[4].text, 'powerpc', 'w+', ['board2', 'board3'],
+        self.assertSummary(lines[4].text, 'sandbox', 'w+', ['board4'],
                            outcome=OUTCOME_WARN)
 
         # Second commit: The warnings should be listed
@@ -226,10 +226,10 @@ class TestBuild(unittest.TestCase):
 
         # Third commit: Still fails
         self.assertEqual(lines[6].text, '03: %s' % commits[2][1])
-        self.assertSummary(lines[7].text, 'sandbox', '+', ['board4'])
+        self.assertSummary(lines[7].text, 'arm', '', ['board1'],
-        self.assertSummary(lines[8].text, 'arm', '', ['board1'],
                            outcome=OUTCOME_OK)
-        self.assertSummary(lines[9].text, 'powerpc', '+', ['board2', 'board3'])
+        self.assertSummary(lines[8].text, 'powerpc', '+', ['board2', 'board3'])
+        self.assertSummary(lines[9].text, 'sandbox', '+', ['board4'])
 
         # Expect a compiler error
         self.assertEqual(lines[10].text, '+%s' %
@@ -237,8 +237,6 @@ class TestBuild(unittest.TestCase):
 
         # Fourth commit: Compile errors are fixed, just have warning for board3
         self.assertEqual(lines[11].text, '04: %s' % commits[3][1])
-        self.assertSummary(lines[12].text, 'sandbox', 'w+', ['board4'],
-                           outcome=OUTCOME_WARN)
         expect = '%10s: ' % 'powerpc'
         expect += ' ' + col.Color(col.GREEN, '')
         expect += '  '
@@ -246,7 +244,9 @@ class TestBuild(unittest.TestCase):
         expect += ' ' + col.Color(col.YELLOW, 'w+')
         expect += '  '
         expect += col.Color(col.YELLOW, ' %s' % 'board3')
-        self.assertEqual(lines[13].text, expect)
+        self.assertEqual(lines[12].text, expect)
+        self.assertSummary(lines[13].text, 'sandbox', 'w+', ['board4'],
+                           outcome=OUTCOME_WARN)
 
         # Compile error fixed
         self.assertEqual(lines[14].text, '-%s' %
@@ -259,9 +259,9 @@ class TestBuild(unittest.TestCase):
 
         # Fifth commit
         self.assertEqual(lines[16].text, '05: %s' % commits[4][1])
-        self.assertSummary(lines[17].text, 'sandbox', '+', ['board4'])
+        self.assertSummary(lines[17].text, 'powerpc', '', ['board3'],
-        self.assertSummary(lines[18].text, 'powerpc', '', ['board3'],
                            outcome=OUTCOME_OK)
+        self.assertSummary(lines[18].text, 'sandbox', '+', ['board4'])
 
         # The second line of errors[3] is a duplicate, so buildman will drop it
         expect = errors[3].rstrip().split('\n')

tools/buildman/toolchain.py

@@ -4,18 +4,19 @@
 
 import re
 import glob
-from HTMLParser import HTMLParser
+from html.parser import HTMLParser
 import os
 import sys
 import tempfile
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 
 import bsettings
 import command
 import terminal
+import tools
 
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
-    PRIORITY_CALC) = range(4)
+    PRIORITY_CALC) = list(range(4))
 
 # Simple class to collect links from a page
 class MyHTMLParser(HTMLParser):
@@ -100,15 +101,15 @@ class Toolchain:
                                      raise_on_error=False)
             self.ok = result.return_code == 0
             if verbose:
-                print 'Tool chain test: ',
+                print('Tool chain test: ', end=' ')
                 if self.ok:
-                    print "OK, arch='%s', priority %d" % (self.arch,
+                    print("OK, arch='%s', priority %d" % (self.arch,
-                                                          self.priority)
+                                                          self.priority))
                 else:
-                    print 'BAD'
+                    print('BAD')
-                    print 'Command: ', cmd
+                    print('Command: ', cmd)
-                    print result.stdout
+                    print(result.stdout)
-                    print result.stderr
+                    print(result.stderr)
         else:
             self.ok = True
 
@@ -138,7 +139,7 @@ class Toolchain:
         value = ''
         for name, value in bsettings.GetItems('toolchain-wrapper'):
             if not value:
-                print "Warning: Wrapper not found"
+                print("Warning: Wrapper not found")
         if value:
             value = value + ' '
 
@@ -227,11 +228,11 @@ class Toolchains:
         """
         toolchains = bsettings.GetItems('toolchain')
         if show_warning and not toolchains:
-            print ("Warning: No tool chains. Please run 'buildman "
+            print(("Warning: No tool chains. Please run 'buildman "
                    "--fetch-arch all' to download all available toolchains, or "
                    "add a [toolchain] section to your buildman config file "
                    "%s. See README for details" %
-                   bsettings.config_fname)
+                   bsettings.config_fname))
 
         paths = []
         for name, value in toolchains:
@@ -272,10 +273,10 @@ class Toolchains:
         if add_it:
             self.toolchains[toolchain.arch] = toolchain
         elif verbose:
-            print ("Toolchain '%s' at priority %d will be ignored because "
+            print(("Toolchain '%s' at priority %d will be ignored because "
                    "another toolchain for arch '%s' has priority %d" %
                    (toolchain.gcc, toolchain.priority, toolchain.arch,
-                    self.toolchains[toolchain.arch].priority))
+                    self.toolchains[toolchain.arch].priority)))
 
     def ScanPath(self, path, verbose):
         """Scan a path for a valid toolchain
@@ -289,9 +290,9 @@ class Toolchains:
         fnames = []
         for subdir in ['.', 'bin', 'usr/bin']:
             dirname = os.path.join(path, subdir)
-            if verbose: print "      - looking in '%s'" % dirname
+            if verbose: print("      - looking in '%s'" % dirname)
             for fname in glob.glob(dirname + '/*gcc'):
-                if verbose: print "         - found '%s'" % fname
+                if verbose: print("         - found '%s'" % fname)
                 fnames.append(fname)
         return fnames
 
@@ -321,9 +322,9 @@ class Toolchains:
         Args:
             verbose: True to print out progress information
         """
-        if verbose: print 'Scanning for tool chains'
+        if verbose: print('Scanning for tool chains')
         for name, value in self.prefixes:
-            if verbose: print "   - scanning prefix '%s'" % value
+            if verbose: print("   - scanning prefix '%s'" % value)
             if os.path.exists(value):
                 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
                 continue
@@ -335,10 +336,10 @@ class Toolchains:
             for f in fname_list:
                 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
             if not fname_list:
-                raise ValueError, ("No tool chain found for prefix '%s'" %
+                raise ValueError("No tool chain found for prefix '%s'" %
                                    value)
         for path in self.paths:
-            if verbose: print "   - scanning path '%s'" % path
+            if verbose: print("   - scanning path '%s'" % path)
             fnames = self.ScanPath(path, verbose)
             for fname in fnames:
                 self.Add(fname, True, verbose)
@@ -346,13 +347,13 @@ class Toolchains:
     def List(self):
         """List out the selected toolchains for each architecture"""
         col = terminal.Color()
-        print col.Color(col.BLUE, 'List of available toolchains (%d):' %
+        print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
-                        len(self.toolchains))
+                        len(self.toolchains)))
         if len(self.toolchains):
-            for key, value in sorted(self.toolchains.iteritems()):
+            for key, value in sorted(self.toolchains.items()):
-                print '%-10s: %s' % (key, value.gcc)
+                print('%-10s: %s' % (key, value.gcc))
         else:
-            print 'None'
+            print('None')
 
     def Select(self, arch):
         """Returns the toolchain for a given architecture
@@ -370,7 +371,7 @@ class Toolchains:
                         return self.toolchains[alias]
 
         if not arch in self.toolchains:
-            raise ValueError, ("No tool chain found for arch '%s'" % arch)
+            raise ValueError("No tool chain found for arch '%s'" % arch)
         return self.toolchains[arch]
 
     def ResolveReferences(self, var_dict, args):
@@ -464,9 +465,9 @@ class Toolchains:
         links = []
         for version in versions:
             url = '%s/%s/%s/' % (base, arch, version)
-            print 'Checking: %s' % url
+            print('Checking: %s' % url)
-            response = urllib2.urlopen(url)
+            response = urllib.request.urlopen(url)
-            html = response.read()
+            html = tools.ToString(response.read())
             parser = MyHTMLParser(fetch_arch)
             parser.feed(html)
             if fetch_arch == 'list':
@@ -488,14 +489,14 @@ class Toolchains:
                 Full path to the downloaded archive file in that directory,
                     or None if there was an error while downloading
         """
-        print 'Downloading: %s' % url
+        print('Downloading: %s' % url)
         leaf = url.split('/')[-1]
         tmpdir = tempfile.mkdtemp('.buildman')
-        response = urllib2.urlopen(url)
+        response = urllib.request.urlopen(url)
         fname = os.path.join(tmpdir, leaf)
         fd = open(fname, 'wb')
         meta = response.info()
-        size = int(meta.getheaders('Content-Length')[0])
+        size = int(meta.get('Content-Length'))
         done = 0
         block_size = 1 << 16
         status = ''
@@ -504,19 +505,19 @@ class Toolchains:
         while True:
             buffer = response.read(block_size)
             if not buffer:
-                print chr(8) * (len(status) + 1), '\r',
+                print(chr(8) * (len(status) + 1), '\r', end=' ')
                 break
 
             done += len(buffer)
             fd.write(buffer)
-            status = r'%10d MiB  [%3d%%]' % (done / 1024 / 1024,
+            status = r'%10d MiB  [%3d%%]' % (done // 1024 // 1024,
-                                             done * 100 / size)
+                                             done * 100 // size)
             status = status + chr(8) * (len(status) + 1)
-            print status,
+            print(status, end=' ')
             sys.stdout.flush()
         fd.close()
         if done != size:
-            print 'Error, failed to download'
+            print('Error, failed to download')
             os.remove(fname)
             fname = None
         return tmpdir, fname
@@ -565,11 +566,11 @@ class Toolchains:
         """
         # Fist get the URL for this architecture
         col = terminal.Color()
-        print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
+        print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
         url = self.LocateArchUrl(arch)
         if not url:
-            print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
+            print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
-                   arch)
+                   arch))
             return 2
         home = os.environ['HOME']
         dest = os.path.join(home, '.buildman-toolchains')
@@ -580,28 +581,28 @@ class Toolchains:
         tmpdir, tarfile = self.Download(url)
         if not tarfile:
             return 1
-        print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
+        print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
         sys.stdout.flush()
         path = self.Unpack(tarfile, dest)
         os.remove(tarfile)
         os.rmdir(tmpdir)
-        print
+        print()
 
         # Check that the toolchain works
-        print col.Color(col.GREEN, 'Testing')
+        print(col.Color(col.GREEN, 'Testing'))
         dirpath = os.path.join(dest, path)
         compiler_fname_list = self.ScanPath(dirpath, True)
         if not compiler_fname_list:
-            print 'Could not locate C compiler - fetch failed.'
+            print('Could not locate C compiler - fetch failed.')
             return 1
         if len(compiler_fname_list) != 1:
-            print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
+            print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
-                            ', '.join(compiler_fname_list))
+                            ', '.join(compiler_fname_list)))
         toolchain = Toolchain(compiler_fname_list[0], True, True)
 
         # Make sure that it will be found by buildman
         if not self.TestSettingsHasPath(dirpath):
-            print ("Adding 'download' to config file '%s'" %
+            print(("Adding 'download' to config file '%s'" %
-                   bsettings.config_fname)
+                   bsettings.config_fname))
             bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
         return 0

tools/dtoc/dtoc.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Copyright (C) 2016 Google, Inc

tools/dtoc/fdt.py

@@ -56,9 +56,6 @@ def BytesToValue(data):
                 is_string = False
                 break
             for ch in string:
-                # Handle Python 2 treating bytes as str
-                if type(ch) == str:
-                    ch = ord(ch)
                 if ch < 32 or ch > 127:
                     is_string = False
                     break
@@ -66,15 +63,9 @@ def BytesToValue(data):
         is_string = False
     if is_string:
         if count == 1: 
-            if sys.version_info[0] >= 3:  # pragma: no cover
             return TYPE_STRING, strings[0].decode()
         else:
-                return TYPE_STRING, strings[0]
-        else:
-            if sys.version_info[0] >= 3:  # pragma: no cover
             return TYPE_STRING, [s.decode() for s in strings[:-1]]
-            else:
-                return TYPE_STRING, strings[:-1]
     if size % 4:
         if size == 1:
             return TYPE_BYTE, tools.ToChar(data[0])
@@ -415,8 +406,8 @@ class Node:
             prop_name: Name of property to set
             val: String value to set (will be \0-terminated in DT)
         """
-        if sys.version_info[0] >= 3:  # pragma: no cover
+        if type(val) == str:
-            val = bytes(val, 'utf-8')
+            val = val.encode('utf-8')
         self._CheckProp(prop_name).props[prop_name].SetData(val + b'\0')
 
     def AddString(self, prop_name, val):

tools/dtoc/test_dtoc.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2012 The Chromium OS Authors.
 #

tools/dtoc/test_fdt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 # Copyright (c) 2018 Google, Inc
 # Written by Simon Glass <sjg@chromium.org>

tools/libfdt/fdt_rw.c

@@ -11,6 +11,7 @@ int fdt_remove_unused_strings(const void *old, void *new)
 	const char *str;
 	int ret;
 	int tag = FDT_PROP;
+	int allocated;
 
 	/* Make a copy and remove the strings */
 	memcpy(new, old, size);
@@ -25,7 +26,7 @@ int fdt_remove_unused_strings(const void *old, void *new)
 		new_prop = (struct fdt_property *)(unsigned long)
 			fdt_get_property_by_offset(new, offset, NULL);
 		str = fdt_string(old, fdt32_to_cpu(old_prop->nameoff));
-		ret = fdt_find_add_string_(new, str);
+		ret = fdt_find_add_string_(new, str, &allocated);
 		if (ret < 0)
 			return ret;
 		new_prop->nameoff = cpu_to_fdt32(ret);

tools/microcode-tool.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Copyright (c) 2014 Google, Inc
@@ -126,15 +126,15 @@ def List(date, microcodes, model):
         microcodes:     Dict of Microcode objects indexed by name
         model:          Model string to search for, or None
     """
-    print 'Date: %s' % date
+    print('Date: %s' % date)
     if model:
         mcode_list, tried = FindMicrocode(microcodes, model.lower())
-        print 'Matching models %s:' % (', '.join(tried))
+        print('Matching models %s:' % (', '.join(tried)))
     else:
-        print 'All models:'
+        print('All models:')
-        mcode_list = [microcodes[m] for m in microcodes.keys()]
+        mcode_list = [microcodes[m] for m in list(microcodes.keys())]
     for mcode in mcode_list:
-        print '%-20s: model %s' % (mcode.name, mcode.model)
+        print('%-20s: model %s' % (mcode.name, mcode.model))
 
 def FindMicrocode(microcodes, model):
     """Find all the microcode chunks which match the given model.
@@ -164,7 +164,7 @@ def FindMicrocode(microcodes, model):
     for i in range(3):
         abbrev = model[:-i] if i else model
         tried.append(abbrev)
-        for mcode in microcodes.values():
+        for mcode in list(microcodes.values()):
             if mcode.model.startswith(abbrev):
                 found.append(mcode)
         if found:
@@ -229,17 +229,17 @@ data = <%s
     args += [mcode.words[i] for i in range(7)]
     args.append(words)
     if outfile == '-':
-        print out % tuple(args)
+        print(out % tuple(args))
     else:
         if not outfile:
             if not os.path.exists(MICROCODE_DIR):
-                print >> sys.stderr, "Creating directory '%s'" % MICROCODE_DIR
+                print("Creating directory '%s'" % MICROCODE_DIR, file=sys.stderr)
                 os.makedirs(MICROCODE_DIR)
             outfile = os.path.join(MICROCODE_DIR, mcode.name + '.dtsi')
-        print >> sys.stderr, "Writing microcode for '%s' to '%s'" % (
+        print("Writing microcode for '%s' to '%s'" % (
-                ', '.join([mcode.name for mcode in mcodes]), outfile)
+                ', '.join([mcode.name for mcode in mcodes]), outfile), file=sys.stderr)
         with open(outfile, 'w') as fd:
-            print >> fd, out % tuple(args)
+            print(out % tuple(args), file=fd)
 
 def MicrocodeTool():
     """Run the microcode tool"""
@@ -289,14 +289,14 @@ def MicrocodeTool():
     if cmd == 'list':
         List(date, microcodes, options.model)
     elif cmd == 'license':
-        print '\n'.join(license_text)
+        print('\n'.join(license_text))
     elif cmd == 'create':
         if not options.model:
             parser.error('You must specify a model to create')
         model = options.model.lower()
         if options.model == 'all':
             options.multiple = True
-            mcode_list = microcodes.values()
+            mcode_list = list(microcodes.values())
             tried = []
         else:
             mcode_list, tried = FindMicrocode(microcodes, model)

tools/moveconfig.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
@@ -304,7 +304,7 @@ import glob
 import multiprocessing
 import optparse
 import os
-import Queue
+import queue
 import re
 import shutil
 import subprocess
@@ -450,8 +450,8 @@ def get_matched_defconfigs(defconfigs_file):
             line = line.split(' ')[0]  # handle 'git log' input
         matched = get_matched_defconfig(line)
         if not matched:
-            print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \
+            print("warning: %s:%d: no defconfig matched '%s'" % \
-                                                 (defconfigs_file, i + 1, line)
+                                                 (defconfigs_file, i + 1, line), file=sys.stderr)
 
         defconfigs += matched
 
@@ -494,11 +494,11 @@ def show_diff(a, b, file_path, color_enabled):
 
     for line in diff:
         if line[0] == '-' and line[1] != '-':
-            print color_text(color_enabled, COLOR_RED, line),
+            print(color_text(color_enabled, COLOR_RED, line), end=' ')
         elif line[0] == '+' and line[1] != '+':
-            print color_text(color_enabled, COLOR_GREEN, line),
+            print(color_text(color_enabled, COLOR_GREEN, line), end=' ')
         else:
-            print line,
+            print(line, end=' ')
 
 def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre,
                          extend_post):
@@ -554,9 +554,9 @@ def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre
 def confirm(options, prompt):
     if not options.yes:
         while True:
-            choice = raw_input('{} [y/n]: '.format(prompt))
+            choice = input('{} [y/n]: '.format(prompt))
             choice = choice.lower()
-            print choice
+            print(choice)
             if choice == 'y' or choice == 'n':
                 break
 
@@ -809,10 +809,10 @@ def try_expand(line):
         val= val.strip('\"')
         if re.search("[*+-/]|<<|SZ_+|\(([^\)]+)\)", val):
             newval = hex(eval(val, SIZES))
-            print "\tExpanded expression %s to %s" % (val, newval)
+            print("\tExpanded expression %s to %s" % (val, newval))
             return cfg+'='+newval
     except:
-        print "\tFailed to expand expression in %s" % line
+        print("\tFailed to expand expression in %s" % line)
 
     return line
 
@@ -838,7 +838,7 @@ class Progress:
 
     def show(self):
         """Display the progress."""
-        print ' %d defconfigs out of %d\r' % (self.current, self.total),
+        print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
         sys.stdout.flush()
 
 
@@ -1312,7 +1312,7 @@ class Slot:
         log += '\n'.join([ '    ' + s for s in self.log.split('\n') ])
         # Some threads are running in parallel.
         # Print log atomically to not mix up logs from different threads.
-        print >> (sys.stdout if success else sys.stderr), log
+        print(log, file=(sys.stdout if success else sys.stderr))
 
         if not success:
             if self.options.exit_on_error:
@@ -1411,8 +1411,8 @@ class Slots:
             msg = "The following boards were not processed due to error:\n"
             msg += boards
             msg += "(the list has been saved in %s)\n" % output_file
-            print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED,
+            print(color_text(self.options.color, COLOR_LIGHT_RED,
-                                            msg)
+                                            msg), file=sys.stderr)
 
             with open(output_file, 'w') as f:
                 f.write(boards)
@@ -1431,8 +1431,8 @@ class Slots:
             msg += "It is highly recommended to check them manually:\n"
             msg += boards
             msg += "(the list has been saved in %s)\n" % output_file
-            print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW,
+            print(color_text(self.options.color, COLOR_YELLOW,
-                                            msg)
+                                            msg), file=sys.stderr)
 
             with open(output_file, 'w') as f:
                 f.write(boards)
@@ -1448,11 +1448,11 @@ class ReferenceSource:
           commit: commit to git-clone
         """
         self.src_dir = tempfile.mkdtemp()
-        print "Cloning git repo to a separate work directory..."
+        print("Cloning git repo to a separate work directory...")
         subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
                                 cwd=self.src_dir)
-        print "Checkout '%s' to build the original autoconf.mk." % \
+        print("Checkout '%s' to build the original autoconf.mk." % \
-            subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()
+            subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
         subprocess.check_output(['git', 'checkout', commit],
                                 stderr=subprocess.STDOUT, cwd=self.src_dir)
 
@@ -1480,14 +1480,14 @@ def move_config(toolchains, configs, options, db_queue):
     """
     if len(configs) == 0:
         if options.force_sync:
-            print 'No CONFIG is specified. You are probably syncing defconfigs.',
+            print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
         elif options.build_db:
-            print 'Building %s database' % CONFIG_DATABASE
+            print('Building %s database' % CONFIG_DATABASE)
         else:
-            print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.',
+            print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
     else:
-        print 'Move ' + ', '.join(configs),
+        print('Move ' + ', '.join(configs), end=' ')
-    print '(jobs: %d)\n' % options.jobs
+    print('(jobs: %d)\n' % options.jobs)
 
     if options.git_ref:
         reference_src = ReferenceSource(options.git_ref)
@@ -1517,7 +1517,7 @@ def move_config(toolchains, configs, options, db_queue):
     while not slots.empty():
         time.sleep(SLEEP_TIME)
 
-    print ''
+    print('')
     slots.show_failed_boards()
     slots.show_suspicious_boards()
 
@@ -1691,15 +1691,15 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added,
     for config in config_list:
         defconfigs = defconfig_db.get(config)
         if not defconfigs:
-            print '%s not found in any defconfig' % config
+            print('%s not found in any defconfig' % config)
             continue
 
         # Get the set of defconfigs without this one (since a config cannot
         # imply itself)
         non_defconfigs = all_defconfigs - defconfigs
         num_defconfigs = len(defconfigs)
-        print '%s found in %d/%d defconfigs' % (config, num_defconfigs,
+        print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
-                                                len(all_configs))
+                                                len(all_configs)))
 
         # This will hold the results: key=config, value=defconfigs containing it
         imply_configs = {}
@@ -1736,7 +1736,7 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added,
             if common_defconfigs:
                 skip = False
                 if find_superset:
-                    for prev in imply_configs.keys():
+                    for prev in list(imply_configs.keys()):
                         prev_count = len(imply_configs[prev])
                         count = len(common_defconfigs)
                         if (prev_count > count and
@@ -1806,15 +1806,15 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added,
                             add_list[fname].append(linenum)
 
             if show and kconfig_info != 'skip':
-                print '%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
+                print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
-                                              kconfig_info, missing_str)
+                                              kconfig_info, missing_str))
 
         # Having collected a list of things to add, now we add them. We process
         # each file from the largest line number to the smallest so that
         # earlier additions do not affect our line numbers. E.g. if we added an
         # imply at line 20 it would change the position of each line after
         # that.
-        for fname, linenums in add_list.iteritems():
+        for fname, linenums in add_list.items():
             for linenum in sorted(linenums, reverse=True):
                 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
 
@@ -1891,11 +1891,11 @@ def main():
             for flag in options.imply_flags.split(','):
                 bad = flag not in IMPLY_FLAGS
                 if bad:
-                    print "Invalid flag '%s'" % flag
+                    print("Invalid flag '%s'" % flag)
                 if flag == 'help' or bad:
-                    print "Imply flags: (separate with ',')"
+                    print("Imply flags: (separate with ',')")
-                    for name, info in IMPLY_FLAGS.iteritems():
+                    for name, info in IMPLY_FLAGS.items():
-                        print ' %-15s: %s' % (name, info[1])
+                        print(' %-15s: %s' % (name, info[1]))
                     parser.print_usage()
                     sys.exit(1)
                 imply_flags |= IMPLY_FLAGS[flag][0]
@@ -1905,7 +1905,7 @@ def main():
         return
 
     config_db = {}
-    db_queue = Queue.Queue()
+    db_queue = queue.Queue()
     t = DatabaseThread(config_db, db_queue)
     t.setDaemon(True)
     t.start()
@@ -1939,7 +1939,7 @@ def main():
 
     if options.build_db:
         with open(CONFIG_DATABASE, 'w') as fd:
-            for defconfig, configs in config_db.iteritems():
+            for defconfig, configs in config_db.items():
                 fd.write('%s\n' % defconfig)
                 for config in sorted(configs.keys()):
                     fd.write('   %s=%s\n' % (config, configs[config]))

tools/patman/command.py

@@ -4,6 +4,7 @@
 
 import os
 import cros_subprocess
+import tools
 
 """Shell command ease-ups for Python."""
 
@@ -31,6 +32,13 @@ class CommandResult:
         self.return_code = return_code
         self.exception = exception
 
+    def ToOutput(self, binary):
+        if not binary:
+            self.stdout = tools.ToString(self.stdout)
+            self.stderr = tools.ToString(self.stderr)
+            self.combined = tools.ToString(self.combined)
+        return self
+
 
 # This permits interception of RunPipe for test purposes. If it is set to
 # a function, then that function is called with the pipe list being
@@ -41,7 +49,7 @@ test_result = None
 
 def RunPipe(pipe_list, infile=None, outfile=None,
             capture=False, capture_stderr=False, oneline=False,
-            raise_on_error=True, cwd=None, **kwargs):
+            raise_on_error=True, cwd=None, binary=False, **kwargs):
     """
     Perform a command pipeline, with optional input/output filenames.
 
@@ -67,7 +75,7 @@ def RunPipe(pipe_list, infile=None, outfile=None,
         else:
             return test_result
         # No result: fall through to normal processing
-    result = CommandResult()
+    result = CommandResult(b'', b'', b'')
     last_pipe = None
     pipeline = list(pipe_list)
     user_pipestr =  '|'.join([' '.join(pipe) for pipe in pipe_list])
@@ -93,29 +101,36 @@ def RunPipe(pipe_list, infile=None, outfile=None,
             if raise_on_error:
                 raise Exception("Error running '%s': %s" % (user_pipestr, str))
             result.return_code = 255
-            return result
+            return result.ToOutput(binary)
 
     if capture:
         result.stdout, result.stderr, result.combined = (
                 last_pipe.CommunicateFilter(None))
         if result.stdout and oneline:
-            result.output = result.stdout.rstrip('\r\n')
+            result.output = result.stdout.rstrip(b'\r\n')
         result.return_code = last_pipe.wait()
     else:
         result.return_code = os.waitpid(last_pipe.pid, 0)[1]
     if raise_on_error and result.return_code:
         raise Exception("Error running '%s'" % user_pipestr)
-    return result
+    return result.ToOutput(binary)
 
 def Output(*cmd, **kwargs):
     kwargs['raise_on_error'] = kwargs.get('raise_on_error', True)
     return RunPipe([cmd], capture=True, **kwargs).stdout
 
 def OutputOneLine(*cmd, **kwargs):
+    """Run a command and output it as a single-line string
+
+    The command us expected to produce a single line of output
+
+    Returns:
+        String containing output of command
+    """
     raise_on_error = kwargs.pop('raise_on_error', True)
-    return (RunPipe([cmd], capture=True, oneline=True,
+    result = RunPipe([cmd], capture=True, oneline=True,
-            raise_on_error=raise_on_error,
+                     raise_on_error=raise_on_error, **kwargs).stdout.strip()
-            **kwargs).stdout.strip())
+    return result
 
 def Run(*cmd, **kwargs):
     return RunPipe([cmd], **kwargs).stdout

tools/patman/func_test.py

@@ -51,7 +51,7 @@ class TestFunctional(unittest.TestCase):
 
     @classmethod
     def GetText(self, fname):
-        return open(self.GetPath(fname)).read()
+        return open(self.GetPath(fname), encoding='utf-8').read()
 
     @classmethod
     def GetPatchName(self, subject):
@@ -160,7 +160,7 @@ class TestFunctional(unittest.TestCase):
                     dry_run, not ignore_bad_tags, cc_file,
                     in_reply_to=in_reply_to, thread=None)
             series.ShowActions(args, cmd, process_tags)
-        cc_lines = open(cc_file).read().splitlines()
+        cc_lines = open(cc_file, encoding='utf-8').read().splitlines()
         os.remove(cc_file)
 
         lines = out[0].splitlines()
@@ -229,14 +229,14 @@ Simon Glass (2):
 2.7.4
 
 '''
-        lines = open(cover_fname).read().splitlines()
+        lines = open(cover_fname, encoding='utf-8').read().splitlines()
         self.assertEqual(
                 'Subject: [RFC PATCH v3 0/2] test: A test patch series',
                 lines[3])
         self.assertEqual(expected.splitlines(), lines[7:])
 
         for i, fname in enumerate(args):
-            lines = open(fname).read().splitlines()
+            lines = open(fname, encoding='utf-8').read().splitlines()
             subject = [line for line in lines if line.startswith('Subject')]
             self.assertEqual('Subject: [RFC %d/%d]' % (i + 1, count),
                              subject[0][:18])

tools/patman/patchstream.py

@@ -511,8 +511,8 @@ def FixPatch(backup_dir, fname, series, commit):
         A list of errors, or [] if all ok.
     """
     handle, tmpname = tempfile.mkstemp()
-    outfd = os.fdopen(handle, 'w')
+    outfd = os.fdopen(handle, 'w', encoding='utf-8')
-    infd = open(fname, 'r')
+    infd = open(fname, 'r', encoding='utf-8')
     ps = PatchStream(series)
     ps.commit = commit
     ps.ProcessStream(infd, outfd)

tools/patman/patman.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0+
 #
 # Copyright (c) 2011 The Chromium OS Authors.

tools/patman/series.py

@@ -223,7 +223,7 @@ class Series(dict):
         col = terminal.Color()
         # Look for commit tags (of the form 'xxx:' at the start of the subject)
         fname = '/tmp/patman.%d' % os.getpid()
-        fd = open(fname, 'w')
+        fd = open(fname, 'w', encoding='utf-8')
         all_ccs = []
         for commit in self.commits:
             cc = []

tools/patman/settings.py

@@ -165,7 +165,7 @@ def ReadGitAliases(fname):
         fname: Filename to read
     """
     try:
-        fd = open(fname, 'r')
+        fd = open(fname, 'r', encoding='utf-8')
     except IOError:
         print("Warning: Cannot find alias file '%s'" % fname)
         return
@@ -259,7 +259,7 @@ def _ReadAliasFile(fname):
     """
     if os.path.exists(fname):
         bad_line = None
-        with open(fname) as fd:
+        with open(fname, encoding='utf-8') as fd:
             linenum = 0
             for line in fd:
                 linenum += 1

tools/patman/test.py

@@ -72,12 +72,12 @@ Signed-off-by: Simon Glass <sjg@chromium.org>
 '''
         out = ''
         inhandle, inname = tempfile.mkstemp()
-        infd = os.fdopen(inhandle, 'w')
+        infd = os.fdopen(inhandle, 'w', encoding='utf-8')
         infd.write(data)
         infd.close()
 
         exphandle, expname = tempfile.mkstemp()
-        expfd = os.fdopen(exphandle, 'w')
+        expfd = os.fdopen(exphandle, 'w', encoding='utf-8')
         expfd.write(expected)
         expfd.close()
 

tools/patman/tools.py

@@ -186,7 +186,7 @@ def PathHasFile(path_spec, fname):
             return True
     return False
 
-def Run(name, *args):
+def Run(name, *args, **kwargs):
     """Run a tool with some arguments
 
     This runs a 'tool', which is a program used by binman to process files and
@@ -201,13 +201,14 @@ def Run(name, *args):
         CommandResult object
     """
     try:
+        binary = kwargs.get('binary')
         env = None
         if tool_search_paths:
             env = dict(os.environ)
             env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
         all_args = (name,) + args
         result = command.RunPipe([all_args], capture=True, capture_stderr=True,
-                                 env=env, raise_on_error=False)
+                                 env=env, raise_on_error=False, binary=binary)
         if result.return_code:
             raise Exception("Error %d running '%s': %s" %
                (result.return_code,' '.join(all_args),
@@ -375,7 +376,7 @@ def ToBytes(string):
     """Convert a str type into a bytes type
 
     Args:
-        string: string to convert value
+        string: string to convert
 
     Returns:
         Python 3: A bytes type
@@ -385,6 +386,18 @@ def ToBytes(string):
         return string.encode('utf-8')
     return string
 
+def ToString(bval):
+    """Convert a bytes type into a str type
+
+    Args:
+        bval: bytes value to convert
+
+    Returns:
+        Python 3: A bytes type
+        Python 2: A string type
+    """
+    return bval.decode('utf-8')
+
 def Compress(indata, algo, with_header=True):
     """Compress some data using a given algorithm
 
@@ -406,14 +419,14 @@ def Compress(indata, algo, with_header=True):
     fname = GetOutputFilename('%s.comp.tmp' % algo)
     WriteFile(fname, indata)
     if algo == 'lz4':
-        data = Run('lz4', '--no-frame-crc', '-c', fname)
+        data = Run('lz4', '--no-frame-crc', '-c', fname, binary=True)
     # cbfstool uses a very old version of lzma
     elif algo == 'lzma':
         outfname = GetOutputFilename('%s.comp.otmp' % algo)
         Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8')
         data = ReadFile(outfname)
     elif algo == 'gzip':
-        data = Run('gzip', '-c', fname)
+        data = Run('gzip', '-c', fname, binary=True)
     else:
         raise ValueError("Unknown algorithm '%s'" % algo)
     if with_header:
@@ -446,13 +459,13 @@ def Decompress(indata, algo, with_header=True):
     with open(fname, 'wb') as fd:
         fd.write(indata)
     if algo == 'lz4':
-        data = Run('lz4', '-dc', fname)
+        data = Run('lz4', '-dc', fname, binary=True)
     elif algo == 'lzma':
         outfname = GetOutputFilename('%s.decomp.otmp' % algo)
         Run('lzma_alone', 'd', fname, outfname)
-        data = ReadFile(outfname)
+        data = ReadFile(outfname, binary=True)
     elif algo == 'gzip':
-        data = Run('gzip', '-cd', fname)
+        data = Run('gzip', '-cd', fname, binary=True)
     else:
         raise ValueError("Unknown algorithm '%s'" % algo)
     return data

tools/rkmux.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Script to create enums from datasheet register tables
 #
@@ -43,8 +43,8 @@ class RegField:
         self.desc.append(desc)
 
     def Show(self):
-        print self
+        print(self)
-        print
+        print()
         self.__init__()
 
     def __str__(self):
@@ -65,11 +65,11 @@ class Printer:
             self.output_footer()
 
     def output_header(self):
-        print '/* %s */' % self.name
+        print('/* %s */' % self.name)
-        print 'enum {'
+        print('enum {')
 
     def output_footer(self):
-        print '};';
+        print('};');
 
     def output_regfield(self, regfield):
         lines = regfield.desc
@@ -97,7 +97,7 @@ class Printer:
             self.first = False
             self.output_header()
         else:
-            print
+            print()
         out_enum(field, 'shift', bit_low)
         out_enum(field, 'mask', mask)
         next_val = -1
@@ -175,7 +175,7 @@ def out_enum(field, suffix, value, skip_val=False):
             val_str = '%d' % value
 
         str += '%s= %s' % ('\t' * tabs, val_str)
-    print '\t%s,' % str
+    print('\t%s,' % str)
 
 # Process a CSV file, e.g. from tabula
 def process_csv(name, fd):