feat: Validate ESD catalog XML responses (#2054)

src/image.sh

@@ -1407,7 +1407,7 @@ extractESD() {
   local installPad=3000000
   local spacePad=1073741824
 
-  local msg="Extracting $desc bootdisk from ESD file"
+  local msg="Extracting bootdisk from ESD file"
   info "$msg..." && html "$msg..."
 
   if ! size=$(stat -c%s -- "$iso"); then
@@ -1507,7 +1507,7 @@ extractESD() {
   bootWim="$dir/sources/boot.wim"
   installWim="$dir/sources/install.esd"
 
-  msg="Extracting $desc environment from ESD file"
+  msg="Extracting environment from ESD file"
   info "$msg..." && html "$msg..."
 
   index="2"
@@ -1523,7 +1523,7 @@ extractESD() {
 
   fKill "progress.sh"
 
-  msg="Extracting $desc setup from ESD file"
+  msg="Extracting setup from ESD file"
   info "$msg..."
 
   index="3"
@@ -1545,7 +1545,7 @@ extractESD() {
     LABEL="CPBA_A64FRE_EN-US_DV9"
   fi
 
-  msg="Extracting $desc image from ESD file"
+  msg="Extracting image from ESD file"
   info "$msg..." && html "$msg..."
 
   edition=$(getCatalog "$version" "name")

src/mido.sh

@@ -716,8 +716,14 @@ getWorCatalog() {
   local ret="$2"
   local build="${3:-}"
 
-  local edition="" filter="" version=""
+  local file="catalog.xml"
-  local file="catalog.xml" url="" name="" 
+  local name="" filter="" url=""
+  local version="" edition="" arch=""
+
+  case "${PLATFORM,,}" in
+    "x64" ) arch="x64" ;;
+    "arm64" ) arch="ARM64" ;;
+  esac
 
   case "${id,,}" in
     "win11${PLATFORM,,}" )
@@ -738,8 +744,8 @@ getWorCatalog() {
       name="Windows 10 Enterprise" ;;
   esac
 
-  if [ -n "$version" ]; then
+  if [ -n "$version" ] && [ -n "$arch" ]; then
-    url="https://worproject.com/dldserv/esd/getcatalog.php?ver=${version}&arch=${PLATFORM^^}&edition=${filter}"
+    url="https://worproject.com/dldserv/esd/getcatalog.php?ver=${version}&arch=${arch}&edition=${filter}"
     [ -n "$build" ] && url+="&build=${build}"
   fi
 
@@ -764,30 +770,32 @@ parseESD() {
   local culture="$6"
 
   local xmlFile="${xml##*/}"
-  local file_path file_sum file_size file_edition
+  local file_size file_edition file_culture
-  local file_culture file_match=0 language_match=0
+  local file_path file_sum file_sha1 file_sha256
+  local file_match=0 language_match=0
   local records architecture language separator=$'\x1f'
+  local upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+  local lower="abcdefghijklmnopqrstuvwxyz"
 
   ESD=""
   ESD_SUM=""
   ESD_SIZE=""
 
-  # Microsoft catalogs have used different XML namespaces. Match elements by
+  # Microsoft catalogs may use different XML namespaces, tag casing, and checksum
-  # local name and flatten the catalog once so selection needs no temporary XML.
+  # algorithms. Flatten the catalog once so all selection logic remains in Bash.
-  if ! records=$(xmlstarlet sel \
+  local upper="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    -T -t \
+  local lower="abcdefghijklmnopqrstuvwxyz"
-    -m "//*[local-name()='File']" \
+  local lname="translate(local-name(), '$upper', '$lower')"
-      -v "normalize-space(*[local-name()='Architecture'])" \
+
-      -o "$separator" \
+  if ! records=$(xmlstarlet sel -T -t \
-      -v "normalize-space(*[local-name()='Edition'])" \
+      -m "//*[$lname='file']" \
-      -o "$separator" \
+        -v "normalize-space(*[$lname='architecture'][1])" -o "$separator" \
-      -v "normalize-space(*[local-name()='LanguageCode'])" \
+        -v "normalize-space(*[$lname='edition'][1])" -o "$separator" \
-      -o "$separator" \
+        -v "normalize-space(*[$lname='languagecode'][1])" -o "$separator" \
-      -v "normalize-space(*[local-name()='FilePath'])" \
+        -v "normalize-space(*[$lname='filepath'][1])" -o "$separator" \
-      -o "$separator" \
+        -v "normalize-space(*[$lname='sha256'][1])" -o "$separator" \
-      -v "normalize-space(*[local-name()='Sha1'])" \
+        -v "normalize-space(*[$lname='sha1'][1])" -o "$separator" \
-      -o "$separator" \
+        -v "normalize-space(*[$lname='size'][1])" \
-      -v "normalize-space(*[local-name()='Size'])" \
         -n \
       "$xml" 2>/dev/null); then
 
@@ -799,9 +807,9 @@ parseESD() {
   # distinguish an unavailable edition from an unavailable translation.
   while IFS="$separator" read -r \
     architecture file_edition file_culture \
-    file_path file_sum file_size; do
+    file_path file_sha256 file_sha1 file_size; do
 
-    [ -n "$architecture$file_path$file_sum$file_size$file_culture$file_edition" ] || continue
+    [ -n "$architecture$file_path$file_sha256$file_sha1$file_size$file_culture$file_edition" ] || continue
 
     [ "${architecture,,}" = "${PLATFORM,,}" ] || continue
 
@@ -815,6 +823,8 @@ parseESD() {
     [ "${file_culture,,}" = "${culture,,}" ] || continue
 
     language_match=1
+    file_sum="${file_sha256:-$file_sha1}"
+
     ESD="$file_path"
     ESD_SUM="$file_sum"
     ESD_SIZE="$file_size"
@@ -853,6 +863,83 @@ parseESD() {
   return 0
 }
 
+getCatalogError() {
+
+  local file="$1"
+  local result=""
+
+  [ -s "$file" ] || return 0
+
+  # Prefer leaf text when the response is parseable XML, separating adjacent
+  # elements so HTML error pages remain readable. Fall back to the beginning
+  # of a plain-text response when XML parsing fails.
+  result=$(xmlstarlet sel -T -t \
+    -m '//*[not(*) and normalize-space()]' \
+      -v 'normalize-space(.)' -o ' ' \
+    "$file" 2>/dev/null) || result=$(head -c 1024 -- "$file" 2>/dev/null || :)
+
+  # Keep upstream error details useful without allowing control sequences,
+  # multiline output, or a complete HTML error page into the application log.
+  result=$(
+    printf '%s' "$result" | \
+      LC_ALL=C tr -cd '\11\12\15\40-\176' | \
+      sed 's/<[^>]*>/ /g' | tr '\r\n\t' '   ' | \
+      sed 's/[[:space:]][[:space:]]*/ /g; s/^ //; s/ $//'
+  )
+
+  if (( ${#result} > 500 )); then
+    result="${result:0:500}..."
+  fi
+
+  printf '%s' "$result"
+  return 0
+}
+
+validateESDCatalog() {
+
+  local xml="$1"
+  local provider="$2"
+
+  local metadata root fileCount 
+  local separator=$'\x1f' response
+  local upper='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+  local lower='abcdefghijklmnopqrstuvwxyz'
+
+  if [ ! -s "$xml" ]; then
+    error "$provider returned an empty ESD catalog!"
+    return 1
+  fi
+
+  # Successful Microsoft and WoR responses contain an MCT document with at
+  # least one File record. Reject API error tokens and unrelated XML before
+  # parseESD can misreport them as a products.xml parsing failure.
+  if metadata=$(xmlstarlet sel \
+      -T -t \
+      -v "translate(local-name(/*), '$upper', '$lower')" -o "$separator" \
+      -v "count(/*[translate(local-name(), '$upper', '$lower')='mct']//*[translate(local-name(), '$upper', '$lower')='file'])" \
+      "$xml" 2>/dev/null); then
+
+    IFS="$separator" read -r root fileCount <<< "$metadata"
+
+    if [ "$root" = "mct" ] &&
+        [[ "$fileCount" =~ ^[0-9]+$ ]] &&
+        (( fileCount > 0 )); then
+      return 0
+    fi
+
+  fi
+
+  response=$(getCatalogError "$xml") || response=""
+
+  if [ -n "$response" ]; then
+    error "$provider returned: $response"
+  else
+    error "$provider returned an invalid ESD catalog!"
+  fi
+
+  return 1
+}
+
 getESD() {
 
   local dir="$1"
@@ -861,9 +948,9 @@ getESD() {
   local desc="$4"
   local source="${5:-microsoft}"
 
-  local file culture log
-  local edition catalog rc=0
   local xmlFile="products.xml"
+  local file culture provider
+  local edition catalog log rc=0
 
   culture=$(getLanguage "$lang" "culture")
   file=$(getCatalog "$version" "file" "$source")
@@ -875,6 +962,9 @@ getESD() {
     return 1
   fi
 
+  provider="Microsoft"
+  [[ "${file,,}" == *.xml ]] && provider="WoR"
+
   local msg="Downloading ESD catalog..."
   info "$msg" && html "$msg"
 
@@ -922,9 +1012,11 @@ getESD() {
 
   rm -f "$log"
 
-  # Normal catalogs arrive as CAB archives, while pinned build catalogs are
+  # Microsoft catalogs arrive as CAB archives. WoR catalogs are returned as
-  # already XML and only need the common filename.
+  # XML and must be validated before they are published as products.xml.
-  if [[ "$file" == *".xml" ]]; then
+  if [[ "${file,,}" == *.xml ]]; then
+
+    validateESDCatalog "$dir/$file" "$provider" || return 1
 
     if ! mv -f "$dir/$file" "$dir/$xmlFile"; then
       error "Failed to rename $file to $xmlFile."
@@ -948,6 +1040,12 @@ getESD() {
     return 1
   fi
 
+  # Validate extracted Microsoft catalogs as well, so parseESD only receives
+  # the catalog format it expects.
+  if [[ "${file,,}" != *.xml ]]; then
+    validateESDCatalog "$dir/$xmlFile" "$provider" || return 1
+  fi
+
   if ! parseESD \
     "$dir/$xmlFile" "$version" "$lang" "$desc" "$edition" "$culture"; then
     return 1
@@ -1299,7 +1397,6 @@ downloadImage() {
     if getESD "$TMP/esd" "$version" "$lang" "$desc"; then
       success="y"
     else
-      delay "$seconds"
       getESD "$TMP/esd" "$version" "$lang" "$desc" "wor" && success="y"
     fi