#! /bin/bash

# malgen regenerates lists in malware pages from mal.rec.

# Usage:  malgen [page1 page2 ...]
#  e.g.,  malgen malware-microsoft.html proprietary-inferference.html

# The script processes all pages when no argument is given.

# Copyright (C) 2018, 2019, 2020, 2021, 2023, 2025 Free Software Foundation, Inc.

# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.  This file is offered as-is,
# without any warranty.

# Written by Therese <thg@gnu.org> and Ineiev.

export LC_ALL=C
set -e

# Output mal.rec items sorted by descending Id.
function sort_recfile () {
  {
    # Output sorted Id lines separated by empty lines.
    grep '^Id: ' mal.rec | sort -r | sed "s/$/\\n/";
    # Mark end of ids, append mal.rec.
    echo end of ids; echo; cat mal.rec;
  } | awk 'BEGIN {
         RS=""; ORS="\n\n"; FS="\n"
         reading_ids = 1; id_no = 1
       }
       reading_ids && /end of ids/ { reading_ids = 0; next }
       reading_ids { ids[id_no++] = $1; next }
       /(.*\n|^)Id: / {
         id=$0; sub(/(.*\n|^)Id:/, "Id:", id); sub(/\n.*/, "", id)
         a[id]=$0
       }
       END {
         for (i = 1; i < id_no; i++) { print a[ids[i]] }
       }' > $sorted
}

# Regenerate targets.rec.
function update_targets () {
  # List English pages in proprietary/.
  pages=$(ls ../*.html |
        grep -Ev '\.([[:alpha:]]{2}-)?[[:alpha:]]{2}\.html$' |
        sed 's,^\.\./,,')
  # Create the new Targets record.
  for p in $pages; do
    targets=$(awk -f list-targets.awk ../$p)
    if test -n "$targets"; then
      cat >> $temp1 <<EOF

Page: $p
Targets: $targets
EOF
    fi
  done
  # Replace the old one.
  sed -i '/^Page:/,$d
          /%type: Targets/r'"$temp1" targets.rec
}

# Convert the blurbs to entries in malware-*.html or proprietary-*.html.
function rec2html () {
  sed    '$ { /^$/d }' $temp1 > $temp2
  sed    '/^[-0-9][-0-9]*$/d' $temp2 |
  sed    '/^Id:/p;/^Blurb:/p;/^+/p;/^$/p;d' |
  sed    's,^Id:[ \t]*\([0-9]\{9\}\(\..\)\?\).*$,  <li id="M\1">,' |
  sed    's,[[:space:]]*<li id="M\(20[012][0-9]\)\([0-9]\{2\}\).*$,'\
'<!-- Copied from workshop/mal.rec. Do not edit in '"$1"'. -->\
&\
    <!--#set var="DATE" value='"'"'<small class="date-tag">\1-\2</small>'"'"'\
    --><!--#echo encoding="none" var="DATE" -->,
          s,date-tag">\([0-9]\{4\}\)-00,date-tag">[\1],
          s,^$,  </li>\n,
          s,^Blurb:[ \t]*,    ,
          s,^+, ,
          s,^[ \t]*$,,
          $s,$,\n  </li>\n</ul>,' > $temp1
}

# Convert the blurbs to entries in all.html.
function rec2all () {
  split="<span class=\"gnun-split\"></span>"
  yr="20[012][0-9]"
  md="[0-9]\\{2\\}"
  sed    '$ { /^$/d }' $temp1 > $temp2
  sed    '/^Added:/p;/^Id:/p;/^Blurb:/p;/^+/p;/^$/p;d' $temp2 |
  sed    -e "/^Added:/! { p; d }" -e "N" \
         -e "s,Added:[ \t]*\\($yr-$md-$md\\),\
<!--#set var='ADD' value='\\1' -->,"\
         -e "s,Id:[ \t]*\\($yr\\)\\($md\\)\\($md\\)\\(.*\\)$,\
<!--#set var='PUB' value='\\1-\\2-\\3' --> \
<!--#set var='ID' value='M\\1\\2\\3\\4' -->," \
         -e "i<!-- Copied from workshop/mal.rec; don't edit in $1. -->" \
         -e "a<li id='<!--#echo encoding='none' var='ID' -->'>" \
         -e "a<small class='date-tag'>Added: $split<!--#echo" \
         -e "aencoding='none' var='ADD' -->$split" \
         -e "a\\&mdash; Latest reference: $split<!--#echo" \
         -e "aencoding='none' var='PUB' --></small>" |
  sed    's,^$,  </li>\n,
          s,^Blurb:[ \t]*,    ,
          s,^+, ,
          s,^[ \t]*$,,
          $s,$,\n  </li>\n</ul>,' > $temp1
}

function insert_list () {
  sed "/id=[\"']$t[\"']/,/^<\/ul>/ {
         /id=[\"']$t[\"']/,/^<ul class=\"blurbs\">/p
         /class=\"blurbs\"/r $temp1
         d
      }" $out > $f.temp && mv $f.temp $out
}

# Check mal.rec for consistency.
if [ `grep '^Id: ' mal.rec | sort | uniq -d | wc -l` -gt 0 ]; then
  echo 1>&2 "Id lines in mal.rec aren't unique:"
  grep '^Id: ' mal.rec |  sort | uniq -dc 
  exit 1
fi

# Remove any trailing spaces from the Target: lines.
sed -i '/Target:/ s,[ \t]*$,,' mal.rec

# List English pages in proprietary/.
pages=$(ls ../*.html |
        grep -Ev '\.([[:alpha:]]{2}-)?[[:alpha:]]{2}\.html$' |
        sed 's,^\.\./,,')

if [ $# = 0 ]; then
  input="$pages"
else
  input="$(while [ $# -gt 0 ]; do echo "x$1" | sed 's,.,,;q'; shift; done)"
  wrong_args=$(echo "x$input" | sed '1s,.,,' | while read arg; do
    if ! echo "x$pages" | sed '1s,.,,' | grep -q "^${arg//./[.]}$"; then
      echo "'$arg'"
    fi
  done)
  if [ -n "$wrong_args" ]; then
    echo 1>&2 "These pages don't exist: $wrong_args"
    exit 1
  fi
fi

# Create temporary files.
temp1=$(mktemp -t mal.XXXXXX) || (echo 1>&2 "Can't make temp1";  exit 1)
temp2=$(mktemp -t mal.XXXXXX) || (echo 1>&2 "Can't make temp2";  exit 1)
sorted=$(mktemp -t mal.XXXXXX) || (echo 1>&2 "Can't make sorted";  exit 1)
trap 'rm -f "$temp1" "$temp2" "$sorted"' EXIT

sort_recfile
update_targets

echo "x$input" | sed '1s,.,,' | while read f; do
 out="../$f"
 if [ "$f" != 'proprietary-menu.html' ]; then
  echo "Regenerating '$f'"
  # List all possible targets.
  targets=$(awk 'BEGIN { RS = ""; FS="\nTargets: " }'"
                 /^Page: ${f//./[.]}\n/"'{$1=""; print $0}' targets.rec \
                 | sed 's,^ *,,;s, *$,,')

  # Check that mal.rec only lists real targets.
  wrong_targets=$(grep "Target:[ \t]*${f//./[.]}" mal.rec | 
    sed -e 's,^.*html[ \t]*,,' | sort | uniq |
    while read t; do
    if ! echo " $targets " | grep -q " $t " ; then
      if test "x$t" = x; then
         t='<empty target>'
      fi
      echo "    * $f $t"
    fi
  done)
  if [ -n "$wrong_targets" ]; then
    echo 1>&2 "!!! Incomplete regeneration - These targets don't exist:
$wrong_targets
    If one of these is a new target, add an empty blurb list with
    proper id to the relevant page, then run malgen again."
    exit 1
  fi
  for t in $targets; do
    if [ "$f" != 'proprietary.html' ] && [ "$f" != 'all.html' ]; then

      awk 'BEGIN { RS=""; ORS="\n\n"; FS="\n" }
           /\nTarget: '"${f//./[.]}[ \t]+${t//./[.]}"'\n/ { print }' \
          $sorted  > $temp1

      if [ -s $temp1 ]; then
        rec2html "$f"
        insert_list
      else
        echo 1>&2 "!!! No item was found for $f#$t."
      fi

    else 
      # List 5 recently added items. Sort by descending date.
      # 20 items are sorted, to allow for variants that will not be displayed
      # (e.g. Id: 201902041.1).
      all=`grep "^Added: " $sorted | sort -r \
              | sed "s/Added: //" | uniq`
      echo "x$all" | sed '1s,.,,' | grep -v "^[-0-9]*$" \
        | while read i; do
            echo 1>&2 "Malformed \"Added\" field found: '$i'"
          done

     if [ "$f" = 'proprietary.html' ]; then
      for i in $all; do
        awk 'BEGIN { RS=""; FS="\n"; ORS="\n\n" }
             /(\n|^)Added: '$i'(\n|$)/ && !/(\n|^)Id: [^\n]*[.]/ { print }
            ' $sorted;
      done | awk 'BEGIN { RS=""; FS="\n"; ORS="\n\n"; i = 1 }
                  { if (i++ > 5) { exit 0; } print }
                 ' > $temp1
      rec2html "$f"
      insert_list
     else
      for i in $all; do
        awk 'BEGIN { RS=""; FS="\n"; ORS="\n\n" }
             /(\n|^)Added: '$i'(\n|$)/ && !/(\n|^)Id: [^\n]*[.]/ { print }
            ' $sorted;
      done | awk 'BEGIN { RS=""; FS="\n"; ORS="\n\n"; i = 1 }
                  { print }
                 ' > $temp1
      rec2all "$f"
      insert_list
     fi
    fi
  done # for t in $targets
 fi
done # for f in $input

./update-item-count