data-run

#!/bin/sh

# supercop/data-run
# D. J. Bernstein
# Public domain.

. ./data-vars

exec <"$top/todo"
exec >/dev/null
exec 2>/dev/null

startseconds=`sinceepoch`
timeout="${1-0}"

while read target
do
  if [ "$timeout" -gt 0 ]
  then
    nowseconds=`sinceepoch`
    if [ `expr $nowseconds - $startseconds` -gt "$timeout" ]
    then
      exit 111
    fi
  fi

  mv "$top/$target/LOCK" "$top/$target/LOCKED" 2>/dev/null || continue

  rm -rf "$top/$target/log.new"
  rm -rf "$top/$target/data.new"
  rm -rf "$top/$target/sources.new"

  exec >"$top/$target/log.new"
  exec 2>&1
  exec 5>"$top/$target/data.new"
  exec 8>"$top/$target/sources.new"

  echo "=== `date` === starting $target"
  startdate=`date +%Y%m%d`

  rm -rf "$top/$target/work"
  if [ ! -f "$top/$target/BUILD" ]
  then
    echo "=== `date` === $target already built"
    rm "$top/$target/LOCKED"
    continue
  fi

  abi=`echo $target | sed 's_/.*__'`
  t1etc=`echo $target | sed 's_^[^/]*/__'`
  t1=`echo $t1etc | sed 's_/.*__'`
  t2etc=`echo $t1etc | sed 's_^[^/]*/__'`
  t2=`echo $t2etc | sed 's_/.*__'`
  t3etc=`echo $t2etc | sed 's_^[^/]*/__'`
  t3=`echo $t3etc | sed 's_/.*__'`
  t4etc=`echo $t3etc | sed 's_^[^/]*/__'`
  t4=`echo $t4etc | sed 's_/.*__'`
  t5etc=`echo $t4etc | sed 's_^[^/]*/__'`
  t5=`echo $t5etc | sed 's_/.*__'`
  t6=`echo $t5etc | sed 's_^[^/]*/__'`

  while read source
  do
    while [ -f "$top/$abi/$source/BUILD" ]
    do
      sleep 1
      if [ "$timeout" -gt 0 ]
      then
        nowseconds=`sinceepoch`
        if [ `expr $nowseconds - $startseconds` -gt "$timeout" ]
        then
          exit 111
        fi
      fi
    done
  done < "$top/$target/dep"

  (
    if [ "$t1" = measure1 -o "$t1" = measure2 -o "$t1" = measure3 -o "$t1" = integrate ]
    then
      grep '^try/' "$top/$target/dep" \
      | while read source
      do
        if [ -s "$top/$abi/$source/CYCLES" ]
        then
          echo `cat "$top/$abi/$source/CYCLES"` "$source"
        fi
      done \
      | sort -n \
      | head -1 \
      | awk '{print $2}' \
      | while read source
      do
        echo "$source" > "$top/$target/best"
	echo "$target/best $source" >&8
	cat "$top/$abi/$source/sources" >&8
      done

      if [ "$t1" = integrate ]
      then
        grep "^$t2 " OPERATIONS \
        | while read o macros prototypes
        do
          p=$t3
          [ -f "$o/$p/used" ] \
	  && echo "used" >&8
        done
      fi

    else
      while read source
      do
	if [ -s "$top/$abi/$source/sources" ]
	then
          cat "$top/$abi/$source/sources" >&8
        fi
      done < "$top/$target/dep"

      if [ "$t1" = gmp ]
      then
        echo "gmp 6.1.2" >&8
      fi
  
      if [ "$t1" = cryptopp ]
      then
        echo "cryptopp 565" >&8
      fi
  
      if [ "$t1" = openssl ]
      then
        rm -rf "$top/$target/work"
        mkdir -p "$top/$target/work"
	(
	  cd "$top/$target/work"
	  (
	    /bin/echo '#include <stdio.h>'
	    /bin/echo '#include <openssl/opensslv.h>'
	    /bin/echo 'main() { printf("openssl/version %s\n",OPENSSL_VERSION_TEXT); return 0; }'
	  ) > v.c
	  libs=`"oklibs-$abi"`
          okc-$abi \
          | while read compiler
          do
	    if $compiler -o v v.c $libs
	    then
	      break
	    fi
          done
          ./v >&8 || :
        )
      fi
  
      if [ "$t1" = compiler ]
      then
        language=$t2
        compilerword=$t3
        compiler=`echo "$compilerword" | tr '_' ' '`
  
        rm -rf "$top/$target/work"
        mkdir -p "$top/$target/work"
        cp -p "measure-anything.c" "$top/$target/work/measure-anything.$language"
        (
          echo 'const char *primitiveimplementation = "";'
          echo 'const char *implementationversion = "";'
          echo 'const char *sizenames[] = { 0 };'
          echo 'extern const long long sizes[];'  # XXX: seems necessary for g++
          echo 'const long long sizes[] = { 0 };'
          echo 'void preallocate(void) { }'
          echo 'void allocate(void) { }'
          echo 'void measure(void) { }'
        ) > "$top/$target/work/measure.$language"
  
        (
          cd "$top/$target/work"
          libs=`"oklibs-$abi"`
          libs="$top/$abi/lib/cpucycles.o $libs"
          libs="$top/$abi/lib/kernelrandombytes.o $libs"
	  if $compiler -D'COMPILER="'"$compiler"'"' \
            -DLOOPS=1 \
            -I. -I"$top/$abi/include" \
            -o measure measure.$language measure-anything.$language \
	    $libs
	  then
	    ./measure || :
	  fi
        ) \
	| ( egrep '^cpuid|^cpucycles_implementation|^compiler' || : ) \
	| while read m
	do
	  /bin/echo "$target/$m"
	done >&8

      fi

      if [ "$t1" = implementation ]
      then
        implementationdir="$t2etc"
	find "$implementationdir" -follow -type f \
	'!' -name implementors \
	'!' -name architectures \
	| sort \
	| while read f
	do
	  /bin/echo "$f" `adequatechecksum < $f`
	done >&8
	for f in "$t2/try.c" "$t2/measure.c"
	do
	  /bin/echo "$f" `adequatechecksum < $f`
	done >&8
      fi
    fi
  )


  if cmp "$top/$target/sources.new" "$top/$target/sources" >/dev/null 2>/dev/null
  then
    echo "=== `date` === $target is up to date"
    exec 5>&-
    exec 8>&-
    rm "$top/$target/sources.new"
    rm "$top/$target/data.new"
    rm "$top/$target/BUILD"
    rm "$top/$target/LOCKED"
    continue
  fi


  (

    if [ "$t1" = gmp ]
    then
      rm -f "$top/$abi/lib/libgmp.a"
      rm -f "$top/$abi/lib/libgmpxx.a"
      okc-$abi | head -1 \
      | while read c copts
      do
        okcpp-$abi | head -1 \
        | while read cpp cppopts
        do
          for gmpabi in 64 32 2.0w 2.0n 1.0 o32 n32 aix64 mode64 mode32 standard
          do
	    [ -s "$top/$abi/lib/libgmp.a" ] && continue
	    echo "=== `date` === trying CC=$c CXX=$cpp CFLAGS=$copts CXXFLAGS=$cppopts ABI=$gmpabi"
	    rm -rf "$top/$target/work"
	    mkdir -p "$top/$target/work"
	    cp -pr gmp-6.1.2/* "$top/$target/work"
	    ( cd "$top/$target/work" \
	      && ./configure --enable-cxx \
	         ABI="$gmpabi" \
	         CC="$c" CXX="$cpp" CFLAGS="$copts" CXXFLAGS="$cppopts" LDFLAGS="$copts" \
	      && make \
	      && make check \
	      && cp gmp.h gmpxx.h gmp-impl.h longlong.h \
	      config.h gmp-mparam.h fac_table.h fib_table.h mp_bases.h \
	      "$top/$abi/include" \
	      && ( ranlib ".libs/libgmp.a" || : ) \
	      && cp .libs/libgmp.a "$top/$abi/lib/libgmp.a" \
	      && chmod 644 "$top/$abi/lib/libgmp.a" \
	      && ( ranlib ".libs/libgmpxx.a" || : ) \
	      && ( cp .libs/libgmpxx.a "$top/$abi/lib/libgmpxx.a" || : ) \
	      && ( chmod 644 "$top/$abi/lib/libgmpxx.a" || : )
	    ) && break || :
          done
        done
      done
      exit 0
    fi

    if [ "$t1" = cryptopp ]
    then
      rm -f "$top/$abi/lib/libcryptopp.a"
      rm -rf "$top/$abi/include/cryptopp"
      mkdir -p "$top/$abi/include/cryptopp"
      okcpp-$abi | head -1 \
      | while read cpp cppopts
      do
        if [ -s "$top/$abi/lib/libcryptopp.a" ]
	then
	  continue
	fi
        echo "=== `date` === trying CXX=$cpp CXXFLAGS=$cppopts"
        rm -rf "$top/$target/work"
        mkdir -p "$top/$target/work"
        cp -pr cryptopp-565/* "$top/$target/work"
        ( cd "$top/$target/work" \
          && make CXX="$cpp" CXXFLAGS="-DNDEBUG $cppopts" LDFLAGS="$cppopts" \
          && cp libcryptopp.a "$top/$abi/lib/libcryptopp.a" \
          && cp *.h "$top/$abi/include/cryptopp/"
        ) && break
      done
    fi
  
    if [ "$t1" = base ]
    then
      rm -rf "$top/$abi/lib/lib${project}.a"
      rm -rf "$top/$target/work"
      mkdir -p "$top/$target/work"
      echo 'void crypto_'"$project"'_base(void) { ; }' > "$top/$target/work/${project}_base.c"
      okc-$abi \
      | while read compiler
      do
        ( cd "$top/$target/work" && $compiler -c ${project}_base.c ) && break
      done
      okar-$abi cr "$top/$abi/lib/lib${project}.a" "$top/$target/work/${project}_base.o"
      ( ranlib "$top/$abi/lib/lib${project}.a" || exit 0 )
    fi
  
    if [ "$t1" = try ]
    then
      grep "^$t4 " OPERATIONS \
      | while read o macros prototypes
      do
        rm -rf "$top/$target/CYCLES"
        rm -rf "$top/$target/errors"
        rm -rf "$top/$target/work"
        rm -rf "$top/$target/compiled"

        language=$t2
        compilerword=$t3
        compiler=`echo "$compilerword" | tr '_' ' '`
        p=$t5
        op="${o}_${p}"
        implementationdir="${o}/${p}/${t6}"
        opi=`echo "$implementationdir" | tr ./- ___`
  
        expectedchecksumbig=''
        [ -f "$o/$p/checksumbig" ] && expectedchecksumbig=`cat "$o/$p/checksumbig"`
        expectedchecksumsmall=''
        [ -f "$o/$p/checksumsmall" ] && expectedchecksumsmall=`cat "$o/$p/checksumsmall"`
  
        libs=`"oklibs-$abi"`
        libs="$top/$abi/lib/cpucycles.o $libs"
        libs="$top/$abi/lib/kernelrandombytes.o $libs"
	if grep '^gmp ' "$top/$abi/implementation/$implementationdir/sources" >/dev/null
	then
          [ -f "$top/$abi/lib/libgmp.a" ] && libs="$top/$abi/lib/libgmp.a $libs"
          [ -f "$top/$abi/lib/libgmpxx.a" ] && libs="$top/$abi/lib/libgmpxx.a $libs"
	fi
	if grep '^cryptopp$' "$top/$abi/implementation/$implementationdir/dep" >/dev/null
	then
          [ -f "$top/$abi/lib/libcryptopp.a" ] && libs="$top/$abi/lib/libcryptopp.a $libs"
	fi
        [ -f "$top/$abi/lib/lib${project}.a" ] && libs="$top/$abi/lib/lib${project}.a $libs"

	trylibs="$libs"
	measurelibs="$libs"
	userandombytes=0
	[ $o = crypto_scalarmult ] && userandombytes=1
	[ $o = crypto_box ] && userandombytes=1
	[ $o = crypto_dh ] && userandombytes=1
	[ $o = crypto_encrypt ] && userandombytes=1
	[ $o = crypto_kem ] && userandombytes=1
	[ $o = crypto_sign ] && userandombytes=1
	if [ $userandombytes = 1 ]
	then
	  trylibs="$top/$abi/lib/knownrandombytes.o $trylibs"
	  measurelibs="$top/$abi/lib/fastrandombytes.o $measurelibs"
	fi
  
        mkdir -p "$top/$target/work"
        cp -pr "$implementationdir"/* "$top/$target/work"
  
        cfiles=`ls "$top/$target/work" | grep '\.c$' || :`
        sfiles=`ls "$top/$target/work" | grep '\.[sS]$' || :`
        ccfiles=`ls "$top/$target/work" | grep '\.cc$' || :`
        cppfiles=`ls "$top/$target/work" | grep '\.cpp$' || :`
  
        cp -p "$o/try.c" "$top/$target/work/try.$language"
        cp -p "$o/measure.c" "$top/$target/work/measure.$language"
        cp -p "try-anything.c" "$top/$target/work/try-anything.$language"
        cp -p "measure-anything.c" "$top/$target/work/measure-anything.$language"

        (
          cd "$top/$target/work"
          (
            echo "#ifndef ${o}_H"
            echo "#define ${o}_H"
            echo ""
            echo "#include \"${op}.h\""
            echo ""
            echo "$macros" | tr : '\012' | while read macro
            do
              echo "#define ${o}${macro} ${op}${macro}"
            done
            echo "#define ${o}_PRIMITIVE \"${p}\""
            echo "#define ${o}_IMPLEMENTATION ${op}_IMPLEMENTATION"
            echo "#define ${o}_VERSION ${op}_VERSION"
            echo ""
            echo "#endif"
          ) > "$o.h"
          (
            echo "#ifndef ${op}_H"
            echo "#define ${op}_H"
            echo ""
            sed 's/[         ]CRYPTO_/ '"${opi}"'_/g' < api.h
	    echo ' '
            echo '#ifdef __cplusplus'
            echo 'extern "C" {'
            echo '#endif'
            echo "$prototypes" | tr : '\012' | while read prototype
            do
              echo "extern int ${opi}${prototype};"
            done
            echo '#ifdef __cplusplus'
            echo '}'
            echo '#endif'
            echo ""
            echo "$macros" | tr : '\012' | while read macro
            do
              echo "#define ${op}${macro} ${opi}${macro}"
            done
            echo "#define ${op}_IMPLEMENTATION \"${implementationdir}\""
            echo "#ifndef ${opi}_VERSION"
            echo "#define ${opi}_VERSION \"-\""
            echo "#endif"
            echo "#define ${op}_VERSION ${opi}_VERSION"
            echo ""
            echo "#endif"
          ) > "$op.h"
  
	  touch OK
          for f in $cfiles $sfiles $ccfiles $cppfiles
          do
            if [ -f OK ]
            then
	      (
                killafter 300 \
		$compiler -DSUPERCOP \
                  -I. -I"$top/$abi/include" \
                  -c "$f" 2>&1 || rm -f OK
	      ) | awk -v PREFIX="$version $shorthostname $abi $startdate $o $p fromcompiler $implementationdir $compilerword $f" '
	        {
		  if (NR > 25) { print PREFIX,"..."; exit }
		  print PREFIX,$0
		}
	      ' >&5
            fi
          done
  
          [ -f OK ] || exit 0
          okar-$abi cr "$op.a" *.o || exit 0
          ( ranlib "$op.a" || : )
  
	  (
            killafter 300 \
            $compiler -DSUPERCOP -DSMALL \
              -I. -I"$top/$abi/include" \
              -o try try.$language try-anything.$language \
              "$op.a" $trylibs 2>&1 || rm -f OK
	  ) | awk -v PREFIX="$version $shorthostname $abi $startdate $o $p fromcompiler $implementationdir $compilerword try.$language" '
	    {
	      if (NR > 25) { print PREFIX,"..."; exit }
	      print PREFIX,$0
	    }
	  ' >&5
          [ -f OK ] || exit 0
  
          if sh -c 'killafter 3600 ./try || exit $?' >../errors 2>&1
          then
            checksumsmall=`awk '{print $1}' < ../errors`
            cycles=`awk '{print $2}' < ../errors`
            checksumcycles=`awk '{print $3}' < ../errors`
            cyclespersecond=`awk '{print $4}' < ../errors`
            impl=`awk '{print $5}' < ../errors`
          else
            echo "$version $shorthostname $abi $startdate $o $p tryfails $implementationdir $compilerword error $?" >&5
            cat ../errors \
            | while read err
            do
              echo "$version $shorthostname $abi $startdate $o $p tryfails $implementationdir $compilerword $err" >&5
            done
            exit 0
          fi

	  [ "x$expectedchecksumsmall" != x ] \
	  && [ "x$expectedchecksumsmall" != "x$checksumsmall" ] \
          && echo "$version $shorthostname $abi $startdate $o $p try $checksumsmall fails $cycles $checksumcycles $cyclespersecond $impl $compilerword" >&5 \
	  && exit 0

	  (
            killafter 300 \
            $compiler -DSUPERCOP \
              -I. -I"$top/$abi/include" \
              -o try try.$language try-anything.$language \
              "$op.a" $trylibs 2>&1 || rm -f OK
	  ) | awk -v PREFIX="$version $shorthostname $abi $startdate $o $p fromcompiler $implementationdir $compilerword try.$language" '
	    {
	      if (NR > 25) { print PREFIX,"..."; exit }
	      print PREFIX,$0
	    }
	  ' >&5
          [ -f OK ] || exit 0
  
          if sh -c 'killafter 3600 ./try || exit $?' >../errors 2>&1
          then
            checksumbig=`awk '{print $1}' < ../errors`
            cycles=`awk '{print $2}' < ../errors`
            checksumcycles=`awk '{print $3}' < ../errors`
            cyclespersecond=`awk '{print $4}' < ../errors`
            impl=`awk '{print $5}' < ../errors`
          else
            echo "$version $shorthostname $abi $startdate $o $p tryfails $implementationdir $compilerword error $?" >&5
            cat ../errors \
            | while read err
            do
              echo "$version $shorthostname $abi $startdate $o $p tryfails $implementationdir $compilerword $err" >&5
            done
            exit 0
          fi

	  [ "x$expectedchecksumbig" != x ] \
	  && [ "x$expectedchecksumbig" != "x$checksumbig" ] \
          && echo "$version $shorthostname $abi $startdate $o $p try
	  $checksumsmall/$checksumbig fails $cycles $checksumcycles $cyclespersecond $impl $compilerword" >&5 \
	  && exit 0

	  checksumok=ok
	  [ "x$expectedchecksumsmall" = x ] && checksumok=unknown
	  [ "x$expectedchecksumbig" = x ] && checksumok=unknown
          echo "$version $shorthostname $abi $startdate $o $p try $checksumsmall/$checksumbig $checksumok $cycles $checksumcycles $cyclespersecond $impl $compilerword" >&5
  
	  (
            killafter 3600 \
            $compiler -D'COMPILER="'"$compiler"'"' \
              -DSUPERCOP -DLOOPS=1 \
              -I. -I"$top/$abi/include" \
              -o measure measure.$language measure-anything.$language \
              "$op.a" $measurelibs 2>&1 || rm -f OK
	  ) | awk -v PREFIX="$version $shorthostname $abi $startdate $o $p fromcompiler $implementationdir $compilerword measure.$language" '
	    {
	      if (NR > 25) { print PREFIX,"..."; exit }
	      print PREFIX,$0
	    }
	  ' >&5
          [ -f OK ] || exit 0

	  rm -rf ../compiled || exit 0
	  mkdir ../compiled || exit 0
  
	  echo "$cycles" > ../CYCLES
          for f in *.o
          do
            cp -p "$f" "../compiled/${opi}-$f"
          done
          cp -p "$op.h" "../compiled/$op.h"
          cp -p "$o.h" "../compiled/$o.h"
          cp -p measure ../compiled/measure
        )
      done
    fi

    if [ "$t1" = knownrandombytes ]
    then
      rm -f "$top/$abi/lib/knownrandombytes.o"
      rm -rf "$top/$target/work"
      mkdir -p "$top/$target/work"
      cp knownrandombytes/* "$top/$target/work"

      include="-I. -I"$top/$abi/include""

      libs=`"oklibs-$abi"`
      [ -f "$top/$abi/lib/libcryptopp.a" ] && libs="$top/$abi/lib/libcryptopp.a $libs"
      [ -f "$top/$abi/lib/lib${project}.a" ] && libs="$top/$abi/lib/lib${project}.a $libs"

      ( cd "$top/$target/work" \
        && env abi="$abi" libs="$libs" include="$include" sh do \
	&& cp lib/$abi/knownrandombytes.o "$top/$abi/lib/knownrandombytes.o"
      )
      exit 0
    fi

    if [ "$t1" = fastrandombytes ]
    then
      rm -f "$top/$abi/lib/fastrandombytes.o"
      rm -rf "$top/$target/work"
      mkdir -p "$top/$target/work"
      cp fastrandombytes/* "$top/$target/work"

      include="-I. -I"$top/$abi/include""

      libs=`"oklibs-$abi"`
      libs="$top/$abi/lib/cpucycles.o $libs"
      libs="$top/$abi/lib/kernelrandombytes.o $libs"
      [ -f "$top/$abi/lib/libcryptopp.a" ] && libs="$top/$abi/lib/libcryptopp.a $libs"
      [ -f "$top/$abi/lib/lib${project}.a" ] && libs="$top/$abi/lib/lib${project}.a $libs"

      ( cd "$top/$target/work" \
        && env abi="$abi" libs="$libs" include="$include" sh do \
	&& cp lib/$abi/fastrandombytes.o "$top/$abi/lib/fastrandombytes.o"
      )
      exit 0
    fi

    if [ "$t1" = integrate ]
    then
      grep "^$t2 " OPERATIONS \
      | while read o macros prototypes
      do
        p=$t3
        op="${o}_${p}"

	cat "$top/$target/best" \
	| while read source
	do
          [ -f "$o/$p/used" ] \
          && (
	    okar-$abi t "$top/$abi/lib/lib${project}.a" > "$top/$abi/lib/lib${project}.index"
	    grep '^'${op}_ "$top/$abi/lib/lib${project}.index" \
	    | while read existing
	    do
	      okar-$abi d "$top/$abi/lib/lib${project}.a" "$existing"
	    done
	    rm -f "$top/$abi/lib/lib${project}.index"
	    okar-$abi cr "$top/$abi/lib/lib${project}.a" "$top/$abi/$source/compiled"/*.o
	  ) \
          && ( ranlib "$top/$abi/lib/lib${project}.a" || exit 0 ) \
          && cp -p "$top/$abi/$source/compiled/$op.h" "$top/$abi/include/$op.h" \
          && [ -f "$o/$p/selected" ] \
          && cp -p "$top/$abi/$source/compiled/$o.h" "$top/$abi/include/$o.h" \
          || :
	done

      done
    fi

    if [ "$t1" = measure1 -o "$t1" = measure2 -o "$t1" = measure3 ]
    then
      grep "^$t2 " OPERATIONS \
      | while read o macros prototypes
      do
        p=$t3
        op="${o}_${p}"

	cat "$top/$target/best" \
	| while read source
	do
	  "$top/$abi/$source/compiled/measure" \
          | while read measurement
          do
            echo "$version $shorthostname $abi $startdate $o $p $measurement" >&5
          done
	done

      done
    fi

  )

  exec 5>&-
  exec 8>&-
  echo "=== `date` === finishing $target"

  rm -rf "$top/$target/work"

  mv "$top/$target/sources.new" "$top/$target/sources"
  mv "$top/$target/data.new" "$top/$target/data"
  mv "$top/$target/log.new" "$top/$target/log"

  cat "$top/$target/data" >> "$top/$target/data.archive"
  cat "$top/$target/log" >> "$top/$target/log.archive"

  rm "$top/$target/BUILD"
  rm "$top/$target/LOCKED"
done