Commit c43e90cc authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Updated script for running the linear solvers benchmark

parent 1b322b86

src/Benchmarks/scripts/run-matrix-solvers-benchmark

deleted100755 → 0
+0 −246
Original line number Diff line number Diff line 
#!/usr/bin/env bash



BASE="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/"

                

MATRIX_SOLVERS_BENCHMARK="tnl-matrix-solvers-benchmark-dbg"

STOP_TIME="1"



SOLVERS="sor cg bicgstab gmres"

#SOLVERS="cg"

SOR_OMEGAS="0.4 0.8 1.2 1.6 2.0"

GMRES_RESTARTINGS="8 16 32 64 128 256"



LOG_FILE="matrix-solvers-benchmark.html"



export CUDA_PROFILE=0

export CUDA_PROFILE_CONFIG="$TNL_SOURCE_DIR/tests/cuda-profiler.conf"

PROCESS_CUDA_PROFILE="$TNL_SOURCE_DIR/tests/process-cuda-profile.pl"

source ./matrix-market

source ./florida-matrix-market



export TNL_SPARSE_MATRIX_CHECK_CFG_DESC_FILE="$TNL_SOURCE_DIR/tests/tnl-sparse-matrix-check.cfg.desc"



#MM_MATRICES=""

#FLORIDA_MM_MATRICES=""





benchmark_matrix()

# $1 input file

# $2 input mtx file

# $3 complete log file

# $4 verbose

{

   echo "          <tr>" >> $3

   echo "             <td>$2</td>" >> $3

   $MATRIX_SOLVERS_BENCHMARK --input-file $1 \

                             --input-mtx-file $2 \

                             --solver-name none\

                             --matrix-stats-file matrix-stats.html \

                             --verbose $4

   cat matrix-stats.html >> $3

   for solver in $SOLVERS;

   do     

      echo "Testing with $solver solver." 

      if [ x$solver == xcg ] || [ x$solver == xbicgstab ];

      then

         logfile="$2.results.$solver.log"

         if test ! -e $logfile;

         then

            $MATRIX_SOLVERS_BENCHMARK --input-file $1 \

                                      --input-mtx-file $2 \

                                      --solver-name $solver \

                                      --log-file $logfile \

                                      --verbose $4

            echo ""

         else

            echo "Test has already passed."

         fi

         cat $logfile >> $3

      fi

      if test x$solver == xsor;

      then

         for omega in $SOR_OMEGAS;

         do

            echo "Setting sor-omega to $omega"

            logfile="$2.results.sor.omega-$omega.log"

            if test ! -e $logfile;

            then

               $MATRIX_SOLVERS_BENCHMARK --input-file $1 \

                                         --input-mtx-file $2 \

                                         --solver-name $solver \

                                         --sor-omega $omega \

                                         --log-file $logfile \

                                         --verbose $4

               echo ""

            else

               echo "Test has already passed."

            fi

            cat $logfile >> $3

         done

      fi

      if test x$solver == xgmres;

      then

         for gmres_restarting in $GMRES_RESTARTINGS;

         do

            echo "Setting GMRES restarting to $gmres_restarting"

            logfile="$2.results.gmres.restarting-$gmres_restarting.log"

            if test ! -e $logfile;

            then

               $MATRIX_SOLVERS_BENCHMARK --input-file $1 \

                                         --input-mtx-file $2 \

                                         --solver-name $solver \

                                         --gmres-restarting $gmres_restarting \

                                         --log-file $logfile \

                                         --verbose $4

               echo ""

            else

               echo "Test has already passed."

            fi

            cat $logfile >> $3

         done

      fi

   done

   echo "          </tr>" >> $3

}



write_log_header()

# $1 log file name

{

   echo "<html>" > $1

   echo "   <body>" >> $1

   echo "      <table border=1>" >> $1

   echo "          <tr>" >> $1

   echo "             <td rowspan=1 colspan=3 align=center>Matrix</td>" >> $1

   echo "             <td rowspan=1 colspan=15 align=center>SOR</td>" >> $1

   echo "             <td rowspan=2 colspan=3 align=center>CG</td>" >> $1

   echo "             <td rowspan=2 colspan=3 align=center>BICGStab</td>" >> $1

   echo "             <td rowspan=1 colspan=18 align=center>GMRES</td>" >> $1

   echo "          </tr>" >> $1

         

   echo "          <tr>" >> $1

   echo "             <td rowspan=2>Name</td>" >> $1                      # Matrix description

   echo "             <td rowspan=2>Size</td>" >> $1

   echo "             <td rowspan=2>NonZeros No.</td>" >> $1

   echo "             <td colspan=3>0.4</td>" >> $1                      # SOR Omegas

   echo "             <td colspan=3>0.8</td>" >> $1                      # SOR Omegas

   echo "             <td colspan=3>1.2</td>" >> $1                      # SOR Omegas

   echo "             <td colspan=3>1.6</td>" >> $1                      # SOR Omegas

   echo "             <td colspan=3>2.0</td>" >> $1                      # SOR Omegas

#   echo "             <td colspan=3></td>" >> $1                         # CG

#   echo "             <td colspan=3></td>" >> $1                         # BICGStab

   echo "             <td colspan=3>8</td>" >> $1                        # GMRES restartings

   echo "             <td colspan=3>16</td>" >> $1                       # GMRES restartings

   echo "             <td colspan=3>32</td>" >> $1                       # GMRES restartings

   echo "             <td colspan=3>64</td>" >> $1                       # GMRES restartings

   echo "             <td colspan=3>128</td>" >> $1                      # GMRES restartings

   echo "             <td colspan=3>256</td>" >> $1                      # GMRES restartings

   echo "          </tr>" >> $1

   

   echo "          <tr>" >> $1

   echo "             <td>Residue</td>" >> $1                           # SOR Results omega = 0.4

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # SOR Results omega = 0.8

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # SOR Results omega = 1.2

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # SOR Results omega = 1.6

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # SOR Results omega = 2.0

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # CG

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # BICGStab

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 8

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 16

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 32

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 64

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 128

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "             <td>Residue</td>" >> $1                           # GMRES Results restarting 256

   echo "             <td>Iterations</td>" >> $1

   echo "             <td>CPU</td>" >> $1

   echo "          </tr>" >> $1

}



close_log()

# $1 log file name

{

      echo "      </table>" >> $1

   echo "   </body>" >> $1

   echo "</html>" >> $1

}



write_log_header $LOG_FILE



for link in $MM_MATRICES;

do

   echo "###############################################################################################"

   matrix=matrices`echo $link | sed 's/ftp:\/\/math.nist.gov\/pub//'`

   unzipped_matrix=`echo $matrix | sed 's/.gz//'`

   if test ! -e $matrix;

   then

      echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first."

      #echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first." >> sparse-matrix-benchmark.log            

   else

      if test ! -e $unzipped_matrix.double.bin.bz2;

      then

         echo "Missing $unzipped_matrix.double.bin.bz2 !!! Run the script 'convert-matrices'."

         #echo "Missing $unzipped_matrix.double.bin.bz2 !!! Run the script 'convert-matrices'."  >> sparse-matrix-benchmark.log

      else

         gunzip -f $matrix

         echo "Checking with the matrix $unzipped_matrix in double precison ..."

         export CUDA_PROFILE_LOG=$unzipped_matrix.double.log 

         benchmark_matrix $unzipped_matrix.double.bin.bz2 $unzipped_matrix $LOG_FILE 1

         gzip $unzipped_matrix

      fi          

   fi

done



for link in $FLORIDA_MM_MATRICES;

do

   matrix=matrices`echo $link | sed 's/http:\/\/www.cise.ufl.edu\/research\/sparse//'`

   if test ! -e $matrix;

   then      

      echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first."

      #echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first." >> sparse-matrix-benchmark.log

   else

     DIRNAME=`dirname $matrix`

     FILENAME=`basename $matrix`

     cd $DIRNAME

     tar zxvf $FILENAME

     cd $IWD

     SUBDIRNAME=`echo $FILENAME | sed 's/.tar.gz//'`

     rm -f $DIRNAME/$SUBDIRNAME/*_b.mtx # these are usualy in array format

     for file in $DIRNAME/$SUBDIRNAME/*.mtx;

     do

         echo "###############################################################################################"

         if test ! -e $file.double.bin.bz2;

         then

            echo "Missing $file.double.bin.bz2 !!! Run the script 'convert-matrices'."

         else   

            echo "Checking with the matrix $file ..."

            benchmark_matrix $file.double.bin.bz2 $file $LOG_FILE 1                        

         fi                           

         rm $file         

      done

   fi

done



close_log matrix-solvers-benchmark.log

src/Benchmarks/scripts/run-tnl-benchmark-linear-solvers

100644 → 100755
+95 −66
Original line number Diff line number Diff line 
#!/usr/bin/env bash

                

DEBUG="no"

STOP_TIME="1"

export CUDA_PROFILE=0



PWD=`pwd`

IWD="$PWD"

BASE="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/"

BENCHMARK="tnl-benchmark-linear-solvers"

BENCHMARK_DBG="tnl-benchmark-linear-solvers-dbg"



export CUDA_PROFILE_CONFIG="$IWD/cuda-profiler.conf"

PROCESS_CUDA_PROFILE="$IWD/process-cuda-profile.pl"

source matrix-market

source florida-matrix-market



for link in $MM_MATRICES;

do

   echo "======================================================================================================"

   matrix=matrices`echo $link | sed 's/ftp:\/\/math.nist.gov\/pub//'`

   unzipped_matrix=`echo $matrix | sed 's/.gz//'`

   if test ! -e $matrix;

   then

      echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first."

      #echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first." >> sparse-matrix-benchmark.log            

   else

      gunzip -c ${matrix} > ${unzipped_matrix}      

      echo "Benchmarking with the matrix $unzipped_matrix ..."

      export CUDA_PROFILE_LOG=$unzipped_matrix.float.log

      if test x$DEBUG = xyes;

      then

         gdb --args ${BENCHMARK_DBG} --test mtx --input-file $unzipped_matrix --log-file sparse-matrix-benchmark.log --verbose 1

#! /bin/bash



# exit whenever a command fails

set -e



shopt -s extglob            # enable extended pattern-matching features

shopt -s globstar           # recursive globbing



function get_cpu_count() {

    echo $(( $(grep -c ^processor /proc/cpuinfo) / 2 ))

}



function get_cpu_affinity() {

#    echo $(python3 -c "print(','.join(str(i) for i in range(0, 2*$omp_threads, 2)))")

    echo $(python3 -c "print(','.join(str(i) for i in range(0, $omp_threads)))")

}



function run_benchmark() {

    echo "==> omp_threads = $omp_threads, matrix = $bench_name"



    log_file=${log_file:=tnl-benchmark-linear-solvers.mtx.log}



    gmresRestartingMax=${gmresRestartingMax:=30}

    gmresRestartingMin=${gmresRestartingMin:=3}

    gmresRestartingStepMax=${gmresRestartingStepMax:=4}

    gmresRestartingStepMin=${gmresRestartingStepMin:=2}



    # set number of OpenMP threads

    omp_threads=${omp_threads:=1}

    if [[ "$omp_threads" == "1" ]]; then

        omp_enabled=false

    else

         $BENCHMARK --test mtx --input-file $unzipped_matrix --log-file sparse-matrix-benchmark.log --verbose 1

        omp_enabled=true

    fi

      #perl $PROCESS_CUDA_PROFILE $unzipped_matrix.float.log sparse-matrix-profiling-float.log          



#    mpirun -np 1 -H localhost:4 ${solver} \

#    ${solver} \

    taskset -c $(get_cpu_affinity) ${solver} \

        --verbose 1 \

        --loops 10 \

        --log-file "$log_file" \

        --output-mode "append" \

        --name "$bench_name" \

        --redirect-mpi-output "false" \

        --devices "$devices" \

        --openmp-enabled "$omp_enabled" \

        --openmp-max-threads "$omp_threads" \

        --input-matrix "$bench_matrix" \

        --convergence-residue 1e-11 \

        --max-iterations 10000 \

        --gmres-restarting-max $gmresRestartingMax \

        --gmres-restarting-min $gmresRestartingMin \

        --gmres-restarting-step-max $gmresRestartingStepMax \

        --gmres-restarting-step-min $gmresRestartingStepMin \

        --ilut-threshold 1e-4 \

        --ilut-p 8 \

        --solvers "$solvers" \

        --gmres-variants "$gmresVariants" \

        --preconditioners "$preconditioners" \

        --with-preconditioner-update "true" \

    || true

}



devices="all"



#max_omp_threads=1

max_omp_threads=$(get_cpu_count)



#solver="tnl-benchmark-linear-solvers"

solver="tnl-benchmark-linear-solvers-cuda"



solvers="gmres,bicgstab,bicgstab-ell,tfqmr"

#solvers="gmres"

#solvers="tfqmr,bicgstab"

#solvers="bicgstab-ell"

#solvers="bicgstab"

gmresVariants="CGS,CGSR,MGS,MGSR,CWY"

preconditioners="jacobi,ilu0,ilut"



matrices_base_path="./mtx_matrices"



if [[ ! -d "$matrices_base_path" ]]; then

    echo "The path '$matrices_base_path' does not exist." >&2

    exit 1

fi

done



for link in $FLORIDA_MM_MATRICES;

do

   matrix=matrices`echo $link | sed 's/http:\/\/www.cise.ufl.edu\/research\/sparse//'`

   if test ! -e $matrix;

   then      

      echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first."

      #echo "Matrix $matrix is missing !!! Run the script 'get-matrices' first." >> sparse-matrix-benchmark.log

for bench_matrix in "$matrices_base_path"/**/*.mtx; do

    bench_name=$(realpath --relative-to="$matrices_base_path" "$bench_matrix")

    for (( omp_threads=1; omp_threads<=$max_omp_threads; omp_threads++ )); do

        if [[ "$omp_threads" == "1" ]]; then

            devices="all"

        else

     DIRNAME=`dirname $matrix`

     FILENAME=`basename $matrix`

     cd $DIRNAME

     tar zxvf $FILENAME

     cd $IWD

     SUBDIRNAME=`echo $FILENAME | sed 's/.tar.gz//'`

     rm -f $DIRNAME/$SUBDIRNAME/*_b.mtx # these are usualy in array format

     for file in $DIRNAME/$SUBDIRNAME/*.mtx;

     do

         echo "======================================================================================================"

         echo "Benchmarking with the matrix $file ..."

         if test x$DEBUG = xyes;

         then

            gdb --args $BENCHMARK --test mtx --input-file $file --log-file sparse-matrix-benchmark.log --verbose 1

         else

            $BENCHMARK --test mtx --input-file $file --log-file sparse-matrix-benchmark.log --verbose 1                        

            devices="host"

        fi

        run_benchmark

    done

   fi

done