Commit 1b322b86 authored by Jakub Klinkovský's avatar Jakub Klinkovský
Browse files

Updated script for downloading matrices from the markets

parent c1ef9623

src/Benchmarks/scripts/florida-matrix-market

deleted100644 → 0
+0 −1454

File deleted.

Preview size limit exceeded, changes collapsed.

src/Benchmarks/scripts/get-matrices

+30 −33
Original line number Diff line number Diff line 
#!/usr/bin/env bash

#!/bin/bash



BASE="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/"

# exit whenever a command fails

set -e



#PWD=`pwd`

#TNL_SPARSE_MATRIX_CHECK="$PWD/tnl-sparse-matrix-check"

#IWD="$PWD"

# enable recursive globbing

shopt -s globstar



#source ../tnl-env-variables

source matrix-market

source florida-matrix-market



for link in $MM_MATRICES;

do

   echo "###############################################################################"

   matrix=matrices`echo $link | sed 's/ftp:\/\/math.nist.gov\/pub//'`

   if test ! -e $matrix;

   then

      echo "Getting $matrix ..."      

      wget -x --cut-dirs=1 -P matrices -nH $link

   else

      echo "Matrix $matrix was already downloaded."

matrices_base_path="./mtx_matrices"

if [[ ! -d "$matrices_base_path" ]]; then

   mkdir "$matrices_base_path"

fi

cd "$matrices_base_path"



done

# Matrix Market: https://math.nist.gov/MatrixMarket/

wget --mirror --continue --force-directories --include-directories=/MatrixMarket/,/pub/MatrixMarket2/ --exclude-directories="*" --accept "*.mtx.gz" "https://math.nist.gov/MatrixMarket/" || true



# SuiteSparse Matrix Collection: https://sparse.tamu.edu/

# note that we need to crawl the site where the matrices are actually stored

# (https://sparse.tamu.edu/ links to https://suitesparse-collection-website.herokuapp.com/

# for all download links)

wget --mirror --continue --no-parent --force-directories -X RB,mat --accept "*.tar.gz" "https://suitesparse-collection-website.herokuapp.com/"



for link in $FLORIDA_MM_MATRICES;

do

   echo "###############################################################################"

   matrix=matrices`echo $link | sed 's/http:\/\/www.cise.ufl.edu\/research\/sparse//'`

   if test ! -e $matrix;

   then

      echo "Getting $matrix ..."      

      wget -x --cut-dirs=2 -P matrices -nH $link

   else

      echo "Matrix $matrix was already downloaded."   

   fi

# delete empty directories

find math.nist.gov suitesparse-collection-website.herokuapp.com -type d -empty -delete



# unzip (keep the original archive)

for f in math.nist.gov/**/*.mtx.gz; do

   gunzip --keep --force "$f"

done

for f in suitesparse-collection-website.herokuapp.com/**/*.tar.gz; do

   pushd $(dirname "$f") >/dev/null

   tar -xaf $(basename "$f")

   rm -f **/*_b.mtx  # these are usualy in array format

   popd >/dev/null

done

src/Benchmarks/scripts/matrix-market

deleted100644 → 0
+0 −157
Original line number Diff line number Diff line 
MM_MATRICES="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/astroph/mcca.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/astroph/mcfe.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk14.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk16.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk17.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk18.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk19.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk20.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk21.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk22.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk23.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk24.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstk25.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm19.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm20.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm21.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm22.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm23.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm24.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc3/bcsstm25.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstk26.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstk27.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstk28.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstm26.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstm27.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemimp/impcol_a.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemimp/impcol_b.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemimp/impcol_c.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemimp/impcol_d.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemimp/impcol_e.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0067.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0132.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0156.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0167.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0381.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0479.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0497.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0655.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west0989.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west1505.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/chemwest/west2021.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/cirphys/jpwh_991.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s1rmq4m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s2rmq4m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s3rmq4m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s1rmt3m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s2rmt3m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s3rmt3m1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s3dkq4m2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s3dkt3m2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/cylshell/s3rmt3m3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0100.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0200.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0300.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0400.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e05r0500.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r0000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r0100.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r0500.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r1000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r2000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r3000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r4000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e20r5000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r0000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r0100.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r0500.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r1000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r2000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r3000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r4000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e30r5000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e40r2000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e40r3000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e40r4000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/SPARSKIT/drivcav/e40r5000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/econaus/orani678.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/econaus/mahindas.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/econiea/mbeacxc.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/econiea/mbeaflw.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/econiea/mbeause.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_183_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_183_3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_183_4.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_183_6.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_680_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_680_2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_680_3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_760_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_760_2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/facsimile/fs_760_3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/gemat/gemat11.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/gemat/gemat12.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/grenoble/gre__115.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/grenoble/gre__185.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/grenoble/gre__343.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/grenoble/gre__512.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/grenoble/gre_1107.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/hamm/add20.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/hamm/add32.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/misc/hamm/memplus.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos7.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos6.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos5.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos4.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lanpro/nos1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/laplace/gr_30_30.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lnsp3937.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lnsp_511.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lnsp_131.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lns_3937.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lns__511.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/lns/lns__131.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/nnceng/hor__131.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/nucl/nnc1374.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/nucl/nnc666.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/nucl/nnc261.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/oilgen/orsreg_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/oilgen/orsirr_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/oilgen/orsirr_2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/sherman/sherman5.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/sherman/sherman4.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/sherman/sherman3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/sherman/sherman2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/sherman/sherman1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/lund_a.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/lund_b.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/shl____0.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/shl__200.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/shl__400.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/str____0.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/str__200.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/str__400.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/str__600.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp_____0.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp___200.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp___400.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp___600.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp___800.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp__1000.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp__1200.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp__1400.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/bp__1600.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/fs_541_1.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/fs_541_2.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/fs_541_3.mtx.gz \

               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/smtape/fs_541_4.mtx.gz"

# Broken links

#               ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk15.mtx.gz \



#MM_MATRICES="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/astroph/mcfe.mtx.gz"

#MM_MATRICES="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/astroph/mcca.mtx.gz"

#MM_MATRICES="ftp://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc4/bcsstk28.mtx.gz"

#MM_MATRICES=""