Rocks install notes
Cluster Specifications
Headnode:
Intel Core i7-3770 3.40GHz CPU per node
16 GB of RAM per node
500GB HDD plus 1TB HDD data disk
7 Compute nodes:
Intel Core i7-3770 3.40GHz CPU per node
16 GB of RAM per node
500GB HDD
Only the frontend is directly connected to the outside network. The compute nodes are interconnected with a dedicated Gigabit ethernet network.
Storage Configuration:
/home -- The user home directories reside on the headnode 500GB disk and are NFS exported to all the compute nodes.
/home/data -- The headnode's 1TB disk is NFS exported to the compute notes at /home/data. Users can create subdirectories under /home/data as on a standard scratch and the contents will be accessible from any compute node.
/state/partition1 -- Each compute nodes has a local working area mounted at /state/partition1. These areas are local to each node and are not visible to the other nodes. If your job is disk intensive, you may wish to have your jobscript copy files onto the local work space at the beginning of the job. If your job writes files into the local work space, be sure to have your jobscript move them out when it is done. This is for two reasons. First you will not necessarily know what node your job ran on, so it may be difficult to find the files later. Secondly, it leaves the node clean for the next user.
Swap space on 1TB disk.
Installing the headnode:
Follow instructions from Rocks 6.0 DVD using build option at prompt at boot.
Select rolls you want to install - in this case all of them.
Fill in details as required.
Manually partition as follows -
/dev/sda
50GB /
20GB /var
400GB /export
/dev/sdb
32GB swap
920GB /export/home/data
Add cluster compute nodes:
insert-ethers
rocks add firewall host=localhost network=public protocol=tcp service=www chain=INPUT action=ACCEPT rulename="ACCEPT-WWW"
rocks report host firewall localhost
rocks sync host firewall localhost
Add user accounts to headnode.
rocks sync users
mkdir /share/apps/matlab
#Setup file shares
vi /etc/exports
###Begin file
/export 10.1.1.1(rw,async,no_root_squash) 10.1.0.0/255.255.0.0(rw,async)
/export/home/data 10.1.1.1(rw,async,no_root_squash) 10.1.0.0/255.255.0.0(rw,async)
###End of file
#Make file shares available on nodes
vi /etc/auto.home
###Begin file
data -nfsvers=3 physplanck.local:/export/home/data
condor -nfsvers=3 physplanck.local:/export/home/condor
###End of file
make -C /var/411
/etc/init.d nfs restart
Install Python, numpy, scipy, ATLAS, lapack, etc
#Get additional software for install
cd /usr/local/src
wget ...
#Python install
yum install readline-devel
tar -xzf Python-2.6.8.tgz
cd Python-2.6.8
./configure --prefix=/share/apps
make -j8
make install
cd ..
#LAPACK install
tar -xzf lapack-3.4.2.tgz
cd lapack-3.4.2
cp INSTALL/make.inc.gfortran make.inc
vi make.inc
###Changes to file
OPTS = -O2 -fPIC
NOOPT = -O0 -fPIC
###End of changes
make -j8
make blaslib lapacklib tmglib
# make lapack_testing #This fails, apparently know and not needed at 12/2012
cd ..
#To install ATLAS software, disable CPU throttling.
cpufreq-selector -g performance
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu2/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu3/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu4/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu5/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu6/cpufreq/scaling_governor
cp /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor /sys/devices/system/cpu/cpu7/cpufreq/scaling_governor
tar -xjf atlas3.10.0.tar.bz2
mkdir ATLAS/atlas_linux
cd ATLAS/atlas_linux/
../configure -Fa alg -fPIC --prefix=/share/apps --with-netlib-lapack-tarfile=/usr/local/src/lapack-3.4.2.tgz
make
cd lib
make shared
make ptshared
cd ..
make install
cd ../..
#Install Sparse software
mkdir sparse
cd sparse
tar -xzf ../UMFPACK-5.5.1.tar.gz
tar -xzf ../UFconfig-3.6.0.tar.gz
tar -xzf ../AMD-2.2.2.tar.gz
vi UFconfig/UFconfig.mk
###Beginning of file
#Modify sparse/UFconfig/UFconfig.mk to have following:
diff UFconfig.mk.orig UFconfig.mk
37c37
< CFLAGS = -O3 -fexceptions
---
> CFLAGS = -O3 -fexceptions -m64 -fPIC
52,53c52,53
< F77 = f77
< F77FLAGS = -O
---
> F77 = gfortran
> F77FLAGS = -O -m64 -fPIC
74,75c74,75
< INSTALL_LIB = /usr/local/lib
< INSTALL_INCLUDE = /usr/local/include
---
> INSTALL_LIB = /share/apps/lib
> INSTALL_INCLUDE = /share/apps/include
97,98c97,98
< BLAS = -lblas
< LAPACK = -llapack
---
> BLAS = -L/usr/lib/gcc/x86_64-redhat-linux/4.1.1 -L/share/apps/lib -llapack -lf77blas -lcblas -latlas -lgfortran
> LAPACK = -L/usr/lib/gcc/x86_64-redhat-linux/4.1.1 -L/share/apps/lib -llapack -lf77blas -lcblas -latlas -lgfortran
135,136c135,136
< # METIS_PATH =
< # METIS =
---
> METIS_PATH =
> METIS =
154c154
< UMFPACK_CONFIG =
---
> UMFPACK_CONFIG = -DNCHOLMOD
192c192
< CHOLMOD_CONFIG =
---
> CHOLMOD_CONFIG = -DNPARTITION
###End of file
cd UMFPACK/
make -j8 library
make install
cd ../AMD/
make install
cd ../..
tar -xzf fftw-3.3.3.tar.gz
cd fftw-3.3.3
./configure --enable-sse2 --enable-threads --with-combined-threads CFLAGS="-O3 -fomit-frame-pointer -fstrict-aliasing -ffast-math -pthread -fPIC -m64" FFLAGS="-g -O2 -fPIC -m64" CXXFLAGS="-g -O2 -fPIC -m64" --prefix=/share/apps
make -j8
make install
cd ..
#numpy install
tar -xzf numpy-1.6.2.tar.gz
cd numpy-1.6.2
vi site.cfg
###Beginning of file
[DEFAULT]
library_dirs = /share/apps/lib
include_dirs = /share/apps/include
[atlas]
atlas_libs = lapack, f77blas, cblas, atlas
[amd]
amd_libs = amd
[umfpack]
umfpack_libs = umfpack, gfortran
[fftw]
libraries = fftw3
### End of file
/share/apps/bin/python setup.py build
/share/apps/bin/python setup.py install --prefix=/share/apps
cd ..
#scipy install
tar -xzf scipy-0.11.0.tar.gz
cd scipy-0.11.0
/share/apps/bin/python setup.py build
/share/apps/bin/python setup.py install --prefix=/share/apps
cd ..
tar -xzf matplotlib-1.1.1.tar.gz
cd matplotlib-1.1.1
/share/apps/bin/python setup.py build
/share/apps/bin/python setup.py install --prefix=/share/apps
cd ..
tar -xzf emcee-1.1.2.tar.gz
cd emcee-1.1.2
/share/apps/bin/python setup.py build
/share/apps/bin/python setup.py install --prefix=/share/apps
#For testing purposes also install the following python modules
setuptools-0.6c11.tar.gz
nose-1.2.1.tar.gz
tools-0.0.3.tar.gz ????
#Add system paths to system reference file
mkdir /share/apps/resources
vi /share/apps/resources/bashrc
###Begin file
export PATH=/share/apps/bin:$PATH
export PYTHONPATH=/share/apps/lib/python2.6/site-packages:$PYTHONPATH
export LD_LIBRARY_PATH=/share/apps/lib:/share/apps/lib/python2.6:$LD_LIBRARY_PATH
export INCLUDE=/share/apps/include:/share/apps/include/atlas:/share/apps/include/python2.6:$INCLUDE
export ATLAS=/share/apps/lib
export BLAS=/share/apps/lib
export LAPACK=/share/apps/lib
###End of file
#Append to /etc/skel/.bashrc
echo "source /shareapps/resources/bashrc" >> /etc/skel/.bashrc
Additional notes to be added in future.