Instalacao do novo Compute Element

Sistema Operacional

[root@sprace ~]# vim /etc/hosts
192.168.1.152   osg-ce.grid     osg-ce
[root@sprace ~]# vim /etc/dhcpd.conf
host osg-ce
{
        hardware ethernet 00:30:48:89:22:88;
        fixed-address osg-ce;
        filename "/tftpboot/pxelinux.0";
}
[root@sprace ~]# vim /tftpboot/pxelinux.cfg/default
    append ks=nfs:192.168.1.200:/export/linux/kickstart/ks_osgce_instalar.cfg initrd=initrd.img  devfs=nomount ksdevice=eth0
[root@sprace ~]# cp /export/linux/kickstart/ks_osgse_instalar.cfg /export/linux/kickstart/ks_osgce_instalar.cfg
[root@sprace ~]# vim /export/linux/kickstart/ks_osgce_instalar.cfg

Configurar como esta no arquivo anexo: * ks_osgce_instalar.cfg: Kickstart para instala a OSG-CE

[root@sprace ~]# /etc/init.d/dhcpd start
[root@sprace ~]# /etc/init.d/xinetd restart

Mudando na spgrid temporariamente o DNS

[root@spgrid ~]# vim /var/named/chroot/var/named/80.136.200.in-addr.arpa.zone
                                       2008012201 ; Serial
25      IN      PTR     osg-ce1.
[root@spgrid ~]# vim /var/named/chroot/var/named/sprace.org.br.zone
                                      2008012201 ; Serial
osg-ce1         IN      A       200.136.80.25
[root@spgrid ~]# /etc/init.d/named restart

Ja no terminal da maquina

[root@osg-ce ~]# mkdir /opt/scratch/
[root@osg-ce ~]# ln -s  /opt/scratch/ /scratch
[root@osg-ce ~]# ln -s /opt  /usr/local/opt
[root@osg-ce ~]# ln -s /home/ /usr/local/home
[root@osg-ce ~]# rpm -Uvh ftp://ftp.scientificlinux.org/linux/scientific/4x/i386/misc/RPMS/yum-conf-latest.SL.noarch.rpm; yum -y clean all; yum -y  update yum; yum -y update

Configuracao do NIS

[root@osg-ce ~]# yum install ypserv
[root@osg-ce ~]# vim /etc/nsswitch.conf
passwd:     files nis
shadow:     files nis
group:      files nis
protocols:  files nis
services:   files nis
netgroup:   files nis
automount:  files nis
aliases:    files nis
[root@osg-ce ~]# vim /etc/yp.conf
yperver 192.168.1.150

[root@osg-ce ~]# vim /etc/xinetd.d/time
        disable         = no
[root@osg-ce ~]# vim /etc/xinetd.d/time-udp
        disable         = no
[root@osg-ce ~]# chkconfig xinetd on
[root@osg-ce ~]# chkconfig ypserv on
[root@osg-ce ~]# chkconfig portmap on
[root@osg-ce ~]# chkconfig yppasswdd on
[root@osg-ce ~]# domainname grid
[root@osg-ce ~]# /usr/lib/yp/ypinit -m
[root@osg-ce ~]# service xinetd start
[root@osg-ce ~]# service portmap start
[root@osg-ce ~]# service ypserv start
[root@osg-ce ~]# service yppasswdd start

Na spgrid

[root@spgrid ~]#  scp /etc/passwd /etc/shadow /etc/group /etc/gshadow root@osg-ce.sprace.org.br:/etc/.

Novamente

[root@osg-ce ~]# cd /var/yp/
[root@osg-ce yp]# make

Verifique se ele esta escutando

[root@osg-ce ~]# rpcinfo -u localhost ypserv

Em outra maquina sem jobs do cluster, teste

[root@node84 ~]# vim /etc/yp.conf
ypserver 192.168.1.152
[root@node84 ~]# /etc/init.d/ypbind restart
[root@node84 ~]# rpcinfo -u localhost ypbind
[root@node84 ~]# ypcat passwd

Copiando o home dos usuários da spgrid

[root@spgrid ~]# scp -pr /home/* osg-ce.sprace.org.br:/home/.

O home na verdade ficara na storage02, no raid0. Depois de feita a copia do /home para este disco, entao criamos um ponto de montagem para ele

[root@osg-ce ~]# vi /etc/auto.master
/osghome    /etc/auto.osghome    --timeout=30
[root@osg-ce ~]# vim /etc/auto.osghome
home      -rw,soft,bg,rsize=8192,wsize=8192,tcp     storage02:/raid0/osghome
[root@osg-ce ~]# mkdir /osghome
[root@osg-ce ~]# ln -s /osghome/home/ /home
[root@osg-ce ~]# chkconfig autofs on

Configuração do NFS

Da spgrid

[root@spgrid ~]# scp /etc/hosts.allow osg-ce.sprace.org.br:/etc/.

Principalmente a linha

192.168.1.              \

Feito isso:

[root@osg-ce ~]# vim /etc/exports 
/usr/local/opt/OSG       192.168.1.0/24(rw,async,no_root_squash)
[root@osg-ce ~]# chkconfig nfs on
[root@osg-ce ~]# service nfs start

De um nó qualquer

[root@node84 ~]# mount 192.168.1.152:/usr/local/home teste

Configuração do NTP

[root@osg-ce ~]# vim /etc/ntp.conf
#Permite acesso ao servidor para sincronizar mas nao permite modificacoes no servico
restrict default nomodify notrap noquery
# Permite acesso completo para a interface local
restrict 127.0.0.1
# Libera acesso a minha rede local
  restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap
#Nosso servidor
server pcdsh05.on.br
server ntp.cais.rnp.br
server ntp.puc-rio.br
server ntp1.pucpr.br
server ntp.pop-rs.rnp.br

driftfile /var/lib/ntp/drift

Para ver as permissões:

[root@osg-ce ~]# ntpdc -nc reslist
   address          mask            count        flags
=====================================================================
0.0.0.0         0.0.0.0               886  noquery, nomodify, notrap
127.0.0.1       255.255.255.255         1  none
127.0.0.1       255.255.255.255         0  ntpport, interface, ignore
192.168.1.152   255.255.255.255         0  ntpport, interface, ignore
200.136.80.25   255.255.255.255         0  ntpport, interface, ignore
::              ::                      0  none

Para ver o status do servidor:

[root@osg-ce ~]#  ntpq -p
     remote           refid      st t when poll reach   delay   offset  jitter
==============================================================================
+b.ntp.br        200.160.7.186    2 u  243 1024  377  162.219   -7.703   0.657
*a.ntp.br        200.160.7.192    2 u  270 1024  377  152.367  -10.902   1.354
+titan.cais.rnp. 69.100.142.49    2 u  728 1024   77   13.516   32.667   1.149
 LOCAL(0)        LOCAL(0)        10 l   47   64  377    0.000    0.000   0.004

Feito isso

[root@osg-ce ~]# service ntpd start
[root@osg-ce ~]# chkconfig ntpd on

De uma maquina qualquer

[root@node84 ~]# ntpdate -d  192.168.1.152

Configuração do DNS(chrooted)

[root@osg-ce ~]# vim /var/named/chroot/etc/named.conf
//Dominio sprace.org.br
zone "sprace.org.br" IN{
        type master;
        file "sprace.org.br.zone";
        allow-update { none; };
};
//Dominio Reverso da sprace.org.br
zone "80.136.200.in-addr.arpa" IN{
        type master;
        file "80.136.200.in-addr.arpa.zone";
        allow-update { none; };
};
//Dominio Grid
zone "grid" IN{
        type master;
        file "grid.zone";
        allow-update { none; };
};
//Dominio reverso grid
zone "1.168.192.in-addr.arpa" IN{
        type master;
        file "1.168.192.in-addr.arpa.zone";
        allow-update { none; };
};
[root@osg-ce ~]# vim /var/named/chroot/var/named/sprace.org.br.zone
$TTL    86400
;Nome do servidor e e-mail do administrador
@       IN      SOA   osg-ce1.sprace.org.br. root.osg-ce1.sprace.org.br.   (

                                      2008012201 ; Serial
                                      28800      ; Refresh
                                      14400      ; Retry
                                      3600000    ; Expire
                                      86400 )    ; Minimum
;Especificando o servidor de nomes
@       IN      NS      osg-ce1.sprace.org.br.
;Especificando o servidor de e-mails
@       IN      MX      5       osg-ce1.sprace.org.br.
;adicionando endereços
osg-ce          IN      A       200.136.80.4
access          IN      A       200.136.80.3
osg-se          IN      A       200.136.80.27
ftp-01          IN      A       200.136.80.14
osg-ce1      IN      A         200.136.80.25
;Adicionando o servidor www
www     IN      CNAME           access
[root@osg-ce ~]# vim /var/named/chroot/var/named/grid.zone
$TTL    86400
@       IN      SOA     osg-ce.grid. root.osg-ce.grid.  (
                                     2008011901 ; Serial
                                      8H      ; Refresh
                                      2H      ; Retry
                                      1W    ; Expire
                                      1D )    ; Minimum
;Servidor de nomes
@       IN      NS      osg-ce.grid.
;Servidor de e-mail
@       IN      MX      0       osg-ce.grid.
;Especificando enderecos
@       IN      A       192.168.1.152 ;(ip do servidor)                  
osg-ce   IN      A       192.168.1.152
;Adicionando apelidos
mail    IN      CNAME   osg-ce 
[root@osg-ce ~]# vim /var/named/chroot/var/named/80.136.200.in-addr.arpa.zone
$TTL    86400
@       IN      SOA   osg-ce1.sprace.org.br. root.osg-ce1.sprace.org.br.   (
                                      2008012201 ; Serial
                                      28800      ; Refresh
                                      14400      ; Retry
                                      3600000    ; Expire
                                      86400 )    ; Minimum
;Especificando o servidor de nomes
@       IN      NS      osg-ce1.sprace.org.br.
;Adicionando os enderecos reversos
3       IN      PTR     access.sprace.org.br.
4       IN      PTR     osg-ce.
14      IN      PTR     ftp-01.
27      IN      PTR     osg-se.
25      IN      PTR     osg-ce1.
[root@osg-ce ~]# vim /var/named/chroot/var/named/1.168.192.in-addr.arpa.zone
$TTL    86400
@       IN      SOA     osg-ce.grid. root.osg-ce.grid.  (
                                      2008011901 ; Serial
                                      8H      ; Refresh
                                      2H      ; Retry
                                      1W    ; Expire
                                      1D )    ; Minimum
;Especificando o servidor de nomes
@               IN      NS      osg-ce.grid.
;Adicionando endereços reversos
152             IN      PTR     osg-ce.grid.
[root@osg-ce ~]# chkconfig named on

Testando em um nó

[root@node84 ~]# vim /etc/resolv.conf
nameserver 192.168.1.152

Monitoramento do Ganglia

[root@osg-ce ~]# wget http://dag.wieers.com/rpm/packages/rrdtool/rrdtool-1.0.50-3.el4.rf.i386.rpm
[root@osg-ce ~]# rpm -ivh rrdtool-1.0.50-3.el4.rf.i386.rpm 
[root@osg-ce tmp]# wget http://downloads.sourceforge.net/ganglia/ganglia-3.0.6-1.src.rpm?modtime=1197841780&big_mirror=0
[root@osg-ce ~]# cd /tmp/
[root@osg-ce tmp]# rpm -i /root/ganglia-3.0.6-1.src.rpm 
[root@osg-ce tmp]# cd /usr/src/redhat/SPECS
[root@osg-ce SPECS]# rpmbuild -ba --target noarch ganglia.spec

Instalar dependências

[root@osg-ce SPECS]# cd /root/
[root@osg-ce ~]# wget http://dag.wieers.com/rpm/packages/rrdtool/rrdtool-devel-1.0.50-3.el4.rf.i386.rpm
[root@osg-ce ~]# rpm -ivh rrdtool-devel-1.0.50-3.el4.rf.i386.rpm
[root@osg-ce ~]# yum install libart_lgpl-devel libpng-devel
[root@osg-ce ~]# cd /usr/src/redhat/SPECS
[root@osg-ce SPECS]# rpmbuild -ba --target noarch ganglia.spec
[root@osg-ce SPECS]# rpmbuild -ba --target i686 ganglia.spec
[root@osg-ce SPECS]# cd /usr/src/                                               
[root@osg-ce src]# rpm -ivh */RPMS/*/ganglia-gmetad-*rpm
[root@osg-ce src]# yum install php-gd.i386
[root@osg-ce src]# rpm -i */RPMS/*/ganglia-web*rpm
[root@osg-ce src]# chkconfig --level 345 gmetad on
[root@osg-ce src]# chkconfig httpd on

Configurando o gmond

[root@osg-ce src]# mount -t nfs 192.168.1.200:/export /teste;
[root@osg-ce src]# cd /teste
[root@osg-ce teste]# rpm -ivh ganglia-monitor-core-gmond-2.5.4-8.i386.rpm 
[root@osg-ce teste]# cd
[root@osg-ce ~]# umount /teste/
[root@osg-ce ~]# rmdir /teste/
[root@osg-ce ~]# vim /etc/gmond.conf
name "SPGRID Cluster"
owner "SPRACE-HEP"
url "http://osg-ce1.sprace.org.br/"
trusted_hosts 200.136.80.25
num_nodes  86
setuid ganglia
location "0,3,0"

Obter Certificado

[root@osg-ce ~]# mkdir /OSG
[root@osg-ce ~]# mount 192.168.1.150:/usr/local/opt/OSG /OSG/
[root@osg-ce ~]# ln -s /OSG/ /usr/local/opt/OSG
[root@osg-ce ~]# . /OSG/setup.sh
[root@osg-ce ~]# cd $VDT_LOCATION
[root@osg-ce OSG]# source  ./setup.sh
[root@osg-ce ~]# date
Thu Jan 17 09:01:07 BRST 2008
[root@osg-ce OSG]# cd /root/
[root@osg-ce ~]# cert-request -ou s -host osg-ce.sprace.org.br -dir . -label osg-ce  -agree  -email mdias1@ift.unesp.br  -phone +55.11.XXXXXX -reason "Instaling a new Computer Element head node for SPRACE site"  -name "Marco Dias"
Choose a registration authority to which you are affiliated.
osg
Choose a virtual organization under your OSG affiliation:
dosar

Recebido o e-mail com o certificado

[root@osg-ce ~]# . /OSG/setup.sh
[root@osg-ce ~]# cert-retrieve  -certnum XXXX -label osg-ce -dir . -prefix osg-ce.sprace.org.br
[root@osg-ce ~]# mv osg-ce.sprace.org.brcert.pem /etc/grid-security/hostcert.pem
[root@osg-ce ~]# mv osg-ce.sprace.org.brkey.pem /etc/grid-security/hostkey.pem 
[root@osg-ce ~]# chmod 400 /etc/grid-security/hostkey.pem
[root@osg-ce ~]# chmod 444 /etc/grid-security/hostcert.pem

Firewall

Não dá para mostrar as regras aqui!! Logo o procedimento será

[root@spgrid ~]# scp /etc/init.d/firewall osg-ce.sprace.org.br:/etc/init.d/.

e na osg-ce

CORPIF="eth1"
CORPIP="200.136.80.25"
SERVICEIF="eth0"
SERVICEIP="192.168.1.152"
[root@osg-ce ~]# chkconfig firewall on

O teste é feito em um node qualquer

[root@node84 ~]# more /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=node84
GATEWAY=192.168.1.152
NISDOMAIN=grid
[root@node84 ~]# /etc/init.d/network restart
[root@node84 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 eth1
169.254.0.0     0.0.0.0         255.255.0.0     U     0      0        0 eth1
0.0.0.0         192.168.1.152   0.0.0.0         UG    0      0        0 eth1

Instalacão do OSG

Instalacao do PACMAN

[root@osg-ce ~]# cd /opt/
[root@osg-ce opt]# wget http://physics.bu.edu/pacman/sample_cache/tarballs/pacman-latest.tar.gz
[root@osg-ce opt]# tar --no-same-owner -xzvf  pacman-latest.tar.gz
[root@osg-ce opt]# cd pacman-3.21
[root@osg-ce pacman-3.21]# source setup.sh

Preparacão

[root@osg-ce ~]# mkdir /opt/osg-0.8.0
[root@osg-ce ~]# ln -s /opt/osg-0.8.0/ /OSG/
[root@osg-ce ~]# . /opt/pacman-3.21/setup.sh
[root@osg-ce ~]# cd /opt/osg-0.8.0/
[root@osg-ce ~]#  pacman -get OSG:ce
[root@osg-ce ~]# cd /OSG/
[root@osg-ce OSG]# source setup.sh
[root@osg-ce OSG]# pacman -get OSG:Globus-Condor-Setup
[root@osg-ce OSG]# pacman -get  OSG:ManagedFork
[root@osg-ce OSG]# $VDT_LOCATION/vdt/setup/configure_globus_gatekeeper --managed-fork y --server y

Post Install

[root@osg-ce1 ~]# visudo
Runas_Alias GLOBUSUSERS = ALL, !root
globus ALL=(GLOBUSUSERS) \
       NOPASSWD: /opt/osg-0.8.0/globus/libexec/globus-gridmap-and-execute \
       -g /etc/grid-security/grid-mapfile \
       /opt/osg-0.8.0/globus/libexec/globus-job-manager-script.pl *
globus ALL=(GLOBUSUSERS) \
       NOPASSWD: /opt/osg-0.8.0/globus/libexec/globus-gridmap-and-execute \
       -g /etc/grid-security/grid-mapfile \
       /opt/osg-0.8.0/globus/libexec/globus-gram-local-proxy-tool *
[root@osg-ce1 ~]# . /OSG/setup.sh
[root@osg-ce1 ~]# $VDT_LOCATION/vdt/setup/configure_monalisa --prompt
monalisa
SPRACE
OSG
[root@osg-ce1 ~]# vim $VDT_LOCATION/MonaLisa/Service/VDTFarm/ml.properties
MonaLisa.Location=Sao Paulo
MonaLisa.Country=Brazil
MonaLisa.LAT=-23.5592
MonaLisa.LONG=-46.7358
[root@osg-ce1 ~]#vdt-register-service --name MLD --enable
[root@osg-ce1 ~]# cd /etc/grid-security/
[root@osg-ce1 grid-security]# cp hostkey.pem containerkey.pem
[root@osg-ce1 grid-security]# cp hostcert.pem containercert.pem
[root@osg-ce1 grid-security]# chown globus: containerkey.pem containercert.pem
[root@osg-ce1 grid-security]# cp /opt/osg-0.8.0/post-install/gsi-authz.conf /etc/grid-security/.
[root@osg-ce1 grid-security]# cp /opt/osg-0.8.0/post-install/prima-authz.conf /etc/grid-security/.
[root@osg-ce1 ~]# . /OSG/setup.sh 
[root@osg-ce1 ~]#  /opt/osg-0.8.0/vdt/sbin/vdt-register-service -name
gums-host-cron --enable
vdt-register-service: updated cron service 'gums-host-cron'
vdt-register-service: desired state = enable
vdt-register-service: cron time     = '12 6,12,18,0 * * *'
vdt-register-service: cron command  =
'/opt/osg-0.8.0/gums/scripts/gums-host-cron'
[root@osg-ce1 ~]#  /opt/osg-0.8.0/vdt/sbin/vdt-control --on gums-host-cron
enabling cron service gums-host-cron... no crontab for root
ok

Pedindo o certificado para o apache

[mdias@spgrid ~]$ . /OSG/setup.sh
[mdias@spgrid ~]$ cert-request -ou s -dir . -host osg-ce.sprace.org.br
-service http -label osg-ce.http
[mdias@spgrid ~]$ . /OSG/setup.sh
[mdias@spgrid ~]$  cert-retrieve  -certnum 21546  -dir . -label osg-ce.http
 using CA doegrids
Checking that the usercert and ./osg-ce.httpkey.pem match
writing RSA key
./usercert.pem and ./userkey.pem now contain your Globus credential
[mdias@spgrid ~]$ scp usercert.pem userkey.pem osg-ce1.sprace.org.br:/home/mdias/.
[root@osg-ce1 mdias]#  mkdir /etc/grid-security/http 
[root@osg-ce1 mdias]# cp ./usercert.pem /etc/grid-security/http/httpcert.pem 
[root@osg-ce1 mdias]# cp ./userkey.pem /etc/grid-security/http/httpkey.pem 
[root@osg-ce1 mdias]# chmod 444 /etc/grid-security/http/httpcert.pem 
[root@osg-ce1 mdias]# chmod 400 /etc/grid-security/http/httpkey.pem 
[root@osg-ce1 mdias]# chown -R daemon.daemon /etc/grid-security/http 

E finalmente

[root@osg-ce1 ~]# hostname osg-ce.sprace.org.br
[root@osg-ce1 ~]#vdt-control --on

TODO List:

-trocar hostname e IP no /etc/sysconfig/network para o mesmo da spgrid

-colocar o IP 192.168.1.150 no lugar de 192.168.1.152 no /etc/sysconfig/network/ifcfg-ethX

-Trocar as interfaces de rede para ficar padrão