SAN Storage

HP Storage

Overall Sequence

  1. Create Aliases for WWPN's on SAN switch
  2. Create Zone on SAN to allow server to see storage & save config
  3. Create Host on EVA storage and add WWPN's
  4. Create vdisk on EVA storage and present LUN to Host created above.
  5. On server, scan for new storage
  6. Configure /etc/multipathd.conf to combine /dev/sd devices into /dev/mapper devices under multipath control
  7. Partition disk and create filesystem.

Find out WWN

RHEL4

[root@test01 ~]# cat /proc/scsi/qla2xxx/* | grep adapter-port
scsi-qla0-adapter-port=500110a00019ac8c;
scsi-qla1-adapter-port=500110a00019ac8e;

RHEL5

[root@db03 ~]# cat /sys/class/fc_host/host*/port_name 
0x5001438009ac0d2c
0x5001438009ac0e1e

RHEL6

[root@satellite02 ~]# cat /sys/class/fc_host/host*/port_name
0x500110a00019a1c8
0x500110a00019a1ca

Multipath info

[root@bi01 ~]# lspci | grep Q
08:00.0 Fibre Channel: QLogic Corp. ISP2432-based 4Gb Fibre Channel to PCI Express HBA (rev 02)
08:00.1 Fibre Channel: QLogic Corp. ISP2432-based 4Gb Fibre Channel to PCI Express HBA (rev 02)

All paths here are via one HBA, one fabric has failed (the 1: in the 1st column)

[root@bi01 ~]# multipath -ll /dev/sda
P_BI_Data_0003 (3600508b4001052920000e00003770000)
[size=250 GB][features="1 queue_if_no_path"][hwhandler="0"]
\_ round-robin 0 [prio=100][active]
 \_ 1:0:0:1 sda 8:0   [active][ready]
 \_ 1:0:3:1 sdm 8:192 [active][ready] 
\_ round-robin 0 [prio=20][enabled]
 \_ 1:0:1:1 sde 8:64  [active][ready]
 \_ 1:0:2:1 sdi 8:128 [active][ready]

After enabling the fabric, paths are seen via both HBA's (different server)

[root@db01 ~]# multipath -ll /dev/sda
ocr1 (3600508b4001052a80000b00001740000)
[size=2 GB][features="1 queue_if_no_path"][hwhandler="0"]
\_ round-robin 0 [prio=200][active]
 \_ 0:0:0:1  sda  8:0     [active][ready]
 \_ 0:0:3:1  sdbu 68:128  [active][ready]
 \_ 1:0:0:1  sdcs 70:0    [active][ready]
 \_ 1:0:3:1  sdfm 130:128 [active][ready]
\_ round-robin 0 [prio=40][enabled]
 \_ 0:0:2:1  sdaw 67:0    [active][ready]
 \_ 1:0:1:1  sddq 71:128  [active][ready]
 \_ 1:0:2:1  sdeo 129:0   [active][ready]
 \_ 0:0:1:1  sdy  65:128  [active][ready]
[root@bi01 ~]# ls -l /sys/block/*/device
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/cciss!c0d0/device -> ../../devices/pci0000:40/0000:40:0c.0/0000:41:00.0/0000:42:08.0/disk0
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sda/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:0/1:0:0:1
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdb/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:0/1:0:0:2
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdc/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:0/1:0:0:3
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdd/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:0/1:0:0:4
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sde/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:1/1:0:1:1
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdf/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:1/1:0:1:2
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdg/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:1/1:0:1:3
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdh/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:1/1:0:1:4
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdi/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:2/1:0:2:1
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdj/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:2/1:0:2:2
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdk/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:2/1:0:2:3
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdl/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:2/1:0:2:4
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdm/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:3/1:0:3:1
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdn/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:3/1:0:3:2
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdo/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:3/1:0:3:3
lrwxrwxrwx  1 root root 0 Aug 26 12:08 /sys/block/sdp/device -> ../../devices/pci0000:00/0000:00:0e.0/0000:08:00.1/host1/target1:0:3/1:0:3:4

Parameters

in /etc/modprobe.conf:-

alias scsi_hostadapter1 qla2xxx_conf
alias scsi_hostadapter2 qla2xxx
options qla2xxx  ql2xmaxqdepth=16 qlport_down_retry=10 ql2xloginretrycount=30 

These settings are more aggressive in failing quickly, this allows multipath to adjust sooner to lost paths.

options qla2xxx ql2xfailover=0 ql2xioctltimeout=10 ql2xprocessnotready=0 ql2xloginretrycount=3 ql2xsuspendcount=3 ql2xretrycount=2  lport_down_retry=1
options lpfc lpfc_lun_queue_depth=16 lpfc_nodev_tmo=30 lpfc_discovery_threads=32

in /etc/multipath.conf (for EVA 8400):-

defaults {
	udev_dir		/dev
	polling_interval	10
	selector		"round-robin 0"
	path_grouping_policy	failover
	getuid_callout		"/sbin/scsi_id -g -u -s /block/%n"
	prio_callout		"/bin/true"
	path_checker		tur
	rr_min_io		100
	rr_weight       	uniform
	failback		immediate
	no_path_retry		12
	user_friendly_names	yes
	bindings_file		"/var/lib/multipath/bindings"
}

blacklist {
	devnode		"^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
	devnode 	"^hd[a-z][[0-9]*]"
	devnode		"^cciss!c[0-9]d[0-9]*"
}

devices {

device {
	vendor			"HP|COMPAQ"
      	product			"HSV1[01]1 \(C\)COMPAQ|HSV2[01]0|HSV300|HSV4[50]0"
	path_grouping_policy	group_by_prio
        getuid_callout		"/sbin/scsi_id -g -u -s /block/%n"
        path_checker		tur
        path_selector		"round-robin 0"
        prio_callout		"/sbin/mpath_prio_alua /dev/%n"
        rr_weight		uniform
        failback		immediate
	hardware_handler	"0"
        no_path_retry		18
	rr_min_io		100
       }
}
# End device

multipaths {

        multipath {
        wwid 36001438005def5bf00006000004e0000
        alias HSV450_FRA_001
        }

        multipath {
        wwid 36001438005def5bf00006000004a0000
        alias HSV450_arch_001
        }

USB drives and multipath

No USB drives should be added by multipath. However, multipath can add a USB drive into it's configuration, if it detects a new drive.

To check if multipath has added a USB drive, it is possible to check by running:

# mount /dev/sda1 /mnt/ext_hdd 
mount: /dev/sda1 already mounted or /mnt/ext_hdd busy


sudo /sbin/multipath -l
/dev/sda1: UUID="da85426e-cffc-4d13-8d7f-64549e7ca0ce" SEC_TYPE="ext2" TYPE="ext3" 
HSV450_log01_arch (36001438005dec6e40000500000060000) dm-6 HP,HSV450
[size=500G][features=1 queue_if_no_path][hwhandler=0][rw]
\_ round-robin 0 [prio=200][active]
 \_ 0:0:0:1 sda 8:0   [active][ready]
 \_ 0:0:1:1 sdb 8:16  [active][ready]
 \_ 1:0:0:1 sdd 8:48  [active][ready]
 \_ 1:0:1:1 sde 8:64  [active][ready]
\_ round-robin 0 [prio=40][enabled]
 \_ 0:0:2:1 sdc 8:32  [active][ready]
 \_ 0:0:3:1 sdf 8:80  [active][ready]
 \_ 1:0:2:1 sdg 8:96  [active][ready]
 \_ 1:0:3:1 sdh 8:112 [active][ready]

If the USB drive is found in multipath, then the device should be blacklisted in /etc/multipath.conf.

An example is below:

blacklist {
        devnode         "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
        devnode         "^hd[a-z][[0-9]*]"
        devnode         "^cciss!c[0-9]d[0-9]*"
        wwid            "da85426e-cffc-4d13-8d7f-64549e7ca0ce"
}

multipath has to be flushed for the new configuration to take place. This is done via:

# multipath -F
# service multipath restart

Linux SAN - removing LUNS

Overall Sequence to remove a LUN

  1. Ensure LUN required to be removed is not mounted anywhere.
  2. Remove WWID stanza from multipath.conf.
  3. Run multipath -F to flush unused LUNS from config.
  4. Unpresent LUN from server on gy-asp-sma01 web interface or cli.
  5. Delete LUN (if required).
  6. If appropriate, remove zone config from SAN.

Check if LUN is mounted

Look in /etc/multipath.conf and record the wwid and alias name for the unwanted LUN. This LUN will appear as /dev/mapper/$ALIAS, run the mount command and check that the LUN is not mounted. If it is mounted, it must be unmounted before you can proceed. Also check if the LUN is refered to in /etc/fstab. This will also require editing to remove the LUN so as not to cause any problems next reboot. It may be good practice to delete the mount point if it is not in use any more.

Edit multipath.conf

Edit /etc/multipath.conf and remove the config relating to the LUN to be removed. Run multipath to rescan LUNS, this will ensure config is up to date. multipath -ll will still show the paths to the LUN to be deleted:-

netbackup (3600508b4001052920000e00003fe0000)
[size=10 GB][features="1 queue_if_no_path"][hwhandler="0"]
\_ round-robin 0 [prio=200][active]
 \_ 0:0:0:25 sda  8:0     [active][ready]
 \_ 0:0:1:25 sdb  8:16    [active][ready]
 \_ 1:0:0:25 sdco 69:192  [active][ready]
 \_ 1:0:1:25 sdcp 69:208  [active][ready]
\_ round-robin 0 [prio=40][enabled]
 \_ 1:0:2:25 sdcq 69:224  [active][ready]
 \_ 1:0:3:25 sdcr 69:240  [active][ready]
 \_ 0:0:2:25 sdc  8:32    [active][ready]
 \_ 0:0:3:25 sdd  8:48    [active][ready]

Run multipath -F to flush unused paths from the config, this will show the maps in use, any not in use will be flushed silently:-

[root@db01 ~]# multipath -F
HSV450_P_vote1: map in use
HSV450_P_RGS_OCFS2_rman: map in use
HSV450_P_RGSPROD_Data_0003: map in use
HSV450_P_RGSPROD_Data_0002: map in use
HSV450_P_RGSPROD_Data_0001: map in use

Running multipath -ll should not show the paths to the LUN to be deleted anymore.

Unpresent and delete LUN on SMA01

Either in the GUI (https://sma01:2372/Login) or CLI, unpresent the LUN from the host. The CLI differs from the GUI in the terminology it uses for doing the same things. In the GUI, you “unpresent” a vdisk from a host, but in the sssu CLI, you “delete a LUN”, note:- not delete a vdisk.

From http://h30499.www3.hp.com/t5/Storage-Area-Networks-SAN/SSSU-unpresent-VDISK-from-given-SAN/td-p/3927720 :- You “DELETE” the LUNS. NOT DELETE VDISKS, but DELETE LUNS.

Array01> ls lun

LUNs available on this Cell:
....edited....
  \Hosts\Database server\Production\db01\25

Array01> delete LUN "\Hosts\Database server\Production\db01\25"
Array01> ls vdisk

Vdisks available on this Cell:
....edited....
  \Virtual Disks\Oracle\RGS_Prod\vDskDB01_Netbackup\ACTIVE

Array01> delete vdisk "\Virtual Disks\Oracle\RGS_Prod\vDskDB01_Netbackup\ACTIVE"

Remove Zoning on SAN

See Documentation elsewhere.

Re-Scan Fibre devices

RHEL 4

[root@bi01 SANsurfer]# /opt/hp/hp_fibreutils/hp_rescan -a
Sending rescan signal to /proc/scsi/qla2xxx/0...
Sending rescan signal to /proc/scsi/qla2xxx/1...
Adding legacy tape devices to /proc/scsi/device_info
Scanning /proc/scsi/qla2xxx/0, target 3, LUN 4      
Scanning /proc/scsi/qla2xxx/1, target 3, LUN 4      
                                                      

scsi0  00 00 00 HP         HSV210     5110       RAID
scsi0  00 00 01 HP         HSV210     5110       Direct-Access
scsi0  00 00 02 HP         HSV210     5110       Direct-Access
scsi0  00 00 03 HP         HSV210     5110       Direct-Access
scsi0  00 00 04 HP         HSV210     5110       Direct-Access
edited
scsi1  00 00 00 HP         HSV210     5110       RAID
scsi1  00 00 01 HP         HSV210     5110       Direct-Access
scsi1  00 00 02 HP         HSV210     5110       Direct-Access
scsi1  00 00 03 HP         HSV210     5110       Direct-Access
scsi1  00 00 04 HP         HSV210     5110       Direct-Access

[root@bi01 SANsurfer]# multipath -ll
P_BI_Data_0003 (3600508b4001052920000e00003770000)
[size=250 GB][features="1 queue_if_no_path"][hwhandler="0"]
\_ round-robin 0 [prio=200][active]
 \_ 1:0:0:1 sda  8:0    [active][ready]
 \_ 1:0:3:1 sdm  8:192  [active][ready]
 \_ 0:0:0:1 sdq  65:0   [active][ready]
 \_ 0:0:1:1 sdu  65:64  [active][ready]
\_ round-robin 0 [prio=40][enabled]
 \_ 1:0:1:1 sde  8:64   [active][ready]
 \_ 1:0:2:1 sdi  8:128  [active][ready]
 \_ 0:0:2:1 sdy  65:128 [active][ready]
 \_ 0:0:3:1 sdac 65:192 [active][ready]

Paths now visible via HBA 0 and 1 (0: and 1:)

RHEL 5

echo "- - -" >  /sys/class/scsi_host/<host listings here>/scan

Get the <host listings here> from:-

[root@r08u6 scsi_host]# ll
total 0
drwxr-xr-x 2 root root 0 Aug  4 15:52 host0
drwxr-xr-x 2 root root 0 Aug  4 15:52 host1
drwxr-xr-x 2 root root 0 Aug  4 15:52 host2

Multipath command output

Output of multipath command

Standard output of multipath command

# multipath -ll
mydev1 (3600a0b800011a1ee0000040646828cc5) dm-1 IBM,1815      FAStT
[size=512M][features=1 queue_if_no_path][hwhandler=1 rdac]
\_ round-robin 0 [prio=6][active]
 \_ 29:0:0:1 sdf 8:80  [active][ready]
 \_ 28:0:1:1 sdl 8:176 [active][ready]
\_ round-robin 0 [prio=0][enabled]
 \_ 28:0:0:1 sdb 8:16  [active][ghost]
 \_ 29:0:1:1 sdq 65:0  [active][ghost]

Annotated output of multipath command

mydev1 (3600a0b800011a1ee0000040646828cc5) dm-1 IBM,1815      FAStT
------  ---------------------------------  ---- --- ---------------
  |               |                         |    |          |-------> Product  
  |               |                         |    |------------------> Vendor
  |               |                         |-----------------------> sysfs name
  |               |-------------------------------------------------> WWID of the device
  |------------------------------------------------------ ----------> User defined Alias name

 [size=512M][features=1 queue_if_no_path][hwhandler=1 rdac]
 ---------  ---------------------------  ----------------
     |                 |                        |--------------------> Hardware Handler, if any
     |                 |---------------------------------------------> Features supported
     |---------------------------------------------------------------> Size of the DM device

Path Group 1:
\_ round-robin 0 [prio=6][active]
-- -------------  ------  ------
 |    |              |      |----------------------------------------> Path group state
 |    |              |-----------------------------------------------> Path group priority
 |    |--------------------------------------------------------------> Path selector and repeat count
 |-------------------------------------------------------------------> Path group level

First path on Path Group 1:
 \_ 29:0:0:1 sdf 8:80  [active][ready]
    -------- --- ----   ------  -----
      |      |     |        |      |---------------------------------> Physical Path state
      |      |     |        |----------------------------------------> DM Path state
      |      |     |-------------------------------------------------> Major, minor numbers
      |      |-------------------------------------------------------> Linux device name
      |--------------------------------------------------------------> SCSI information: host, channel, scsi_id and lun

Second path on Path Group 1:
 \_ 28:0:1:1 sdl 8:176 [active][ready]

Path Group 2:
\_ round-robin 0 [prio=0][enabled]
 \_ 28:0:0:1 sdb 8:16  [active][ghost]
 \_ 29:0:1:1 sdq 65:0  [active][ghost]

rb/linuxsan.txt · Last modified: 20/08/2019 12:58 by andrew