/ filled

[satish]

Dear Erman,

we have 2 node RAC,on node1 we have ample of space but on node 2 / is almost full

[root@erpprod2 .oracle]# df -h
Filesystem                                Size  Used Avail Use% Mounted on
/dev/mapper/rhel-root                      50G   48G  2.8G  95% /
devtmpfs                                  126G     0  126G   0% /dev
tmpfs                                     126G  663M  126G   1% /dev/shm
tmpfs                                     126G  811M  126G   1% /run
tmpfs                                     126G     0  126G   0% /sys/fs/cgroup
/dev/sda2                                  10G  166M  9.9G   2% /boot
/dev/sda1                                  10G  9.5M   10G   1% /boot/efi
/dev/mapper/prddb02_vgu02-prddb02--lvu02  197G   68G  120G  37% /u02
/dev/mapper/prddb02_vgu01-prddb02--lvu01  197G   35G  153G  19% /u01
/dev/mapper/prddb02_vgu03-prddb02--lvu03  2.0T  1.4T  587G  70% /u03
/dev/mapper/rhel-tmp                       30G   89M   30G   1% /tmp
/dev/mapper/rhel-u04                      200G  109G   92G  55% /u04
10.25.1.202:/vol/ERP                      973G  649G  325G  67% /ERPShareddrive
tmpfs                                      26G     0   26G   0% /run/user/54321
tmpfs                                      26G   20K   26G   1% /run/user/0
[root@erpprod2 .oracle]#


/ is almost full,have 2.8GB free..

[root@erpprod2 /]# lsof -s|grep -i deleted |sort -k 8
crsd.bin   9001  9003           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9010           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9040           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9041           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9042           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9043           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9045           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9046           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9049           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9050           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9051           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9052           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9053           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9054           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9055           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9056           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9057           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
crsd.bin   9001  9058           root  185uw     REG              253,0         0            126071357 /var/tmp/.oracle/sCRSD_UI_SOCKET_lock (deleted)
:
:
:
In progress

[root@erpproddb1 tmp]# cd /
[root@erpproddb1 /]# du -sh *
0       1
0       bin
143M    boot

[satish]

[root@erpproddb2 ~]# cd /
[root@erpproddb2 /]# ls
1    boot       dev             etc   hpmnt  lib64  mnt  proc  run   srv  TFA.txt  u01  u03  u05  var
bin  core.7094  ERPShareddrive  home  lib    media  opt  root  sbin  sys  tmp      u02  u04  usr
[root@erpproddb2 /]#

[root@erpproddb1 /]# du -sh *
0       1
0       bin
143M    boot
5.8M    core.31044
8.6M    core.6892
663M    dev
646G    ERPShareddrive
46M     etc
148M    home
0       hpmnt
0       lib
0       lib64
0       media
0       mnt
392M    opt
du: cannot access âproc/15662/task/15662/fd/4â: No such file or directory
du: cannot access âproc/15662/task/15662/fdinfo/4â: No such file or directory
du: cannot access âproc/15662/fd/4â: No such file or directory
du: cannot access âproc/15662/fdinfo/4â: No such file or directory
0       proc
1.8G    root
427M    run
0       sbin
0       srv
0       sys
4.0K    TFA.txt
49M     tmp
35G     u01
29G     u02
1.4T    u03---backup mountpoint
3.9G    u04
3.9G    usr
2.5G    var
[root@erpproddb1 /]#

[ErmanArslansOracleBlog]

You say your problem is on node 2, but you sent du -sh outputs from node1?

Anyways,

login with root into "node2" and send me the following output-

1)
df -h

2)
cd /
du -sh * |grep G

[satish]

please excuse me for wrong update

Here is the requested information

[root@erpprod2 tmp]# df -h
Filesystem                                Size  Used Avail Use% Mounted on
/dev/mapper/rhel-root                      50G   48G  2.8G  95% /
devtmpfs                                  126G     0  126G   0% /dev
tmpfs                                     126G  663M  126G   1% /dev/shm
tmpfs                                     126G  811M  126G   1% /run
tmpfs                                     126G     0  126G   0% /sys/fs/cgroup
/dev/sda2                                  10G  166M  9.9G   2% /boot
/dev/sda1                                  10G  9.5M   10G   1% /boot/efi
/dev/mapper/prddb02_vgu02-prddb02--lvu02  197G   68G  120G  37% /u02
/dev/mapper/prddb02_vgu01-prddb02--lvu01  197G   35G  153G  19% /u01
/dev/mapper/prddb02_vgu03-prddb02--lvu03  2.0T  1.4T  587G  70% /u03
/dev/mapper/rhel-tmp                       30G   88M   30G   1% /tmp
/dev/mapper/rhel-u04                      200G  109G   92G  55% /u04
10.25.1.202:/vol/ERP                      973G  649G  325G  67% /ERPShareddrive
tmpfs                                      26G     0   26G   0% /run/user/54321
tmpfs                                      26G   20K   26G   1% /run/user/0


[root@erpprod2 tmp]# cd /
[root@erpprod2 /]# du -sh * |grep G
646G    ERPShareddrive
du: cannot access âproc/16124/task/16124/fd/4â: No such file or directory
du: cannot access âproc/16124/task/16124/fdinfo/4â: No such file or directory
du: cannot access âproc/16124/fd/4â: No such file or directory
du: cannot access âproc/16124/fdinfo/4â: No such file or directory
1.8G    root
35G     u01
68G     u02
109G    u04
38G     u05
3.8G    usr
2.6G    var
[root@erpprod2/]#



ERPShareddrive-this is shared drive between node1 and node2

This is from node1

[oraprod@erpprod1 ~]$ df -h
Filesystem                                Size  Used Avail Use% Mounted on
/dev/mapper/rhel-root                      50G  8.7G   42G  18% /
devtmpfs                                  126G     0  126G   0% /dev
tmpfs                                     126G  663M  126G   1% /dev/shm
tmpfs                                     126G  427M  126G   1% /run
tmpfs                                     126G     0  126G   0% /sys/fs/cgroup
/dev/mapper/prddb01_vgu01-prddb01--lvu01  196G   36G  151G  19% /u01
/dev/sda2                                  10G  166M  9.9G   2% /boot
/dev/sda1                                  10G  9.5M   10G   1% /boot/efi
/dev/mapper/prddb01_vgu02-prddb01--lvu02  197G   29G  159G  16% /u02
/dev/mapper/rhel-u04                      200G  3.9G  197G   2% /u04
/dev/mapper/rhel-tmp                       30G   82M   30G   1% /tmp
172.16.111.77:/u03                        2.0T  1.4T  587G  70% /u03
10.25.1.202:/vol/ERP                      973G  649G  325G  67% /ERPShareddrive
tmpfs                                      26G   24K   26G   1% /run/user/0
tmpfs                                      26G     0   26G   0% /run/user/54321
[oraprod@erpprod1 ~]$

[ErmanArslansOracleBlog]

your problem is u05..
It is 38G and it is not mounted from anywhere.. I mean it is not a storage or a remote filesystem.. It is not a seperate partition or it is not located on a seperate disk.

It is counted under the / filesystem and it occupies 38 GB from there..

So concantrate on u05.

[satish]

Dear Erman,

I have deleted some files under /u05 and issue was fixed.But not sure how /u05 was created.Thanks a ton.
we have breaking our heads since saturday and finally it was fixed with your help

recently we have copied a file to /u05 and before that nothing is available in /u05.Not sure how this was created.Do you think it was wrongly created

[root@erpprod2 u05]# df -h
Filesystem                                Size  Used Avail Use% Mounted on
/dev/mapper/rhel-root                      50G  9.3G   41G  19% /
devtmpfs                                  126G     0  126G   0% /dev
tmpfs                                     126G  663M  126G   1% /dev/shm
tmpfs                                     126G  811M  126G   1% /run
tmpfs                                     126G     0  126G   0% /sys/fs/cgroup
/dev/sda2                                  10G  166M  9.9G   2% /boot
/dev/sda1                                  10G  9.5M   10G   1% /boot/efi
/dev/mapper/prddb02_vgu02-prddb02--lvu02  197G   68G  120G  37% /u02
/dev/mapper/prddb02_vgu01-prddb02--lvu01  197G   35G  153G  19% /u01
/dev/mapper/prddb02_vgu03-prddb02--lvu03  2.0T  1.4T  586G  70% /u03
/dev/mapper/rhel-tmp                       30G   88M   30G   1% /tmp
/dev/mapper/rhel-u04                      200G  109G   92G  55% /u04
10.25.1.202:/vol/ERP                      973G  649G  325G  67% /ERPShareddrive
tmpfs                                      26G     0   26G   0% /run/user/54321
tmpfs                                      26G   20K   26G   1% /run/user/0
[root@erpprod2 u05]#

[satish]

erman,

Can we check it out when this u05 was created?

[ErmanArslansOracleBlog]

Check with stat command ->

stat /u05

[satish]

[root@erpprod2 /]# stat /u05
  File: â/u05â
  Size: 6               Blocks: 0          IO Block: 4096   directory
Device: fd00h/64768d    Inode: 67661379    Links: 2
Access: (0755/drwxr-xr-x)  Uid: (    0/    root)   Gid: (    0/    root)
Access: 2018-06-25 15:01:37.394060898 +0530
Modify: 2018-06-25 15:01:30.084800292 +0530
Change: 2018-06-25 15:01:30.084800292 +0530
 Birth: -
[root@erpproddb2 /]

[ErmanArslansOracleBlog]

stat command can display the creation time as "Birth".
but you have a "-" there..
It changes according to the filesystem and in your case, in your filesystem it is not stored.
so there is no way to know that..

[satish]

Thanks for the information erman and for the great support