To copy the directory tree from filesystem1 to filesystem2 use a combination of the find and cpio commands.
# find . -print -depth | cpio -pdmv /var/test/
. starts in the current working directory.
– print Prints the file names.
– depth Descends the directory hierarchy and prints file names from the bottom up.
-p Creates a list of files.
-d Creates directories as needed.
-m Sets the correct modification times on directories.
Find the files older that 15 days and save in a list
find /data/files -type f -mtime +15 > /tmp/list
Review the list to ensure you’re happy to delete the files
for i in `cat /tmp/list `; do ls -l $i ; done | more
Delete all files in the list above
for i in `cat /tmp/list `; do rm -f $i ; done
=(B54+(“1/1/1970 1:00 AM”-“01/01/1900 12:00 AM”+1)*86400)/86400
where B54 is the cell with the epoch data
Here is the output from strace for an echo of Hello World .
The actual output is in black and my description (interpretation ) of the system call is in red italics above the output.
system call (ARGUMENT ) = RETURN CODE
[root@server1 ~]# strace -f echo “Hello world”
Execute (program,argument list, its environment list) = success
execve("/usr/bin/echo", ["echo", "Hello world"], [/* 25 vars */]) = 0
Determine the location of the pointer in the process memory address space
brk(0) = 0x1f73000
Allocate 4kB of memory at the location of the kernel‘s choice ( NULL) ,
allow reads and writes, anonymous memory , not backed by a file fd =-1 ,
no offset from start address = the address of the start of the mapped memory
mmap(NULL,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0)=0x7ff9d54ea000
Checks if the file exists with read access = FAIL
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
Open the file as read only and assign FD as 3
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
The above file fd=3 is a regular file with permissions 0644
and 85.8kbytes in size
fstat(3, {st_mode=S_IFREG|0644, st_size=85800, ...}) = 0
Allocate 85.8Kbytes of memory for FD3 at any location ( NULL) ,
allow reads and writes , private process memory with no
offset from the start address = start of mapped mem
mmap(NULL, 85800, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff9d54d5000
Close the FD and return integer 3 to the OS
close(3) = 0
Open this new file as read only and assign FD as 3
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
Read 832 bytes from FD 3 and store at the buffer below = 322 bytes read
read(3, "\177ELF\2\1\1\3\3>\1@\30\2G<"..., 832) = 832
FD 3 is a regular file with permissions 0755 and 2MB in size
fstat(3, {st_mode=S_IFREG|0755, st_size=2065552, ...}) = 0
Allocate 3.9MB of memory for FD 3 at location 0x3c47000000,
ballow reads and execute, private process memory with no
offset from the start address = start of mapped mem
mmap(0x3c47000000,3892376,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_DENYWRITE,3,0)=0x3c47000000
Set no protection (all access to rwx ) on 2MB or memory starting at 0x3c471ac000,
mprotect(0x3c471ac000, 2097152, PROT_NONE) = 0
Allocate 24.5kB of memory for FD 3 at location 0x3c473ac000, allow reads
and write , private process memory with an offset of 0x1ac000 from the
start address = start of mapped memory
mmap(0x3c473ac000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE,3,0x1ac000)=0x3c473ac000
Allocate 17.5kB of memory at location 0x3c473b2000, allow reads and write,
private process memory , anonymous memory , not backed by a file fd =-1
with no offset from the start address = start of mapped memory
mmap(0x3c473b2000,17560,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS,-1,0)=0x3c473b2000
Close the FD and return it integer 3 to the OS
close(3) = 0b
Allocate 4096 bytes at the location of the kernel‘s choice (NULL) , allow read
and writes, anonymous private memory, not backed by a file fd =-1 , no offset
from start address = the address of the start of the mapped memory
mmap(NULL,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0)=0x7ff9d54d4000
Allocate 8292 bytes at the location of the kernel‘s choice (NULL) , allow read
and writes,anonymous private memory , not backed by a file fd =-1 , no offset
from start address = the address of the start of the mapped memory
mmap(NULL,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0)=0x7ff9d54d2000
Set the 64-bit base for the FS register at 0x7ff9d54d2740
arch_prctl(ARCH_SET_FS, 0x7ff9d54d2740) = 0
Set read only protection on 4kB off memory starting at 0x605000
mprotect(0x605000, 4096, PROT_READ) = 0
Set read only protection on 16kB of memory starting at 0x3c473ac000
mprotect(0x3c473ac000, 16384, PROT_READ) = 0
Set read only protection on 4kB of memory starting at 0x3c46e1f000
mprotect(0x3c46e1f000, 4096, PROT_READ) = 0
Unmap or release 85.8kB of memory starting at 0x7ff9d54d5000
munmap(0x7ff9d54d5000, 85800) = 0
Determine the current location of the pointer to the memory
allocated to the data segment of the process
brk(0) = 0x1f73000
Set the end of memory allocated to the data segment of the process
to 0x1f94000
brk(0x1f94000) = 0x1f94000
Determine the location of the pointer in the process memory address space
brk(0) = 0x1f94000
Open this new file as read only and assign FD as 3
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
FD 3 is a regular file with permissions 0644 and 105MB in size
fstat(3, {st_mode=S_IFREG|0644, st_size=105038240, ...}) = 0
Allocate 105MB of memory for FD3 at any location ( NULL) ,
allow read , private process memory with no offset from the start
address = start of mapped mem
mmap(NULL, 105038240, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff9cf0a5000
Close the FD and return integer 3 to the OS
close(3) = 0
FD1 ie std output is a character device with permission 620 with device id
( inode) 136,0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
Allocate 4kB of memory at any location ( NULL) , allow read and write
private anonymous process memory , not backed by a file fd =-1,
with no offset from the start address = start of mapped mem
mmap(NULL,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0)=0x7ff9d54e9000
Write the 12 byte phrase “hello world “ to std output = 12 bytes written
write(1, "Hello world\n", 12Hello world) = 12
Close the std output FD and return it integer 1 to the OS
close(1) = 0
Unmap or release 4kB for memory starting from address 0x7ff9d54e9000
munmap(0x7ff9d54e9000, 4096) = 0
Close the std error FD and return it integer 2 to the OS
close(2) = 0
Exit all threads in a process
exit_group(0) = ?
+++ exited with 0 +++
Strace is a tool that traces the execution of system calls. System calls are the translation mechanism that provides interface between a process and the operating system (kernel). The operating system functionality that allows tracing is called ptrace. Strace calls on ptrace and reads the process behavior, reporting back. Here are my most used strace commands
strace -s 512 -f -t -T -o my_output <command|script> strace -s 512 -f -t -T -o my_output -p <PID> strace -s 512 -f -c <command|script> strace -s 512 -f -C -p <PID> strace -s 512 -f -e open <command|script>
-s 512 increases the output string length to 512 bytes; the default is 32
-f trace child process that may be created for the fork() system call . If -ff is used with
-o filename option is in effect, each processes trace is written to filename.pid
-t add the time stamp at the beginging of each line. Use -tt for microseconds
-T the time spent in the system call ie end time – start time
-c Summary table of all system calls used by the process, including errors.
-C Same a -c plus but also includes the normal strace output
-p attach to a PID and begin tracing
-o write the output to a file rather than the screen
-e to trace only the specified set of system calls. The -c option can be used to identify relevant calls
-e open,close,read,write :trace these four system calls
-e process Trace all system calls which involve process management.
-e network Trace all the network related system calls.
-e signal Trace all signal related system calls.
-e ipc Trace all IPC related system calls.
-e desc Trace all file descriptor related system calls.
System calls are the translation mechanism that provides interface between a process and the operating system (kernel). Linux currently provides about 200 different system calls. A listing of system calls is in /usr/include/asm/unistd.h. System calls enable the transition from userspace to kernel space. System calls are used by program to access process, files, I/O, threads, network sockets and IPCs. When a program makes a system call, the arguments are packaged up and handed to the kernel, which takes over execution of the program until the call completes.
open: To open or create a file and return a file descriptor (f)
access: It determines whether the calling process has access permission to a file. It can check any combination of read, write, and execute permission, and it can also check for a file’s existence.
read: To copy bytes from a file to memory. Reads are done from disk to OS buffer then to process memory
f= open(‘/tmp/file’)
data = read(f)
close(f)
write: To copy bytes from memory to a file. Writes are done from the process to OS buffer to disk. It produces output, FD 1 corresponds to standard output. The third argument is the number of characters to write,
f= open(‘/tmp/file’)
write(f,’data1′,13)
write (f,’data2′,15)
close(f)
An strace of the write call will return the value of the number of characters that were actually written. The system call’s name is listed, followed by its arguments and its return value.
write(1, “myhostname\n”, 13) = 13
sendfile: To provide an efficient mechanism for copying data from one file descriptor to another. The file descriptors may be open to disk files, sockets, or other devices. Call sendfile, passing the file descriptor to write to; the descriptor to read from; a pointer to an offset variable; and the number of bytes to transfer. The offset variable contains the offset in the input file from which the read should start (0 indicates the beginning of the file) and is updated to the position in the file after the transfer. The return value is the number of bytes transferred.
mmap: To allocate pages of memory to a process, these pages are mapped to actual address in physical memory. You should specify how many bytes of memory is required
munmap: The opposite mmap and you specify the address of the first byte to be returned
close:– To release a file descriptor
For other calls – man 2 <system call>
A file descriptor is an object that a process uses to read or write to an open file and open network sockets (although there are other uses).
It’s a num to uniquely identify an open file
|
Integer value |
Name |
| 0 | Standard input (stdin) |
| 1 | Standard output (stdout) |
| 2 | Standard error (stderr) |
Beyond the standard file descriptors there are 3-1024. These can be created in scripts with:
exec 10<> my_file
From this point on, anything written to file descriptor 10 gets written to my_file
Operating Systems place limits on the number of file descriptors that a process may open. In addition to per-process limits, an OS will also have a global limit on the amount of file descriptors that all its processes together may consume.
Global FD Limits
The aximum number of open file descriptors allowed:
cat /proc/sys/fs/file-max sysctl fs.file-max
User Level FD Limits
To limit httpd (or any other users) user to specific limits by editing /etc/security/limits.conf
# vi /etc/security/limits.conf
Number Of Open File Descriptors
cat /proc/sys/fs/file-nr 1500 210 65535 where: 1500 is total alocated file descriptors 210 is total of free alocated file descriptors 65535 is the total limit of the system
A port is a number between 1 and 65536 that signifies a logical gate in a device. Every connection between a client and server requires a unique socket. TCP connection is defined by two endpoints aka sockets.
A socket consists of three things:
Sockets come in two primary flavors. An active socket is connected to a remote active socket via an open data connection. Closing the connection destroys the active sockets at each endpoint. A passive socket is not connected, but rather awaits an incoming connection, which will spawn a new active socket. Each port can have a single passive socket, awaiting incoming connections, and multiple active sockets, each corresponding to an open connection on the port
This program will consume all the server’s Virtual Memory. The sleep(1) near the end of the script slows the process you can remove this line for a faster result. The OOM killer should kill this program once you are out of RAM and swap. I’d avoid running this on a production box
Enable maximium swappiness
echo 100 > /proc/sys/vm/swappiness
In a different window monitor memory and swap usage
watch "free"
Dowload the file ram_eater
Rename the file and verify there are no control characters ( I use dos2unix).
mv ram_eater.doc ram_eater.c
Compile with gcc
gcc ram_eater.c -o my_ram_eater_appRun the newly created program
./my_ram_eater_app
In your 2nd window the memory and swap “used” should increase
To list each process and it’s swap usage :
[root@server1 tmp]# for file in /proc/*/status ;
do awk '/VmSwap|Name/{printf $2 " " $3}END{ print ""}' $file;
done | sort -k 2 -n -r | head
Xorg 36460 kB
mysqld 22948 kB
gnome-shell 20468 kB
gnome-session 4288 kB
httpd 3980 kB
grep Swap /proc/<PID>/smaps
If you’d like a pretty output use the following script my_swap_usage
[root@server1 tmp]# ./my_swap_usage ........................................ Overall swap used: 194232 kB ======================================== kB pid name ======================================== 36292 623 Xorg 22944 1360 mysqld 20364 1654 gnome-shell 4180 1433 gnome-session 3980 1327 httpd
To test remote ports
nmap -p 80 192.1.2.15
telnet 192.1.2.15 80
To view local ports
netstat -lnp | grep :80
To view local ports
lsof -i :25
Linux provides tools that allow you to probe CPU and motherboard temperatures, and, in some cases, the temperatures of PCI devices and even fan speeds. All of this support is provided by the lm_sensors package. Once the lm-sensors package is installed, run the sensors-detect program
# sensors-detect
This interactive script will probe the hardware on your system so it knows how to query for temperature. If you don’t know how to respond to some of the questions it asks, just hit Enter to accept the default. Once the sensors-detect script is completed, you can pull data about your server by running the command sensors:
# sensors nouveau-pci-0100 Adapter: PCI adapter temp1: +56.0°C (high = +100.0°C, crit = +120.0°C) coretemp-isa-0000 Adapter: ISA adapter Core 0: +36.0°C (high = +74.0°C, crit = +100.0°C) Core 1: +36.0°C (high = +74.0°C, crit = +100.0°C) Core 2: +36.0°C (high = +74.0°C, crit = +100.0°C) Core 3: +36.0°C (high = +74.0°C, crit = +100.0°C)
Once you have identified the load is high and that your web server processes are the issue, if the load is CPU-bound, then you will likely need to troubleshoot any CGIs, PHP code, and so on, that your web server executes to generate dynamic content
If the load seems RAM-bound and you notice you are using more and more swap storage and may even completely run out of RAM, then you may be facing the dreaded web server swap death spiral. When you configure your web server, you can configure the maximum number of web server instances the server will spawn in response to traffic. In Apache prefork, this is known as the MaxClient setting. When a server gets so much traffic that it spawns more web server processes than can fit in RAM, processes end up using the much slower swap space instead. This causes those processes to respond much more slowly than processes residing in RAM, which causes the requests to take longer, which in turn causes more processes to be needed to handle the current load until, ultimately, both RAM and swap are consumed.
If the load is I/O bound, and the web server has a database back-end on the same machine, you might imply be saturating your disk I/O with database requests.. Even if the database server is on a separate machine, each web server process that is waiting on a response from the database over the network may still generate a high load average.
Understanding access logs
# cat /var/log/httpd/access_log
192.168.1.62 - - [30/Dec/2012:20:51:01 +0000] "GET / HTTP/1.1" 200 28 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11"
First, the request came from the IP 192.168.1.62. Second, we can identify a date stamp for the request. After that, we can see the exact HTTP request the server processed (GET / HTTP/1.1) and the following 200 in the log corresponds to the HTTP return code. Finally, at the end of the log is the User-Agent string the client passed to the server; in this case, it identifies the client as chrome
.
Summarizing access logs
# perl -e 'while(<>){ if( m|(^\d+\.\d+\.\d+\.\d+).*?30/Dec/2012| ){ $v{$1}++; } }
foreach( keys %v ){ print "$v{$_}\t$_\n"; }' /var/log/httpd/access_log | sort -n
12 192.168.1.65
5 192.168.1.62
2 192.168.1.69
It pulls out the IP and a specifi c date from the logs and keeps a tally of each IP it finds. It outputs the complete list of IPs and their tally:
.
telnet server1 80
Trying 127.0.0.1…
Connected to server1.
Escape character is ‘^]’.
GET / HTTP/1.1
host:
HTTP/1.1 200 OK
Date: Sun, 30 Dec 2012 19:39:55 GMT
Server: Apache/2.2.22 (Fedora)
Last-Modified: Tue, 02 Oct 2012 13:03:23 GMT
ETag: “a042f-1c-4cb1325aaecbe”
Accept-Ranges: bytes
Content-Length: 28
Connection: close
Content-Type: text/html; charset=UTF-8
Apache default hello world
Connection closed by foreign host.
.
telnet ibm.com 80
Trying 129.42.38.1…
Connected to ibm.com.
Escape character is ‘^]’.
HEAD / HTTP/1.0
HTTP/1.1 301 Moved Permanently
Date: Sun, 30 Dec 2012 20:15:08 GMT
Server: IBM_HTTP_Server
Content-Type: text/html
Location: http://www.ibm.com/
epKe-Alive: timeout=10, max=88
Connection: Keep-Alive
Connection closed by foreign host.
Bird's Blog 