23 very useful shell scripts

shell Scripting is a powerful tool to help programmers and system administrators complete the tedious work that takes time and effort , It is an effective way to interact with computers and manage files and system operations . Just a few lines of code , You can get the computer close to doing what you want .

I have arranged for you 23 An example , adopt 23 A real classic script example , It shows shell Script programming practical technology and common tool usage . You just need to be on your own , These common tasks and portable automation scripts are extended to other similar problems , It can solve the troubles that happen in two or three days .

Check the consistency of files in the specified directory of two servers

#!/bin/bash
#####################################
# Check the consistency of files in the specified directory of two servers 
#####################################
# By comparing the files on the two servers md5 value , Achieve the purpose of testing consistency 
dir=/data/web
b_ip=192.168.88.10
# Traverse all the files in the specified directory as md5sum Arguments to the command , And then get all the files md5 value , And write to the specified file 
find $dir -type f|xargs md5sum > /tmp/md5_a.txt
ssh $b_ip "find $dir -type f|xargs md5sum > /tmp/md5_b.txt"
scp $b_ip:/tmp/md5_b.txt /tmp
# Compare the file names one by one as traversal objects 
for f in `awk '{print 2} /tmp/md5_a.txt'`
do
# With a The machine is the standard , When b When the machine does not exist, it directly outputs the result that does not exist when traversing the file in the object 
if grep -qw "$f" /tmp/md5_b.txt
then
md5_a=`grep -w "$f" /tmp/md5_a.txt|awk '{print 1}'`
md5_b=`grep -w "$f" /tmp/md5_b.txt|awk '{print 1}'`
# When a file exists , If md5 If the value is inconsistent, the result of the file change will be output 
if [ $md5_a != $md5_b ]
then
echo "$f changed."
fi
else
echo "$f deleted."
fi
done

Clear the contents of the file regularly , Record the file size regularly

#!/bin/bash
################################################################
# Execute scripts every hour （ Task plan ）, When the time is 0 Point or 12 a.m. , Put all files in the target directory 
# Rong qingkong , But do not delete files , At other times, only the size of each file is counted , One file, one line , When output to # Between and date named in the file , We need to consider the second level of the target directory 、 Third level subdirectory files 
################################################################
logfile=/tmp/`date +%H-%F`.log
n=`date +%H`
if [ $n -eq 00 ] || [ $n -eq 12 ]
then
# adopt for loop , With find Command as traversal condition , Traverse all the files in the target directory and do the corresponding operation 
for i in `find /data/log/ -type f`
do
true > $i
done
else
for i in `find /data/log/ -type f`
do
du -sh $i >> $logfile
done
fi

Check network card traffic , And record in the log according to the prescribed format

#!/bin/bash
#######################################################
# Check network card traffic , And record in the log according to the prescribed format 
# Make it a minute 
# The log format is as follows :
#2019-08-12 20:40
#ens33 input: 1234bps
#ens33 output: 1235bps
######################################################3
while :
do
# Set the language to English , Ensure that the output is in English , Otherwise bug
LANG=en
logfile=/tmp/`date +%d`.log
# Redirect the output of the following command to logfile In the log 
exec >> $logfile
date +"%F %H:%M"
#sar The traffic unit of command statistics is kb/s, The format of the log is bps, So we have to *1000*8
sar -n DEV 1 59|grep Average|grep ens33|awk '{print $2,"\t","input:","\t",$5*1000*8,"bps","\n",$2,"\t","output:","\t",$6*1000*8,"bps"}'
echo "####################"
# Because execution sar Orders require 59 second , So you don't need sleep
done

Count the number of digits per line of the document , And calculate the total number of the whole document

#!/bin/bash
#########################################################
# Count the number of digits per line of the document , And calculate the total number of the whole document 
########################################################
# Use awk Only output the number of document lines （ Intercept the first paragraph ）
n=`wc -l a.txt|awk '{print $1}'`
sum=0
# There may be spaces on every line in the document , So you can't directly traverse with the content of the document 
for i in `seq 1 $n`
do
# When the output line is represented by a variable , Need double quotes 
line=`sed -n "$i"p a.txt`
#wc -L Options , Count the length of the longest line 
n_n=`echo $line|sed s'/[^0-9]//'g|wc -L`
echo $n_n
sum=$[$sum+$n_n]
done
echo "sum:$sum"

Kill all scripts

#!/bin/bash
################################################################
# There are some scripts added to cron In , There are situations where the script has not finished running yet and new tasks need to be executed ,
# Causes the system load to rise , So you can write a script , Select the processes that affect the load and kill them all at once .
################################################################
ps aux|grep  Specify the process name |grep -v grep|awk '{print $2}'|xargs kill -9

from FTP Server downloads files

#!/bin/bash
if [ $# -ne 1 ]; then
    echo "Usage: $0 filename"
fi
dir=$(dirname $1)
file=$(basename $1)
ftp -n -v << EOF   # -n  automatic logon 
open 192.168.1.10  # ftp The server 
user admin password
binary   #  Set up ftp The transmission mode is binary , avoid MD5 Value is different or .tar.gz Compressed package format error 
cd $dir
get "$file"
EOF

Continuous input 5 individual 100 Numbers within , Statistics and 、 Minimum and maximum

#!/bin/bash
COUNT=1
SUM=0
MIN=0
MAX=100
while [ $COUNT -le 5 ]; do
    read -p " Please enter 1-10 It's an integer ：" INT
    if [[ ! $INT =~ ^[0-9]+$ ]]; then
        echo " The input must be an integer ！"
        exit 1
    elif [[ $INT -gt 100 ]]; then
        echo " The input must be 100 within ！"
        exit 1
    fi
    SUM=$(($SUM+$INT))
    [ $MIN -lt $INT ] && MIN=$INT
    [ $MAX -gt $INT ] && MAX=$INT
    let COUNT++
done
echo "SUM: $SUM"
echo "MIN: $MIN"
echo "MAX: $MAX"

Users guess numbers

#!/bin/bash  #  The script generates a  100  Random number within , Prompt users to guess numbers , According to the user's input , Prompt the user to guess correctly ,#  Guess small or guess big , Until the end of the script .# RANDOM  The system variable that comes with the system , The value is  0‐32767 The random number #  Use the remainder algorithm to change the random number to  1‐100  The random number num=$[RANDOM%100+1]echo "$num" #  Use  read  Prompt users to guess numbers #  Use  if  Judge the size of the number the user guesses :‐eq( be equal to ),‐ne( It's not equal to ),‐gt( Greater than ),‐ge( Greater than or equal to ),# ‐lt( Less than ),‐le( Less than or equal to )while :do     read -p " The computer generated a  1‐100  The random number , Have a guess : " cai    if [ $cai -eq $num ]    then        echo " Congratulations , Guessed it "           exit        elif [ $cai -gt $num ]        then            echo "Oops, Guess the "         else            echo "Oops, Guess a little "     fidone

monitoring Nginx Access log 502 situation , And do the corresponding action

Suppose the server environment is lnmp, Recent visits often appear 502 The phenomenon , And 502 Error in restart php-fpm Disappear after service , So you need to write monitoring scripts , Once it appears 502, It will restart automatically php-fpm service .

# scene ：
#1. The path to access the log file ：/data/log/access.log
#2. Script loop , Every time 10 Check once per second ,10 The number of logs per second is 300 strip , appear 502 Not less than 10%（30 strip ） You need to restart php-fpm service 
#3. The restart command is ：/etc/init.d/php-fpm restart
#!/bin/bash
###########################################################
# monitoring Nginx Access log 502 situation , And do the corresponding action 
###########################################################
log=/data/log/access.log
N=30 # Set threshold 
while :
do
 # Check the access log for the latest 300 strip , And statistics 502 The number of times 
    err=`tail -n 300 $log |grep -c '502" '`
 if [ $err -ge $N ]
 then
 /etc/init.d/php-fpm restart 2> /dev/null
 # Set up 60s Delay prevention scripts bug Cause infinite restart php-fpm service 
     sleep 60
 fi
 sleep 10
done

Assign the results to variables

 Application scenarios ： You want to assign execution results or location parameters to variables , For later use .

 Method 1：

for i in $(echo "4 5 6"); do
   eval a$i=$i
done
echo $a4 $a5 $a6
 Method 2： Set the position parameter 192.168.1.1{1,2} Split into variables 

num=0
for i in $(eval echo $*);do   #eval take {1,2} Decompose into 1 2
   let num+=1
   eval node${num}="$i"
done
echo $node1 $node2 $node3
# bash a.sh 192.168.1.1{1,2}
192.168.1.11 192.168.1.12
 Method 3：

arr=(4 5 6)
INDEX1=$(echo ${arr[0]})
INDEX2=$(echo ${arr[1]})
INDEX3=$(echo ${arr[2]})

Batch modify file name

 Example ：

# touch article_{1..3}.html
# ls
article_1.html  article_2.html  article_3.html
 Purpose ： hold article Change it to bbs

 Method 1：

for file in $(ls *html); do
    mv $file bbs_${file#*_}
    # mv $file $(echo $file |sed -r 's/.*(_.*)/bbs\1/')
    # mv $file $(echo $file |echo bbs_$(cut -d_ -f2)
done
 Method 2：

for file in $(find . -maxdepth 1 -name "*html"); do
     mv $file bbs_${file#*_}
done
 Method 3：

# rename article bbs *.html

Delete the first five lines of a document that contain letters , At the same time to delete 6 To 10 All the letters contained in the row

1） Prepare test files , The file named 2.txt

 The first 1 That's ok 1234567 No letters 
 The first 2 That's ok 56789BBBBBB
 The first 3 That's ok 67890CCCCCCCC
 The first 4 That's ok 78asdfDDDDDDDDD
 The first 5 That's ok 123456EEEEEEEE
 The first 6 That's ok 1234567ASDF
 The first 7 That's ok 56789ASDF
 The first 8 That's ok 67890ASDF
 The first 9 That's ok 78asdfADSF
 The first 10 That's ok 123456AAAA
 The first 11 That's ok 67890ASDF
 The first 12 That's ok 78asdfADSF
 The first 13 That's ok 123456AAAA

2） The script is as follows ：

#!/bin/bash
##############################################################
# Delete the first five lines of a document that contain letters , At the same time to delete 6 To 10 All the letters contained in the row 
##############################################################
sed -n '1,5'p 2.txt |sed '/[a-zA-Z]/'d
sed -n '6,10'p 2.txt |sed s'/[a-zA-Z]//'g
sed -n '11,$'p 2.txt
# The end result is just a print on the screen , If you want to change the file directly , The output can be written to a temporary file , Replace with 2.txt Or use -i Options 

Count the current directory with .html Total size of files at the end

 Method 1：

# find . -name "*.html" -exec du -k {} \; |awk '{sum+=$1}END{print sum}'

 Method 2：

for size in $(ls -l *.html |awk '{print $5}'); do
    sum=$(($sum+$size))
done
echo $sum

Scan host port status

#!/bin/bash
HOST=$1
PORT="22 25 80 8080"
for PORT in $PORT; do
    if echo &>/dev/null > /dev/tcp/$HOST/$PORT; then
        echo "$PORT open"
    else
        echo "$PORT close"
    fi
done

use shell The number of letters in the print sample statement is less than 6 's words

# Sample statements ：
#Bash also interprets a number of multi-character options.
#!/bin/bash
##############################################################
#shell The number of letters in the print sample statement is less than 6 's words 
##############################################################
for s in Bash also interprets a number of multi-character options.
do
 n=`echo $s|wc -c`
 if [ $n -lt 6 ]
 then
 echo $s
 fi
done

Enter the number and run the corresponding command

#!/bin/bash
##############################################################
# Enter the number and run the corresponding command 
##############################################################
echo "*cmd menu* 1-date 2-ls 3-who 4-pwd 0-exit "
while :
do
# Capture the value the user typed 
 read -p "please input number :" n
 n1=`echo $n|sed s'/[0-9]//'g`
# Null input detection  
 if [ -z "$n" ]
 then
 continue
 fi
# Non digital input detection  
 if [ -n "$n1" ]
 then
 exit 0
 fi
 break
done
case $n in
 1)
 date
 ;;
 2)
 ls
 ;;
 3)
 who
 ;;
 4)
 pwd
 ;;
 0)
 break
 ;;
    # Enter the number not 1-4 A hint of 
 *)
 echo "please input number is [1-4]"
esac

Expect Realization SSH No interactive command execution

Expect Is a tool for automatic interactive applications , Such as telnet,ftp,passwd etc. .

 It needs to be installed first expect software package .

 Method 1：EOF Standard output as expect The standard input 

#!/bin/bash
USER=root
PASS=123.com
IP=192.168.1.120
expect << EOF
set timeout 30
spawn ssh [email protected]$IP   
expect {
    "(yes/no)" {send "yes\r"; exp_continue}
    "password:" {send "$PASS\r"}
}
expect "[email protected]*"  {send "$1\r"}
expect "[email protected]*"  {send "exit\r"}
expect eof
EOF
 Method 2：

#!/bin/bash
USER=root
PASS=123.com
IP=192.168.1.120
expect -c "
    spawn ssh [email protected]$IP
    expect {
        \"(yes/no)\" {send \"yes\r\"; exp_continue}
        \"password:\" {send \"$PASS\r\"; exp_continue}
        \"[email protected]*\" {send \"df -h\r exit\r\"; exp_continue}
    }"
 Method 3： take expect The script is independent 

 Login script ：

# cat login.exp
#!/usr/bin/expect
set ip [lindex $argv 0]
set user [lindex $argv 1]
set passwd [lindex $argv 2]
set cmd [lindex $argv 3]
if { $argc != 4 } {
puts "Usage: expect login.exp ip user passwd"
exit 1
}
set timeout 30
spawn ssh [email protected]$ip
expect {
    "(yes/no)" {send "yes\r"; exp_continue}
    "password:" {send "$passwd\r"}
}
expect "[email protected]*"  {send "$cmd\r"}
expect "[email protected]*"  {send "exit\r"}
expect eof
 Execute command script ： Write a loop to operate multiple servers in batch 

#!/bin/bash
HOST_INFO=user_info.txt
for ip in $(awk '{print $1}' $HOST_INFO)
do
    user=$(awk -v I="$ip" 'I==$1{print $2}' $HOST_INFO)
    pass=$(awk -v I="$ip" 'I==$1{print $3}' $HOST_INFO)
    expect login.exp $ip $user $pass $1
done
Linux host SSH Connection information ：

# cat user_info.txt
192.168.1.120 root 123456

establish 10 Users , And set the password separately , Password requirements 10 Bit and contains uppercase and lowercase letters and numbers , Finally, the password of each user needs to be stored in the specified file

#!/bin/bash
##############################################################
# establish 10 Users , And set the password separately , Password requirements 10 Bit and contains uppercase and lowercase letters and numbers 
# Finally, the password of each user needs to be stored in the specified file 
# Prerequisite ： install mkpasswd command 
##############################################################
# Generate 10 A sequence of users （00-09）
for u in `seq -w 0 09`
do
 # Create user 
 useradd user_$u
 # Generate the password 
 p=`mkpasswd -s 0 -l 10`
 # Read the password from the standard input and modify it （ unsafe ）
 echo $p|passwd --stdin user_$u
 # General password modification 
 echo -e "$p\n$p"|passwd user_$u
 # Record the created user and the corresponding password in the log file 
 echo "user_$u $p" >> /tmp/userpassword
done

monitor httpd The number of processes , Deal with it according to the monitoring situation

#!/bin/bash
###############################################################################################################################
# demand ：
#1. every other 10s monitor httpd The number of processes , If the number of processes is greater than or equal to 500, It will restart automatically Apache service , And check whether the service is restarted successfully 
#2. If not, it needs to be started again , If restart 5 The first time is still unsuccessful , Send an alert email to the administrator , And exit the test 
#3. If the startup is successful , Is waiting for 1 Check again in minutes httpd Number of processes , If the number of processes is normal , Normal detection is restored （10s once ）, Otherwise, give up the restart and send an alert email to the administrator , And exit the test 
###############################################################################################################################
# Counter functions 
check_service()
{
 j=0
 for i in `seq 1 5` 
 do
 # restart Apache The order of 
 /usr/local/apache2/bin/apachectl restart 2> /var/log/httpderr.log
    # Judge whether the service is restarted successfully 
 if [ $? -eq 0 ]
 then
 break
 else
 j=$[$j+1]
 fi
    # Determine if the service has attempted to restart 5 Time 
 if [ $j -eq 5 ]
 then
 mail.py
 exit
 fi
 done 
}
while :
do
 n=`pgrep -l httpd|wc -l`
 # Judge httpd Whether the number of service processes exceeds 500
 if [ $n -gt 500 ]
 then
 /usr/local/apache2/bin/apachectl restart
 if [ $? -ne 0 ]
 then
 check_service
 else
 sleep 60
 n2=`pgrep -l httpd|wc -l`
 # Judge whether it still exceeds 500
             if [ $n2 -gt 500 ]
 then 
 mail.py
 exit
 fi
 fi
 fi
 # every other 10s Test once 
 sleep 10
done

Batch change server user password

Linux host SSH Connection information ： Old password 

# cat old_pass.txt 
192.168.18.217  root    123456     22
192.168.18.218  root    123456     22
 Content format ：IP User Password Port

SSH Change password script remotely ： New passwords are randomly generated 
https://www.linuxprobe.com/books
#!/bin/bash
OLD_INFO=old_pass.txt
NEW_INFO=new_pass.txt
for IP in $(awk '/^[^#]/{print $1}' $OLD_INFO); do
    USER=$(awk -v I=$IP 'I==$1{print $2}' $OLD_INFO)
    PASS=$(awk -v I=$IP 'I==$1{print $3}' $OLD_INFO)
    PORT=$(awk -v I=$IP 'I==$1{print $4}' $OLD_INFO)
    NEW_PASS=$(mkpasswd -l 8)  #  Random cipher 
    echo "$IP   $USER   $NEW_PASS   $PORT" >> $NEW_INFO
    expect -c "
    spawn ssh -p$PORT [email protected]$IP
    set timeout 2
    expect {
        \"(yes/no)\" {send \"yes\r\";exp_continue}
        \"password:\" {send \"$PASS\r\";exp_continue}
        \"[email protected]*\" {send \"echo \'$NEW_PASS\' |passwd --stdin $USER\r exit\r\";exp_continue}
    }"
done
 Generate new password file ：

# cat new_pass.txt 
192.168.18.217  root    n8wX3mU%      22
192.168.18.218  root    c87;ZnnL      22

iptables Automatically block frequent visits to websites IP

 scene ： Malicious access , Safety precautions 

1） Block access more than... Per minute 200 Of IP

 Method 1： According to the access log （Nginx For example ）

#!/bin/bash
DATE=$(date +%d/%b/%Y:%H:%M)
ABNORMAL_IP=$(tail -n5000 access.log |grep $DATE |awk '{a[$1]++}END{for(i in a)if(a[i]>100)print i}')
# First tail Prevent files from being too large , Slow reading , The number can adjust the maximum number of visits per minute .awk Can't filter logs directly , Because it contains special characters .
for IP in $ABNORMAL_IP; do
    if [ $(iptables -vnL |grep -c "$IP") -eq 0 ]; then
        iptables -I INPUT -s $IP -j DROP
    fi
done
 Method 2： adopt TCP Established connection 

#!/bin/bash
ABNORMAL_IP=$(netstat -an |awk '$4~/:80$/ && $6~/ESTABLISHED/{gsub(/:[0-9]+/,"",$5);{a[$5]++}}END{for(i in a)if(a[i]>100)print i}')
#gsub It's the fifth column （ client IP） Remove the colon and port of 
for IP in $ABNORMAL_IP; do
    if [ $(iptables -vnL |grep -c "$IP") -eq 0 ]; then
        iptables -I INPUT -s $IP -j DROP
    fi
done

2） Shielding per minute SSH Try to log in more than 10 Time of IP

 Method 1： adopt lastb Get login status :

#!/bin/bash
DATE=$(date +"%a %b %e %H:%M") # It's Sunday, month and Sunday   %e Single digit display 7, and %d Show 07
ABNORMAL_IP=$(lastb |grep "$DATE" |awk '{a[$3]++}END{for(i in a)if(a[i]>10)print i}')
for IP in $ABNORMAL_IP; do
    if [ $(iptables -vnL |grep -c "$IP") -eq 0 ]; then
        iptables -I INPUT -s $IP -j DROP
    fi
done
 Method 2： Get login status through log 

#!/bin/bash
DATE=$(date +"%b %d %H")
ABNORMAL_IP="$(tail -n10000 /var/log/auth.log |grep "$DATE" |awk '/Failed/{a[$(NF-3)]++}END{for(i in a)if(a[i]>5)print i}')"
for IP in $ABNORMAL_IP; do
    if [ $(iptables -vnL |grep -c "$IP") -eq 0 ]; then
        iptables -A INPUT -s $IP -j DROP
        echo "$(date +"%F %T") - iptables -A INPUT -s $IP -j DROP" >>~/ssh-login-limit.log
    fi
done

according to web Access log , The blocking request amount is abnormal IP, Such as IP It's back to normal in half an hour , Then lift the ban

#!/bin/bash
####################################################################################
# according to web Access log , The blocking request amount is abnormal IP, Such as IP It's back to normal in half an hour , Then lift the ban 
####################################################################################
logfile=/data/log/access.log
# Show the hours and minutes of a minute ago 
d1=`date -d "-1 minute" +%H%M`
d2=`date +%M`
ipt=/sbin/iptables
ips=/tmp/ips.txt
block()
{
 # Filter and extract all the logs one minute ago IP And count the number of visits 
 grep '$d1:' $logfile|awk '{print $1}'|sort -n|uniq -c|sort -n > $ips
 # utilize for The number of cycles will exceed 100 Of IP Go through it one by one and ban it 
 for i in `awk '$1>100 {print $2}' $ips`
 do
 $ipt -I INPUT -p tcp --dport 80 -s $i -j REJECT
 echo "`date +%F-%T` $i" >> /tmp/badip.log
 done
}
unblock()
{
 # What will happen after the ban pkts Quantity less than 10 Of IP In turn to be unsealed 
 for a in `$ipt -nvL INPUT --line-numbers |grep '0.0.0.0/0'|awk '$2<10 {print $1}'|sort -nr`
 do 
 $ipt -D INPUT $a
 done
 $ipt -Z
}
# When time is 00 Points and 30 Time sharing execution of the unsealing function 
if [ $d2 -eq "00" ] || [ $d2 -eq "30" ]
 then
 # You have to untie it before you seal it , Because it just came into being when it was banned pkts Very few 
 unblock
 block
 else
 block
fi

Judge whether the user input is IP Address

 Method 1:

#!/bin/bash
function check_ip(){
    IP=$1
    VALID_CHECK=$(echo $IP|awk -F. '$1< =255&&$2<=255&&$3<=255&&$4<=255{print "yes"}')
    if echo $IP|grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$">/dev/null; then
        if [ $VALID_CHECK == "yes" ]; then
            echo "$IP available."
        else
            echo "$IP not available!"
        fi
    else
        echo "Format error!"
    fi
}
check_ip 192.168.1.1
check_ip 256.1.1.1
 Method 2：

#!/bin/bash
function check_ip(){
    IP=$1
    if [[ $IP =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
        FIELD1=$(echo $IP|cut -d. -f1)
        FIELD2=$(echo $IP|cut -d. -f2)
        FIELD3=$(echo $IP|cut -d. -f3)
        FIELD4=$(echo $IP|cut -d. -f4)
        if [ $FIELD1 -le 255 -a $FIELD2 -le 255 -a $FIELD3 -le 255 -a $FIELD4 -le 255 ]; then
            echo "$IP available."
        else
            echo "$IP not available!"
        fi
    else
        echo "Format error!"
    fi
}
check_ip 192.168.1.1
check_ip 256.1.1.1
 Added version ：

 Add a dead cycle , If IP Exit when available , Do not use the prompt to continue , And use awk Judge .

#!/bin/bash
function check_ip(){
    local IP=$1
    VALID_CHECK=$(echo $IP|awk -F. '$1< =255&&$2<=255&&$3<=255&&$4<=255{print "yes"}')
    if echo $IP|grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" >/dev/null; then
        if [ $VALID_CHECK == "yes" ]; then
            return 0
        else
            echo "$IP not available!"
            return 1
        fi
    else
        echo "Format error! Please input again."
        return 1
    fi
}
while true; do
    read -p "Please enter IP: " IP
    check_ip $IP
    [ $? -eq 0 ] && break || continue
done