Shell Script Syntax
Last Updated 2018-11-16
The first line of every script
#!/bin/bash
# "#!"" is referred to as the Shebang
# /bin/bash is the path to the interpreter
# This must be the first line of the file
# with no spaces in between
Comments
<<COMMENT1
This is one way to comment out multiple lines.
This is using the HERE document feature
However, variables will be substituted,
so it only works for pure text comments
COMMENT1
: '
This is another way to comment
out multiple lines. The space between
the colon and the quote is important.
However, variables will be substituted,
so it only works for pure text comments
'
PLACE AT THE START OF EVERY SCRIPT
set -u # stop the script if a variable is not assigned
set -e # stop the script if an error is encountered
Debugging
set -x # turn on debugging
set +x # turn off debugging
# Other sets
set +u # turn off set -u
set +e # turn off set -e
Functions
# Notes: 1. Must be defined before used
# 2. Two possible formats
# 3. Outputs can be used in command subsitutions
# 4. By default variables within a function have global scope
# 5. By default arrays declared within a function have local scope
# 6. The Unix "quietness" philosphy:
# A function should not print to the console unless it is
# necessary, either (1) to inform the user of a value (pwd),
# (2) to display the result of a computation (bc),
# or (3) to warn the user that an error has occurred.
# template_function: A template function to modify from
# $1 = TODO
# $2 (opt) = TODO
# Usage:
# template_function [arg1] [arg2 (opt)]
# Requires:
# TODO
# Used by:
# TODO
# History:
# 2018-XX-XX Created by TODO
function template_function {
# Check the number of arguments TODO: Might need to be changed
[[ $# -lt 1 ]] && echo "USAGE: $FUNCNAME [arg1] [arg2 (opt)]" && return 1
# Read in required arguments
local arg1=$1
# Read in optional arguments
local arg2
[[ $# -ge 2 ]] && arg2=$2 || arg2="$(whoami)"
local var=$@ # make a variable (global by default) local scope
declare -g arr # make an array (local by default) global scope
# Read in standard input
local stdin="$(cat)"
# TODO: Code here; can use $0, $1, $2, $#, "$@"
# and/or a command that reads from stdin
echo "${var}" # TODO: echo results so they can be used
return 0 # successful completion
}
# Alternative format: the () is for decoration only
functionName () {
# Read in required arguments
local arg1="$1"
# Remove the first argument from the argument list
shift
# Store the rest of the arguments in an array
declare -a arr=("$@")
local var # var is an empty array
return ${returnStatus} # return status can be a variable
}
# Command wrappers
ls () {
command ls -lart # Run a command as opposed to the function
# the command keyword prevents an infinite loop
}
Line breaks: Split lines with \ (the concatenation operator)
sudo apt-get install \
gedit-plugins # for gedit
Variables
# Assignments
# Note: the scope is the current script/shell by default
var=1 # Warning: no space before and after =
# Recall a variable with $
echo $var
# Modify numeric variables with let
let var=$var+1
# Export variables to child processes
# Note: This makes a copy of the variable when the child process is created,
# so the original variables are not affected
export var
# Special variables for a shell (see man page for bash)
$? # the exit status of the last foreground job
$! # the process ID of the last background job
$$ # the process ID of the current shell/script
$- # the current command line option flags
$0 # the name of the shell
# or the name of the command
$_ # the absolute path name of the shell
# Special variables for a script/function
$0 # name of the shell script
$FUNCNAME # name of the function
$1, $2, $3, ..., $9 # 1st to 9th argument
${10}, ${11}, ${12}, ... # 10th argument and beyond
$# # number of arguments, not counting $0
$* # expands to $1 $2 $3 … ${N}
$@ # expands to $1 $2 $3 … ${N}
"$*" # all arguments as a single string
# i.e., expands to "$1c$2c$3c…c${N}"
# where c is the first character in IFS
"$@" # all arguments as separate strings
# i.e., expands to "$1" "$2" "$3" … "${N}"
"${@:2}" # all arguments from the 2nd argument
"${@: -1}" # all arguments from the last argument
"${@:START:COUNT}" # expand COUNT arguments beginning at START
# Note: If START is negative,
# the arguments are numbered in
# reverse starting with the last one
# Arguments can be altered
shift # $1 is discarded; $2 becomes $1, ..., ${N} becomes ${N-1}
shift 4 # $1-$4 are discarded; $5 becomes $1, etc.
# Environmental variables (duplicated in Linux_notes.txt)
$USER # user name
$HOSTNAME # hostname of the machine
$SECONDS # the number of seconds since the shell/script was started
$LINENO # the current line number in the shell/script
$RANDOM # a random number from the Linux kernel's
# built-in random number generator,
# by the OpenSSL library function,
# using the FIPS140 algorithm
$HOME # home directory of current user
$PWD # present working directory
$PATH # ordered list of directories scanned upon command
$PS1 # the prompt statement
$SHELL # user's default command shell
$HISTFILE # the location of the history file
$HISTFILESIZE # the maximum number of lines in the history file
# (default 500)
$HISTSIZE # the maximum number of commands in the history file
$HISTCONTROL # how commands are stored (ignoreboth, ignordups)
$HISTIGNORE # which command lines can be unsaved
Command substitution
# Note: if the output is more than one line, newline characters are removed
`inner command` # older method
$(inner command) # newer method, allows nesting
ls /lib/modules/$(uname -r)/
echo $(ls) # note the output is one single line
# Command chains (conditional execution)
command1 ; command2 ; command3 # execute regardless
command1 && command2 && command3 # execute until false
command1 || command2 || command3 # execute until true
make ; make install ; make clean # all commands will execute regardless
# of the exit status of previous command
make && make install && make clean # commands will execute if prev one succeeds
make || make install || make clean # stops once a command is successful
Booleans
# Simple functions
true # has exit code 0
false # has exit code 1
# Ways to test Booleans
bool=true
if [ "$bool" = true ]; then
if [ "$bool" = "true" ]; then
if [[ "$bool" = true ]]; then
if [[ "$bool" = "true" ]]; then
if [[ "$bool" == true ]]; then
if [[ "$bool" == "true" ]]; then
if test "$bool" = true; then
if test "$bool" = "true"; then
# Test files: Type man 1 test for full list
[[ -e /etc/passwd ]] # test if the file exists
[[ -d /etc/passwd ]] # test if the file is a directory
[[ -f /etc/passwd ]] # test if the file is a regular file
[[ -h /etc/passwd ]] # test if the file is a symbolic link
[[ -s /etc/passwd ]] # test if the file is of nonzero size
[[ -g /etc/passwd ]] # test if the file has sgid set
[[ -u /etc/passwd ]] # test if the file has suid set
[[ -r /etc/passwd ]] # test if the file is readable
[[ -w /etc/passwd ]] # test if the file is writable
[[ -x /etc/passwd ]] # test if the file is executable
# Test strings
[[ -z string ]] OR [ -z "string" ] # test if string is NULL
[[ -n string ]] OR [ -n "string" ] # test if string has non-zero length
[[ $string == *"substr"* ]] # test if string contains a substring
# Test expressions
[[ ... ]] && echo t || echo f # test if an expressions is true or false
# Operators
&& # AND operator
|| # OR operator
! # NOT operator
= # equal to operator for numbers
-eq # equal to operator for numbers
-ne # not equal to operator
-gt # greater than operator
-lt # less than operator
-ge # greater than or equal to operator
-le # less than or equal to operator
== # equal operator for strings
=~ # regex match operator for strings
> # compares sorting order for strings
-nt # newer than operator for files
-ot # older than operator for files
Sequences
# Using brace expansion (separates values by spaces)
{1..5} # sequences of numbers
{1..5..2} # sequences of numbers with increment
{a..e} # sequences of letters
{a..e..2} # sequences of letters with increment
{Y..b} # sequences of characters
b{0..4}c # modify sequences
{ {1..3},{A..C} } # concatenate sequences
{Who,What,Why,When,How} # concatenate strings
{1..3}{A..C} # sequences of all combinations
eval echo {$start..$end} # evaluate variables in braces
# Using seq (separates values by newlines)
seq 0.1 0.2 5 # sequence of numbers from 0.1 to <= 5
# with a step size of 0.2
Strings
# Basics
'string' # treat every character literally
"${var}" # allow variable substitution
string=$(echo "part1" "part2") # concatenate strings joined by spaces
# Properties of strings
${#string} # length of a string
${string:0:n} # first n characters of a string
${string:1} # substring starting from 2nd character
${string:1:-1} # remove first and last characters
${string%%+(0)} # remove trailing zeros
${string##+(0)} # remove leading zeros
${string%%.*} # all characters in a string before first .
${string%.*} # all characters in a string before last .
${string#*.} # all characters in a string after first .
${string##*.} # all characters in a string after last .
IFS=':'; arr=($str); unset IFS; # split a string into an array
# with a custom delimiter
${string/'substr1'/"${substr2}"} # replace first occurrence of
# substr1 with substr2
${string//'substr1'/"${substr2}"} # replace globally substr1 with substr2
grep '[a-z]\{3\}' <<< "${string}" # search for a pattern in a string
sed 's/^ *//g' <<< "${string}" # remove leading spaces
# Evaluate a string as a command
eval ${commandstr}
# Print a string
echo "${str}" >> "${file}"
# Regular expressions for grep or egrep
# http://www.robelle.com/smugbook/regexpr.html
^ # match expression at the start of a line, as in ^A
$ # match expression at the end of a line, as in A$
# Mnemonic: "First you get the power, then you get the money."
\ # turn off the special meaning of the next character, as in \^
[ ] # match any one of the enclosed characters, as in [aeiou]
# Use Hyphen "-" for a range, as in [0-9]
[^ ] # match any one character except those enclosed in [ ], as in [^0-9]
. # match a single character of any value, except end of line
* # match zero or more of the preceding character or expression
? # match zero or one of the preceding character or expression
# Regular expressions for grep
\{x\} # match exactly x occurrences of the preceding
\{x,y\} # match x to y occurrences of the preceding
\{x,\} # match x or more occurrences of the preceding
# Regular expressions for egrep
+ # match one or more of the preceding character or expression
| # match either the left expression or the right expression
"{x}" # match exactly x occurrences of the preceding
"{x,y}" # match x to y occurrences of the preceding
"{x,}" # match x or more occurrences of the preceding
"( )" # capturing group
"\w" # match "word" characters: all letters, numbers,
# and even the underscore character
"\d" # match "digit" characters (not sure if this works)
"\s" # match "whitespace" characters
"\W" # match "non-word" characters
"\D" # match "non-digit" characters (not sure if this works)
"\S" # match "non-whitespace" characters
# Regular expression examples
grep smug files # search files for lines with 'smug'
grep '^smug' files # 'smug' at the start of a line
grep 'smug$' files # 'smug' at the end of a line
grep '^smug$' files # lines containing only 'smug'
grep '\^s' files # lines starting with '^s', "\" escapes the ^
grep '[Ss]mug' files # search for 'Smug' or 'smug'
grep 'B[oO][bB]' files # search for BOB, Bob, BOb or BoB
grep '^$' files # search for blank lines
grep '[0-9][0-9]' file # search for pairs of numeric digits
grep '^From: ' /usr/mail/$USER # list your mail
grep '[a-zA-Z]' # any line with at least one letter
grep '[^a-zA-Z0-9]' # anything not a letter or number
grep '[0-9]\{3\}-[0-9]\{4\}' # 999-9999, like phone numbers
grep '^.$' # lines with exactly one character
grep '^.\{5\}$' # lines with exactly 5 characters
# egrep "^.{$n}$" # lines with exactly $n characters
grep '"smug"' # 'smug' within double quotes
grep '"*smug"*' # 'smug', with or without quotes
grep '^\.' # any line that starts with a Period "."
grep '^\.[a-z][a-z]' # line start with "." and 2 lc letters
String arrays
# Declare array variables
declare -a arr1 arr2
declare -a arr=(element1 element2 element3)
# Other ways to create an array
arr=("element1" "element2" "element3")
arr=($(ls *.pdf)) # command output on one line (assume no spaces)
IFS=$'\n'
arr=($(ls *.pdf)) # command output on one line (assume spaces)
unset IFS
arr=({1..3}{A..C}) # by brace expansion
arr=(~/myDir/*) # by wildcard expansion
# Read into an array with a specific IFS
IFS=$'\n'; read -r -a arr <<< $(find /path/to/dir -name "*.pdf"); unset IFS
# Change an individual element in the array
arr[2]="newElement"
# Length of the array
nElements=${#arr[@]} # number of elements (ever defined?)
# Append to an array
arr=("${arr[@]}" "newElement")
arr+=("newElement")
arr[${#arr[*]}]="newElement"
# Access elements of the array
element1="${arr[0]}" # the first element (also $arr or ${arr})
element1="${#arr[0]}" # length of the first element (also ${#arr})
elements="${arr[$n - 1]}" # use variables and simple arithmetic
elements=${arr[*]} # all the elements
elements=${arr[@]} # all the elements
elements=${arr[*]:$(($m-1))} # all elements starting from the mth element
elements=${arr[*]:$(($m-1)):$n} # n elements starting from the mth element
elements=${arr[@]:$(($m-1)):$n} # n elements starting from the mth element
elements=${arr[@]/#/$str} # all the elements prepended by str
elements=${arr[@]/%/$str} # all the elements appended by str
# Print each string in an array with a space in between
echo "${arr[@]}"
# Print each string in an array with a space in between
echo "${arr[@]}"
# Print each string in an array with nothing in between
printf '%s' "${arr[@]}"
# Print each string in an array on a different line and write to a file
printf '%s\n' "${arr[@]}" >> "${file}"
Input/Output (see more in Common_Unix_commands.txt)
# General commands
cat /dev/stdin # read from standard input
exec < $fileName # redirect standard input to a file
# here document
program << here
commands
here
# Read input stream (separated by whitespace) into var1, var2, var3
# Note: If # of input items > # of variable names,
# then the remaining items will all be added to the last var
# If # of input items < # of variable names,
# then the remaining variable names will be set to blank or null.
read var1 var2 var3 # -p - specify a prompt
# -s - suppress display of user input
# Perform commands on each line of output
command1 | xargs -I @ command2 @
# Perform commands on each line of output with while loop
command1 | \
while IFS= read -a arr
do
echo ${arr[@]}
done
# Format output into columns
command | column -t -s $'\t'
If-else statements
# KornShell, Zsh & BASH
if [[ ]] ; then
elif [[ ]] ; then
else
fi
# posh, yash, dash, BourneShell
if [ ] ; then
elif [ ] ; then
else
fi
# mywiki.wooledge.org/BashFAQ/031
Case statements
case expression in
pattern1) execute commands ;;
pattern2) execute commands ;;
pattern3) execute commands ;;
pattern4) execute commands ;;
*) execute default commands or nothing ;;
esac
For loops
# http://www.cyberciti.biz/faq/bash-for-loop/
for VARIABLE in 1 2 3 4 5
do
echo $VARIABLE
break
continue
done
# Only bash v3.0+
# Note: 1. must be integers
# 2. no spaces between the integer and ..
for VARIABLE in {1..N}
do
echo $VARIABLE
done
for VARIABLE in $( eval echo {1..${num}} )
do
echo $VARIABLE
done
# for idx in $( eval echo {0..$((${num}-1))} ) # Not needed
for idx in $( eval echo {0..$((num-1))} )
do
echo ${idx}
echo $arr
echo ${arr[$idx]}
done
# Only bash v4.0+
for VARIABLE in {0..10..2}
do
echo $VARIABLE
done
for file in file1 file2 file3
do
echo $file
done
for element in "${arr[@]}"
do
# Do whatever with individual element of the array
# You can access array elements using echo "${arr[0]}", "${arr[1]}" too
echo "$element"
done
for element in ${arr[@]} # ${arr[*]} works too but not preferred
do
echo "$element"
done
for OUTPUT in $(command)
do
echo $OUTPUT
done
# Without a word list, the for loop uses the argument list
for arg
do
echo "$arg"
done
# C-style for-loop
for (( i = 1 ; i <= N ; i++ ))
do
commands
done
# Run through arguments with C-style for loop
numargs=$#
for (( i=1 ; i <= ${numargs} ; i++ ))
do
echo "$1"
shift
done
While loops executes until condition is false
while [[ ]]
do
commands
done
A typical while loop
count=$N
while [[ $count -gt 0 ]]
do
echo "$count"
let count=$count-1
done
# Read lines from a file into an array
while IFS= read -a arr # Set IFS to null so lines are not separated
do
echo ${arr[@]}
done < $fileName
# Run through arguments with while loop and parameter expansion
while [ "${1+defined}" ]
do
echo "$1"
shift
done
Until loops executes until condition is true
until [[ ]]
do
commands
done
Arithmetics
# Evaluate expressions
echo $(( $num1 + $num2 )) # built-in shell format (recommended)
echo $(expr $num1 + $num2) # the expr utility
# Note: will return exit status 1 if
# evaluated to 0
let x=( $num1 + $num2 ); echo $x # the built-in shell command let
# Note: will return exit status 1 if
# evaluated to 0
echo $(bc -l <<< "$num1 / $num2") # compute floating point arithmetics
Useful commands
sleep 5 # pause execution for 5 seconds
sleep 2m # pause execution for 2 minutes
sleep 3h # pause execution for 3 hours
sleep 5d # pause execution for 5 days
Useful loops
for i in {1..10} # Print 10 empty lines
do
echo ""
done
# Check if a pattern of files exists
for f in /path/to/your/files*; do
if [[ -e "${f}" ]] ; then
command
fi
break
done
Useful if/else
if [[ ${userName} == "root" ]] ; then
# Do not need sudo as root
command
else
# Need sudo if not root
sudo command
fi
Useful files
accountFile='/etc/passwd' # file with all accounts on the server
TEMP=$( mktemp /tmp/tmpfile.XXXXXXXX )
# create a temporary file
TEMPDIR=$( mktemp -d /tmp/tmpdir.XXXXXXXX )
# create a temporary directory
Useful variables
# Information
today=$(date '+%Y%m%d') # current date
timeStamp=$(date '+%Y%m%dT%H%M%S') # current date and time
echo "The date-time is: ${timeStamp}" > "${logFileName}"
userName=$(whoami) # user name
userID=$(id -u) # effective user ID
groupID=$(id -g) # effective group ID
serverName=$(hostname) # server name
userHome="${HOME}" # home directory
# From a file name
# Note: the double quotes are necessary if names contain spaces
path="$(dirname "$(readlink -f $0)")"
# path to current file
pwdName=$(basename $(pwd)) # name of present working directory
dirBase="${fullPath##*/}" # basename of a directory
filePath="$(dirname "$(readlink -f "${file}")")"
# path to a file
fileBase="$(basename "${file%.*}")" # basename of a file
# (get rid of the path and the extension)
fileExt="$(basename "${file##*.}")" # extension of a file
# (get rid of the path and the base name)
fileType=$(file --mime -b "${file}" | cut -d "/" -f 1)
fileType=$(file --mime -b "${file}" | cut -d "/" -f 1 | sed 's/^ *//g')
# get the first part of the MIME file type
# "text", "inode", "image", "application"
fileType=$(file -b file) # get the file type
owner=$(ls -l "${outFile}" | cut -d ' ' -f 3)
# owner for a file
group=$(ls -l "${outFile}" | cut -d ' ' -f 4)
# group for a file
# From files
userID=$(grep ${userName}:x ${accountFile} | cut -d ':' -f 3)
# user ID (3rd field of /etc/passwd)
groupID=$(grep ${userName}:x ${accountFile} | cut -d ':' -f 4)
# group ID (4th field of /etc/passwd)
thisLine=$(sed -n "${iLine}p" "${file}")
# a specific line of a file
nLines=$(sed -n '$=' "${file}") # number of lines of a file (faster)
nLines=$(wc -l "${file}") # number of lines of a file
# Elapsed time
start=$(date -u +%s) # used to compute elapsed time
finish=$(date -u +%s) # used to compute elapsed time
elapsed=$(( $finish - $start ))
echo "The elapsed time is: $elapsed seconds"
The exit statement allows communication with a parent process
# This return value is stored in the environmental variable $?
# Convention: 0 - success
exit 0