Common Unix Commands
Last Updated 2020-12-24
Filesystems
lsb_release -a # Check Ubuntu version
fsck.ext4 /dev/sda # File system check
# If mounting on a non-empty directory, the contents of that directory
# will be inaccessible until the filesystem is unmounted
mount [-l] # list all mounted filesystems
sudo umount -l /dev/sdb # unmount a disk
sudo mount -a # mount all disks listed in the /etc/fstab file
sudo mount <device node> <mount point> # mount a disk
sudo mount /dev/sdb /media/markX # mount
df -Th # (disk-free) display info about mounted filesystems
# including disk usage
du -sh * # return a list of sizes from a directory
# -s - (summarize) display only a total
# -h - print in human-readable format
ncdu # open the disk usage analyzer
sudo systemctl start nfs # start NFS daemons
exportfs -av # notify Linux about the dirs to be remotely mounted
sudo systemctl enable nfs # start NFS daemons whenver system is booted
sudo mount servername:/projects /mnt/nfs/projects
lsblk # list all block devices
ls -l /dev/disk/by-label # list all detected disks by their label
sudo e2label /dev/disk label# change disk label
System Control
sudo systemctl start/stop/restart nfsd
sudo systemctl enable/disable nfsd
sudo systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target
# disable sleep, suspend, hibernate
sudo systemctl unmask sleep.target suspend.target hibernate.target hybrid-sleep.target
# re-enable sleep, suspend, hibernate
Documentation
help # display list of built-in commands
man printf # display manual page for printf
# man pages are divided into chapters 1~9
man 3 printf # display manual page for printf from chapter 3
man -a printf # display all manual pages for printf
# from each chapter sequentially
man 1 test # display list of file test operations
type NAME # display information about command type
# -a display all locations containing
# an executable named NAME
# -p returns the name of the disk file
# that would be executed
# -t output a single word, one of:
# 'alias', 'keyword', 'function',
# 'builtin', 'file'
whatis string # same as man -f string
# view all pages containing a string
apropos subject # same as man -k subject
# view all pages that discuss a subject
info subject # view info pages about subject
program --help # display short description of program
help echo # same as --help for built-in commands
yelp man:cat # open the man page of cat with
# the graphical help system
Package management
sudo apt-get install gedit-plugins # for gedit
dpkg --install foo.deb # install or update package
# RPM: rpm -i foo.rpm OR rpm -U foo.rpm
apt-get install foo # install or update package, dependencies
# RPM: yum install foo OR yum update foo
dpkg --remove foo.deb # remove package
# RPM: rpm -e foo.rpm
apt-get autoremove foo # remove package, dependencies
# RPM: yum remove foo
apt-get dist-upgrade # update entire system
# RPM: yum update
dpkg --list OR dpkg -l # show all installed packages
# RPM: rpm -qa OR yum list installed
dpkg --listfiles foo # get information on package
# RPM: rpm -qil foo
apt-cache search foo # show packages named foo
# RPM: yum list "foo"
apt-cache policy foo # check status of packages named foo
# RPM: yum info foo
apt-cache dumpavail # show all available packages
# RPM: yum list
dpkg --search file # what package is file part of?
# RPM: rpm -qf file
Run & stop program
bash script.sh # run script with bash
./script # provide path to run executable
<command> & # run job in the background
bg # put a process in the background
fg # put a process in the foreground
sleep 5 # pause execution for 5 seconds
sleep 2m # pause execution for 2 minutes
sleep 3h # pause execution for 3 hours
sleep 5d # pause execution for 5 days
#### Schedule commands
```bash
at now + 2 days # schedule a command 2 days from now
<command> # the command to schedule
<EOT> # Press CTRL + D
atq # view job queue
crontab -e # open the crontab editor for current user
sudo crontab -u username -e # open the crontab editor for a specific user
* * * * * XXX.sh # execute XXX.sh (in home directory) every minute
# Note: must provide full path if script
# is not in the home directory
30 08 10 06 * backup.sh # execute backup.sh at 8:30 am, 10-June
# irrespective of the day of the week
inotifywait # wait for changes to files or directories
# -m - (monitor) execute indefinitely
# -e - event
Troubleshoot
startx # initialize an X session
bash -x myscript.sh # run script in debug mode
time command # time a simple command
w # show who's logged on and what they are doing
top # display all running tasks
top -u adam # display all running tasks of a specific user
top -p $(pgrep -d ',' -f MATLAB)
# display all running tasks of a specific command
htop OR atop # TODO
free # display memory usage
uptime # tell how long the system has been running
# and display load averages over
# the last min, last 5 mins and last 15 mins
jobs -l # display all jobs running in the background
grep CRON /var/log/syslog # check if CRON jobs ran
ps # display processes running under current shell
ps -A # display all running processes
ps -u # display processes for the current user
ps -fu adam # display all processes for a given user
ps -ef # display all processes in the system
ps -eLf # display all threads in the system
ps aux # display all processes of all users
ps axo stat,priority,pid,pcpu,comm # display specific attributes for processes
pidof dd # display the process number of dd
pgrep ^dd # list process IDs which matches selection criteria
pstree # display processes as a tree diagram
# threads are in curly braces
pkill -STOP -u adam # logout user
ps -dN|grep pts/3 # find parent process of session
kill -SIGKILL <pid> # terminate a process
kill -9 <pid> # terminate a process
sudo systemctl stop gdm # stop the GUI desktop (CentOS and SUSE)
sudo systemctl stop lightdm # stop the GUI desktop (Ubuntu)
sudo telinit 3 # stop the GUI desktop
sudo systemctl start gdm # start the GUI desktop (CentOS and SUSE)
sudo systemctl start lightdm# start the GUI desktop (Ubuntu)
sudo telinit 5 # start the GUI desktop
sudo systemctl restart network
OR sudo service network restart
# restart the network
sudo systemctl restart NetworkManager
OR sudo NetworkManager restart
# restart the NetworkManager
sudo systemctl status webmin
# check webmin status
sudo systemctl restart webmin
# restart webmin
sudo dhclient eth0 # get a fresh IP address
sudo mtr --report-cycles 3 8.8.8.8
# trace route to 8.8.8.8
gdb -c core # find out what program caused a core dump file
nvidia-smi # print the NVIDIA driver version
nvidia-settings # open the NVIDIA driver settings GUI
ubuntu-drivers devices # show recommended nvidia drivers
Network & Remote login
ifconfig # information about network interfaces (older)
ip # information about network interfaces (newer)
ip addr show # view IP address
ip route show # view routing information
ping -c 172.28.77.1 # check network status
route -n or ip route # show current routing table
route add -net address OR ip route add
# add static route
route del -net address OR ip route del
# delete static route
traceroute google.com # inspect the route to google.com
host google.com # get IP address of specified host
nslookup google.com # look up name servers interactively
dig google.com # look up domain name info from name server
sudo ethtool eth0 # querie network interfaces and set parameters
netstat -r # display all active connections and routing tables
# -r - display kernel routing tables
sudo nmap -sP 10.0.2.15/24 # scans open ports on the network
tcpdump # dumps network traffic for analysis
iptraf # monitors network traffic in text mode
mtr # combines ping and traceroute
sudo ufw allow proto tcp from 172.25.228.11 to any port 445
22 # secureCRT
445 # windows samba server
5900:5920 # vnc server
10000 # webmin
sudo ufw status numbered
sudo ufw delete ####
sudo ufw delete allow 80/tcp
ssh -Y adam@172.28.77.1 # Secure Shell (SSH) to chalkboard
ssh -Y adam@172.28.76.139 # fishfish
ssh -Y al4ng@sp-udc-led-cs01.hpc.virginia.edu
# Rivanna udc-ba-36-37
ssh -Y al4ng@rivanna1.hpc.virginia.edu
# Rivanna udc-ba-36-36
ssh -Y al4ng@rivanna2.hpc.virginia.edu
# Rivanna udc-ba-34-36
ssh -Y al4ng@rivanna3.hpc.virginia.edu
# Rivanna udc-ba-35-36
ssh -Y al4ng@rivanna.hpc.virginia.edu
# Rivanna, random?
ssh al4ng@rivanna.hpc.virginia.edu command
# perform command remotely on rivanna
ssh -l al4ng rivanna.hpc.virginia.edu whoami
# perform whoami remotely on rivanna
scp -p al4ng@sp-udc-led-cs01.hpc.virginia.edu:/home/al4ng/test.text /media/adamX/.
scp -p /media/adamX/test.text al4ng@interactive.hpc.virginia.edu:/home/al4ng/.
scp -p Common_Unix_commands.txt al4ng@sp-udc-led-cs01.hpc.virginia.edu:/scratch/al4ng/Miscellaneous_backup
# upload stuff to Rivanna
rsync -au /media/adamX/m3ha/data_dclamp/*.slurm al4ng@sp-udc-led-cs01.hpc.virginia.edu:/scratch/al4ng/m3ha/data_dclamp/
# upload stuff to Rivanna
wget -O file.html <URL> # download webpage to file.html
curl -o file.html <URL> # read information about a URL and save to file
curl -O <URL> # read and save with same name as URL
curl "http://httpbin.org/get?city=Baltimore&state=Maryland"
# send a general HTTP GET request
ftp ftp.gnu.org # start ftp
# get FILE - download a file
ssh-keygen # generate a public/private rsa key pair
# for the current user in the ~/.ssh/ directory
ssh-copy-id -i ~/.ssh/id_rsa.pub 128.143.16.164
ssh-copy-id -i ~/.ssh/id_rsa.pub al4ng@rivanna.hpc.virginia.edu
# place the public key in the ~/.ssh/authorized_keys
# file of the remote host
VNC server
vncserver -geometry 1366x760 # open a VNC server session with
# a certain window resolution
vncserver -kill :1 # kill a VNC server session
vncconfig & # show clipboard options
vncconfig -nowin & # allow copy and paste without showing
# an options window
ps -ef | grep vnc # get info for all the active vnc sessions
Printing
sudo /etc/init.d/cups start # start the CUPS daemon manually
sudo service cups start # ditto in CentOS
sudo /etc/init.d/cups restart # restart the CUPS daemon manually
sudo service cups restart # ditto in CentOS
sudo /etc/init.d/cups status # check status of CUPS daemon
sudo service cups status # ditto in CentOS
sudo /etc/init.d/cups stop # stop the CUPS daemon manually
sudo service cups stop # ditto in CentOS
sudo update-rc.d cups -f defaults # enable CUPS daemon after installation
sudo update-rc.d cups enable # start CUPS daemon at boot time
sudo chkconfig cups on # ditto in openSUSE or CentOS
sudo update-rc.d cups disable # do not start CUPS daemon at boot time
sudo chkconfig cups off # ditto in openSUSE or CentOS
System date and time
sudo ntpdate -q ntp.ubuntu.com
# compare time with ntp
sudo ntpdate-debian # sync time with ntp
Rebooting and Shutting Down
shutdown # preferred method from shutting down;
# sends a warning message,
# then prevents further users from logging in
sudo shutdown -h now
sudo shutdown -h 10:00 "Shutting down for scheduled maintenance"
sudo halt # same as sudo shutdown -h
sudo poweroff # same as sudo shutdown -h
sudo reboot # same as sudo shutdown -r
System information
uname -a # print all system information
uname -r # print Linux kernel version
xdpyinfo | grep dim # get screen resolution
lscpu # prints CPU architecture information from
# sysfs and /proc/cpuinfo
cpuid # dumps complete information about the CPU(s)
# collected from the CPUID instruction,
# and also discover the exact model of
# x86 CPU(s) from that information
sudo update-pciids # grab the current version of the pci.ids file
# from the Internet
lspci # list PCI devices
# -v - (verbose) list detailed info for all devices
hostname # get system hostname
sudo hostname fishfish # change system hostname
date # get current date and time
date +"%Y%m%dT%H%M" # display date and time in YYYYMMDDTHHMM format
date -u +%s # display number of seconds
# since 1970-01-01 00:00:00 UTC
date -u +%s.%N # display number of seconds with decimal points
# since 1970-01-01 00:00:00 UTC
date +%u # display the day of the week (1..7)
date --date='-2 day' # display date two days in the past
date --date='2 day ago' # display date two days in the past
date --date='yesterday' # display yesterday's date
date --date='next day' # display tomorrow's date
date --date='2 day' # display date two days in the future
cal # display a calendar for the current date
Shell formatting
bash --version # get the version of bash
/etc/bashrc # bash shell configuration for all users
~/.bashrc # bash shell configuration for current user
source script.sh # use function definitions in a bash script
# as command line commands
source ~/.bashrc OR . ~/.bashrc
# refresh bash shell configuration
clear # clear previous outputs
Environmental variables (see more in shell_script_syntax.sh)
set # list environmental variables
env # list environmental variables
printenv # list environmental variables
export # list all exported environmental variables
echo $SHELL # show the value of an environmental variable
$RANDOM # get a random number
export VARIABLE=value OR VARIABLE=value; export VARIABLE
# export a new variable value to make available
# for a child process
# put this is in ~/.bashrc to make permanent
export PATH=$HOME/bin:$PATH
# export a private bin directory to search path
echo $PS1 # display default command line prompt
PS1="\u@\h:\w[\t]\$" # format the default command line prompt
# \u - User name
# \h - Host name
# \w - Current working directory
# \! - History number of this command
# \d - Date
# \t - Time
PS2=">" # format the second-tier command line prompt
LS_COLORS # coloring of different file or dir types
Command line history
history # view the list of previously executed commands
# of the current terminal
cat ~/.bash_history # view the list of previously executed commands
# of previous terminals
!! OR !-1 # execute the previous command
! # start a history substitution
!$ # refer to the last argument in a line
!n # refer to the nth command line
!str # refer to the most recent command starting with str
User accounts
who # list the currently logged on users
# -a - more detailed information
# -m - only the user associated with stdin
who am i # alias for who -m
whoami # identify the current user
id # give information about current user
id -u # print effective user ID
id -g # print effective group ID
last # show listing of last logged in users
sudo vi /etc/passwd # edit user account info
sudo vi /etc/group # edit user group info
sudo useradd bjmoose # add user bjmoose
sudo /usr/sbin/useradd bjmoose # for openSUSE
sudo userdel bjmoose # delete user bjmoose without deleting home directory
sudo userdel -r bjmoose # delete user bjmoose along with its home directory
sudo groupadd anewgroup # add group
sudo groupdel anewgroup # delete group
groups bjmoose # list all groups that include bjmoose
newgrp BarrettRivanna # make BarrettRivanna the primary group
# Note: this only lasts for the current session
sudo usermod -a -G anewgroup bjmoose
# add bjmoose to anewgroup
# -a - append
sudo usermod -G group1 group2 bjmoose
# make bjmoose a member of group1, group2
# and remove from all other groups
sudo groupmod -g 1004 anewgroup
# change group id of anewgroup to 1004
sudo groupmod -n testgroup anewgroup
# change name of anewgroup to testgroup
sudo visudo # edit the /etc/sudoers file
# who where = (as_whom) what
su # launch a shell as root
su adam2 # launch a shell as adam2 (substitute user)
alias # list currently defined aliases
alias +='pushd' # add an alias
unalias # remove an alias
vi ~/.bash_aliases # add aliases here for all future shells
Set passwords
sudo passwd <username>
sudo smbpasswd <username>
sudo vncpasswd <username>
echo -n <passwd> | sha512sum
# print password in SHA-512 encrypted form
chage --list <username> # see the last time password was changed
Filesystem management
which diff # locate where the diff program resides
whereis diff # similar to which but with broader range, and locates
# the source and man files packaged with the program
echo $HOME # displays path to home directory
pwd # displays the present working directory
# -P avoid symbolic links
readlink -f file.txt # get path to a file
readlink -f . # get path to current directory
dirname file.txt # get the directory containing a file
file * # determine file type for all files in directory
file --mime * # determine the MIME file type for all files
file -b file # determine file type without prepending with filename
cd subdir # change to a subdirectory of the present working directory
cd /path/to/dir # change to a directory using the full path
cd OR cd ~ # change to home directory ($HOME)
cd .. # change to parent directory
cd - # change to previous directory
cd / # change to root directory
pushd /tmp/dir1 # change to /tmp/dir1 and add it to a
# list of recently viewed directories
popd # change to last directory on the
# list of recently viewed directories
dirs # display list of recently viewed directories
////user//bin # seen as /usr/bin by the system
ls # list the contents of the present working directory
# -1 - list each file on a separate line
# -F - directories end with / & executables end with *
# -l - long listing format
# -a - show hidden files & directories
# -r - reverse order while sorting
# -t - sort by newest first
# -R - recursively show all files within subdirectories
# -i - prints the inode number in the first column
# -d - show directories only and not their contents
# --full-time - list full time info
ls -1 | wc -l # count files in current directory
ls -d */ # list subdirectories only
ls -l | grep '^d' # list subdirectories only, long-listing format
ls -l | grep -v '^d' # list files only, long-listing format
ls -li file? # list all files that begin with file
ls -lart | grep "Feb" # list all files that were created in February
find -maxdepth 1 -type d | sort | while IFS= read -r dir; do n=$(find "$dir" -type f | wc -l); printf "%s\t%s\n" "$dir" "$n"; done | column -t -s $'\t'
# list subdirectories with file count
tree # displays a tree view of the filesystem
tree -d # just the directories
Links
ln </path/to/file-name> <link-name> # create hard link
ln -s </path/to/file-name> <link-name> # create soft (symbolic) link
Directories
mkdir sampdir # make directory
mv sampdir sampdir2 # rename a directory
rename p_d d p_d* # rename many directories (Rivanna version)
# -v - list what directories were renamed
rename 's/p_d/d/' p_d* # rename many directories (fishfish version)
# -v - list what directories were renamed
# -n - see what directories would be renamed
rmdir sampdir # remove an empty directory
rm -rf sampdir # forcefully remove a directory recursively
Files
touch Top.out # create file
touch -t 03201600 file # set the date and time stamp of file to
# 4 p.m., March 20th
cat Top.out # display file content
column -t -s $'\t' file # format tab-separated file into columns
mktemp /tmp/tmpfile.XXXXXXXX
# create a temporary file
# mktemp will replace XXXXXXXX with a number
mktemp -d /tmp/tmpdir.XXXXXXXX
# create a temporary directory
cat > file1 # write to file1
cat >> file2 # append to file2
cat 1>> file3 # append stdout to file3
cat 2>> file3 # append stderr to file3
echo "line1" > file.txt # write to file
echo "line2" >> file.txt # append to file
# cat << EOF > file2.txt # get standard input before EOF and write to file
tac Top.out # display file content backwards
less Top.out # view file content with scroll-back capabilities
# / to search for a pattern in the forward direction
# ? to search for a pattern in the backward direction
head Top.out # display the first 0 lines of a file
head -15 Top.out # display the first 15 lines of a file
head -n 20 Top.out # display the first 20 lines of a file
tail Top.out # display the last 10 lines of a file
tail -n +2 Top.out # display file starting from the 2nd line
sed -n '6,10p' file # display the file from line numbers 6 through 10
sed -n '$=' "${file}" # display the number of lines in a file
zcat compressed.txt.gz # view a compressed file
zless file.gz # page through a compressed file
zgrep -i less file.gz # search inside a compressed file
zdiff file1.gz file2.gz # compare two compressed files
select-editor # select default editor
sudo update-alternatives --config editor
# configure default editor
nano file.txt # edit a file with nano
vi Top.out # open file using the vi editor
vi +100 Top.out # open file at the 100th line
vi +/str Top.out # open file at the first occurrence of str
vi -r Top.out # edit Top.out in recovery mode from a system crash
mv Top.out Top2.out # rename a file
rename .out .txt *.out # rename many files
rm Top.out # remove a file
unlink symlink # unlink a symbolic link
rm -f Top.out # forcefully remove a file
rm -i Top.out # interactively remove a file
Input/Output of files (see more in shell_script_syntax.sh)
command < input-file
command > output-file
command 2> error-file
command > all-output-file 2>&1
# put anything written to file descriptor 2
# in the same place as file descriptor 1
command >& all-output-file
# same as command > all-output-file 2>&1 in bash
command1 | command2 # command2 doesn't have to wait until command1
# completes to read in the latter's output as input
command > /dev/null # ignore standard output
command >& /dev/null # ignore standard output and standard error
read var # read input stream
Manipulating files
wc file # print number of lines, words, characters
wc -l file # print number of lines
wc -w file # print number of words
wc -c file # print number of bytes
cat file1 file2 > file3 # combine file1 and file2 into file3
cat file1 >> file3 # append file1 to file3
paste file1 file2 # combine columns of file1 and file2
# default delimiter is \t
paste -d, file1 file2 # use , as the delimiter
cut -f 2 tabfile # cut out 2nd column
# -d - delimiter
join file1 file2 # combine two files on a common field
sort file # sort the lines in file according to the first field
sort -k 3 file # sort the lines in file according to the 3rd field
# -r - sort in reverse order
# -n - use numeric order
# -t - delimiter
# -u - remove duplicate lines
uniq file # remove consecutive duplicate lines in file
uniq -c filename # count consecutive duplicate lines in file
split infile # split file into files of 1000 lines
Search for files or directories
locate zip | grep bin # list all the files and directories
# with both "zip" and "bin" in their name
updatedb # update the database for locate
find . -name "*.log"
# find all .log files in current directory recursively
# -name - only list files with pattern in name
# -iname - also ignore the case of file names
# -type - restrict the results to a certain type
# -size - restrict the results to a certain size
# -maxdepth - maximum number of levels down the tree
# -not -path - exclude a path
find -L . -name "*.log" # find all .log files in current directory recursively
# following symbolic links
find . -name "*.log" -not -path "./backup/*"
find . -path ./backup -prune -o -name '*.log' -print
# two ways of excluding entire directories
sudo find / -type d -name "nrnoc"
# find directories containing "nrnoc" in root
find . -maxdepth 1 -type f -name "*.m"
# find all files with a given name non-recursively
find . -type l # find symbolic links recursively
find -L . -type l # find broken symbolic links recursively
# Note: use symlinks -dr instead
find / -ctime 3 # find files when the inode metadata
# was last changed 3 days ago
find / -atime +3 # find files accessed/last read greater than 3 days ago
find / -mtime -3 # find files modified/last written less than 3 days ago
find / -size 1 # find files with size greater than 0
# but less than 512 bytes
find / -size 6 # find files with size greater than 5 * 512 bytes
# but less than 6 * 512 bytes
find / -size -7c # find files with size less than 7 bytes
find / -size +7k # find files with size greater than 7 kilobytes
find / -size +7M # find files with size greater than 7 megabytes
find / -size +7G # find files with size greater than 7 gigabytes
symlinks -v . # show all symbolic links in current directory
symlinks -d . # delete all broken symbolic links
symlinks -dr . # delete all broken symbolic links recursively
Search and perform actions
find . -maxdepth 1 -type f -name "*.m" | xargs -I @ basename @ '.m'
# find all files with a given name and grep basenames
find . -maxdepth 1 -type f -exec mv {} Introduction\ To\ Linux/ \;
# find and move files found
find . -maxdepth 1 -type f | grep test12 | xargs -I @ mv @ 20180124_singleneuronfitting17_test12_Rivanna/
# find and move files found
find -name "*.swp" -exec rm {} ':'
OR find -name "*.swp" -ok rm {} "\;"
# find and remove all files found
# -exec - run command on search results
# -ok - prompt user for permission before execution
find /media/shareX/ -type f -name "*.wmv" -size +1M -size -10M -exec ls -lh {} \;
# find all files smaller than a particular size
find /media/shareX/ -type f -name "*.wmv" -size -512k | tail -1 | xargs -I @ cp @ /media/shareX/share/Vignesh/
# find and copy
find /media/shareX/ -type f -name "*.wmv" -size +10M -size -2G | tail -1 | xargs -I @ cp @ /media/shareX/share/Vignesh/
# find and copy
grep [regex] file # search for a pattern in file and print matching lines
# -v - print lines that do not match the pattern
# -C n - print n lines above and below pattern
# -r - recursively find
# -n - prefix with line number
# -w - must match PATTERN as a whole word
# -i - case-insensitive
# -l - print file names with matches
# -m - maximum number of lines to match
# -e PATTERN - use PATTERN as pattern
# -o - extract only the matching parts of each line
egrep "regex" file # search for a pattern in file and print matching lines
grep [regex] <<< "str" # search for a pattern in a string
grep -n "#REMOVE" template.sh | while read line ; do echo ${line%%:*} ; done
# find and print line numbers only
grep -n "regex" file | cut -f 1 -d ":"
# find and print line numbers only
grep --include="*.m" -rlw '/media/adamX/m3ha/optimizer4gabab/' -e "addpath" | grep -v backup | grep -v singleneuronfitting
# find all files under a directory that contain string
grep --include=*.m -rnw '/home/Matlab/' -e "make_time_column_NEW"
# find all files under a directory that contain string
# and print it
grep --include="*.m" -rlw '/home/Matlab/' -e "make_time_column_NEW" | xargs -I @ sed -i.backup 's/make_time_column_NEW/make_time_column/g' @
# replace string in many files and create backup for original
strings book1.xls # extract all printable character strings in a file
strings book1.xls | grep my_string
# search for my_string inside a binary file
strings --print-file-name /usr/bin/a* | grep GPL
# print all programs withe GPL licenses
strings a.url | grep URL# get web address from link
Comparing files and file types
diff [options] <filename1> <filename2>
# compare text files and directories
# -c - list 3 lines of context
# -r - recursively compare subdirectories
# -i - ignore case of letters
# -w - ignore differences in white space
# -q - be quiet
diff -arq folder1 folder2
# compare two directories recursively and only print out differences
cmp file1 file2 # compare binary files, returns 0 if the same
cmp --silent $old $new || echo "files are different"
# check if files are different
diff3 MY-FILE COMMON-FILE YOUR-FILE
# compare three files at once
diff -Nur originalfile newfile > patchfile
# create a patch file
sdiff <filename1> <filename2>
# compare files side by side
md5sum <filename> # get the MD5 hash of a file
shasum <filename> # get the SHA-1 hash of a file
vimdiff <filename1> <filename2>
diffuse <filename1> <filename2>
patch -p1 < patchfile # apply a patch to an entire directory tree
patch orginialfile patchfile
# apply a patch to a file
Copy files
dd if=textFrom of=textTo conv=ucase
# copy a text file while converting to upper case
dd if=/dev/sda of=sda.mbr bs=512 count=1
# backup Master Boot Record (MBR)
cp -p path/to/source path/to/dest
# copy a file;
scp -p [user@]host:pathFrom [user@]host:pathTo
# copy a file; [user@] is only needed if different
rsync -avhu --progress project-X archive-machine:archives/project-X
# sync recursively down the directory tree
# Note: use -dry-run option to test rsync
rsync -avhu --progress path/to/source path/to/dest
# rsync by creating the source directory
# in the dest directory
rsync -avhu --progress path/to/source/ path/to/dest
# rsync by syncing source directory contents with
# the dest directory contents
# --dry-run - used for testing
# --prune-empty-dirs - do not copy empty directories
# --include='pattern' - always include these files
# --exclude='pattern' - always exclude these files
# Note: use --include BEFORE --exclude
Compress files
gzip * # compress all files in current directory; each file is
# compressed and renamed with a .gz extension
gzip -r projectX # compress all files in the projectX directory
# and its subdirectories
gunzip foo # decompresses foo found in the file foo.gz
# gunzip == gzip -d
bzip2 * # compress all files in current directory; each file is
# compressed and renamed with a .bz2 extension
bunzip2 *.bz2 # decompresses all files with the .bz2 extension
# bunzip == bzip2 -d
$ xz * # compress all files in current directory; each file is
# compressed and renamed with a .xz extension
xz foo # compress the file foo into foo.xz using the default
# compression level (-6) and removes foo
# if compression succeeds
xz -dk bar.xz # decompress bar.xz into bar and does not remove
# bar.xz even if decompression is successful
xz -dcf a.txt b.txt.xz > abcd.txt
# decompress files into standard output
$ xz -d *.xz # decompress files compressed using xz
zip -r foo foo # compress all files and subdirectories under foo to foo.zip
zip backup * # compress all files and place into file backup.zip
zip -r backup.zip ~ # compress all files and subdirectories under ~
unzip backup.zip # extract all files and place under current directory
tar # group files into an archive and compress archive
tar Jcvf mydir.tar.xz mydir
# compress files in mydir directory to mydir.tar.xz
find /home/Matlab -name "*.m" -o -name "*.txt" | tar zcvf AllmFiles20170427.tar.gz -T -
find /home/Matlab -name "*.m" -o -name "*.txt" | tar jcvf AllmFiles20170427.tar.bz2 -T -
find /home/Matlab -name "*.m" -o -name "*.txt" | tar Jcvf AllmFiles20170427.tar.xz -T -
# compress files of the same type
# z means use gzip;
# j means use bzip2;
# J means use xz;
# c means create;
# v means verbosely list files processed;
# f means use archive file
# -T FILE means get names to extract from FILE
tar xvf mydir.tar # extract files in mydir.tar to the mydir directory
Manipulating text
dos2unix -n a.txt e.txt # convert a Windows text file to a Linux one
echo "Print this" # display string
# -n - omit trailing newline character
sed -e s/deb/bed/ file # replace first occurrence of deb with bed in file
sed -i "s:$1:$2:" file # replace with parameters
sed -e s/deb/bed/g file # replace all occurrences of deb with bed in file
sed -e 1,3s/deb/bed/g file
# replace between lines 1 and 3 in file
# -i - save changes to file (NOTE: not reversible)
sed s/'\/sbin\/nologin'/'\/bin\/bash'/g /etc/passwd
OR sed s:'/sbin/nologin':'/bin/bash':g /etc/passwd
# colon can be used as the delimiter instead
sed s/\r\(\n\)/\1/g file
# replace Windows carriage returns with Linux ones
# \( \) - captures a pattern
# \1 - the first captured group of characters
find . -type f | xargs sed -i "s/test10/test11/g"
# replace all files in a directory
ls *test12* | xargs sed -i 's/test11/test12/g'
# replace certain files in a directory
sed -n '/PATTERN/,/^$/p' file
# display the lines from a pattern to the first empty line
awk '$0 !~ /your_pattern/'
# display all the lines not matching a pattern
awk 'command' var=value file
# specify a command directly at the command line
awk -f scriptfile var=value file1
# specify a file that contains the script to be
# executed along with file1
awk '{ print $0 }' file # print entire file
awk -F: '{ print $1 $7 }' /etc/passwd
# print 1st and 7th field of every line
awk -F: '{ print $1 "X" }' /etc/passwd
# print 1st field and append a string
awk -F: '{ print substr($3, 2, 5) }' /etc/passwd
# print the 2nd to 6th character of every 3rd field
tr set1 [set2] # translate characters in set1 to set2
tr [a-z] [A-Z] # convert lower case to upper case
tr '{}' '()' < inputfile > outputfile
# translate braces to parentheses
tr [:space:] '\t' # translate white-space to tabs
# -s - squeeze repetition of characters
# -d - delete specified characters
# -c - complement the sets
tr -cd [:print:] < file # remove all nonprintable characters from a file
tr -cd [:digit:] < file # remove all characters except digits from a file
tr -s '\n' ' ' < file # join all the lines in a file into a single line
tee newfile # save input as newfile
Wildcards
? # matches any single character
* # matches any string of characters
[adf] # matches any occurrence of "a", "d", or "f"
[a-s] # matches any occurrence of chars from "a" to "s"
[!adf] # matches any occurrence not one of "a", "d", or "f"
Regular expressions (see more in shell_script_syntax.sh)
. # match any single character
a|z # match a or z
a-z # match a to z
^ # match beginning of string
$ # match end of string
+ # match preceding item 1 or more times
* # match preceding item 0 or more times
? # match preceding item 0 or 1 time
Regular expressions for egrep (see more in shell_script_syntax.sh)
"{x}" # match exactly x occurrences of the preceding
"{x,y}" # match x to y occurrences of the preceding
"{x,}" # match x or more occurrences of the preceding
"( )" # capturing group
Other bash terminology (see more in shell_script_syntax.sh)
let var=$var+1 # increment a variable
eval echo {$v1..$v2} # evaluate variables in brace expansions
eval ${commandstr} # evaluate a string as a command
Arithmetics
bc -l <<< "3.3 - 2.2" # compute floating point arithmetics
# with the bench calculator (bc)
echo "22 / 7" | bc -l # compute floating point arithmetics
Image files
convert File.tif File.jpg # convert from an image format to another
Video files
# Convert video files
ffmpeg -i 2012-04-12-10-53-55.3542.wmv -strict experimental 2012-04-12-10-53-55.3542.mp4
ffmpeg -i truncated_absence_movie.wmv -strict experimental -r 10 truncated_absence_movie_fps10.mp4
# Split video files
MP4Box -splits 262144 2016-09-14_16-55-52.mp4
# Count frames
ffmpeg -i 2019-09-03_11-56-54.mp4 -map 0:v:0 -c copy -f null -
Printing
lp <filename> # print the file to default printer
# -d printer - use a specific printer
# -n number - print multiple copies
lpoptions -d printer # set the default printer
lpq -a # show the queue status
lpadmin # configure printer queues
lpstat -p -d # get a list of available printers with their status
lpstat -a # check the status of all connected printers,
# including job numbers
cancel job-id OR lprm job-id
# cancel a print job
lpmove job-id newprinter# move a print job to new printer
enscript file.txt # print a textfile to default printer
PostScripts and PDFs
evince file.ps OR evince file.pdf
# open PS or PDF file with evince
pdfinfo readme.pdf # Collect information on a PDF file
enscript -p file.ps file.txt
# convert a text file to PostScript
# -4 - place text in 4 columns (can be 1~9)
# -r - rotate to landscape mode
pdf2ps file.pdf # converts file.pdf to file.ps
ps2pdf file.ps # converts file.ps to file.pdf
pdftops input.pdf output.ps OR convert input.pdf output.ps
# converts input.pdf to output.ps
pstopdf input.ps output.pdf OR convert input.ps output.pdf
# converts input.ps to output.pdf
pdftk 1.pdf 2.pdf cat output 3.pdf
# concatenates 1.pdf & 2.pdf into 3.pdf
pdftk A=old.pdf cat A1-2 output new.pdf
# extract pages 1-2 of PDF file
pdftk A=old.pdf cat A1-endright output new.pdf
# rotate all pages of a PDF file 90 degrees clockwise
pdftk public.pdf output private.pdf user_pw PROMPT
# Add a password to encrypt a PDF file
gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite \
-sOutputFile=all.pdf 1.pdf 2.pdf 3.pdf
# combine three PDF files into one
gs -sDEVICE=pdfwrite -dNOPAUSE -dBATCH -dDOPDFMARKS=false \
-dFirstPage=10 -dLastPage=20 -sOutputFile=split.pdf file.pdf
# extract pages 10-20 of PDF file
Markdowns and HTMLs
pandoc --from html --to markdown doc.html -o doc.md
# converts an HTML file to a markdown file
File ownership
umask # check current file permissions mask (default 0022)
umask 0002 # change file permissions to disallow other-write only
# Note: this is currently the default for
# all users on chalkboard and fishfish
chown root file # change ownership of a file or directory
chgrp LabFolks file # change group ownership
chown adam:LabFolks file # change ownership and group at the same time
chmod uo+x, g-w file # change permissions for a file
# u - user
# g - group
# o - everyone else
# a - u + g + o
# + - add permission
# - - remove permission
# = - set permission
# r - read
# w - write to or edit
# x - execute
sudo chmod -R 775 /adamX/
# change permissions recursively for a directory
# 1 - execute
# 2 - write
# 4 - read
# 5 - read/execute
# 6 - read/write
# 7 - read/write/execute
chmod +x testat.sh # make testat.sh executable
getfacl file # get file access control list
setfacl -bn file # remove access control list from file
# without recalculating the effective rights mask
Rivanna
whois -h whois.virginia.edu chen, andrew
allocations # checks allocation
sfsq # checks amount of scratch space available
queues # checks partitions that are available
module load matlab
module avail
module list
module load mymod/N.M
module load mymod/compiler</N.M>
module remove mymod
module purge
sbatch job_script.slurm
sbatch --array=0-30 myjob.sh # job array, numbered from 0 to 30
sbatch --array=1-7:2 myjob.sh # numbers them 1, 3, 5, 7
sbatch --array=1,3,4,5,7,9 myjob.sh
sbatch --dependency=afterok:18375 job_2.slurm # Suppose job_1.slurm got the job ID 18375
# afterok/after/afterany/afternotok
squeue -u al4ng # PD - pending; R - running; S - suspended; CG - exiting
queues
jobq
scancel 18316
./myexec myinput.${SLURM_ARRAY_TASK_ID}.in
jobe -v # checks usage/efficiency
sacct
sacct --format=jobID --format=jobname --format=Elapsed --format=state
/usr/bin/firefox # opens a Firefox browswer
Must do for fastX if ~/.bashrc and ~/.bash_profile do not already exist
cp /share/apps/templates/bash_files/bash_profile ~/.bash_profile
cp /share/apps/templates/bash_files/bashrc ~/.bashrc
Interactive jobs in SLURM
salloc # request interactive resources on frontend
ijob -c <cores> -A <account> -p <partition> -t <minutes>
# run ijob with no options to see all possible options
MPI
module load openmpi/gcc/4.8.5/2.0.0
gfortran mycode.f90
gcc mycode.c
gxx mycode.cxx
mpirun -np 4 ./a.out
git
git --version # get the version of git
git help status # get the manual page of git status
git init # initialize a .git directory under current directory
git status # get status of the repository
git config --global user.name "luadam4c"
# add user name to account
git config --global user.name
# check user name
git config --global user.email "al4ng@virginia.edu"
# add user email to account
git config --global user.email
# check user email
git config --global credential.helper cache
# Set git to use the credential memory cache
git config --global core.editor "vim"
# Set the default git editor to be vim
git submodule add url/to/submodule.git path/to/submodule
# add a submodule
git submodule init # initialize all submodules
git submodule update # update all submodules from the current project
git submodule foreach git pull origin master
# update all submodules from its own repositories
git branch # show all branches
git branch newbranch # create a new branch
git branch -d otherbranch # delete a branch
git checkout -b newbranch # create and change to new branch
git checkout otherbranch # change to an existing branch
git merge otherbranch # merge the changes in otherbranch to
# the current branch
git add . # add all files in the directory to repository
git add -A # add all files
echo "*.png" >> .gitignore # never track png files
git reset # unstage all files
git reset HEAD <file> # unstage a file
git rm --cached <file> # untrack a file
git rm -r --cached <file> # untrack a directory
git commit -a # commit all files recursively
git commit -m 'message' # commit with a message
git reset --soft HEAD~ # undo the most recent commit
git checkout -- <file> # revert a file back to the last commited version
git log # view git log
git log -3 # view the last three commits
git diff <file> # view the difference between the file
# and the last commited version
git diff <SHA-1 hash> # view the differences between the given commit
# and the current version
git diff <SHA-1 hash> <file># view the differences between the given commit
# and the current version for a file
git diff HEAD HEAD^ # view the difference between the last commit
# and the second last commit
git diff branch1..branch2 # view the difference between branch1 and branch2
git diff branch1...branch2 # view the difference between branch1, branch2
# and common ancestor
git clone http://github.com/uvasomrc/git-intro-somrc
# fork from github to a directory with the same name
git remote add origin https://github.com/luadam4c/minEASE.git
# link the local git repository to
# a remote git repository (named origin locally)
git push -u origin master # push the existing local git repository to
# the remote git repository
git push origin master # push to the master branch of the remote repository
git push # push to the default branch of the remote repository
git pull origin master # pull from the master branch of the remote repository
git pull # pull from the default branch of the remote repository
git remote -v # list the current configured remote repository for your fork
git remote add upstream https://github.com/uvasomrc/git-intro-somrc.git
# specify a new remote upstream repository that
# will be synced with the fork
git fetch upstream # sync the local repository with the upstream
Matlab commands
mcc -m -v parallelpcalc1 # Matlab Compiler
AWS commands
s3cmd ls # list all buckets
s3cmd sync --delete-removed <localPath> <bucketPath>
# sync to bucket
s3cmd sync --skip-existing --delete-removed <localPath> <bucketPath>
# sync to bucket without checking all files
aws s3 sync --delete <localPath> <bucketPath>
# sync to bucket
Hugo commands
hugo help # get documentation on all commands
hugo -v # build the website (public directory) in verbose mode
hugo server # build the website on localhost:1313
hugo -D server # build the website on localhost:1313 drafts-included
For the vim settings file ()
set nobackup " no backup files
set nowritebackup " no backup files while editing
set noswapfile " no swap files
set swapfile " use swap files
set dir=~/tmp " set the directory to store swap files