View on GitHub

Settings_Linux

Useful bash scripts and settings for managing a Linux server

Common Unix Commands

Last Updated 2020-12-24


Filesystems

lsb_release -a              # Check Ubuntu version
fsck.ext4 /dev/sda          # File system check

#   If mounting on a non-empty directory, the contents of that directory 
#   will be inaccessible until the filesystem is unmounted
mount [-l]                  # list all mounted filesystems
sudo umount -l /dev/sdb     # unmount a disk
sudo mount -a               # mount all disks listed in the /etc/fstab file
sudo mount <device node> <mount point>  # mount a disk
sudo mount /dev/sdb /media/markX        # mount
df -Th                      # (disk-free) display info about mounted filesystems
                            #   including disk usage
du -sh *                    # return a list of sizes from a directory
                            #   -s - (summarize) display only a total
                            #   -h - print in human-readable format
ncdu                        # open the disk usage analyzer
sudo systemctl start nfs    # start NFS daemons
exportfs -av                # notify Linux about the dirs to be remotely mounted
sudo systemctl enable nfs   # start NFS daemons whenver system is booted
sudo mount servername:/projects /mnt/nfs/projects
lsblk                       # list all block devices
ls -l /dev/disk/by-label    # list all detected disks by their label
sudo e2label /dev/disk label# change disk label

System Control

sudo systemctl start/stop/restart nfsd
sudo systemctl enable/disable nfsd

sudo systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target     
                            # disable sleep, suspend, hibernate
sudo systemctl unmask sleep.target suspend.target hibernate.target hybrid-sleep.target     
                            # re-enable sleep, suspend, hibernate

Documentation

help                        # display list of built-in commands
man printf                  # display manual page for printf
                            #   man pages are divided into chapters 1~9
man 3 printf                # display manual page for printf from chapter 3
man -a printf               # display all manual pages for printf 
                            #   from each chapter sequentially
man 1 test                  # display list of file test operations
type NAME                   # display information about command type
                            #   -a	display all locations containing 
                            #       an executable named NAME
                            #   -p	returns the name of the disk file 
                            #       that would be executed
                            #   -t	output a single word, one of:
                            #       'alias', 'keyword', 'function', 
                            #       'builtin', 'file'
whatis string               # same as man -f string
                            #   view all pages containing a string
apropos subject             # same as man -k subject
                            #   view all pages that discuss a subject
info subject                # view info pages about subject
program --help              # display short description of program
help echo                   # same as --help for built-in commands
yelp man:cat                # open the man page of cat with 
                            #   the graphical help system

Package management

sudo apt-get install gedit-plugins  # for gedit
dpkg --install foo.deb      # install or update package
                            #   RPM: rpm -i foo.rpm OR rpm -U foo.rpm
apt-get install foo         # install or update package, dependencies
                            #   RPM: yum install foo OR yum update foo
dpkg --remove foo.deb       # remove package
                            #   RPM: rpm -e foo.rpm
apt-get autoremove foo      # remove package, dependencies
                            #   RPM: yum remove foo
apt-get dist-upgrade        # update entire system
                            #   RPM: yum update
dpkg --list OR dpkg -l      # show all installed packages
                            #   RPM: rpm -qa OR yum list installed
dpkg --listfiles foo        # get information on package
                            #   RPM: rpm -qil foo
apt-cache search foo        # show packages named foo
                            #   RPM: yum list "foo"
apt-cache policy foo        # check status of packages named foo
                            #   RPM: yum info foo
apt-cache dumpavail         # show all available packages
                            #   RPM: yum list
dpkg --search file          # what package is file part of?
                            #   RPM: rpm -qf file

Run & stop program

bash script.sh              # run script with bash
./script                    # provide path to run executable
<command> &                 # run job in the background
bg                          # put a process in the background
fg                          # put a process in the foreground
sleep 5                     # pause execution for 5 seconds
sleep 2m                    # pause execution for 2 minutes
sleep 3h                    # pause execution for 3 hours
sleep 5d                    # pause execution for 5 days

#### Schedule commands
‚Äč```bash
at now + 2 days             # schedule a command 2 days from now
    <command>               # the command to schedule
    <EOT>                   # Press CTRL + D   
atq                         # view job queue
crontab -e                  # open the crontab editor for current user
sudo crontab -u username -e # open the crontab editor for a specific user
* * * * * XXX.sh            # execute XXX.sh (in home directory) every minute
                            #   Note: must provide full path if script 
                            #       is not in the home directory
30 08 10 06 * backup.sh     # execute backup.sh at 8:30 am, 10-June
                            #   irrespective of the day of the week
inotifywait                 # wait for changes to files or directories
                            #   -m - (monitor) execute indefinitely
                            #   -e - event

Troubleshoot

startx                      # initialize an X session
bash -x myscript.sh         # run script in debug mode
time command                # time a simple command
w                           # show who's logged on and what they are doing
top                         # display all running tasks
top -u adam                 # display all running tasks of a specific user
top -p $(pgrep -d ',' -f MATLAB)
                            # display all running tasks of a specific command                           
htop OR atop                # TODO
free                        # display memory usage
uptime                      # tell how long the system has been running
                            #   and display load averages over 
                            #   the last min, last 5 mins and last 15 mins
jobs -l                     # display all jobs running in the background
grep CRON /var/log/syslog   # check if CRON jobs ran
ps                          # display processes running under current shell
ps -A                       # display all running processes 
ps -u                       # display processes for the current user
ps -fu adam                 # display all processes for a given user
ps -ef                      # display all processes in the system
ps -eLf                     # display all threads in the system
ps aux                      # display all processes of all users
ps axo stat,priority,pid,pcpu,comm  # display specific attributes for processes
pidof dd                    # display the process number of dd
pgrep ^dd                   # list process IDs which matches selection criteria
pstree                      # display processes as a tree diagram
                            #   threads are in curly braces
pkill -STOP -u adam         # logout user
ps -dN|grep pts/3           # find parent process of session
kill -SIGKILL <pid>         # terminate a process
kill -9 <pid>               # terminate a process
sudo systemctl stop gdm     # stop the GUI desktop (CentOS and SUSE)
sudo systemctl stop lightdm # stop the GUI desktop (Ubuntu)
sudo telinit 3              # stop the GUI desktop
sudo systemctl start gdm    # start the GUI desktop (CentOS and SUSE)
sudo systemctl start lightdm# start the GUI desktop (Ubuntu)
sudo telinit 5              # start the GUI desktop
sudo systemctl restart network
    OR  sudo service network restart
                            # restart the network
sudo systemctl restart NetworkManager
    OR  sudo NetworkManager restart
                            # restart the NetworkManager
sudo systemctl status webmin
                            # check webmin status
sudo systemctl restart webmin
                            # restart webmin
sudo dhclient eth0          # get a fresh IP address
sudo mtr --report-cycles 3 8.8.8.8
                            # trace route to 8.8.8.8
gdb -c core                 # find out what program caused a core dump file
nvidia-smi                  # print the NVIDIA driver version
nvidia-settings             # open the NVIDIA driver settings GUI
ubuntu-drivers devices      # show recommended nvidia drivers

Network & Remote login

ifconfig                    # information about network interfaces (older)
ip                          # information about network interfaces (newer)
ip addr show                # view IP address
ip route show               # view routing information
ping -c 172.28.77.1       # check network status
route -n or ip route        # show current routing table
route add -net address OR ip route add
                            # add static route
route del -net address OR ip route del
                            # delete static route
traceroute google.com       # inspect the route to google.com
host google.com             # get IP address of specified host
nslookup google.com         # look up name servers interactively
dig google.com              # look up domain name info from name server
sudo ethtool eth0           # querie network interfaces and set parameters
netstat -r                  # display all active connections and routing tables
                            #   -r  - display kernel routing tables
sudo nmap -sP 10.0.2.15/24  # scans open ports on the network
tcpdump                     # dumps network traffic for analysis
iptraf                      # monitors network traffic in text mode
mtr                         # combines ping and traceroute
sudo ufw allow proto tcp from 172.25.228.11 to any port 445
22                          # secureCRT
445                         # windows samba server
5900:5920                   # vnc server
10000                       # webmin
sudo ufw status numbered
sudo ufw delete ####
sudo ufw delete allow 80/tcp
ssh -Y adam@172.28.77.1   # Secure Shell (SSH) to chalkboard
ssh -Y adam@172.28.76.139  # fishfish
ssh -Y al4ng@sp-udc-led-cs01.hpc.virginia.edu
                            # Rivanna udc-ba-36-37
ssh -Y al4ng@rivanna1.hpc.virginia.edu
                            # Rivanna udc-ba-36-36
ssh -Y al4ng@rivanna2.hpc.virginia.edu
                            # Rivanna udc-ba-34-36
ssh -Y al4ng@rivanna3.hpc.virginia.edu
                            # Rivanna udc-ba-35-36
ssh -Y al4ng@rivanna.hpc.virginia.edu
                            # Rivanna, random?
ssh al4ng@rivanna.hpc.virginia.edu command
                            # perform command remotely on rivanna
ssh -l al4ng rivanna.hpc.virginia.edu whoami
                            # perform whoami remotely on rivanna
scp -p al4ng@sp-udc-led-cs01.hpc.virginia.edu:/home/al4ng/test.text /media/adamX/.
scp -p /media/adamX/test.text al4ng@interactive.hpc.virginia.edu:/home/al4ng/. 
scp -p Common_Unix_commands.txt al4ng@sp-udc-led-cs01.hpc.virginia.edu:/scratch/al4ng/Miscellaneous_backup
                            # upload stuff to Rivanna 
rsync -au /media/adamX/m3ha/data_dclamp/*.slurm al4ng@sp-udc-led-cs01.hpc.virginia.edu:/scratch/al4ng/m3ha/data_dclamp/
                            # upload stuff to Rivanna 
wget -O file.html <URL>     # download webpage to file.html
curl -o file.html <URL>     # read information about a URL and save to file
curl -O <URL>               # read and save with same name as URL
curl "http://httpbin.org/get?city=Baltimore&state=Maryland"
                            # send a general HTTP GET request
ftp ftp.gnu.org             # start ftp
                            #   get FILE - download a file
ssh-keygen                  # generate a public/private rsa key pair
                            #   for the current user in the ~/.ssh/ directory
ssh-copy-id -i ~/.ssh/id_rsa.pub 128.143.16.164
ssh-copy-id -i ~/.ssh/id_rsa.pub al4ng@rivanna.hpc.virginia.edu
                            # place the public key in the ~/.ssh/authorized_keys
                            #   file of the remote host

VNC server

vncserver -geometry 1366x760        # open a VNC server session with
                                    #   a certain window resolution
vncserver -kill :1                  # kill a VNC server session
vncconfig &                         # show clipboard options
vncconfig -nowin &                  # allow copy and paste without showing
                                    #   an options window
ps -ef | grep vnc                   # get info for all the active vnc sessions

Printing

sudo /etc/init.d/cups start         # start the CUPS daemon manually
sudo service cups start             # ditto in CentOS
sudo /etc/init.d/cups restart       # restart the CUPS daemon manually
sudo service cups restart           # ditto in CentOS
sudo /etc/init.d/cups status        # check status of CUPS daemon 
sudo service cups status            # ditto in CentOS
sudo /etc/init.d/cups stop          # stop the CUPS daemon manually
sudo service cups stop              # ditto in CentOS
sudo update-rc.d cups -f defaults   # enable CUPS daemon after installation
sudo update-rc.d cups enable        # start CUPS daemon at boot time
sudo chkconfig cups on              # ditto in openSUSE or CentOS
sudo update-rc.d cups disable       # do not start CUPS daemon at boot time
sudo chkconfig cups off             # ditto in openSUSE or CentOS

System date and time

sudo ntpdate -q ntp.ubuntu.com
                        # compare time with ntp
sudo ntpdate-debian     # sync time with ntp

Rebooting and Shutting Down

shutdown                # preferred method from shutting down;
                        #   sends a warning message, 
                        #   then prevents further users from logging in
sudo shutdown -h now
sudo shutdown -h 10:00 "Shutting down for scheduled maintenance"
sudo halt               # same as sudo shutdown -h
sudo poweroff           # same as sudo shutdown -h
sudo reboot             # same as sudo shutdown -r

System information

uname -a                # print all system information 
uname -r                # print Linux kernel version
xdpyinfo | grep dim     # get screen resolution
lscpu                   # prints CPU architecture information from 
                        #   sysfs and /proc/cpuinfo
cpuid                   # dumps complete information about the CPU(s) 
                        #   collected from the CPUID instruction, 
                        #   and also discover the exact model of 
                        #   x86 CPU(s) from that information
sudo update-pciids      # grab the current version of the pci.ids file 
                        #   from the Internet
lspci                   # list PCI devices
                        #   -v - (verbose) list detailed info for all devices
hostname                # get system hostname
sudo hostname fishfish  # change system hostname
date                    # get current date and time
date +"%Y%m%dT%H%M"     # display date and time in YYYYMMDDTHHMM format
date -u +%s             # display number of seconds 
                        #   since 1970-01-01 00:00:00 UTC
date -u +%s.%N          # display number of seconds with decimal points
                        #   since 1970-01-01 00:00:00 UTC
date +%u                # display the day of the week (1..7)
date --date='-2 day'    # display date two days in the past
date --date='2 day ago' # display date two days in the past
date --date='yesterday' # display yesterday's date
date --date='next day'  # display tomorrow's date
date --date='2 day'     # display date two days in the future
cal                     # display a calendar for the current date

Shell formatting

bash --version          # get the version of bash
/etc/bashrc             # bash shell configuration for all users
~/.bashrc               # bash shell configuration for current user
source script.sh        # use function definitions in a bash script 
                        #   as command line commands
source ~/.bashrc OR . ~/.bashrc
                        # refresh bash shell configuration
clear                   # clear previous outputs

Environmental variables (see more in shell_script_syntax.sh)

set                     # list environmental variables
env                     # list environmental variables
printenv                # list environmental variables
export                  # list all exported environmental variables
echo $SHELL             # show the value of an environmental variable
$RANDOM                 # get a random number
export VARIABLE=value OR VARIABLE=value; export VARIABLE
                        # export a new variable value to make available 
                        #   for a child process
                        #   put this is in ~/.bashrc to make permanent
export PATH=$HOME/bin:$PATH
                        # export a private bin directory to search path
echo $PS1               # display default command line prompt
PS1="\u@\h:\w[\t]\$"    # format the default command line prompt
                        #   \u - User name
                        #   \h - Host name
                        #   \w - Current working directory
                        #   \! - History number of this command
                        #   \d - Date
                        #   \t - Time
PS2=">"                 # format the second-tier command line prompt
LS_COLORS               # coloring of different file or dir types

Command line history

history                 # view the list of previously executed commands 
                        #   of the current terminal
cat ~/.bash_history     # view the list of previously executed commands 
                        #   of previous terminals
!! OR !-1               # execute the previous command
!                       # start a history substitution
!$                      # refer to the last argument in a line
!n                      # refer to the nth command line
!str                    # refer to the most recent command starting with str

User accounts

who                     # list the currently logged on users
                        #   -a  - more detailed information
                        #   -m  - only the user associated with stdin
who am i                # alias for who -m
whoami                  # identify the current user
id                      # give information about current user
id -u                   # print effective user ID
id -g                   # print effective group ID
last                    # show listing of last logged in users
sudo vi /etc/passwd     # edit user account info
sudo vi /etc/group      # edit user group info
sudo useradd bjmoose    # add user bjmoose
sudo /usr/sbin/useradd bjmoose # for openSUSE
sudo userdel bjmoose    # delete user bjmoose without deleting home directory
sudo userdel -r bjmoose # delete user bjmoose along with its home directory
sudo groupadd anewgroup # add group
sudo groupdel anewgroup # delete group
groups bjmoose          # list all groups that include bjmoose
newgrp BarrettRivanna   # make BarrettRivanna the primary group
                        #   Note: this only lasts for the current session
sudo usermod -a -G anewgroup bjmoose
                        # add bjmoose to anewgroup
                        #   -a  - append
sudo usermod -G group1 group2 bjmoose
                        # make bjmoose a member of group1, group2 
                        #   and remove from all other groups
sudo groupmod -g 1004 anewgroup    
                        # change group id of anewgroup to 1004
sudo groupmod -n testgroup anewgroup
                        # change name of anewgroup to testgroup
sudo visudo             # edit the /etc/sudoers file
                        #   who where = (as_whom) what
su                      # launch a shell as root
su adam2                # launch a shell as adam2 (substitute user)
alias                   # list currently defined aliases
alias +='pushd'         # add an alias
unalias                 # remove an alias
vi ~/.bash_aliases      # add aliases here for all future shells

Set passwords

sudo passwd <username>
sudo smbpasswd <username>
sudo vncpasswd <username>
echo -n <passwd> | sha512sum
                        # print password in SHA-512 encrypted form 
chage --list <username> # see the last time password was changed

Filesystem management

which diff              # locate where the diff program resides
whereis diff            # similar to which but with broader range, and locates
                        #   the source and man files packaged with the program
echo $HOME              # displays path to home directory
pwd                     # displays the present working directory
                        #   -P avoid symbolic links
readlink -f file.txt    # get path to a file
readlink -f .           # get path to current directory
dirname file.txt        # get the directory containing a file
file *                  # determine file type for all files in directory
file --mime *           # determine the MIME file type for all files
file -b file            # determine file type without prepending with filename
cd subdir               # change to a subdirectory of the present working directory
cd /path/to/dir         # change to a directory using the full path
cd OR cd ~              # change to home directory ($HOME)
cd ..                   # change to parent directory
cd -                    # change to previous directory
cd /                    # change to root directory
pushd /tmp/dir1         # change to /tmp/dir1 and add it to a 
                        #   list of recently viewed directories
popd                    # change to last directory on the 
                        #   list of recently viewed directories
dirs                    # display list of recently viewed directories
////user//bin           # seen as /usr/bin by the system
ls                      # list the contents of the present working directory
                        #   -1 - list each file on a separate line
                        #   -F - directories end with / & executables end with *
                        #   -l - long listing format
                        #   -a - show hidden files & directories
                        #   -r - reverse order while sorting
                        #   -t - sort by newest first
                        #   -R - recursively show all files within subdirectories
                        #   -i - prints the inode number in the first column
                        #   -d - show directories only and not their contents
                        #   --full-time - list full time info
ls -1 | wc -l           # count files in current directory
ls -d */                # list subdirectories only
ls -l | grep '^d'       # list subdirectories only, long-listing format
ls -l | grep -v '^d'    # list files only, long-listing format
ls -li file?            # list all files that begin with file
ls -lart | grep "Feb"   # list all files that were created in February
find -maxdepth 1 -type d | sort | while IFS= read -r dir; do n=$(find "$dir" -type f | wc -l); printf "%s\t%s\n" "$dir" "$n"; done | column -t -s $'\t'
                        # list subdirectories with file count
tree                    # displays a tree view of the filesystem
tree -d                 # just the directories
ln </path/to/file-name> <link-name>     # create hard link 
ln -s </path/to/file-name> <link-name>  # create soft (symbolic) link 

Directories

mkdir sampdir           # make directory
mv sampdir sampdir2     # rename a directory
rename p_d d p_d*       # rename many directories (Rivanna version)
                        #   -v - list what directories were renamed
rename 's/p_d/d/' p_d*  # rename many directories (fishfish version)
                        #   -v - list what directories were renamed
                        #   -n - see what directories would be renamed
rmdir sampdir           # remove an empty directory
rm -rf sampdir          # forcefully remove a directory recursively

Files

touch Top.out           # create file
touch -t 03201600 file  # set the date and time stamp of file to 
                        #   4 p.m., March 20th
cat Top.out             # display file content
column -t -s $'\t' file # format tab-separated file into columns
mktemp /tmp/tmpfile.XXXXXXXX
                        # create a temporary file
                        #   mktemp will replace XXXXXXXX with a number
mktemp -d /tmp/tmpdir.XXXXXXXX
                        # create a temporary directory
cat > file1             # write to file1
cat >> file2            # append to file2
cat 1>> file3           # append stdout to file3
cat 2>> file3           # append stderr to file3
echo "line1" > file.txt # write to file
echo "line2" >> file.txt # append to file
# cat << EOF > file2.txt  # get standard input before EOF and write to file
tac Top.out             # display file content backwards
less Top.out            # view file content with scroll-back capabilities
                        #   / to search for a pattern in the forward direction
                        #   ? to search for a pattern in the backward direction
head Top.out            # display the first 0 lines of a file
head -15 Top.out        # display the first 15 lines of a file
head -n 20 Top.out      # display the first 20 lines of a file
tail Top.out            # display the last 10 lines of a file
tail -n +2 Top.out      # display file starting from the 2nd line
sed -n '6,10p' file     # display the file from line numbers 6 through 10
sed -n '$=' "${file}"   # display the number of lines in a file
zcat compressed.txt.gz  # view a compressed file
zless file.gz           # page through a compressed file
zgrep -i less file.gz   # search inside a compressed file
zdiff file1.gz file2.gz # compare two compressed files
select-editor           # select default editor
sudo update-alternatives --config editor
                        # configure default editor
nano file.txt           # edit a file with nano
vi Top.out              # open file using the vi editor
vi +100 Top.out         # open file at the 100th line
vi +/str Top.out        # open file at the first occurrence of str
vi -r Top.out           # edit Top.out in recovery mode from a system crash
mv Top.out Top2.out     # rename a file
rename .out .txt *.out  # rename many files
rm Top.out              # remove a file
unlink symlink          # unlink a symbolic link
rm -f Top.out           # forcefully remove a file
rm -i Top.out           # interactively remove a file

Input/Output of files (see more in shell_script_syntax.sh)

command < input-file
command > output-file
command 2> error-file
command > all-output-file 2>&1 
                        # put anything written to file descriptor 2 
                        #   in the same place as file descriptor 1
command >& all-output-file
                        # same as command > all-output-file 2>&1 in bash
command1 | command2     # command2 doesn't have to wait until command1
                        #   completes to read in the latter's output as input
command > /dev/null     # ignore standard output
command >& /dev/null    # ignore standard output and standard error
read var                # read input stream

Manipulating files

wc file                 # print number of lines, words, characters
wc -l file              # print number of lines
wc -w file              # print number of words
wc -c file              # print number of bytes
cat file1 file2 > file3 # combine file1 and file2 into file3
cat file1 >> file3      # append file1 to file3
paste file1 file2       # combine columns of file1 and file2 
                        #   default delimiter is \t
paste -d, file1 file2   # use , as the delimiter
cut -f 2 tabfile        # cut out 2nd column
                        #   -d  - delimiter
join file1 file2        # combine two files on a common field
sort file               # sort the lines in file according to the first field
sort -k 3 file          # sort the lines in file according to the 3rd field
                        #   -r  - sort in reverse order
                        #   -n  - use numeric order
                        #   -t  - delimiter
                        #   -u  - remove duplicate lines
uniq file               # remove consecutive duplicate lines in file
uniq -c filename        # count consecutive duplicate lines in file
split infile            # split file into files of 1000 lines

Search for files or directories

locate zip | grep bin   # list all the files and directories 
                        #   with both "zip" and "bin" in their name
updatedb                # update the database for locate
find . -name "*.log"
                        # find all .log files in current directory recursively
                        #   -name  - only list files with pattern in name
                        #   -iname - also ignore the case of file names
                        #   -type  - restrict the results to a certain type
                        #   -size  - restrict the results to a certain size
                        #   -maxdepth   - maximum number of levels down the tree
                        #   -not -path  - exclude a path
find -L . -name "*.log" # find all .log files in current directory recursively
                        #   following symbolic links
find . -name "*.log" -not -path "./backup/*"
find . -path ./backup -prune -o -name '*.log' -print
                        # two ways of excluding entire directories
sudo find / -type d -name "nrnoc"   
                        # find directories containing "nrnoc" in root
find . -maxdepth 1 -type f -name "*.m"
                        # find all files with a given name non-recursively
find . -type l          # find symbolic links recursively
find -L . -type l       # find broken symbolic links recursively
                        #   Note: use symlinks -dr instead
find / -ctime 3         # find files when the inode metadata 
                        #   was last changed 3 days ago
find / -atime +3        # find files accessed/last read greater than 3 days ago
find / -mtime -3        # find files modified/last written less than 3 days ago
find / -size 1          # find files with size greater than 0 
                        #   but less than 512 bytes
find / -size 6          # find files with size greater than 5 * 512 bytes 
                        #   but less than 6 * 512 bytes
find / -size -7c        # find files with size less than 7 bytes
find / -size +7k        # find files with size greater than 7 kilobytes
find / -size +7M        # find files with size greater than 7 megabytes
find / -size +7G        # find files with size greater than 7 gigabytes
symlinks -v .           # show all symbolic links in current directory
symlinks -d .           # delete all broken symbolic links
symlinks -dr .          # delete all broken symbolic links recursively

Search and perform actions

find . -maxdepth 1 -type f -name "*.m" | xargs -I @ basename @ '.m'
                        # find all files with a given name and grep basenames
find . -maxdepth 1 -type f -exec mv {} Introduction\ To\ Linux/ \;
                        # find and move files found
find . -maxdepth 1 -type f | grep test12 | xargs -I @ mv @ 20180124_singleneuronfitting17_test12_Rivanna/
                        # find and move files found
find -name "*.swp" -exec rm {} ':'
OR find -name "*.swp" -ok rm {} "\;"
                        # find and remove all files found
                        #   -exec - run command on search results
                        #   -ok   - prompt user for permission before execution
find /media/shareX/ -type f -name "*.wmv" -size +1M -size -10M -exec ls -lh {} \;
                        # find all files smaller than a particular size
find /media/shareX/ -type f -name "*.wmv" -size -512k | tail -1 | xargs -I @ cp @ /media/shareX/share/Vignesh/
                        # find and copy
find /media/shareX/ -type f -name "*.wmv" -size +10M -size -2G | tail -1 | xargs -I @ cp @ /media/shareX/share/Vignesh/
                        # find and copy
grep [regex] file       # search for a pattern in file and print matching lines
                        #   -v - print lines that do not match the pattern
                        #   -C n - print n lines above and below pattern 
                        #   -r - recursively find
                        #   -n - prefix with line number 
                        #   -w - must match PATTERN as a whole word
                        #   -i - case-insensitive
                        #   -l - print file names with matches
                        #   -m - maximum number of lines to match
                        #   -e PATTERN - use PATTERN as pattern
                        #   -o - extract only the matching parts of each line
egrep "regex" file      # search for a pattern in file and print matching lines
grep [regex] <<< "str"  # search for a pattern in a string
grep -n "#REMOVE" template.sh | while read line ; do echo ${line%%:*} ; done
                        # find and print line numbers only
grep -n "regex" file | cut -f 1 -d ":"
                        # find and print line numbers only
grep --include="*.m" -rlw '/media/adamX/m3ha/optimizer4gabab/' -e "addpath" | grep -v backup | grep -v singleneuronfitting
                        # find all files under a directory that contain string 
grep --include=*.m -rnw '/home/Matlab/' -e "make_time_column_NEW"
                        # find all files under a directory that contain string
                        #   and print it 
grep --include="*.m" -rlw '/home/Matlab/' -e "make_time_column_NEW" | xargs -I @ sed -i.backup 's/make_time_column_NEW/make_time_column/g' @
                        # replace string in many files and create backup for original
strings book1.xls       # extract all printable character strings in a file
strings book1.xls | grep my_string
                        # search for my_string inside a binary file
strings --print-file-name /usr/bin/a* | grep GPL
                        # print all programs withe GPL licenses
strings a.url | grep URL# get web address from link

Comparing files and file types

diff [options] <filename1> <filename2>
                        # compare text files and directories
                        #   -c - list 3 lines of context
                        #   -r - recursively compare subdirectories
                        #   -i - ignore case of letters
                        #   -w - ignore differences in white space
                        #   -q - be quiet
diff -arq folder1 folder2 	
						# compare two directories recursively and only print out differences
cmp file1 file2         # compare binary files, returns 0 if the same
cmp --silent $old $new || echo "files are different"
                        # check if files are different
diff3 MY-FILE COMMON-FILE YOUR-FILE
                        # compare three files at once
diff -Nur originalfile newfile > patchfile
                        # create a patch file
sdiff <filename1> <filename2>
                        # compare files side by side
md5sum <filename>       # get the MD5 hash of a file
shasum <filename>       # get the SHA-1 hash of a file
vimdiff <filename1> <filename2>
diffuse <filename1> <filename2>
patch -p1 < patchfile   # apply a patch to an entire directory tree
patch orginialfile patchfile
                        # apply a patch to a file

Copy files

dd if=textFrom of=textTo conv=ucase
                        # copy a text file while converting to upper case
dd if=/dev/sda of=sda.mbr bs=512 count=1
                        # backup Master Boot Record (MBR)
cp -p path/to/source path/to/dest
                        # copy a file;
scp -p [user@]host:pathFrom [user@]host:pathTo
                        # copy a file; [user@] is only needed if different
rsync -avhu --progress project-X archive-machine:archives/project-X
                        # sync recursively down the directory tree
                        # Note: use -dry-run option to test rsync
rsync -avhu --progress path/to/source path/to/dest
                        # rsync by creating the source directory 
                        #   in the dest directory
rsync -avhu --progress path/to/source/ path/to/dest
                        # rsync by syncing source directory contents with 
                        #   the dest directory contents
                        # --dry-run           - used for testing
                        # --prune-empty-dirs  - do not copy empty directories
                        # --include='pattern' - always include these files
                        # --exclude='pattern' - always exclude these files
                        # Note: use --include BEFORE --exclude

Compress files

gzip *                  # compress all files in current directory; each file is 
                        #   compressed and renamed with a .gz extension
gzip -r projectX        # compress all files in the projectX directory 
                        #   and its subdirectories
gunzip foo              # decompresses foo found in the file foo.gz
                        #   gunzip == gzip -d
bzip2 *                 # compress all files in current directory; each file is 
                        #   compressed and renamed with a .bz2 extension
bunzip2 *.bz2           # decompresses all files with the .bz2 extension
                        #   bunzip == bzip2 -d
$ xz *                  # compress all files in current directory; each file is 
                        #   compressed and renamed with a .xz extension
xz foo                  # compress the file foo into foo.xz using the default 
                        #   compression level (-6) and removes foo 
                        #   if compression succeeds
xz -dk bar.xz           # decompress bar.xz into bar and does not remove
                        #   bar.xz even if decompression is successful
xz -dcf a.txt b.txt.xz > abcd.txt
                        # decompress files into standard output
$ xz -d *.xz            # decompress files compressed using xz

zip -r foo foo     		# compress all files and subdirectories under foo to foo.zip
zip backup *            # compress all files and place into file backup.zip
zip -r backup.zip ~     # compress all files and subdirectories under ~
unzip backup.zip        # extract all files and place under current directory
tar                     # group files into an archive and compress archive
tar Jcvf mydir.tar.xz mydir      
                        # compress files in mydir directory to mydir.tar.xz
find /home/Matlab -name "*.m" -o -name "*.txt" | tar zcvf AllmFiles20170427.tar.gz -T -
find /home/Matlab -name "*.m" -o -name "*.txt" | tar jcvf AllmFiles20170427.tar.bz2 -T -
find /home/Matlab -name "*.m" -o -name "*.txt" | tar Jcvf AllmFiles20170427.tar.xz -T -
                        # compress files of the same type
                        # z means use gzip; 
                        # j means use bzip2; 
                        # J means use xz; 
                        # c means create; 
                        # v means verbosely list files processed; 
                        # f means use archive file
                        # -T FILE means get names to extract from FILE
tar xvf mydir.tar       # extract files in mydir.tar to the mydir directory

Manipulating text

dos2unix -n a.txt e.txt # convert a Windows text file to a Linux one
echo "Print this"       # display string
                        #   -n  - omit trailing newline character
sed -e s/deb/bed/ file  # replace first occurrence of deb with bed in file
sed -i "s:$1:$2:" file  # replace with parameters
sed -e s/deb/bed/g file # replace all occurrences of deb with bed in file
sed -e 1,3s/deb/bed/g file
                        # replace between lines 1 and 3 in file
                        #   -i  - save changes to file (NOTE: not reversible)
sed s/'\/sbin\/nologin'/'\/bin\/bash'/g /etc/passwd
    OR sed s:'/sbin/nologin':'/bin/bash':g /etc/passwd
                        # colon can be used as the delimiter instead
sed s/\r\(\n\)/\1/g file
                        # replace Windows carriage returns with Linux ones
                        #   \( \)   - captures a pattern
                        #   \1      - the first captured group of characters
find . -type f | xargs sed -i "s/test10/test11/g"
                        # replace all files in a directory
ls *test12* | xargs sed -i 's/test11/test12/g'
                        # replace certain files in a directory
sed -n '/PATTERN/,/^$/p' file
						# display the lines from a pattern to the first empty line
awk '$0 !~ /your_pattern/'
						# display all the lines not matching a pattern
awk 'command' var=value file
                        # specify a command directly at the command line
awk -f scriptfile var=value file1
                        # specify a file that contains the script to be 
                        #   executed along with file1
awk '{ print $0 }' file # print entire file
awk -F: '{ print $1 $7 }' /etc/passwd
                        # print 1st and 7th field of every line
awk -F: '{ print $1 "X" }' /etc/passwd
                        # print 1st field and append a string
awk -F: '{ print substr($3, 2, 5) }' /etc/passwd
                        # print the 2nd to 6th character of every 3rd field
tr set1 [set2]          # translate characters in set1 to set2
tr [a-z] [A-Z]          # convert lower case to upper case
tr '{}' '()' < inputfile > outputfile
                        # translate braces to parentheses
tr [:space:] '\t'       # translate white-space to tabs
                        #   -s  - squeeze repetition of characters
                        #   -d  - delete specified characters
                        #   -c  - complement the sets
tr -cd [:print:] < file # remove all nonprintable characters from a file
tr -cd [:digit:] < file # remove all characters except digits from a file
tr -s '\n' ' ' < file   # join all the lines in a file into a single line
tee newfile             # save input as newfile

Wildcards

?                       # matches any single character
*                       # matches any string of characters
[adf]                   # matches any occurrence of "a", "d", or "f"
[a-s]                   # matches any occurrence of chars from "a" to "s"
[!adf]                  # matches any occurrence not one of "a", "d", or "f"

Regular expressions (see more in shell_script_syntax.sh)

.                       # match any single character
a|z                     # match a or z
a-z                     # match a to z
^                       # match beginning of string
$                       # match end of string
+                       # match preceding item 1 or more times
*                       # match preceding item 0 or more times
?                       # match preceding item 0 or 1 time

Regular expressions for egrep (see more in shell_script_syntax.sh)

"{x}"                   # match exactly x occurrences of the preceding
"{x,y}"                 # match x to y occurrences of the preceding
"{x,}"                  # match x or more occurrences of the preceding
"( )"                   # capturing group

Other bash terminology (see more in shell_script_syntax.sh)

let var=$var+1          # increment a variable
eval echo {$v1..$v2}    # evaluate variables in brace expansions
eval ${commandstr}      # evaluate a string as a command

Arithmetics

bc -l <<< "3.3 - 2.2"   # compute floating point arithmetics 
                        #   with the bench calculator (bc)
echo "22 / 7" | bc -l   # compute floating point arithmetics

Image files

convert File.tif File.jpg   # convert from an image format to another

Video files

# Convert video files
ffmpeg -i 2012-04-12-10-53-55.3542.wmv -strict experimental 2012-04-12-10-53-55.3542.mp4
ffmpeg -i truncated_absence_movie.wmv -strict experimental -r 10 truncated_absence_movie_fps10.mp4

# Split video files
MP4Box -splits 262144 2016-09-14_16-55-52.mp4

# Count frames
ffmpeg -i 2019-09-03_11-56-54.mp4 -map 0:v:0 -c copy -f null -

Printing

lp <filename>           # print the file to default printer
                        #   -d printer  - use a specific printer
                        #   -n number   - print multiple copies
lpoptions -d printer    # set the default printer
lpq -a                  # show the queue status
lpadmin                 # configure printer queues
lpstat -p -d            # get a list of available printers with their status
lpstat -a               # check the status of all connected printers,
                        #   including job numbers
cancel job-id OR lprm job-id
                        # cancel a print job
lpmove job-id newprinter# move a print job to new printer
enscript file.txt       # print a textfile to default printer

PostScripts and PDFs

evince file.ps OR evince file.pdf
                        # open PS or PDF file with evince
pdfinfo readme.pdf      # Collect information on a PDF file
enscript -p file.ps file.txt
                        # convert a text file to PostScript
                        #   -4  - place text in 4 columns (can be 1~9)
                        #   -r  - rotate to landscape mode
pdf2ps file.pdf         # converts file.pdf to file.ps
ps2pdf file.ps          # converts file.ps to file.pdf
pdftops input.pdf output.ps OR convert input.pdf output.ps
                        # converts input.pdf to output.ps
pstopdf input.ps output.pdf OR convert input.ps output.pdf
                        # converts input.ps to output.pdf
pdftk 1.pdf 2.pdf cat output 3.pdf
                        # concatenates 1.pdf & 2.pdf into 3.pdf
pdftk A=old.pdf cat A1-2 output new.pdf
                        # extract pages 1-2 of PDF file
pdftk A=old.pdf cat A1-endright output new.pdf
                        # rotate all pages of a PDF file 90 degrees clockwise
pdftk public.pdf output private.pdf user_pw PROMPT
                        # Add a password to encrypt a PDF file
gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite \
    -sOutputFile=all.pdf 1.pdf 2.pdf 3.pdf
                        # combine three PDF files into one
gs -sDEVICE=pdfwrite -dNOPAUSE -dBATCH -dDOPDFMARKS=false \
    -dFirstPage=10 -dLastPage=20 -sOutputFile=split.pdf file.pdf
                        # extract pages 10-20 of PDF file

Markdowns and HTMLs

pandoc --from html --to markdown doc.html -o doc.md
                        # converts an HTML file to a markdown file

File ownership

umask                   # check current file permissions mask (default 0022)
umask 0002              # change file permissions to disallow other-write only
                        #   Note: this is currently the default for 
                        #           all users on chalkboard and fishfish
chown root file         # change ownership of a file or directory
chgrp LabFolks file     # change group ownership
chown adam:LabFolks file    # change ownership and group at the same time
chmod uo+x, g-w file    # change permissions for a file
                        #   u - user
                        #   g - group
                        #   o - everyone else
                        #   a - u + g + o
                        #   + - add permission
                        #   - - remove permission
                        #   = - set permission
                        #   r - read
                        #   w - write to or edit
                        #   x - execute
sudo chmod -R 775 /adamX/ 
                        # change permissions recursively for a directory
                        #   1 - execute
                        #   2 - write
                        #   4 - read
                        #   5 - read/execute
                        #   6 - read/write
                        #   7 - read/write/execute
chmod +x testat.sh      # make testat.sh executable
getfacl file            # get file access control list
setfacl -bn file        # remove access control list from file
                        #   without recalculating the effective rights mask

Rivanna

whois -h whois.virginia.edu chen, andrew
allocations             # checks allocation
sfsq                    # checks amount of scratch space available
queues                  # checks partitions that are available
module load matlab
module avail
module list
module load mymod/N.M
module load mymod/compiler</N.M>
module remove mymod
module purge
sbatch job_script.slurm
sbatch --array=0-30 myjob.sh            # job array, numbered from 0 to 30
sbatch --array=1-7:2 myjob.sh           # numbers them 1, 3, 5, 7
sbatch --array=1,3,4,5,7,9 myjob.sh
sbatch --dependency=afterok:18375 job_2.slurm    # Suppose job_1.slurm got the job ID 18375
                        # afterok/after/afterany/afternotok
squeue -u al4ng         # PD - pending; R - running; S - suspended; CG - exiting
queues
jobq
scancel 18316
./myexec myinput.${SLURM_ARRAY_TASK_ID}.in
jobe -v                 # checks usage/efficiency
sacct
sacct --format=jobID --format=jobname --format=Elapsed --format=state
/usr/bin/firefox        # opens a Firefox browswer

Must do for fastX if ~/.bashrc and ~/.bash_profile do not already exist

cp /share/apps/templates/bash_files/bash_profile ~/.bash_profile
cp /share/apps/templates/bash_files/bashrc ~/.bashrc

Interactive jobs in SLURM

salloc                  # request interactive resources on frontend
ijob -c <cores> -A <account> -p <partition> -t <minutes>
                        # run ijob with no options to see all possible options

MPI

module load openmpi/gcc/4.8.5/2.0.0
gfortran mycode.f90
gcc mycode.c
gxx mycode.cxx
mpirun -np 4 ./a.out

git

git --version               # get the version of git
git help status             # get the manual page of git status
git init                    # initialize a .git directory under current directory
git status                  # get status of the repository
git config --global user.name "luadam4c"
                            # add user name to account
git config --global user.name
                            # check user name
git config --global user.email "al4ng@virginia.edu"
                            # add user email to account
git config --global user.email
                            # check user email
git config --global credential.helper cache
                            # Set git to use the credential memory cache
git config --global core.editor "vim"
                            # Set the default git editor to be vim
git submodule add url/to/submodule.git path/to/submodule          
							# add a submodule
git submodule init          # initialize all submodules
git submodule update        # update all submodules from the current project
git submodule foreach git pull origin master
                            # update all submodules from its own repositories
git branch                  # show all branches
git branch newbranch        # create a new branch
git branch -d otherbranch   # delete a branch
git checkout -b newbranch   # create and change to new branch
git checkout otherbranch    # change to an existing branch
git merge otherbranch       # merge the changes in otherbranch to 
                            #   the current branch 
git add .                   # add all files in the directory to repository
git add -A                  # add all files
echo "*.png" >> .gitignore  # never track png files
git reset                   # unstage all files
git reset HEAD <file>       # unstage a file
git rm --cached <file>      # untrack a file
git rm -r --cached <file>   # untrack a directory
git commit -a               # commit all files recursively
git commit -m 'message'     # commit with a message
git reset --soft HEAD~      # undo the most recent commit
git checkout -- <file>      # revert a file back to the last commited version
git log                     # view git log
git log -3                  # view the last three commits
git diff <file>             # view the difference between the file 
                            #   and the last commited version
git diff <SHA-1 hash>       # view the differences between the given commit 
                            #   and the current version
git diff <SHA-1 hash> <file># view the differences between the given commit 
                            #   and the current version for a file
git diff HEAD HEAD^         # view the difference between the last commit 
                            #   and the second last commit
git diff branch1..branch2   # view the difference between branch1 and branch2
git diff branch1...branch2  # view the difference between branch1, branch2
                            #   and common ancestor
git clone http://github.com/uvasomrc/git-intro-somrc
                            # fork from github to a directory with the same name
git remote add origin https://github.com/luadam4c/minEASE.git
                            # link the local git repository to 
                            #   a remote git repository (named origin locally)
git push -u origin master   # push the existing local git repository to 
                            #   the remote git repository
git push origin master      # push to the master branch of the remote repository
git push                    # push to the default branch of the remote repository
git pull origin master      # pull from the master branch of the remote repository
git pull                    # pull from the default branch of the remote repository
git remote -v               # list the current configured remote repository for your fork
git remote add upstream https://github.com/uvasomrc/git-intro-somrc.git
                            # specify a new remote upstream repository that 
                            #   will be synced with the fork
git fetch upstream          # sync the local repository with the upstream

Matlab commands

mcc -m -v parallelpcalc1    # Matlab Compiler

AWS commands

s3cmd ls                    # list all buckets
s3cmd sync --delete-removed <localPath> <bucketPath>
                            # sync to bucket
s3cmd sync --skip-existing --delete-removed <localPath> <bucketPath>
                            # sync to bucket without checking all files
aws s3 sync --delete <localPath> <bucketPath>
                            # sync to bucket

Hugo commands

hugo help                   # get documentation on all commands
hugo -v                     # build the website (public directory) in verbose mode
hugo server                 # build the website on localhost:1313
hugo -D server              # build the website on localhost:1313 drafts-included

For the vim settings file ()

set nobackup        " no backup files
set nowritebackup   " no backup files while editing
set noswapfile      " no swap files
set swapfile        " use swap files
set dir=~/tmp       " set the directory to store swap files