AWK

• FILE SPACING
• NUMBERING AND CALCULATIONS
  • Random number generation
• STRING CREATION
  • FORMATED OUTPUT (Space padding)
• ARRAY CREATION
• TEXT CONVERSION AND SUBSTITUTION
• REGEX BACKREFERENCING
• SELECTIVE DELETION OF CERTAIN LINES
• Snippets
  • Kill all remote user sessions

FILE SPACING

# double space a file
awk '1;{print ""}'
awk 'BEGIN{ORS="\n\n"};1'

# double space a file which already has blank lines in it. Output file
# should contain no more than one blank line between lines of text.
# NOTE: On Unix systems, DOS lines which have only CRLF (\r\n) are
# often treated as non-blank, and thus 'NF' alone will return TRUE.
awk 'NF{print $0 "\n"}'

# triple space a file
awk '1;{print "\n"}'

NUMBERING AND CALCULATIONS

# precede each line by its line number FOR THAT FILE (left alignment).
# Using a tab (\t) instead of space will preserve margins.
awk '{print FNR "\t" $0}' files*

# precede each line by its line number FOR ALL FILES TOGETHER, with tab.
awk '{print NR "\t" $0}' files*

# number each line of a file (number on left, right-aligned)
# Double the percent signs if typing from the DOS command prompt.
awk '{printf("%5d : %s\n", NR,$0)}'

# number each line of file, but only print numbers if line is not blank
# Remember caveats about Unix treatment of \r (mentioned above)
awk 'NF{$0=++a " :" $0};1'
awk '{print (NF? ++a " :" :"") $0}'

# count lines (emulates "wc -l")
awk 'END{print NR}'

# print the sums of the fields of every line
awk '{s=0; for (i=1; i<=NF; i++) s=s+$i; print s}'

# add all fields in all lines and print the sum
awk '{for (i=1; i<=NF; i++) s=s+$i}; END{print s}'

# print every line after replacing each field with its absolute value
awk '{for (i=1; i<=NF; i++) if ($i < 0) $i = -$i; print }'
awk '{for (i=1; i<=NF; i++) $i = ($i < 0) ? -$i : $i; print }'

# print the total number of fields ("words") in all lines
awk '{ total = total + NF }; END {print total}' file

# print the total number of lines that contain "Beth"
awk '/Beth/{n++}; END {print n+0}' file

# print the largest first field and the line that contains it
# Intended for finding the longest string in field #1
awk '$1 > max {max=$1; maxline=$0}; END{ print max, maxline}'

# print the number of fields in each line, followed by the line
awk '{ print NF ":" $0 } '

# print the last field of each line
awk '{ print $NF }'

# print the last field of the last line
awk '{ field = $NF }; END{ print field }'

# print every line with more than 4 fields
awk 'NF > 4'

# print every line where the value of the last field is > 4
awk '$NF > 4'

# print the last X fields
# if you have e.g. a filename test.file.tar.gz
awk -F. '{s=NF-1; print $s" "$NF}'

Random number generation

# generate a pseudo-random seed
SEED=`head -1 /dev/urandom | od -N 1 | head -1 | awk '{ print $2 }'`

# paranoid!! ;-)
# not really needed...
let SEED=$SEED*$$

# use this seed to initialize rand()-function of awk
# and print out NON-negative integers with a maximum value of 15
echo | awk " { srand($SEED); print int(15 * rand()) } "

STRING CREATION

# create a string of a specific length (e.g., generate 513 spaces)
awk 'BEGIN{while (a++<513) s=s " "; print s}'

# insert a string of specific length at a certain character position
# Example: insert 49 spaces after column #6 of each input line.
gawk --re-interval 'BEGIN{while(a++<49)s=s " "};{sub(/^.{6}/,"&" s)};1'

FORMATED OUTPUT (Space padding)

do create some formated output of lines/fields

awk '{X=50-length($3); printf "%s", $3; printf "%*s", X, " "; print "--> "$4}'

ARRAY CREATION

# These next 2 entries are not one-line scripts, but the technique
# is so handy that it merits inclusion here.

# create an array named "month", indexed by numbers, so that month[1]
# is 'Jan', month[2] is 'Feb', month[3] is 'Mar' and so on.
split("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", month, " ")

# create an array named "mdigit", indexed by strings, so that
# mdigit["Jan"] is 1, mdigit["Feb"] is 2, etc. Requires "month" array
for (i=1; i<=12; i++) mdigit[month[i]] = i

TEXT CONVERSION AND SUBSTITUTION

# IN UNIX ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format
awk '{sub(/\r$/,"")};1'   # assumes EACH line ends with Ctrl-M

# IN UNIX ENVIRONMENT: convert Unix newlines (LF) to DOS format
awk '{sub(/$/,"\r")};1'

# IN DOS ENVIRONMENT: convert Unix newlines (LF) to DOS format
awk 1

# IN DOS ENVIRONMENT: convert DOS newlines (CR/LF) to Unix format
# Cannot be done with DOS versions of awk, other than gawk:
gawk -v BINMODE="w" '1' infile >outfile

# Use "tr" instead.
tr -d \r <infile >outfile            # GNU tr version 1.22 or higher

# delete leading whitespace (spaces, tabs) from front of each line
# aligns all text flush left
awk '{sub(/^[ \t]+/, "")};1'

# delete trailing whitespace (spaces, tabs) from end of each line
awk '{sub(/[ \t]+$/, "")};1'

# delete BOTH leading and trailing whitespace from each line
awk '{gsub(/^[ \t]+|[ \t]+$/,"")};1'
awk '{$1=$1};1'           # also removes extra space between fields

# insert 5 blank spaces at beginning of each line (make page offset)
awk '{sub(/^/, "     ")};1'

# align all text flush right on a 79-column width
awk '{printf "%79s\n", $0}' file*

# center all text on a 79-character width
awk '{l=length();s=int((79-l)/2); printf "%"(s+l)"s\n",$0}' file*

# substitute (find and replace) "foo" with "bar" on each line
awk '{sub(/foo/,"bar")}; 1'           # replace only 1st instance
gawk '{$0=gensub(/foo/,"bar",4)}; 1'  # replace only 4th instance
awk '{gsub(/foo/,"bar")}; 1'          # replace ALL instances in a line

# substitute "foo" with "bar" ONLY for lines which contain "baz"
awk '/baz/{gsub(/foo/, "bar")}; 1'

# substitute "foo" with "bar" EXCEPT for lines which contain "baz"
awk '!/baz/{gsub(/foo/, "bar")}; 1'

# change "scarlet" or "ruby" or "puce" to "red"
awk '{gsub(/scarlet|ruby|puce/, "red")}; 1'

# reverse order of lines (emulates "tac")
awk '{a[i++]=$0} END {for (j=i-1; j>=0;) print a[j--] }' file*

# if a line ends with a backslash, append the next line to it (fails if
# there are multiple lines ending with backslash...)
awk '/\\$/ {sub(/\\$/,""); getline t; print $0 t; next}; 1' file*

# print and sort the login names of all users
awk -F ":" '{print $1 | "sort" }' /etc/passwd

# print the first 2 fields, in opposite order, of every line
awk '{print $2, $1}' file

# switch the first 2 fields of every line
awk '{temp = $1; $1 = $2; $2 = temp}' file

# print every line, deleting the second field of that line
awk '{ $2 = ""; print }'

# print in reverse order the fields of every line
awk '{for (i=NF; i>0; i--) printf("%s ",$i);print ""}' file

# concatenate every 5 lines of input, using a comma separator
# between fields
awk 'ORS=NR%5?",":"\n"' file

# substitute " in special fields here: last field in the line
# or ; in e.g. second field 
awk '{gsub("\"","", $NF); print $NF}'
awk '{gsub(";","", $2); print $2}'

REGEX BACKREFERENCING

echo "Paul loves Mary" | gawk '{print gensub(/(.*) (.*) (.*)/,"\\3 \\2 \\1","")}'
Mary loves Paul

SELECTIVE DELETION OF CERTAIN LINES

# delete ALL blank lines from a file (same as "grep '.' ")
awk NF
awk '/./'

# remove duplicate, consecutive lines (emulates "uniq")
awk 'a !~ $0; {a=$0}'

# remove duplicate, nonconsecutive lines
awk '!a[$0]++'                     # most concise script
awk '!($0 in a){a[$0];print}'      # most efficient script

# delete a line that matches PATTERN
# and additionaly the following 2 lines
# modify c=3 to delete more/less lines
awk '/PATTERN-To-Be-Deleted/{c=3} c&&c--{next} 1'

Snippets

Some snippets

Kill all remote user sessions

w | awk '$3 !~ /:0(\.0)?|-/ && $2 ~ /pts/ {print "/dev/"$2}' | xargs fuser -k

w -h | awk '$3 !~ /:0(\.0)?|-/ {print "/dev/"$2} | xargs fuser -k'

in regelmaessigen abstaenden einfach in ‘watch’ oder in ‘while true’ verpacken erklaerung zum awk-command:

1. matched in feld 3 auf :0 UND :0.0 oder auf –
2. und matched feld 2 auf pts (nur noetig wenn ‘w’ benutzt wird NICHT bei ‘w -h’ schadet aber auch nicht!! ;-))
3. trifft beides zu wird das devicefile ausgegeben

mit fuser wird der user dann gekickt… oder was immer man mit dem user tun will! 😉