# Add an empty line after each line
sed G
awk '{printf("%s\n\n",$0)}'
# Delete all original empty lines and add one empty line after each line.
# In this way, there will be only one empty line after each line in the output text.
sed '/^$/d;G'
awk '!/^$/{printf("%s\n\n",$0)}'
# Add two empty lines after each line
sed 'G;G'
awk '{printf("%s\n\n\n",$0)}'
# Delete all empty lines generated by the first script (that is, delete all even lines)
sed 'n;d'
awk '{f=!f;if(f)print $0}'
# Insert an empty line before matching the line of the style "regex"
sed '/regex/{x;p;x;}'
awk '{if(/regex/)printf("\n%s\n",$0);else print $0}'
# Insert an empty line after matching the line of the style "regex"
sed '/regex/G'
awk '{if(/regex/)printf("%s\n\n",$0);else print $0}'
# Insert an empty line before and after the line matching the style "regex"
sed '/regex/{x;p;x;G;}'
awk '{if(/regex/)printf("\n%s\n\n",$0);else print $0}'
serial number:
# Number each line in the file (simple left alignment). "Tab" is used here
# (tab, see the description of the usage of '\t' at the end of this article) Instead of spaces to align edges.
sed = filename | sed 'N;s/\n/\t/'
awk '{i++;printf("%d\t%s\n",i,$0)}'
# Number all lines in the file (line numbers are on the left and the text is aligned at the right end).
sed = filename | sed 'N; s/^/ /; s/ *\(.\{6,\}\)\n/\1 /'
awk '{i++;printf("%6d %s\n",i,$0)}'
# Number all lines in the file, but only the line numbers of non-blank lines are displayed.
sed '/./=' filename | sed '/./N; s/\n/ /'
awk '{i++;if(!/^$/)printf("%d %s\n",i,$0);else print}'
# Calculate the number of rows (simulate "wc -l")
sed -n '$='
awk '{i++}END{print i}'
Text conversion and substitution:
# Unix environment: convert the new line characters (CR/LF) of DOS to Unix format.
sed 's/.$//' # Assume that all rows end in CR/LF
sed 's/^M$//' # In bash/tcsh, change Ctrl-M to Ctrl-V
sed 's/\x0D$//' # ssed, gsed 3.02.80, and later
awk '{sub(/\x0D$/,"");print $0}'
# Unix environment: convert Unix's new line characters (LF) to DOS format.
sed "s/$/`echo -e \\\r`/" # Commands used under ksh
sed 's/$'"/`echo \\\r`/" # Commands used under bash
sed "s/$/`echo \\\r`/" # Commands used under zsh
sed 's/$/\r/' # gsed 3.02.80 and later
awk '{printf("%s\r\n",$0)}'
# DOS environment: convert Unix new line characters (LF) to DOS format.
sed "s/$//" # Method 1
sed -n p # Method 2
# DOS environment: convert DOS new line characters (CR/LF) to Unix format.
# The following script is only valid for UnxUtils sed 4.0.7 and later. To identify the UnxUtils version
# sed can be accessed through its unique “–text” option. You can use the help option ("-help") to see
# Are there any "-text" items in it to determine whether the UnxUtils version is used. Other DOS
# version of sed cannot perform this conversion. But this transformation can be achieved with "tr".
sed "s/\r//" infile >outfile # UnxUtils sed v4.0.7 or later
tr -d \r <infile>outfile # GNU tr 1.22 or later
# Delete the "whitespace characters" (spaces, tabs) leading to each line
# Align it left
sed 's/^[ \t]*//' # See the description of the usage of '\t' at the end of this article
awk '{sub(/^[ \t]+/,"");print $0}'
# Delete the "whitespace characters" (spaces, tabs) that are trailing on each line
sed 's/[ \t]*$//' # See the description of the usage of '\t' at the end of this article
awk '{sub(/[ \t]+$/,"");print $0}'
# Delete leading and trailing whitespace characters in each line
sed 's/^[ \t]*//;s/[ \t]*$//'
awk '{sub(/^[ \t]+/,"");sub(/[ \t]+$/,"");print $0}'
# Insert 5 spaces at the beginning of each line (moving the full text by 5 characters to the right)
sed 's/^/ /'
awk '{printf(" %s\n",$0)}'
# Right-align all text with 79 characters as width
# 78 characters plus the last space
sed -e :a -e 's/^.\{1,78\}$/ &/;ta'
awk '{printf("%79s\n",$0)}'
# Center all text with 79 characters as width. In Method 1, in order to center the text before each line
# Both the header and the back are filled with spaces. In Method 2, only fill in front of the text during the centering text
# spaces, and eventually half of these spaces will be deleted. In addition, no spaces are filled after each line.
sed -e :a -e 's/^.\{1,77\}$/ & /;ta' # Method 1
sed -e :a -e 's/^.\{1,77\}$/ &/;ta' -e 's/\( *\)\1/\1/' # Method 2
awk '{for(i=0;i<39-length($0)/2;i++)printf(" ");printf("%s\n",$0)}' #Equivalent to the above method two
# Find the string "foo" in each line and replace the found "foo" with "bar"
sed 's/foo/bar/' # Replace only the first "foo" string in each line
sed 's/foo/bar/4' # Replace only the fourth "foo" string in each line
sed 's/foo/bar/g' # Change all "foo" in each line to "bar"
sed 's/\(.*\)foo\(.*foo\)/\1bar\2/' # Replace the penultimate "foo"
sed 's/\(.*\)foo/\1bar/' # Replace the last "foo"
awk '{gsub(/foo/,"bar");print $0}' # Change all "foo" in each line to "bar"
# Replace "foo" with "bar" only if the string "baz" appears in the line
sed '/baz/s/foo/bar/g'
awk '{if(/baz/)gsub(/foo/,"bar");print $0}'
# Replace "foo" with "bar", and only if the string "baz" does not appear in the line
sed '/baz/!s/foo/bar/g'
awk '{if(/baz$/)gsub(/foo/,"bar");print $0}'
# Whether it is "scarlet", "ruby" or "puce", it will be replaced with "red"
sed 's/scarlet/red/g;s/ruby/red/g;s/puce/red/g' #Effect for most sed
gsed 's/scarlet\|ruby\|puce/red/g' # only valid for GNU sed
awk '{gsub(/scarlet|ruby|puce/,"red");print $0}'
# Invert all rows, the first row becomes the last row, and so on (simulate "tac").
# For some reason, HHsed v1.5 will delete empty lines in the file when using the following command
sed '1!G;h;$!d' # Method 1
sed -n '1!G;h;$p' # Method 2
awk '{A[i++]=$0}END{for(j=i-1;j>=0;j--)print A[j]}'
# Arrange the characters in the line in reverse order, the first word becomes the last word,... (Simulate "rev")
sed '/\n/!G;s/\(.\)\(.*\n\)/&\2\1/;//D;s/.//'
awk '{for(i=length($0);i>0;i--)printf("%s",substr($0,i,1));printf("\n")}'
# Concatenate every two lines into one line (similar to "paste")
sed '$!N;s/\n/ /'
awk '{f=!f;if(f)printf("%s",$0);else printf(" %s\n",$0)}'
# If the current line ends with a backslash "\", the next line is merged to the end of the current line
# and remove the backslash at the end of the line
sed -e :a -e '/\\$/N; s/\\\n//; ta'
awk '{if(/\\$/)printf("%s",substr($0,0,length($0)-1));else printf("%s\n",$0)}'
# If the current line starts with an equal sign, merge the current line to the end of the previous line
# and replace the "=" of the original line with a single space
sed -e :a -e '$!N;s/\n=/ /;ta' -e 'P;D'
awk '{if(/^=/)printf(" %s",substr($0,2));else printf("%s%s",a,$0);a="\n"}END{printf("\n")}'
# Add comma-separated symbols to the numeric string and change "1234567" to "1,234,567"
gsed ':a;s/\B[0-9]\{3\}\>/,&/;ta' # GNU sed
sed -e :a -e 's/\(.*[0-9]\)\([0-9]\{3\}\)/\1,\2/;ta' # Other sed
#awk's regularity has no backward matching or reference, which makes it a bit embarrassing, haha.
awk '{while(match($0,/[0-9][0-9][0-9][0-9]+/)){$0=sprintf("%s,%s",substr($0,0,RSTART+RLENGTH-4),substr($0,RSTART+RLENGTH-3))}print $0}'
# Add comma separator (GNU sed) to values with decimal points and negative signs
gsed -r ':a;s/(^|[^0-9.])([0-9]+)([0-9]{3})/\1\2,\3/g;ta'
#It's similar to the above example
awk '{while(match($0,/[^\.0-9][0-9][0-9][0-9][0-9]+/)){$0=sprintf("%s,%s",substr($0,0,RSTART+RLENGTH-4),substr($0,RSTART+RLENGTH-3))}print $0}'
# Add a blank line after every 5 lines (add a blank line after every 5 lines)
gsed '0~5G' # only valid for GNU sed
sed 'n;n;n;n;G;' # Other sed
awk '{print $0;i++;if(i==5){printf("\n");i=0}}'
Selectively display specific rows:
# Show the first 10 lines in the file (simulate the behavior of "head")
sed 10q
awk '{print;if(NR==10)exit}'
# Display the first line in the file (simulate the "head -1" command)
sed q
awk '{print;exit}'
# Show the last 10 lines in the file (simulate "tail")
sed -e :a -e '$q;N;11,$D;ba'
#Use awk to do this is a bit of a loss, so you need to cache the full text, which is definitely very slow for large files
awk '{A[NR]=$0}END{for(i=NR-9;i<=NR;i++)print A[i]}'
# Show the last 2 lines in the file (simulate the "tail -2" command)
sed '$!N;$!D'
awk '{A[NR]=$0}END{for(i=NR-1;i<=NR;i++)print A[i]}'
# Show the last line in the file (simulate "tail -1")
sed '$!d' # Method 1
sed -n '$p' # Method 2
#This is easier to deal with, only the last line is saved.
awk '{A=$0}END{print A}'
# Show the penultimate line in the file
sed -e '$!{h;d;}' -e x # When there is only one line in the file, output empty lines
sed -e '1{$q;}' -e '$!{h;d;}' -e x # When there is only one line in the file, the line will be displayed
sed -e '1{$d;}' -e '$!{h;d;}' -e x # When there is only one line in the file, no output
#Save two lines (when there is only one line in the file, output empty lines)
awk '{B=A;A=$0}END{print B}'
# Show only lines matching regular expressions (simulate "grep")
sed -n '/regexp/p' # Method 1
sed '/regexp/!d' # Method 2
awk '/regexp/{print}'
# Show only lines that "not" match regular expressions (simulate "grep -v")
sed -n '/regexp/!p' # Method 1, corresponding to the previous command
sed '/regexp/d' # Method 2, similar syntax
awk '!/regexp/{print}'
# Find "regexp" and display the previous line of the matching row, but does not display the matching row.
sed -n '/regexp/{g;1!p;};h'
awk '/regexp/{print A}{A=$0}'
# Find "regexp" and display the next row of the matching row, but does not display the matching row.
sed -n '/regexp/{n;p;}'
awk '{if(A)print;A=0}/regexp/{A=1}'
# Show the line containing "regexp" and its pre and subsequent lines, and precede the first line with the line number of the line where "regexp" is located (similar to "grep -A1 -B1")
sed -n -e '/regexp/{=;x;1!p;g;$!N;p;D;}' -e h
awk '{if(F)print;F=0}/regexp/{print NR;print b;print;F=1}{b=$0}'
# Show rows containing "AAA", "BBB", and "CCC" (any order)
sed '/AAA/!d; /BBB/!d; /CCC/!d' # The order of the strings does not affect the result
awk '{if(match($0,/AAA/) && match($0,/BBB/) && match($0,/CCC/))print}'
# Show rows containing "AAA", "BBB" and "CCC" (fixed order)
sed '/AAA.*BBB.*CCC/!d'
awk '{if(match($0,/AAA.*BBB.*CCC/))print}'
# Show rows containing "AAA", "BBB" or "CCC" (simulate "egrep")
sed -e '/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d # Most sed
gsed '/AAA\|BBB\|CCC/!d' # works for GNU sed
awk '/AAA/{print;next}/BBB/{print;next}/CCC/{print}'
awk '/AAA|BBB|CCC/{print}'
# Show paragraphs containing "AAA" (paragraphs are separated by blank lines)
# HHsed v1.5 must add "G;" after "x;", and the next 3 scripts are like this
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;'
awk 'BEGIN{RS=""}/AAA/{print}'
awk -vRS= '/AAA/{print}'
# Display paragraphs containing three strings "AAA", "BBB" and "CCC" (any order)
sed -e '/./{H;$!d;}' -e 'x;/AAA/!d;/BBB/!d;/CCC/!d'
awk -vRS= '{if(match($0,/AAA/) && match($0,/BBB/) && match($0,/CCC/))print}'
# Display paragraphs containing any string of "AAA", "BBB", and "CCC" (any order)
sed -e '/./{H;$!d;}' -e 'x;/AAA/b' -e '/BBB/b' -e '/CCC/b' -e d
gsed '/./{H;$!d;};x;/AAA\|BBB\|CCC/b;d' # Only valid for GNU sed
awk -vRS= '/AAA|BBB|CCC/{print "";print}'
# Show lines containing 65 or more characters
sed -n '/^.\{65\}/p'
cat | awk '{if(length($0)>=65)print}'
# Show lines containing less than 65 characters
sed -n '/^.\{65\}/!p' # Method 1, corresponding to the above script
sed '/^.\{65\}/d' # Method 2, a simpler method
awk '{if(length($0)<=65)print}'
# Show part of text—start from the line containing the regular expression to the end of the last line
sed -n '/regexp/,$p'
awk '/regexp/{F=1}{if(F)print}'
# Display part of text—Specify line number range (from line 8 to line 12, including line 8 and 12)
sed -n '8,12p' # Method 1
sed '8,12!d' # Method 2
awk '{if(NR>=8 && NR<12)print}'
# Show line 52
sed -n '52p' # Method 1
sed '52!d' # Method 2
sed '52q;d' # Method 3, more efficient when processing large files
awk '{if(NR==52){print;exit}}'
# Starting from line 3, display every 7 lines
gsed -n '3~7p' # Only valid for GNU sed
sed -n '3,${p;n;n;n;n;n;n;}' # Other sed
awk '{if(NR==3)F=1}{if(F){i++;if(i%7==1)print}}'
# Display text between two regular expressions (included)
sed -n '/Iowa/,/Montana/p' # case sensitive
awk '/Iowa/{F=1}{if(F)print}/Montana/{F=0}'
Selectively delete a specific row:
# Show the entire document except the content between two regular expressions
sed '/Iowa/,/Montana/d'
awk '/Iowa/{F=1}{if(!F)print}/Montana/{F=0}'
# Delete adjacent duplicate lines in the file (simulate "uniq")
# Only the first line in the duplicate line is retained, and other lines are deleted.
sed '$!N; /^\(.*\)\n\1$/!P; D'
awk '{if($0!=B)print;B=$0}'
# Delete duplicate lines in the file, regardless of whether they are adjacent or not. Note the cache size that hold space can support, or use GNU sed.
sed -n 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' #bones7456 Note: This command does not work properly here
awk '{if(!($0 in B))print;B[$0]=1}'
# Delete all rows except duplicate rows (simulate "uniq -d")
sed '$!N; s/^\(.*\)\n\1$/\1/; t; D'
awk '{if($0==B && $0!=l){print;l=$0}B=$0}'
# Delete the beginning of the 10 lines in the file
sed '1,10d'
awk '{if(NR>10)print}'
# Delete the last line in the file
sed '$d'
#awk does not know how many lines there are in the file in total, so it can only be cached throughout the file. Large files may not be suitable, and the following two are the same
awk '{B[NR]=$0}END{for(i=0;i<=NR-1;i++)print B[i]}'
# Delete the last two lines in the file
sed 'N;$!P;$!D;$d'
awk '{B[NR]=$0}END{for(i=0;i<=NR-2;i++)print B[i]}'
# Delete the last 10 lines in the file
sed -e :a -e '$d;N;2,10ba' -e 'P;D' # Method 1
sed -n -e :a -e '1,10!{P;N;D;};N;ba' # Method 2
awk '{B[NR]=$0}END{for(i=0;i<=NR-10;i++)print B[i]}'
# Delete multiple rows of 8
gsed '0~8d' # Only valid for GNU sed
sed 'n;n;n;n;n;n;n;d;' # Other sed
awk '{if(NR%8!=0)print}' |head
# Delete the rows that match styles
sed '/pattern/d' # Delete the line with pattern. Of course pattern can be replaced with any valid regular expression
awk '{if(!match($0,/pattern/))print}'
# Delete all empty lines in the file (the same effect as "grep '.' ")
sed '/^$/d' # Method 1
sed '/./!d' # Method 2
awk '{if(!match($0,/^$/))print}'
# Only the first row of multiple adjacent blank rows are retained. And delete empty lines at the top and end of the file.
# (Simulate "cat -s")
sed '/./,/^$/!d' #Method 1, delete the empty line at the top of the file, allowing the tail to keep an empty line
sed '/^$/N;/\n$/D' #Method 2, allowing a blank line to be kept at the top and no blank line at the end
awk '{if(!match($0,/^$/)){print;F=1}else{if(F)print;F=0}}' #Same as above method 2
# Only the first two rows of multiple adjacent blank rows are retained.
sed '/^$/N;/\n$/N;//D'
awk '{if(!match($0,/^$/)){print;F=0}else{if(F<2)print;F++}}'
# Delete all empty lines at the top of the file
sed '/./,$!d'
awk '{if(F || !match($0,/^$/)){print;F=1}}'
# Delete all empty lines at the end of the file
sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' # Valid for all sed
sed -e :a -e '/^\n*$/N;/\n$/ba' # Same as above, but only valid for gsed 3.02.*
awk '/^.+$/{for(i=l;i<NR-1;i++)print "";print;l=NR}'
# Delete the last line of each paragraph
sed -n '/^$/{p;h;};/./{x;/./p;}'
#It's very long, very ugly, there should be a better way
awk -vRS= '{B=$0;l=0;f=1;while(match(B,/\n/)>0){print substr(B,l,RSTART-l-f);l=RSTART;sub(/\n/,"",B);f=0};print ""}'
Special Applications:
# Remove nroff tag from man page. Used under Unix System V or bash shell
# When using the 'echo' command, the -e option may be added.
sed "s/.`echo \\\b`//g" # Double brackets on the outer layer are required (Unix environment)
sed 's/.^H//g' # In bash or tcsh, press Ctrl-V and then Ctrl-H
sed 's/.\x08//g' # sed 1.5, GNU sed, ssed hexadecimal representation method used
awk '{gsub(/.\x08/,"",$0);print}'
# Extract the header of a newsgroup or e-mail
sed '/^$/q' # Delete all contents after the first empty line
awk '{print}/^$/{exit}'
# Extract the text section of a newsgroup or e-mail
sed '1,/^$/d' # Delete everything before the first empty line
awk '{if(F)print}/^$/{F=1}'
# Extract "Subject" (title bar field) from the email header and remove the word "Subject:" at the beginning
sed '/^Subject: */!d; s///;q'
awk '/^Subject:.*/{print substr($0,10)}/^$/{exit}'
# Get reply address from the email header
sed '/^Reply-To:/q; /^From:/h; /./d;g;q'
#It seems to be outputting the first line starting with Reply-To:? What is From for? Not clear about the rules. .
awk '/^Reply-To:.*/{print;exit}/^$/{exit}'
# Get the email address. The non-email address is further shaved based on the email header generated by the previous script. (See the previous script)
sed 's/ *(.*)//; s/>.*//; s/.*[:<] *//'
#Pick the things in angle brackets?
awk -F'[<>]+' '{print $2}'
# Add an angle bracket and space at the beginning of each line (quote information)
sed 's/^/> /'
awk '{print "> " $0}'
# Remove angle brackets and spaces at the beginning of each line (dereference)
sed 's/^> //'
awk '/^> /{print substr($0,3)}'
# Remove most HTML tags (including cross-row tags)
sed -e :a -e 's/<[^>]*>//g;/</N;//ba'
awk '{gsub(/<[^>]*>/,"",$0);print}'
# Decode the uuencode file divided into multiple volumes. Remove file header information and only the uuencode encoding part is retained.
# Files must be passed to sed in a specific order. The following first version of the script can be entered directly on the command line;
# The second version can be placed in a shell script with execution permissions. (One by Rahul Dhesi
# scripts are modified. )
sed '/^end/,/^begin/d' file1 file2 ... fileX | uudecode # vers. 1
sed '/^end/,/^begin/d' "$@" | uudecode # vers. 2
#I don't want to install uudecode verification, just write it roughly
awk '/^end/{F=0}{if(F)print}/^begin/{F=1}' file1 file2 ... fileX
# Sort the paragraphs in the file alphabetical order. Paragraphs are separated by (one or more lines) blank lines. GNU sed
# The character "\v" represents a vertical tab, and here it is used as a placeholder for newlines - of course you can
# Replace it with other characters that are not used in the file.
sed '/./{H;d;};x;s/\n/={NL}=/g' file | sort | sed '1s/={NL}=//;s/={NL}=/\n/g'
gsed '/./{H;d};x;y/\n/\v/' file | sort | sed '1s/\v//;y/\v/\n/'
awk -vRS= '{gsub(/\n/,"\v",$0);print}' | sort | awk '{gsub(/\v/,"\n",$0);print;print ""}'
# Compress each .TXT file separately, delete the original file after compression and convert the compressed .ZIP file
# Name the same name as the original one (just different extension). (DOS environment: "dir /b"
# Show filename without path).
echo @echo off >
dir /b *.txt | sed "s/^\(.*\)\.TXT/pkzip -mo \1 \/" >>
over!