Convert small base64 encoded string to hexadecimal

1. Reducing forks

Accepted answer do offer a solution with a lot of forks! I hate useless forks!

There is my short base64 to hexadecimal converter:

b64toHex() {
    local _arLines
    mapfile -t _arLines < <(base64 -d <<< "$2" | xxd -p)
    printf -v "${1:-hexString}" %s "${_arLines[@]}"
}

In order to avoid fork like myVar=$(myFunc args), this function will only populate a variable and won't print out anything.

b64toHex myVar "OQbb8rXnj/DwvglW018uP/1tqldwiJMbjxBhX7ZqwTw="
echo $myVar

3906dbf2b5e78ff0f0be0956d35f2e3ffd6daa577088931b8f10615fb66ac13c

Then you could use:

b64toHex iv_hex "${iv}"
b64toHex key_hex "${key}"

See at bottom of this for execution time comparison.

2. Pure bash way:

This don't depend on xxd or base64 to be installed. (And without forks, will be significantly quicker than running 3 forks! Keep in mind, if you plan to run this repetitively!)

From this: Bash script - decode encoded string to byte array, on my website: base64decoder.sh.txt, base64decoder.sh, with a very small modification**:

**At line 41:
- ((_ar==0)) && printf -v _res %b "${_res[@]/#/\\x}"
+ ((_ar==0)) && printf -v _res %s "${_res[@]}"

First prepare a read-only array as:

declare -a B64=( {A..Z} {a..z} {0..9} + / '=' )
declare -Ai 'B64R=()'
for i in "${!B64[@]}"; do B64R["${B64[i]}"]=i%64; done
declare -r B64R
unset B64

Then b64ToHex function:

b64ToHex() {
    local _4B _Tail _hVal _v _opt OPTIND
    local -i iFd _24b _ar
    while getopts "av:" _opt; do case $_opt in
        a) _ar=1;; v) _v=${OPTARG};; *) return 1;; esac; done
    shift $((OPTIND-1))
    if [[ $_v ]];then local -n _res=${_v}; else local _res; fi
    if [[ $1 ]]; then    exec {iFd}<<<"$1"          # Open Input FD from string
    else                 exec {iFd}<&0        ; fi  # Open Input FD from STDIN
    _res=()
    while read -rn4 -u $iFd _4B; do
        if [[ "$_4B" ]]; then
            _Tail=$_4B
            _24b=" B64R['${_4B::1}'] << 18 | B64R['${_4B:1:1}'] << 12 |
                   B64R['${_4B:2:1}'] << 6 | B64R['${_4B:3:1}'] "
            printf -v _hval %02x\  $((_24b>>16)) $((_24b>>8&255)) $((_24b&255))
            read -ra _hval <<<"$_hval"
            _res+=("${_hval[@]}")
        fi
    done
    exec {iFd}<&-
    _Tail=${_Tail##*([^=])}
    while [[ $_Tail ]]; do
        unset "_res[-1]"
        _Tail=${_Tail:1}
    done
    ((_ar==0)) && printf -v _res %s "${_res[@]}" && _res=("${_res[0]}")
    [[ -z $_v ]] && echo "${_res[@]}"
}

If bash loop are known to be slow, doing a loop over only 32 byte will be significantly quicker and less system expansive than running four forks!

Usage from STDIN:

b64ToHex <<<"OQbb8rXnj/DwvglW018uP/1tqldwiJMbjxBhX7ZqwTw="

3906dbf2b5e78ff0f0be0956d35f2e3ffd6daa577088931b8f10615fb66ac13c

From an argument:

b64ToHex "OQbb8rXnj/DwvglW018uP/1tqldwiJMbjxBhX7ZqwTw="

3906dbf2b5e78ff0f0be0956d35f2e3ffd6daa577088931b8f10615fb66ac13c

Assign a variable:

b64ToHex -v someVar "OQbb8rXnj/DwvglW018uP/1tqldwiJMbjxBhX7ZqwTw="
echo "$someVar"

3906dbf2b5e78ff0f0be0956d35f2e3ffd6daa577088931b8f10615fb66ac13c

Then from, to variables:

b64ToHex -v iv_hex "${iv}"
b64ToHex -v key_hex "${key}"

Note: this is done without any fork.

For fun: retrieving original base64 string, with bash V5.1+, you could:

shopt -s extglob
printf %b ${someVar//??/\\x& } | base64

OQbb8rXnj/DwvglW018uP/1tqldwiJMbjxBhX7ZqwTw=

3. Pure bash way, but usign bash V5.2+

Same script, but by using patsub_replacement and mapfile, I could do this without any bash loop!

declare -a B64=( {A..Z} {a..z} {0..9} + / '=' )
printf -v _b64_tstr '["\44{B64[%d]}"]=%%d%%%%64 ' {0..64}
# shellcheck disable=SC2059  # format is variable.
printf -v _b64_tstr "$_b64_tstr" {0..64}
declare -Ai "B64R=($_b64_tstr)"
unset B64 _b64_tstr
declare -r B64R

b64ToHex52() {
    local _line _Tail _v _opt OPTIND _resArry
    local -i iFd _ar
    while getopts "av:" _opt; do case $_opt in
        a) _ar=1;; v) _v=${OPTARG};; *) return 1;; esac; done
    shift $((OPTIND-1))
    if [[ $_v ]];then local -n _res=${_v}; else local _res; fi
    if [[ $1 ]]; then    exec {iFd}<<<"$1"     # Open Input FD from string
    else                 exec {iFd}<&0   ; fi  # Open Input FD from STDIN
    mapfile -tu $iFd _lines
    read -ra _resArry <<<"${_lines[*]//?/& }"
    printf -v _tmpStr '"B64R[%s]<<18|B64R[%s]<<12|B64R[%s]<<6|B64R[%s]" ' \
           "${_resArry[@]}"
    local -ia "_tmpArry=($_tmpStr)"
    printf -v _tmpStr '%06x' "${_tmpArry[@]}"
    read -ra _res <<<"${_tmpStr//??/& }"
    exec {iFd}<&-
    _Tail=${_lines[-1]##*([^=])}
    _res=("${_res[@]::${#_res[@]}-${#_Tail}}")
    ((_ar==0)) && printf -v _res %s "${_res[@]}" && _res=("${_res[0]}")
    [[ -z $_v ]] && echo "${_res[@]}"
}

4. Execution time comparison

Well, now a little comparison test by doing repetitively same conversion to compute execution time.

I now have 4 functions:

1. b64toHex My version using base64 and xxd
2. b64ToHex My pure bash version (notice upper T)
3. b64ToHex52 My pure bash using bash version 5.2+
4. base64_to_hex from accepted answer.

Here's my little test function:

testB64decoders(){ 
    local TIMEFORMAT='r %3lR,  u %3lU,  s %3lS, p %P' bunch \
        inString="${2:-SGVsbG8gd29ybGQhIFRoaXMgaXMgYSB0ZXN0IHN0cmluZy4=}"
    printf -v bunch '%*s' ${1:-100} ''
    mapfile -t bunch <<<"${bunch// /$'\n'}"
    printf ' - %-29s: ' "3 fork (base64 | xxd)";
    time for i in "${bunch[@]}"; do b64toHex hx "$inString"; done
    printf ' - %-29s: ' "Pure bash";
    time for i in "${bunch[@]}"; do b64ToHex -v Hx "$inString"; done;
    printf ' - %-29s: ' "Pure bash V5.2+";
    time for i in "${bunch[@]}"; do b64ToHex52 -v Hx5 "$inString"; done;
    printf ' - %-29s: ' "5 fork =\$(echo| base64 | xxd)";
    time for i in "${bunch[@]}"; do hex=$(base64_to_hex "$inString"); done;
    [[ $hx == "$hex" ]] && [[ $Hx == "$hx" ]] && [[ $Hx5 == "$hx" ]] &&
        printf -v inString '%b' ${hx//??/\\x&} &&
        printf 'Hopefully result strings are same (%s).\n' "${inString@Q}"
}

Let's show with a small thousand of operation:

testB64decoders 1000

Could produce something like:

 - 3 fork (base64 | xxd)        : r 0m2.032s,  u 0m2.315s,  s 0m0.699s, p 148.32
 - Pure bash                    : r 0m0.965s,  u 0m0.735s,  s 0m0.213s, p 98.14
 - Pure bash V5.2+              : r 0m0.571s,  u 0m0.477s,  s 0m0.088s, p 98.91
 - 5 fork =$(echo| base64 | xxd): r 0m2.433s,  u 0m3.242s,  s 0m0.989s, p 173.92

Where r for real, u: user, s: system time and p: cpu percentage is: 100 * ( u + s ) / r

• pure bash method is quicker,
• pure bash method using bash 5.2+ is significantly quicker,
• version using xxd and base64 with only two fork will even be a little quicker than
• version using four forks (a subshell to run three more forks to xxd, base64 and tr). They are the slowest, and yes: two more forks do have system footprint.

Note: on a multicore system, user time is bigger than real time, thanks to parallelization

On my Raspberry-Pi II model B, I had to reduce my test down to 20 loops.

testB64decoders 20

Did produce on my RPi II:

 - 3 fork (base64 | xxd)      …

echo '0A' produces three characters: 0 A NL; xxd -b will then print those three characters in binary. If you wanted just the single byte whose value is 10 (i.e. hexadecimal A), you could write (in bash):

echo -n $'\x0A'
      ^ ^  ^
      | |  |
      | |  +-- `\x` indicates a hexadecimal escape
      | +----- Inside a $' string, escapes are interpreted
      +------- -n suppresses the trailing newline

A better alternative would be printf '\x0A'; printf interprets escape sequences in the format string, and does not output implicit newlines. (For a completely Posix-compatible solution, you would need an octal escape: printf '\012'. printf should work on any Posix-compatible shell but hexadecimal escapes are an extension.) Yet another bash possibility is echo -n -e '\x0A'; the (non-standard) -e flag asks echo to interpret escape sequences.

echo '0A' | xxd -b won't output the equivalent of hex 0A, because xxd doesn't know that you intend 0A to be a hex number rather than two characters. It just takes its input as a series of characters, regardless of what those characters are.

Endianness does not affect bytes. The order of bits inside a byte is entirely conceptual until the byte is transmitted over a serial line and even then it is only visible with an oscilloscope or something similar.

hexdump -ve '1/1 "%02x"'

xxd -p | tr -d '\n'

If you get tired of writing this every time, create an alias.

This worked for me.

$ echo -n 54657374696e672031203220330 | xxd -r -p
Testing 1 2 3$

echo -n prevents a newline from being added to the end

-r tells it to convert hex to ascii as opposed to its normal mode of doing the opposite

-p tells it to use a plain format.

hexdump is able to produce this output, and provides much more options, once you figure out the somewhat arcane, yet powerful, format syntax.

$ hexdump -v -e '"%08_ax" 16/1 " %02X" "\n"' en.gif 
00000000 47 49 46 38 37 61 10 00 0A 00 A1 03 00 CC 00 00
00000010 44 44 CC FF FF FF FF BB 66 2C 00 00 00 00 10 00
00000020 0A 00 00 02 25 8C 20 61 A8 97 BA 92 8B 10 BA CB
00000030 AA 45 FC 59 01 82 C0 18 96 E3 89 A6 64 79 AC 6C
00000040 E5 74 DA 26 D6 F3 84 67 41 01 00 3B

The format in detail:

• "%08_ax" is the offset in hexadecimal, 8 digits with leading zeroes.
• 16/1 " %02X" displays 16 bytes per iteration (line), one byte per block, each block formatted as 2 uppercase hexadecimal digits with leading zeroes and preceeding space.
• "\n" finishes each iteration (line) with a newline.

Try adding the -p (plain) option to xxd like this:

cat /home/alice/flag22 | xxd -r -p

The output I get when I do that in Bash on macOS 10.5.6 (Catalina) seems to be exactly what you are looking for:

9d1ae8d569c83e03d8a8f61568a0fa7d

According to man xxd the -p option does the following:

Output in postscript continuous hexdump style. Also known as plain hexdump style.

And deeper in the man xxd stuff under -r (revert) option explains it as so; bold emphasis is mine:

Reverse operation: convert (or patch) hexdump into binary. If not writing to stdout, xxd writes into its output file without truncating it. Use the combination -r -p to read plain hexadecimal dumps without line number information and without a particular column layout. Additional Whitespace and line-breaks are allowed anywhere.

Since that long string hex dump you have has no line or column info, the -p option is what is needed.