Ethereum addresses are 20 bytes, so you convert hex address to bytes and then pad it to 32 bytes from left.

web3.utils.padLeft(web3.utils.hexToBytes(yourAddressString), 32);

Fixed, answer is use padding

example -> 0x0000000000000000000000000D81d9E21BD7C5bB095535624DcB0759E64B3899

Sometimes the simplest answers are right infront of you

This works fine. Perhaps you were actually including single quotes in your code? If so, remove them.

pragma solidity ^0.4.24;

contract Test {
    function test(bytes32 data) external pure returns (address) {
        return address(data);
    }
}

To be even more efficient:

function toBytes(address a) public pure returns (bytes memory b){
    assembly {
        let m := mload(0x40)
        a := and(a, 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)
        mstore(add(m, 20), xor(0x140000000000000000000000000000000000000000, a))
        mstore(0x40, add(m, 52))
        b := m
   }
}

Takes just 695 gas vs 2500 for Gokulnath's answer and 5000 for Eth's

Edit for solidity ^0.5.0:

This is almost as efficient and much more readable:

function toBytes(address a) public pure returns (bytes memory) {
    return abi.encodePacked(a);
}