Ethereum addresses are 20 bytes, so you convert hex address to bytes and then pad it to 32 bytes from left.

web3.utils.padLeft(web3.utils.hexToBytes(yourAddressString), 32);

Fixed, answer is use padding

example -> 0x0000000000000000000000000D81d9E21BD7C5bB095535624DcB0759E64B3899

Sometimes the simplest answers are right infront of you

It was implicitly convertible in older versions of the compiler.

address a = bytes32(0);

It is explicitly convertible through uint160 in recent versions of the compiler.

address a = address(uint160(uint(bytes32(0))));

In both cases, it is a truncating conversion.

In case parsing that expression is a little cryptic.

bytes32 => uint256 is a non-truncating change of type casting, to unsigned integer

uint256 => uint160 is an allowable, truncating conversion that reduces it to 20 bytes

uint160 => address is a non-truncating change of type casting, to address

Hope it helps.

To be even more efficient:

function toBytes(address a) public pure returns (bytes memory b){
    assembly {
        let m := mload(0x40)
        a := and(a, 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)
        mstore(add(m, 20), xor(0x140000000000000000000000000000000000000000, a))
        mstore(0x40, add(m, 52))
        b := m
   }
}

Takes just 695 gas vs 2500 for Gokulnath's answer and 5000 for Eth's

Edit for solidity ^0.5.0:

This is almost as efficient and much more readable:

function toBytes(address a) public pure returns (bytes memory) {
    return abi.encodePacked(a);
}

This works fine. Perhaps you were actually including single quotes in your code? If so, remove them.

pragma solidity ^0.4.24;

contract Test {
    function test(bytes32 data) external pure returns (address) {
        return address(data);
    }
}

If your goal is to convert your bytes to a bytes32[], maybe you can try something like:

function bytesToBytes32Array(bytes memory data)
    public
    pure
    returns (bytes32[] memory)
{
    // Find 32 bytes segments nb
    uint256 dataNb = data.length / 32;
    // Create an array of dataNb elements
    bytes32[] memory dataList = new bytes32;
    // Start array index at 0
    uint256 index = 0;
    // Loop all 32 bytes segments
    for (uint256 i = 32; i <= data.length; i = i + 32) {
        bytes32 temp;
        // Get 32 bytes from data
        assembly {
            temp := mload(add(data, i))
        }
        // Add extracted 32 bytes to list
        dataList[index] = temp;
        index++;
    }
    // Return data list
    return (dataList);
}