After a lot of playing around I managed to produce the wanted result with rollup. The only downside of the current configuration is that it does not support newly added files in the --watch mode. The magic setting is under the output.preserveModules

Config:

// rollup.config.js
import commonjs from '@rollup/plugin-commonjs';
import typescript from '@rollup/plugin-typescript';
import postcss from 'rollup-plugin-postcss';
import postcssUrl from 'postcss-url';
import resolve from "@rollup/plugin-node-resolve";

import peerDepsExternal from "rollup-plugin-peer-deps-external";

export default {
    input: 'src/index.tsx',
    output: {
        dir: 'dist',
        format: 'es',
        preserveModules: true,
        sourcemap: true,
    },
    plugins: [
        resolve(),
        peerDepsExternal(),
        commonjs(),

        typescript({
            tsconfig: 'tsconfig.build.json'
        }),
        postcss({
            minimize: true,
            modules: {
                generateScopedName: "[hash:base64:5]"
            },
            plugins: [
                postcssUrl({
                    url: "inline"
                })
            ]
        }),

    ],
};

I hope this config can help others as well