you need to split the "worker" code into a separate function and use pool.apply_async to run your functions in parallel.

for writing to log files, make sure you do that in the main process to avoid race conditions, you can do it in the workers, but you'll need a lock for it to work properly.

the following demonstrates how to move all the worker logic to a separate function.

from multiprocessing import Pool

def worker_func(batch_num, batch):
    batch_results = []
    batch_logs = []

    print(f'----Batch {batch_num + 1} of {len(batches)}-------')
    batch_logs.append(f'\nBatch no {batch_num + 1} of {len(batches)} \n')

    # run loops and get results for the current batch
    for i, (k, v) in enumerate(batch):
        print(f'----Iteration {i + 1} of {len(batch)}-------')

        result, msg = run_model(k, v, param)
        batch_results.append(result)
        batch_logs.append(msg)

    # Combine the results and save the output files
    batch_results = pd.concat(batch_results)
    batch_results = batch_results.reset_index(drop=True)
    return batch_logs

if __name__ == '__main__':
    # Create a list of tuples with each key and the corresponding df for each key
    site_data = [(site, df) for site, df in df.groupby('site')]

    # set batch size
    batch_size = 1000

    # split dictionary into batches
    batches = [
        site_data[i:i + batch_size] for i in range(0, len(site_data), batch_size)
    ]

    with Pool() as pool:
        futures = []
        # loop through each batch
        for batch_num, batch in enumerate(batches):
            futures.append(pool.apply_async(worker_func, batch_num, batch))

        for future in futures:
            batch_logs = future.get()  # wait for work to be done
            # Save logs to the file
            log_file = f"logs/logs_{today}"
            # write to log from main process to avoid race conditions
            save_logs(log_file, batch_logs)