Task Queues

Celery

from celery import Celery
app = Celery('tasks')

@app.task
def process_batch(records):
    return compiled_transform(records)

RQ

from redis import Redis
from rq import Queue

q = Queue(connection=Redis())
job = q.enqueue(process_batch, records)

Huey

from huey import RedisHuey
huey = RedisHuey()

@huey.task()
def analyze(data):
    return compiled_analyze(data)

Worker Warm-up

Compile functions when workers start:

@worker_process_init.connect
def warm_up(**kwargs):
    pyvorin.compile('tasks.py', function_name='process_batch')