promptfoo/examples/docker-code-generation-sandbox/validate_and_run_code.py

import re

import epicbox

# Replace with your preferred Docker image
DOCKER_IMAGE = "python:3.9-alpine"


def get_assert(output, context):
    # Extract the Python function from the LLM output
    function_match = re.search(r"```python\s*\n(def\s+.*?)\n```", output, re.DOTALL)
    if not function_match:
        return {"pass": False, "score": 0, "reason": "No function definition found"}

    function_code = function_match.group(1)

    # Configure epicbox
    epicbox.configure(profiles=[epicbox.Profile("python", DOCKER_IMAGE)])

    # Get the function name, test input, and expected output from the context
    function_name = context["vars"]["function_name"]
    test_input = context["vars"]["test_input"]
    expected_output = context["vars"]["expected_output"]

    # Prepare the code to run in the sandbox
    test_code = f"""
{function_code}

# Test the function
result = {function_name}({test_input})
print(result)
"""

    files = [{"name": "main.py", "content": test_code.encode("utf-8")}]
    limits = {"cputime": 1, "memory": 64}

    # Run the code in the sandbox
    result = epicbox.run("python", "python main.py", files=files, limits=limits)

    if result["exit_code"] != 0:
        return {
            "pass": False,
            "score": 0,
            "reason": f"Execution error: {result['stderr'].decode('utf-8')}",
        }

    # Compare the output with the expected result
    actual_output = result["stdout"].decode("utf-8").strip()
    if actual_output == str(expected_output):
        return {
            "pass": True,
            "score": 1,
            "reason": f"Correct output: got {expected_output}",
        }
    else:
        return {
            "pass": False,
            "score": 0,
            "reason": f"Incorrect output. Expected: {expected_output}, Got: {actual_output}",
        }