Skip to content

dspy.MIPROv2

MIPROv2 (提示指令提案优化器 版本2) 是一种能够同时优化指令和少样本示例的提示优化器。它通过自举少样本候选示例、基于任务不同动态提出指令,并使用贝叶斯优化找到这些选项的最优组合来实现这一目标。它可用于联合优化少样本示例和指令,或仅用于零样本优化的指令。

dspy.MIPROv2(metric: Callable, prompt_model: Any | None = None, task_model: Any | None = None, teacher_settings: dict | None = None, max_bootstrapped_demos: int = 4, max_labeled_demos: int = 4, auto: Literal['light', 'medium', 'heavy'] | None = 'light', num_candidates: int | None = None, num_threads: int | None = None, max_errors: int | None = None, seed: int = 9, init_temperature: float = 1.0, verbose: bool = False, track_stats: bool = True, log_dir: str | None = None, metric_threshold: float | None = None)

基类: Teleprompter

Source code in dspy/teleprompt/mipro_optimizer_v2.py
def __init__(
    self,
    metric: Callable,
    prompt_model: Any | None = None,
    task_model: Any | None = None,
    teacher_settings: dict | None = None,
    max_bootstrapped_demos: int = 4,
    max_labeled_demos: int = 4,
    auto: Literal["light", "medium", "heavy"] | None = "light",
    num_candidates: int | None = None,
    num_threads: int | None = None,
    max_errors: int | None = None,
    seed: int = 9,
    init_temperature: float = 1.0,
    verbose: bool = False,
    track_stats: bool = True,
    log_dir: str | None = None,
    metric_threshold: float | None = None,
):
    # Validate 'auto' parameter
    allowed_modes = {None, "light", "medium", "heavy"}
    if auto not in allowed_modes:
        raise ValueError(f"Invalid value for auto: {auto}. Must be one of {allowed_modes}.")
    self.auto = auto
    self.num_fewshot_candidates = num_candidates
    self.num_instruct_candidates = num_candidates
    self.num_candidates = num_candidates
    self.metric = metric
    self.init_temperature = init_temperature
    self.task_model = task_model if task_model else dspy.settings.lm
    self.prompt_model = prompt_model if prompt_model else dspy.settings.lm
    self.max_bootstrapped_demos = max_bootstrapped_demos
    self.max_labeled_demos = max_labeled_demos
    self.verbose = verbose
    self.track_stats = track_stats
    self.log_dir = log_dir
    self.teacher_settings = teacher_settings or {}
    self.prompt_model_total_calls = 0
    self.total_calls = 0
    self.num_threads = num_threads
    self.max_errors = max_errors
    self.metric_threshold = metric_threshold
    self.seed = seed
    self.rng = None

函数

compile(student: Any, *, trainset: list, teacher: Any = None, valset: list | None = None, num_trials: int | None = None, max_bootstrapped_demos: int | None = None, max_labeled_demos: int | None = None, seed: int | None = None, minibatch: bool = True, minibatch_size: int = 35, minibatch_full_eval_steps: int = 5, program_aware_proposer: bool = True, data_aware_proposer: bool = True, view_data_batch_size: int = 10, tip_aware_proposer: bool = True, fewshot_aware_proposer: bool = True, requires_permission_to_run: bool | None = None, provide_traceback: bool | None = None) -> Any

Source code in dspy/teleprompt/mipro_optimizer_v2.py
def compile(
    self,
    student: Any,
    *,
    trainset: list,
    teacher: Any = None,
    valset: list | None = None,
    num_trials: int | None = None,
    max_bootstrapped_demos: int | None = None,
    max_labeled_demos: int | None = None,
    seed: int | None = None,
    minibatch: bool = True,
    minibatch_size: int = 35,
    minibatch_full_eval_steps: int = 5,
    program_aware_proposer: bool = True,
    data_aware_proposer: bool = True,
    view_data_batch_size: int = 10,
    tip_aware_proposer: bool = True,
    fewshot_aware_proposer: bool = True,
    requires_permission_to_run: bool | None = None, # deprecated
    provide_traceback: bool | None = None,
) -> Any:
    if requires_permission_to_run == False:
        logger.warning(
            "'requires_permission_to_run' is deprecated and will be removed in a future version."
        )
    elif requires_permission_to_run == True:
        raise ValueError("User confirmation is removed from MIPROv2. Please remove the 'requires_permission_to_run' argument.")

    effective_max_errors = (
        self.max_errors
        if self.max_errors is not None
        else dspy.settings.max_errors
    )
    zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)

    # If auto is None, and num_trials is not provided (but num_candidates is), raise an error that suggests a good num_trials value
    if self.auto is None and (self.num_candidates is not None and num_trials is None):
        raise ValueError(
            f"If auto is None, num_trials must also be provided. Given num_candidates={self.num_candidates}, we'd recommend setting num_trials to ~{self._set_num_trials_from_num_candidates(student, zeroshot_opt, self.num_candidates)}."
        )

    # If auto is None, and num_candidates or num_trials is None, raise an error
    if self.auto is None and (self.num_candidates is None or num_trials is None):
        raise ValueError("If auto is None, num_candidates must also be provided.")

    # If auto is provided, and either num_candidates or num_trials is not None, raise an error
    if self.auto is not None and (self.num_candidates is not None or num_trials is not None):
        raise ValueError(
            "If auto is not None, num_candidates and num_trials cannot be set, since they would be overridden by the auto settings. Please either set auto to None, or do not specify num_candidates and num_trials."
        )

    # Set random seeds
    seed = seed or self.seed
    self._set_random_seeds(seed)

    # Update max demos if specified
    if max_bootstrapped_demos is not None:
        self.max_bootstrapped_demos = max_bootstrapped_demos
    if max_labeled_demos is not None:
        self.max_labeled_demos = max_labeled_demos

    # Set training & validation sets
    trainset, valset = self._set_and_validate_datasets(trainset, valset)

    # Set hyperparameters based on run mode (if set)
    num_trials, valset, minibatch = self._set_hyperparams_from_run_mode(
        student, num_trials, minibatch, zeroshot_opt, valset
    )

    if self.auto:
        self._print_auto_run_settings(num_trials, minibatch, valset)

    if minibatch and minibatch_size > len(valset):
        raise ValueError(f"Minibatch size cannot exceed the size of the valset. Valset size: {len(valset)}.")

    # Initialize program and evaluator
    program = student.deepcopy()
    evaluate = Evaluate(
        devset=valset,
        metric=self.metric,
        num_threads=self.num_threads,
        max_errors=effective_max_errors,
        display_table=False,
        display_progress=True,
        provide_traceback=provide_traceback,
    )

    # Step 1: Bootstrap few-shot examples
    demo_candidates = self._bootstrap_fewshot_examples(program, trainset, seed, teacher)

    # Step 2: Propose instruction candidates
    instruction_candidates = self._propose_instructions(
        program,
        trainset,
        demo_candidates,
        view_data_batch_size,
        program_aware_proposer,
        data_aware_proposer,
        tip_aware_proposer,
        fewshot_aware_proposer,
    )

    # If zero-shot, discard demos
    if zeroshot_opt:
        demo_candidates = None

    # Step 3: Find optimal prompt parameters
    best_program = self._optimize_prompt_parameters(
        program,
        instruction_candidates,
        demo_candidates,
        evaluate,
        valset,
        num_trials,
        minibatch,
        minibatch_size,
        minibatch_full_eval_steps,
        seed,
    )

    return best_program

get_params() -> dict[str, Any]

获取提示器的参数。

返回:

类型 描述
dict[str, Any]

提示器的参数。

Source code in dspy/teleprompt/teleprompt.py
def get_params(self) -> dict[str, Any]:
    """
    Get the parameters of the teleprompter.

    Returns:
        The parameters of the teleprompter.
    """
    return self.__dict__

:::

示例用法

以下程序展示了使用MIPROv2优化数学程序

import dspy
from dspy.datasets.gsm8k import GSM8K, gsm8k_metric

# Import the optimizer
from dspy.teleprompt import MIPROv2

# Initialize the LM
lm = dspy.LM('openai/gpt-4o-mini', api_key='YOUR_OPENAI_API_KEY')
dspy.configure(lm=lm)

# Initialize optimizer
teleprompter = MIPROv2(
    metric=gsm8k_metric,
    auto="medium", # Can choose between light, medium, and heavy optimization runs
)

# Optimize program
print(f"Optimizing program with MIPROv2...")
gsm8k = GSM8K()
optimized_program = teleprompter.compile(
    dspy.ChainOfThought("question -> answer"),
    trainset=gsm8k.train,
)

# Save optimize program for future use
optimized_program.save(f"optimized.json")

MIPROv2 的工作原理

在高层面上,MIPROv2通过为你的LM程序中的每个预测器创建少量样本示例和新指令来工作,然后使用贝叶斯优化搜索这些变量,为你的程序找到这些变量的最佳组合。如果你想了解视觉解释,请查看这个twitter thread

以下更详细地分解了这些步骤:

1) 引导式少样本示例: 从训练集中随机采样示例,并通过您的语言模型程序运行它们。如果程序对此示例的输出正确,则将其保留为有效的少样本示例候选。否则,我们会尝试另一个示例,直到收集到指定数量的少样本示例候选。此步骤创建num_candidates组包含max_bootstrapped_demos个引导示例和max_labeled_demos个从训练集中采样的基础示例。

2) 提出指令候选。指令提议器包含:(1) 生成的训练数据集属性摘要,(2) 生成的LM程序代码及特定预测器的摘要,(3) 先前引导的少量示例以展示给定预测器的参考输入/输出,以及(4) 随机采样的生成提示(例如"要有创意"、"要简洁"等)以帮助探索潜在指令的特征空间。该上下文提供给一个prompt_model,用于编写高质量的指令候选。

3) 寻找优化的小样本示例与指令组合。最后,我们使用贝叶斯优化来选择哪些指令和演示的组合在我们的程序中对每个预测器效果最佳。这通过运行一系列num_trials试验来实现,每次试验都会在验证集上评估一组新的提示。每次试验中,新提示集仅在大小为minibatch_size的小批量上进行评估(当minibatch=True时)。然后,每minibatch_full_eval_steps次会在完整验证集上评估最佳平均提示集。在优化过程结束时,返回在完整验证集上表现最佳的那组提示的LM程序。

对于想要了解更多细节的人,可以在这篇论文中找到关于MIPROv2的更多信息,以及一项将MIPROv2与其他DSPy优化器进行比较的研究。

优云智算