optimizer CLI and doc (#1151)

* optimizer CLI and doc

Signed-off-by: Mandana Vaziri <[email protected]>

Author: vazirim

.gitignore

@@ -154,6 +154,9 @@ pdl-live/package-lock.json
 # Demo files
 pdl-rag-demo.db
 test.jsonl
+train.jsonl
+validation.jsonl
+experiments/
 
 # Built docs
 _site

README.md

@@ -31,6 +31,12 @@ To install the `pdl` command line tool:
 pip install prompt-declaration-language
 ```
 
+## What's New
+
+Check out AutoPDL, PDL's prompt optimizer tool [Spiess et al. (2025)](https://openreview.net/forum?id=CAeISyE3aR)! AutoPDL can be used to optimize any part of a PDL program. This includes few-shots examples and textual prompts, but also prompting patterns. It outputs an optimized PDL program with optimal values.
+
+For a tutorial on how to use AutoPDL, see [AutoPDL](https://ibm.github.io/prompt-declaration-language/autopdl/)
+
 ## Example Program: A Basic LLM Call
 
 <img src="docs/assets/pdl-ui-3.png" width="500" align="right" alt="PDL GUI"/>

docs/autopdl.md

@@ -0,0 +1,28 @@
+defs:
+  max_tokens: 1024
+lastOf:
+  - "Here are examples of grammatically incorrect sentences and their corrected versions:\n\n"
+  - for:
+      example: ${ demonstrations }
+    repeat:
+      text: "${ example.input } -> ${ example.output }"
+    join:
+      with: "\n\n"
+  - |+
+    Correct the following sentence:
+    
+    ${ input }
+    Here's the corrected sentence:
+
+  - model: ${ model }
+    def: response
+    parameters:
+      max_tokens: ${ max_tokens }
+      temperature: 0
+      
+  - if: ${ verify }
+    then:
+      lastOf:
+      - Do you think this was a correct answer? If not, generated a correct answer.
+      - model: ${ model }
+    else: ${ response }

examples/optimizer/bea19.pdl

@@ -0,0 +1,41 @@
+pdl_path: grammar_correction.pdl # Path to the PDL file to optimize
+dataset: 
+  train: grammar_correction_jsonl/train.jsonl # Path to the training split in JSONL format
+  test: grammar_correction_jsonl/test.jsonl # Path to the test split in JSONL format
+  validation: grammar_correction_jsonl/validation.jsonl # Path to the validation split in JSONL format
+
+demonstrations_variable_name: demonstrations # variable name to insert demonstrations into
+demonstration_columns:
+  - input # column name for the question in the dataset
+  - output # column name for the answer in the dataset
+
+instance_columns:
+  - input # column name for the question in the dataset
+
+groundtruth_column: output # column name for the ground truth in the dataset
+
+eval_pdl: eval_levenshtein.pdl # Path to the PDL file for evaluation
+
+#budget: 2h # Set a budget, can be number of iterations, or a duration string e.g. "2h"
+#budget_growth: double # double validation set size each iteration. ## 
+# or to_max: reach max_test_set_size by final iteration
+initial_validation_set_size: 2 # size of test set in first iteration
+max_validation_set_size: 10 # maximum test set size. 
+max_test_set_size: 10
+num_candidates: 10 # how many candidates to evaluate
+parallelism: 5 # how many threads to run evaluations across
+#shuffle_test: false # shuffling of test set
+#test_set_name: test # name of test set
+#train_set_name: train # name of train set
+#validation_set_name: validation # name of validation set
+variables: # define discrete options to sample from
+  model: # set ${ model } variable
+    - ollama_chat/granite3.3:8b
+    - ollama_chat/gpt-oss:20b
+  num_demonstrations: # overrides num demonstrations above
+    - 0
+    - 3
+    - 5
+  verify:
+    - true 
+    - false

examples/optimizer/bea19_example.yml

@@ -0,0 +1,48 @@
+defs:
+  max_tokens: 1024
+  model: ollama_chat/gpt-oss:20b
+  num_demonstrations:
+    data: 5
+  verify:
+    data: false
+  demonstrations:
+    data:
+    - input: Related and Entities found using configured use relation direction. and Relation Type.
+      output: Related Entities found using configured relation direction and Relation Type.
+    - input: Thanks to Naumann IT Security Consulting's for reporting challenging the XSS got vulnerability.
+      output: Thanks to Naumann IT Security Consulting for reporting the XSS vulnerability.
+    - input: Besides he hates school, he is exhausted all the time, has no appetite, he has penalty of violent, depression, was not happy.
+      output: He hated school, he was exhausted all the time, had no appetite, he had outbreaks of violence, depression, and he was not happy.
+    - input: If your primary ID does not contain a signature, you can present a supplemental ID with photo and signature or a supple government ID with a photograph, as long as they are in the same name you used when you registerd.
+      output: If your primary ID does not contain a signature, you can present a supplemental ID with photo and signature or a supplemental government-issued ID with a photograph, as long as they are in the same name you used when you registered.
+    - input: We want to begin consultatiaon with public-use organisations who are users of these services to help brings experience, knowledge and information on user needs to shape the solution.
+      output: We want to begin consultations with public sector organisations who are users of these services to help bring experience, knowledge and information on user needs to shape the solution.
+lastOf:
+- |+
+  Here are examples of grammatically incorrect sentences and their corrected versions:
+
+- for:
+    example: ${ demonstrations }
+  repeat:
+    text: ${ example.input } -> ${ example.output }
+  join:
+    with: |2+
+
+
+- |+
+  Correct the following sentence:
+
+  ${ input }
+  Here's the corrected sentence:
+
+- def: response
+  model: ${ model }
+  parameters:
+    temperature: 0.0
+    max_tokens: ${ max_tokens }
+- if: ${ verify }
+  then:
+    lastOf:
+    - Do you think this was a correct answer? If not, generated a correct answer.
+    - model: ${ model }
+  else: ${ response }

examples/optimizer/grammar_correction.pdl

@@ -0,0 +1,35 @@
+import json
+from pathlib import Path
+
+from datasets.dataset_dict import DatasetDict
+from datasets.load import load_dataset
+
+# Load dataset
+grammar_correction = load_dataset("agentlans/grammar-correction")
+if not isinstance(grammar_correction, DatasetDict):
+    raise TypeError(
+        f"Expected grammar_correction to be a DatasetDict, but got: {type(grammar_correction)}"
+    )
+
+# Create validation split from train (1024 examples)
+new_split = grammar_correction["train"].train_test_split(test_size=1024)
+grammar_correction["test"] = new_split["test"]
+
+val_split = new_split["train"].train_test_split()
+grammar_correction["train"] = val_split["train"]
+grammar_correction["validation"] = val_split["test"]
+
+# Output dir
+out_dir = Path("grammar_correction_jsonl")
+out_dir.mkdir(parents=True, exist_ok=True)
+
+
+# Save to JSONL
+def save_jsonl(dataset, path: Path) -> None:
+    with path.open("w") as f:
+        for item in dataset:
+            f.write(json.dumps(item) + "\n")
+
+
+for split in ["train", "validation", "test"]:
+    save_jsonl(grammar_correction[split], out_dir / f"{split}.jsonl")