## ----include=FALSE------------------------------------------------------------ knitr::opts_chunk$set( collapse = TRUE, comment = "#>", eval = TRUE ) library(pairwiseLLM) library(dplyr) ## ----------------------------------------------------------------------------- data("example_writing_samples", package = "pairwiseLLM") dplyr::slice_head(example_writing_samples, n = 3) ## ----------------------------------------------------------------------------- pairs <- example_writing_samples |> make_pairs() dplyr::slice_head(pairs, n = 5) ## ----------------------------------------------------------------------------- pairs_small <- sample_pairs(pairs, n_pairs = 10, seed = 123) ## ----------------------------------------------------------------------------- pairs_small <- randomize_pair_order(pairs_small, seed = 99) ## ----------------------------------------------------------------------------- td <- trait_description("overall_quality") td ## ----------------------------------------------------------------------------- td_custom <- trait_description( custom_name = "Clarity", custom_description = "How clearly and effectively ideas are expressed." ) ## ----------------------------------------------------------------------------- tmpl <- set_prompt_template() cat(substr(tmpl, 1, 300)) ## ----eval=FALSE--------------------------------------------------------------- # set_prompt_template(file = "my_template.txt") ## ----eval=FALSE--------------------------------------------------------------- # # Example using parallel processing and incremental saving # res_list <- submit_llm_pairs( # pairs = pairs_small, # backend = "openai", # also "anthropic", "gemini", "vertex", "together", "ollama" # model = "gpt-4o", # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # # New features: # parallel = TRUE, # workers = 2, # save_path = "live_results.csv" # ) ## ----eval=FALSE--------------------------------------------------------------- # # Gemini Developer API live request # res_gemini <- submit_llm_pairs( # pairs = pairs_small, # backend = "gemini", # model = "gemini-3.5-flash-lite", # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # service_tier = "priority" # ) # # # Vertex AI Gemini API live request # res_vertex <- submit_llm_pairs( # pairs = pairs_small, # backend = "vertex", # model = "gemini-3.8-flash", # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # service_tier = "flex" # ) ## ----eval=FALSE--------------------------------------------------------------- # # Successes are in the $results tibble # dplyr::slice_head(res_list$results, 5) # # # Failures (if any) are in $failed_pairs # if (nrow(res_list$failed_pairs) > 0) { # print(res_list$failed_pairs) # } # # # Attempt-level failures (if any) are in $failed_attempts # if (nrow(res_list$failed_attempts) > 0) { # print(res_list$failed_attempts) # } ## ----eval=FALSE--------------------------------------------------------------- # # res_list: output list from submit_llm_pairs() # # We extract the $results tibble for modeling # bt_data <- build_bt_data(res_list$results) # dplyr::slice_head(bt_data, 5) ## ----eval=FALSE--------------------------------------------------------------- # # res_list: output from submit_llm_pairs() # elo_data <- build_elo_data(res_list$results) ## ----eval=FALSE--------------------------------------------------------------- # bt_fit <- fit_bt_model(bt_data) ## ----eval=FALSE--------------------------------------------------------------- # summarize_bt_fit(bt_fit) ## ----eval=FALSE--------------------------------------------------------------- # elo_fit <- fit_elo_model(elo_data, runs = 5) # elo_fit ## ----eval=FALSE--------------------------------------------------------------- # batch <- llm_submit_pairs_batch( # backend = "gemini", # model = "gemini-3.8-flash", # pairs = pairs_small, # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # service_tier = "priority" # ) ## ----eval=FALSE--------------------------------------------------------------- # res_batch <- llm_download_batch_results(batch) # head(res_batch) ## ----eval=FALSE--------------------------------------------------------------- # # Generate a small set of pairs # pairs_small <- example_writing_samples |> # make_pairs() |> # sample_pairs(n_pairs = 10, seed = 4321) |> # randomize_pair_order(seed = 8765) # # td <- trait_description("overall_quality") # tmpl <- set_prompt_template() # # # Split into two batches and include reasoning/chain-of-thought # multi_job <- llm_submit_pairs_multi_batch( # pairs = pairs_small, # backend = "openai", # model = "gpt-5.1", # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # n_segments = 2, # output_dir = "myjob", # write_registry = TRUE, # include_thoughts = TRUE # ) # # # Poll and merge results. Combined results are written to # # "myjob/combined_results.csv" or the directory you specify. # res <- llm_resume_multi_batches( # jobs = multi_job$jobs, # interval_seconds = 30, # write_combined_csv = TRUE # ) # # head(res$combined) ## ----eval=FALSE--------------------------------------------------------------- # # Create a moderate set of pairs # pairs_big <- example_writing_samples |> # make_pairs() |> # sample_pairs(n_pairs = 200, seed = 123) |> # randomize_pair_order(seed = 456) # # td <- trait_description("overall_quality") # tmpl <- set_prompt_template() # # est <- estimate_llm_pairs_cost( # pairs = pairs_big, # backend = "anthropic", # "openai", "anthropic", "gemini", "together" # model = "claude-sonnet-4-5", # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl, # mode = "batch", # batch_discount = 0.5, # set to 1 for no discount # n_test = 10, # paid pilot calls (live) # budget_quantile = 0.9, # p90 output tokens # cost_per_million_input = 3.0, # fill in your provider pricing # cost_per_million_output = 15.0 # ) # # est$summary ## ----eval=FALSE--------------------------------------------------------------- # remaining_pairs <- est$remaining_pairs # # # Example: submit only the remaining pairs as a batch # # batch <- llm_submit_pairs_batch( # backend = "anthropic", # model = "claude-sonnet-4-5", # pairs = remaining_pairs, # trait_name = td$name, # trait_description = td$description, # prompt_template = tmpl) # # results <- llm_download_batch_results(batch) ## ----------------------------------------------------------------------------- check_llm_api_keys()