## ----include=FALSE------------------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment  = "#>"
)

## ----setup, message=FALSE-----------------------------------------------------
library(pairwiseLLM)
library(dplyr)
library(tidyr)
library(purrr)
library(readr)
library(stringr)

## -----------------------------------------------------------------------------
check_llm_api_keys()

## -----------------------------------------------------------------------------
data("example_writing_samples", package = "pairwiseLLM")

td <- trait_description("overall_quality")
td

## -----------------------------------------------------------------------------
tmpl <- set_prompt_template()
cat(substr(tmpl, 1, 400), "...
")

## -----------------------------------------------------------------------------
set.seed(123)

pairs_all <- example_writing_samples |>
  make_pairs()

n_pairs <- min(40L, nrow(pairs_all))

pairs_forward <- pairs_all |>
  sample_pairs(n_pairs = n_pairs, seed = 123) |>
  randomize_pair_order(seed = 456)

pairs_reverse <- sample_reverse_pairs(
  pairs_forward,
  reverse_pct = 1.0,
  seed        = 789
)

get_pairs_for_direction <- function(direction = c("forward", "reverse")) {
  direction <- match.arg(direction)
  if (identical(direction, "forward")) {
    pairs_forward
  } else {
    pairs_reverse
  }
}

## -----------------------------------------------------------------------------
anthropic_models <- c(
  "claude-haiku-4-5-20251001"
)

gemini_models <- c(
  "gemini-3.8-flash"
)

openai_models <- c(
  "gpt-4.1",
  "gpt-5.6-luna"
)

thinking_levels <- c("no_thinking", "with_thinking")
directions <- c("forward", "reverse")

anthropic_grid <- tidyr::expand_grid(
  provider  = "anthropic",
  model     = anthropic_models,
  thinking  = "no_thinking",
  direction = directions
)

gemini_grid <- tidyr::expand_grid(
  provider  = "gemini",
  model     = gemini_models,
  thinking  = "with_thinking",
  direction = directions
)

openai_grid <- tidyr::expand_grid(
  provider  = "openai",
  model     = openai_models,
  thinking  = "no_thinking",
  direction = directions
)

batch_grid <- dplyr::bind_rows(
  anthropic_grid,
  gemini_grid,
  openai_grid
)

batch_grid

## -----------------------------------------------------------------------------
templates_tbl <- tibble::tibble(
  template_id     = c("test1", "test2", "test3", "test4", "test5"),
  prompt_template = list(tmpl, tmpl, tmpl, tmpl, tmpl)
)

templates_tbl

## ----eval=FALSE---------------------------------------------------------------
# out_root <- "dev-output/advanced-multi-batch"
# dir.create(out_root, recursive = TRUE, showWarnings = FALSE)
# 
# run_plan <- tidyr::crossing(
#   templates_tbl |> tidyr::unnest(prompt_template),
#   batch_grid
# ) |>
#   mutate(
#     run_id = paste(template_id, provider, model, thinking, direction, sep = "__"),
#     run_id = gsub("[^A-Za-z0-9_.-]+", "-", run_id),
#     run_dir = file.path(out_root, run_id)
#   )
# 
# run_plan |> dplyr::select(run_id, template_id, provider, model, thinking, direction, run_dir)

## ----eval=FALSE---------------------------------------------------------------
# submit_one_run <- function(template_id, prompt_template, provider, model, thinking, direction, run_dir) {
#   pairs_use   <- get_pairs_for_direction(direction)
#   is_thinking <- identical(thinking, "with_thinking")
# 
#   # Provider-specific knobs (passed through via ...)
#   extra_args <- list()
# 
#   if (identical(provider, "openai")) {
#     # Only request thoughts for models that support them in this workflow
#     extra_args$include_thoughts <- is_thinking && grepl("^gpt-5", model)
#     extra_args$include_raw      <- TRUE
#   } else if (identical(provider, "anthropic")) {
#     extra_args$reasoning        <- if (is_thinking) "enabled" else "none"
#     extra_args$include_thoughts <- is_thinking
#     extra_args$include_raw      <- TRUE
#     # Optional: set deterministic temperature when not using reasoning
#     if (!is_thinking) extra_args$temperature <- 0
#   } else if (identical(provider, "gemini")) {
#     extra_args$include_thoughts <- TRUE
#     extra_args$thinking_level   <- "low"   # example
#     extra_args$include_raw      <- TRUE
#   }
# 
#   message(
#     "Submitting: ", template_id, " | ", provider, " / ", model,
#     " / ", thinking, " / ", direction
#   )
# 
#   # Split strategy:
#   # - For real jobs, use batch_size (e.g., 500–5000) or n_segments (e.g., 10–50)
#   # - Here we keep it simple and submit a single segment per run
#   do.call(
#     llm_submit_pairs_multi_batch,
#     c(
#       list(
#         pairs             = pairs_use,
#         backend           = provider,
#         model             = model,
#         trait_name        = td$name,
#         trait_description = td$description,
#         prompt_template   = prompt_template,
#         n_segments        = 1L,
#         output_dir        = run_dir,
#         write_registry    = TRUE,
#         verbose           = TRUE
#       ),
#       extra_args
#     )
#   )
# }
# 
# run_results <- purrr::pmap(
#   run_plan |>
#     dplyr::select(
#       template_id, prompt_template, provider, model, thinking, direction,
#       run_dir
#     ),
#   submit_one_run
# )
# 
# # Store a lightweight manifest so you can resume later without rebuilding run_plan
# manifest <- run_plan |>
#   mutate(registry_path = file.path(run_dir, "jobs_registry.csv"))
# 
# manifest_path <- file.path(out_root, "run_manifest.csv")
# readr::write_csv(manifest, manifest_path)
# 
# manifest_path

## ----eval=FALSE---------------------------------------------------------------
# manifest_path <- file.path(out_root, "run_manifest.csv")
# manifest <- readr::read_csv(manifest_path, show_col_types = FALSE)
# 
# poll_one_run <- function(run_dir) {
#   llm_resume_multi_batches(
#     jobs               = NULL,   # load from jobs_registry.csv in run_dir
#     output_dir         = run_dir,
#     interval_seconds   = 60,
#     per_job_delay      = 2,
#     write_results_csv  = TRUE,   # writes batch_XX_results.csv files
#     write_registry     = TRUE,   # refreshes jobs_registry.csv with done flags
#     keep_jsonl         = TRUE,
#     verbose            = TRUE,
#     write_combined_csv = TRUE,   # writes combined_results.csv inside run_dir
#     combined_csv_path  = "combined_results.csv"
#   )
# }
# 
# polled <- purrr::map(manifest$run_dir, poll_one_run)

## ----eval=FALSE---------------------------------------------------------------
# combined_all <- purrr::map2_dfr(
#   polled,
#   seq_len(nrow(manifest)),
#   function(res, i) {
#     meta <- manifest[i, ]
#     if (is.null(res$combined)) return(NULL)
# 
#     res$combined |>
#       mutate(
#         template_id = meta$template_id,
#         provider    = meta$provider,
#         model       = meta$model,
#         thinking    = meta$thinking,
#         direction   = meta$direction,
#         run_id      = meta$run_id
#       )
#   }
# )
# 
# combined_path <- file.path(out_root, "combined_all_runs.csv")
# readr::write_csv(combined_all, combined_path)
# 
# combined_path

## ----eval=FALSE---------------------------------------------------------------
# manifest <- readr::read_csv(file.path(out_root, "run_manifest.csv"), show_col_types = FALSE)
# 
# needs_poll <- function(run_dir) {
#   reg_path <- file.path(run_dir, "jobs_registry.csv")
#   if (!file.exists(reg_path)) return(FALSE)
#   reg <- readr::read_csv(reg_path, show_col_types = FALSE)
#   any(!as.logical(reg$done))
# }
# 
# unfinished_dirs <- manifest$run_dir[vapply(manifest$run_dir, needs_poll, logical(1))]
# 
# polled <- purrr::map(unfinished_dirs, poll_one_run)