Phase A: constraint benchmark and solve rate reporting

docs/solve-rate-results.md

@@ -0,0 +1,40 @@
+# Solve rate results
+
+Written by `./gradlew benchmark -Pargs="--solve-rate=true"`; see [new-strategies-plan.md](new-strategies-plan.md).
+Each run stops when it solves the problem, so the time is the time to a solution. The unsolved fitness is the
+median number of constraints still broken in the runs that did not. Only compare runs made on the same machine.
+
+Java 21.0.12.1, 2 processors, 20 seeds, 2000 ms or 5000000 evaluations per run
+
+| Problem | Strategy | Solved | Median ms (solved) | Median evals (solved) | Median broken (unsolved) |
+| --- | --- | ---: | ---: | ---: | ---: |
+| queens-100 | SIMULATED_ANNEALING | 20/20 | 60 | 17,517 | - |
+| queens-100 | GLOBAL_SIMULATED_ANNEALING | 20/20 | 62 | 18,800 | - |
+| queens-100 | HILL_CLIMBING | 0/20 | - | - | 39.0 |
+| queens-100 | GLOBAL_HILL_CLIMBING | 0/20 | - | - | 36.0 |
+| queens-100 | GENETIC_SEARCH | 0/20 | - | - | 2.0 |
+| queens-100 | CONCURRENT_GENETIC_SEARCH | 0/20 | - | - | 2.0 |
+| queens-100 | ITERATED_LOCAL_SEARCH | 0/20 | - | - | 20.0 |
+| queens-100 | VARIABLE_NEIGHBORHOOD_SEARCH | 0/20 | - | - | 20.0 |
+| queens-400 | SIMULATED_ANNEALING | 20/20 | 818 | 97,017 | - |
+| queens-400 | GLOBAL_SIMULATED_ANNEALING | 20/20 | 993 | 94,068 | - |
+| queens-400 | HILL_CLIMBING | 0/20 | - | - | 206.0 |
+| queens-400 | GLOBAL_HILL_CLIMBING | 0/20 | - | - | 199.0 |
+| queens-400 | GENETIC_SEARCH | 0/20 | - | - | 155.5 |
+| queens-400 | CONCURRENT_GENETIC_SEARCH | 0/20 | - | - | 155.0 |
+| queens-400 | ITERATED_LOCAL_SEARCH | 0/20 | - | - | 155.0 |
+| queens-400 | VARIABLE_NEIGHBORHOOD_SEARCH | 0/20 | - | - | 153.0 |
+| coloring-200 | SIMULATED_ANNEALING | 0/20 | - | - | 141.0 |
+| coloring-200 | HILL_CLIMBING | 0/20 | - | - | 107.5 |
+| coloring-200 | GENETIC_SEARCH | 0/20 | - | - | 116.0 |
+| coloring-200 | CONCURRENT_GENETIC_SEARCH | 0/20 | - | - | 115.0 |
+| coloring-200 | ITERATED_LOCAL_SEARCH | 0/20 | - | - | 131.0 |
+| coloring-200 | VARIABLE_NEIGHBORHOOD_SEARCH | 0/20 | - | - | 132.0 |
+| all-interval-14 | SIMULATED_ANNEALING | 1/20 | 2 | 1,771 | 1.0 |
+| all-interval-14 | GLOBAL_SIMULATED_ANNEALING | 1/20 | 9 | 9,764 | 1.0 |
+| all-interval-14 | HILL_CLIMBING | 0/20 | - | - | 2.0 |
+| all-interval-14 | GLOBAL_HILL_CLIMBING | 0/20 | - | - | 1.0 |
+| all-interval-14 | GENETIC_SEARCH | 10/20 | 205 | 58,450 | 1.0 |
+| all-interval-14 | CONCURRENT_GENETIC_SEARCH | 10/20 | 219 | 58,451 | 1.0 |
+| all-interval-14 | ITERATED_LOCAL_SEARCH | 5/20 | 3 | 2,436 | 1.0 |
+| all-interval-14 | VARIABLE_NEIGHBORHOOD_SEARCH | 4/20 | 17 | 9,609 | 1.0 |

scala-test/com/barrybecker4/optimization/benchmark/AllIntervalProblem.scala

@@ -0,0 +1,50 @@
+// Copyright by Barry G. Becker, 2026. Licensed under MIT License: http://www.opensource.org/licenses/MIT
+package com.barrybecker4.optimization.benchmark
+
+import com.barrybecker4.optimization.optimizee.optimizees.OptimizeeProblem
+import com.barrybecker4.optimization.parameter.types.IntegerParameter
+import com.barrybecker4.optimization.parameter.{ParameterArray, ParameterArrayWithFitness, PermutedParameterArray}
+
+import scala.util.Random
+
+/**
+  * The all-interval series: arrange the numbers 0 to n - 1 so that the differences between neighbors are all
+  * different, which makes them exactly 1 to n - 1. It comes from music (a twelve tone row that uses every interval).
+  * The fitness is the number of those differences that no pair of neighbors has, and 0 is a solution.
+  *
+  * It is a good counterweight to N-queens for a benchmark: there are many plateaus, because changing one number moves
+  * two differences at once and rarely helps, and there is no conflict to repair directly.
+  * @param size how many numbers; at least 3
+  * @param seed decides the order in the initial guess
+  */
+class AllIntervalProblem(size: Int, seed: Long) extends OptimizeeProblem {
+
+  require(size >= 3, s"The series needs at least 3 numbers, but got $size")
+
+  override def getName: String = s"All-interval series of $size"
+
+  override def evaluateByComparison: Boolean = false
+
+  /** The number of differences from 1 to n - 1 that are missing. */
+  override def evaluateFitness(series: ParameterArray): Double = {
+    val present = new Array[Boolean](size)
+    (1 until size).foreach(i => present(Math.abs(numberAt(series, i) - numberAt(series, i - 1))) = true)
+    (1 until size).count(!present(_)).toDouble
+  }
+
+  private def numberAt(series: ParameterArray, position: Int): Int = series.get(position).getValue.toInt
+
+  override def getInitialGuess: ParameterArray = seriesOf(new Random(seed + 1).shuffle((0 until size).toList), seed + 2)
+
+  /** Alternating the smallest and the largest number that is left gives the differences n - 1, n - 2, ..., 1. */
+  override def getExactSolution: ParameterArrayWithFitness = {
+    val series = seriesOf((0 until size).map(i => if (i % 2 == 0) i / 2 else size - 1 - i / 2), seed)
+    ParameterArrayWithFitness(series, evaluateFitness(series))
+  }
+
+  private def seriesOf(numbers: Seq[Int], randomSeed: Long): ParameterArray =
+    PermutedParameterArray(numbers.indices.map(i => new IntegerParameter(numbers(i), 0, size - 1, "n" + i)), new Random(randomSeed))
+
+  /** A random series misses about a third of the differences. */
+  override def getFitnessRange: Double = size.toDouble
+}

scala-test/com/barrybecker4/optimization/benchmark/AllIntervalProblemSuite.scala

@@ -0,0 +1,36 @@
+// Copyright by Barry G. Becker, 2026. Licensed under MIT License: http://www.opensource.org/licenses/MIT
+package com.barrybecker4.optimization.benchmark
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class AllIntervalProblemSuite extends AnyFunSuite {
+
+  private def arrayOf(values: Seq[Int]) = com.barrybecker4.optimization.parameter.PermutedParameterArray(
+    values.indices.map(i => new com.barrybecker4.optimization.parameter.types.IntegerParameter(values(i), 0, values.size - 1, "p" + i)),
+    new scala.util.Random(1))
+
+  test("the known series has every difference once, for any size") {
+    (3 to 40).foreach(size => assertResult(0.0, s"size $size")(new AllIntervalProblem(size, 1).getExactSolution.fitness))
+  }
+
+  test("numbers in order have just one difference, so all the others are missing") {
+    val problem = new AllIntervalProblem(10, 1)
+    assertResult(8.0)(problem.evaluateFitness(arrayOf(0 until 10)))
+  }
+
+  test("the initial guess has each number once, and is not already a solution") {
+    val problem = new AllIntervalProblem(14, 2)
+    val guess = problem.getInitialGuess
+    assertResult((0 until 14).toList)((0 until 14).map(guess.get(_).getValue.toInt).sorted.toList)
+    assert(problem.evaluateFitness(guess) > 0)
+  }
+
+  test("the same seed gives the same initial guess") {
+    def values(seed: Long) = { val g = new AllIntervalProblem(14, seed).getInitialGuess; (0 until 14).map(g.get(_).getValue) }
+    assertResult(values(3))(values(3))
+  }
+
+  test("a series needs at least 3 numbers") {
+    intercept[IllegalArgumentException](new AllIntervalProblem(2, 1))
+  }
+}

scala-test/com/barrybecker4/optimization/benchmark/BenchmarkConfig.scala

@@ -13,6 +13,11 @@ object BenchmarkConfig {
   val DEFAULT_REPORT_FILE = "docs/benchmark-results.md"
   private val NO_REPORT = "none"
 
+  /** The solve rate run is separate, with more seeds and a shorter limit, since a failed run takes the whole limit. */
+  val DEFAULT_SOLVE_RATE_REPORT_FILE = "docs/solve-rate-results.md"
+  val DEFAULT_SOLVE_RATE_SEEDS = 20
+  val DEFAULT_SOLVE_RATE_TIME_LIMIT_MILLIS = 2000L
+
   /** The strategies that make sense on permutations and other discrete problems, and finish on their own. */
   val DEFAULT_STRATEGIES: Seq[OptimizationStrategyType] = Seq(
     SIMULATED_ANNEALING, GLOBAL_SIMULATED_ANNEALING, HILL_CLIMBING, GLOBAL_HILL_CLIMBING,
@@ -23,29 +28,41 @@ object BenchmarkConfig {
     * (the last two are comma separated lists), report (the Markdown summary file, or `none`) and out (a file to
     * write the tab separated results of every run to).
     *
+    * The option `solve-rate=true` runs the constraint problems instead (see [[BenchmarkProblems.CONSTRAINT]]), with
+    * 20 seeds and 2 seconds as the defaults, and reports how many runs solved each, and how fast. Its full run writes
+    * [[DEFAULT_SOLVE_RATE_REPORT_FILE]].
+    *
     * A full run (all problems and the default strategies) writes the summary to [[DEFAULT_REPORT_FILE]]. A run
     * restricted to some problems or strategies writes no summary unless `report` is given, so it cannot replace
     * the full one with a partial one.
     */
   def parse(args: Seq[String]): BenchmarkConfig = {
     val options = args.filter(_.startsWith("--")).map(_.drop(2).split("=", 2)).collect { case Array(k, v) => k -> v }.toMap
+    val solveRate = options.get("solve-rate").contains("true")
     val problemNames = options.get("problems").map(_.split(",").toSeq).getOrElse(Seq.empty)
     val strategies = options.get("strategies").map(_.split(",").toSeq.map(OptimizationStrategyType.valueOf))
       .getOrElse(DEFAULT_STRATEGIES)
+    val isFullRun = problemNames.isEmpty && strategies == DEFAULT_STRATEGIES
     BenchmarkConfig(
-      timeLimitMillis = options.get("time-ms").map(_.toLong).getOrElse(DEFAULT_TIME_LIMIT_MILLIS),
+      timeLimitMillis = options.get("time-ms").map(_.toLong).getOrElse(defaultTimeLimit(solveRate)),
       maxEvaluations = options.get("max-evals").map(_.toLong).getOrElse(DEFAULT_MAX_EVALUATIONS),
-      numSeeds = options.get("seeds").map(_.toInt).getOrElse(DEFAULT_NUM_SEEDS),
-      problemNames = problemNames,
+      numSeeds = options.get("seeds").map(_.toInt).getOrElse(defaultSeeds(solveRate)),
+      problemNames = if (solveRate && problemNames.isEmpty) BenchmarkProblems.CONSTRAINT.map(_.name) else problemNames,
       strategies = strategies,
       outputFile = options.get("out"),
-      reportFile = reportFile(options.get("report"), isFullRun = problemNames.isEmpty && strategies == DEFAULT_STRATEGIES))
+      reportFile = reportFile(options.get("report"), isFullRun, solveRate),
+      solveRate = solveRate)
   }
 
-  private def reportFile(requested: Option[String], isFullRun: Boolean): Option[String] = requested match {
+  private def defaultTimeLimit(solveRate: Boolean): Long =
+    if (solveRate) DEFAULT_SOLVE_RATE_TIME_LIMIT_MILLIS else DEFAULT_TIME_LIMIT_MILLIS
+
+  private def defaultSeeds(solveRate: Boolean): Int = if (solveRate) DEFAULT_SOLVE_RATE_SEEDS else DEFAULT_NUM_SEEDS
+
+  private def reportFile(requested: Option[String], isFullRun: Boolean, solveRate: Boolean): Option[String] = requested match {
     case Some(NO_REPORT) => None
     case Some(file) => Some(file)
-    case None => Option.when(isFullRun)(DEFAULT_REPORT_FILE)
+    case None => Option.when(isFullRun)(if (solveRate) DEFAULT_SOLVE_RATE_REPORT_FILE else DEFAULT_REPORT_FILE)
   }
 }
 
@@ -57,7 +74,8 @@ object BenchmarkConfig {
   * @param strategies the strategies to run
   * @param outputFile where to write the results of every run as tab separated values, if anywhere
   * @param reportFile where to write the summary as a Markdown document, if anywhere
+  * @param solveRate true to report how many runs reached the optimal fitness, for the constraint problems
   */
 case class BenchmarkConfig(timeLimitMillis: Long, maxEvaluations: Long, numSeeds: Int,
                            problemNames: Seq[String], strategies: Seq[OptimizationStrategyType],
-                           outputFile: Option[String], reportFile: Option[String])
+                           outputFile: Option[String], reportFile: Option[String], solveRate: Boolean = false)

scala-test/com/barrybecker4/optimization/benchmark/BenchmarkConfigSuite.scala

@@ -53,4 +53,21 @@ class BenchmarkConfigSuite extends AnyFunSuite {
     val error = intercept[IllegalArgumentException](BenchmarkProblems.named(Seq("nope")))
     assert(error.getMessage.contains("tour-50"))
   }
+
+  test("a solve rate run is of the constraint problems, with more seeds and a shorter limit") {
+    val config = BenchmarkConfig.parse(Seq("--solve-rate=true"))
+    assert(config.solveRate)
+    assertResult(BenchmarkProblems.CONSTRAINT.map(_.name))(config.problemNames)
+    assertResult((20, 2000L))((config.numSeeds, config.timeLimitMillis))
+    assertResult(Some("docs/solve-rate-results.md"))(config.reportFile)
+  }
+
+  test("the solve rate defaults can be overridden, and some problems can be chosen") {
+    val config = BenchmarkConfig.parse(Seq("--solve-rate=true", "--seeds=3", "--time-ms=100", "--problems=queens-100"))
+    assertResult((3, 100L, Seq("queens-100"), None))((config.numSeeds, config.timeLimitMillis, config.problemNames, config.reportFile))
+  }
+
+  test("an ordinary run is not a solve rate run") {
+    assert(!BenchmarkConfig.parse(Seq.empty).solveRate)
+  }
 }

scala-test/com/barrybecker4/optimization/benchmark/BenchmarkProblems.scala

@@ -6,8 +6,10 @@ import com.barrybecker4.optimization.optimizee.optimizees.problems.*
 
 /**
   * A problem to benchmark. `create` makes a fresh problem for the seed, so that repeated runs start from the same place.
+  * @param isConstraint true when the problem has a solution that is known to exist and the goal is to reach it. Such a
+  *                     problem is judged by how many runs solve it and how quickly, and a run stops when it does.
   */
-case class BenchmarkProblem(name: String, create: Long => OptimizeeProblem)
+case class BenchmarkProblem(name: String, create: Long => OptimizeeProblem, isConstraint: Boolean = false)
 
 /** The problems that are benchmarked. The tours of increasing length show how a strategy scales with permutation size. */
 object BenchmarkProblems {
@@ -21,9 +23,22 @@ object BenchmarkProblems {
     BenchmarkProblem("dominating-set", seed => new DominatingSetProblem(TYPICAL_DS))
   )
 
+  /**
+    * Problems where the goal is to find a solution that is known to exist. They are run on their own (see the
+    * `--solve-rate` option), with more seeds, since a solve rate needs more samples than a median does.
+    * The sizes are ones where the existing strategies solve some runs and not others in a couple of seconds.
+    */
+  val CONSTRAINT: Seq[BenchmarkProblem] = Seq(
+    BenchmarkProblem("queens-100", seed => new NQueensProblem(100, seed), isConstraint = true),
+    BenchmarkProblem("queens-400", seed => new NQueensProblem(400, seed), isConstraint = true),
+    BenchmarkProblem("coloring-200", seed => new GraphColoringProblem(200, 4, 7.0, seed), isConstraint = true),
+    BenchmarkProblem("all-interval-14", seed => new AllIntervalProblem(14, seed), isConstraint = true)
+  )
+
+  /** @return the problems with these names, or all of the ordinary ones if there are no names */
   def named(names: Seq[String]): Seq[BenchmarkProblem] =
-    if (names.isEmpty) ALL else names.map(n => ALL.find(_.name == n).getOrElse(throw new IllegalArgumentException(
-      s"Unknown problem '$n'. The problems are ${ALL.map(_.name).mkString(", ")}")))
+    if (names.isEmpty) ALL else names.map(n => (ALL ++ CONSTRAINT).find(_.name == n).getOrElse(throw new IllegalArgumentException(
+      s"Unknown problem '$n'. The problems are ${(ALL ++ CONSTRAINT).map(_.name).mkString(", ")}")))
 
   /** The TSP variations pick their initial guess from one shared random generator, so make it start the same. */
   private def resetSharedRandom(seed: Long): Unit = TravelingSalesmanVariation.RANDOM.setSeed(seed)

scala-test/com/barrybecker4/optimization/benchmark/BenchmarkReport.scala

@@ -6,6 +6,16 @@ case class Summary(problem: String, strategy: String, runs: Int, evaluationsPerS
                    bytesPerEvaluation: Option[Double], medianFitness: Double, bestFitness: Double,
                    medianEvaluations: Double, stoppedBy: String)
 
+/**
+  * How often one strategy solved one constraint problem.
+  * @param solved the number of runs that reached the optimal fitness
+  * @param medianMillis the median time of the runs that solved it, or NaN if none did
+  * @param medianEvaluations the median number of evaluations of the runs that solved it, or NaN if none did
+  * @param medianUnsolvedFitness the median best fitness of the runs that did not, or NaN if all did
+  */
+case class SolveSummary(problem: String, strategy: String, runs: Int, solved: Int, medianMillis: Double,
+                        medianEvaluations: Double, medianUnsolvedFitness: Double)
+
 /** Turns the results of runs into tables. */
 object BenchmarkReport {
 
@@ -32,6 +42,16 @@ object BenchmarkReport {
     }
   }
 
+  /** One summary for each problem and strategy, in the order they first appear. */
+  def summarizeSolves(results: Seq[RunResult]): Seq[SolveSummary] = {
+    val groups = results.groupBy(r => (r.problem, r.strategy.toString))
+    results.map(r => (r.problem, r.strategy.toString)).distinct.map { key =>
+      val (solved, unsolved) = groups(key).partition(_.solved)
+      SolveSummary(key._1, key._2, solved.size + unsolved.size, solved.size, median(solved.map(_.millis)),
+        median(solved.map(_.evaluations.toDouble)), median(unsolved.map(_.fitness)))
+    }
+  }
+
   private def bytesPerEvaluation(runs: Seq[RunResult]): Option[Double] = {
     val values = runs.flatMap(_.bytesPerEvaluation)
     Option.when(values.nonEmpty)(median(values))
@@ -60,14 +80,36 @@ object BenchmarkReport {
   def document(settings: String, summaries: Seq[Summary]): String =
     Seq(DOCUMENT_TITLE, DOCUMENT_INTRO, settings, markdown(summaries)).mkString("\n\n") + "\n"
 
+  private val SOLVE_TITLE = "# Solve rate results"
+  private val SOLVE_INTRO =
+    "Written by `./gradlew benchmark -Pargs=\"--solve-rate=true\"`; see [new-strategies-plan.md](new-strategies-plan.md).\n" +
+      "Each run stops when it solves the problem, so the time is the time to a solution. The unsolved fitness is the\n" +
+      "median number of constraints still broken in the runs that did not. Only compare runs made on the same machine."
+
+  /** The solve rates as a document to keep in git, with no date, like [[document]]. */
+  def solveDocument(settings: String, summaries: Seq[SolveSummary]): String =
+    Seq(SOLVE_TITLE, SOLVE_INTRO, settings, solveMarkdown(summaries)).mkString("\n\n") + "\n"
+
+  def solveMarkdown(summaries: Seq[SolveSummary]): String = {
+    val header = "| Problem | Strategy | Solved | Median ms (solved) | Median evals (solved) | Median broken (unsolved) |\n" +
+      "| --- | --- | ---: | ---: | ---: | ---: |"
+    val rows = summaries.map { s =>
+      s"| ${s.problem} | ${s.strategy} | ${s.solved}/${s.runs} | ${number(s.medianMillis, "%,.0f")} | " +
+        s"${number(s.medianEvaluations, "%,.0f")} | ${number(s.medianUnsolvedFitness, "%.1f")} |"
+    }
+    (header +: rows).mkString("\n")
+  }
+
+  private def number(value: Double, format: String): String = if (value.isNaN) "-" else format.format(value)
+
   private def bytes(value: Option[Double]): String = value.map(v => f"$v%,.0f").getOrElse("n/a")
 
   /** One line for every run, so results can be compared with a spreadsheet or a script. */
   def tabSeparated(results: Seq[RunResult]): String = {
-    val header = Seq("problem", "strategy", "seed", "evaluations", "millis", "fitness", "allocatedBytes", "stoppedBy")
+    val header = Seq("problem", "strategy", "seed", "evaluations", "millis", "fitness", "allocatedBytes", "stoppedBy", "solved")
     val rows = results.map { r =>
       Seq(r.problem, r.strategy, r.seed, r.evaluations, f"${r.millis}%.1f", r.fitness,
-        r.allocatedBytes.getOrElse(""), r.stoppedBy).mkString("\t")
+        r.allocatedBytes.getOrElse(""), r.stoppedBy, r.solved).mkString("\t")
     }
     (header.mkString("\t") +: rows).mkString("\n")
   }