Optimize combines by using a lookup table for layout pairs

This is significantly faster than having to go through lower bound hash tables all the time (20% of time was spent in there).

src/combine.rs

@@ -37,6 +37,18 @@ impl CompatibilityCache {
     }
 }
 
+fn choose_layouts<TDB: LayoutDB>(db: &TDB, city: &City, top_layout_count: usize) -> Vec<SavedLayout> {
+    // We do a secondary sort by ID to make the result stable in case of ties
+    let top_deduplicated_layouts: Vec<SavedLayout> = db.layouts().iter()
+        .sorted_by_key(|x| (city::get_price(&city, x.houses()), x.id()))
+        .unique_by(|x| x.houses().iter().sorted_by_key(|x| (x.x, x.y)).collect::<Vec<_>>())
+        .map(|layout| (*layout).clone())
+        .take(top_layout_count)
+        .collect();
+
+    top_deduplicated_layouts
+}
+
 pub fn iterate_combines<TDB: LayoutDB>(mut db: &mut TDB, top_layout_count: usize, city: &City, mut cache: &mut CompatibilityCache, print_progress: bool) {
     #[derive(Eq, PartialEq)]
     enum LastStep {
@@ -52,11 +64,8 @@ pub fn iterate_combines<TDB: LayoutDB>(mut db: &mut TDB, top_layout_count: usize
     loop {
         if last_improve_step == LastStep::Vertical { break; }
         if print_progress { eprintln!("Starting to combine {} top houses DB; vertical cuts", top_layout_count); }
-        let sorted: Vec<SavedLayout> = db.layouts().iter()
+
-            .sorted_by(|x, y| city::get_price(&city, x.houses()).cmp(&city::get_price(&city, y.houses())))
+        let chosen_layouts = choose_layouts(db, &city, top_layout_count);
-            .map(|layout| (*layout).clone())
-            .collect();
-        let chosen_layouts: Vec<_> = sorted.into_iter().take(top_layout_count).collect();
         if create_new_best_combination(&city, &chosen_layouts, &chosen_layouts, db, &mut cache, false, print_progress) {
             last_improve_step = LastStep::Vertical;
         }
@@ -64,11 +73,7 @@ pub fn iterate_combines<TDB: LayoutDB>(mut db: &mut TDB, top_layout_count: usize
 
         if last_improve_step == LastStep::Horizontal { break; }
 
-        let sorted: Vec<SavedLayout> = db.layouts().iter()
+        let chosen_layouts = choose_layouts(db, &city, top_layout_count);
-            .sorted_by(|x, y| city::get_price(&city, x.houses()).cmp(&city::get_price(&city, y.houses())))
-            .map(|layout| (*layout).clone())
-            .collect();
-        let chosen_layouts: Vec<_> = sorted.into_iter().take(top_layout_count).collect();
         let transposed_chosen_layouts: Vec<_> = chosen_layouts.iter().map(|x| transpose_saved_layout(x)).collect();
 
         if print_progress { eprintln!("Starting to combine {} top houses DB; horizontal cuts", top_layout_count); }
@@ -176,6 +181,29 @@ pub fn create_new_best_combination<TDB: LayoutDB>(city: &City, left_layouts: &Ve
 
     let axis = if transposed { "y" } else { "x" };
 
+    // We construct a quick lookup table for skipped pairs of layouts - these are kept the same
+    // the entire run. We also assume that the best price does not get worse throughout the run.
+    let checked_pair_table: Vec<Vec<bool>> = lefts.iter()
+        .map(|left| rights.iter().map(|right| {
+            if left.layout.id() == right.layout.id() {
+                // We do not want to combine a layout with itself, it would never improve the score.
+                return false;
+            }
+            if let Some(bound) = db.get_merge_lower_bound(left.layout, right.layout, transposed) {
+                if bound >= best_price.expect("No best price set while lower bounds exist") {
+                    // This combination can never attain a price better than the current best.
+                    return false;
+                }
+            }
+            return true;
+        }).collect()).collect();
+
+    if print_progress {
+        let total_pairs = checked_pair_table.iter().flatten().count();
+        let checked_pairs = checked_pair_table.iter().flatten().filter(|x| **x).count();
+        eprintln!("Checking {}/{} layout pairs", checked_pairs, total_pairs);
+    }
+
     // x is the last left coordinate, x+1 is right
     for x in 0..city.width() {
         if print_progress { eprintln!("Starting {} {}", axis, x); }
@@ -204,21 +232,14 @@ pub fn create_new_best_combination<TDB: LayoutDB>(city: &City, left_layouts: &Ve
         }
 
         let pairs = iproduct!(lefts.iter().enumerate(), rights.iter().enumerate())
-            .filter(|((left_i, _), (right_i, _))| left_i != right_i)
+            .filter(|((left_i, _), (right_i, _))| checked_pair_table[*left_i as usize][*right_i as usize])
-            .filter(|((_, left), (_, right))| {
-                if let Some(bound) = db.get_merge_lower_bound(left.layout, right.layout, transposed) {
-                    if bound >= best_price.expect("No best price set while lower bounds exist") {
-                        return false;
-                    }
-                }
-                return true;
-            })
             .map(|((left_i, left), (right_i, right))| (left, right, left.line.price + right.line.price, left_i, right_i))
             .sorted_by(|(_, _, price1, _, _), (_, _, price2, _, _)| price1.cmp(&price2));
 
         let mut compatibles = 0;
         let mut incompatibles = 0;
         let mut cache_hits = 0;
+        let mut best_incompatible_price = best_price;
         for (left, right, price, left_i, right_i) in pairs {
             if let Some(min_price) = best_price {
                 if price >= min_price {
@@ -263,19 +284,20 @@ pub fn create_new_best_combination<TDB: LayoutDB>(city: &City, left_layouts: &Ve
                 // All other pairs would be more expensive
                 break;
             } else {
+                best_incompatible_price = Some(best_incompatible_price.unwrap_or(u32::MAX).min(price));
                 incompatibles += 1;
             }
         }
 
-        if print_progress { eprintln!("{} incompatibles checked before {} compatible ({} cache hits)", incompatibles, compatibles, cache_hits); }
+        if print_progress { eprintln!("{} incompatibles checked before {} compatible ({} cache hits) [{}-{}]", incompatibles, compatibles, cache_hits, best_incompatible_price.unwrap_or(u32::MAX), best_price.unwrap_or(u32::MAX)); }
     }
 
     if let Some(lowest_price) = best_price {
         let mut new_bounds = Vec::new();
 
-        for left_layout in left_layouts {
+        for left in &lefts {
-            for right_layout in right_layouts {
+            for right in &rights {
-                new_bounds.push(MergeLowerBound::new(left_layout.id(), right_layout.id(), transposed, lowest_price));
+                new_bounds.push(MergeLowerBound::new(left.layout.id(), right.layout.id(), transposed, lowest_price));
             }
         }
         db.add_merge_lower_bounds(new_bounds);