~ntietz/isabella-db

05442a6539905efc47afae1fd3be2f08c74297ab — Nicole Tietz-Sokolskaya 1 year, 2 months ago b1825f6
Add sorting of games to improve performance of index lookups

Implements: https://todo.sr.ht/~ntietz/isabella-db/10
M isabella/src/bin/idb.rs => isabella/src/bin/idb.rs +9 -1
@@ 34,9 34,12 @@ fn main() {
    match args.command {
        Commands::Convert { pgn_filename } => {
            let file = PgnFile::new(pgn_filename).expect("should open the file");
            let db = GameDB::from_pgn(file);
            let mut db = GameDB::from_pgn(file);
            db.sort();

            tracing::info!("constructed database");
            let buf = bincode::serialize(&db).expect("serializing should work");
            tracing::info!("serialized database");

            let mut outfile: File = File::options()
                .write(true)


@@ 44,6 47,11 @@ fn main() {
                .open(args.gamedb_filename)
                .expect("should open file to write");
            outfile.write_all(&buf).expect("should write to file");
            tracing::info!("wrote database");

            // To speed up exit times, we just forget about the memory without dropping it.
            std::mem::forget(buf);
            std::mem::forget(db);
        }
        Commands::Index => {
            let mut file: File = File::open(&args.gamedb_filename).expect("should open the file");

M isabella/src/db/mod.rs => isabella/src/db/mod.rs +14 -4
@@ 4,7 4,7 @@ use pgn::PgnFile;
use serde::{Deserialize, Serialize};
use shakmaty::Chess;

use crate::game::{Game, StartingPosition};
use crate::game::{Game, GameResult, StartingPosition};
use crate::strings::StringsTable;

const TRACE_CHUNK_SIZE: usize = 500_000;


@@ 52,6 52,12 @@ impl GameDB {
        self.games.get(id)
    }

    pub fn sort(&mut self) {
        self.games.sort_by_key(|game| game.sort_key());
        //self.games.sort_by_key(|game| &game.moves);
        //self.games.sort_by_key(|game| game.result);
    }

    pub fn from_pgn(f: PgnFile) -> Self {
        let mut db = GameDB::new();



@@ 64,9 70,13 @@ impl GameDB {
            }
            game.moves = record.moves;
            for (key, value) in record.tags {
                let kid = db.strings.insert(key);
                let vid = db.strings.insert(value);
                game.tags.push((kid, vid));
                if key == "Result" {
                    game.result = GameResult::from(value.as_ref());
                } else {
                    let kid = db.strings.insert(key);
                    let vid = db.strings.insert(value);
                    game.tags.push((kid, vid));
                }
            }
            db.games.push(game);


M isabella/src/game/mod.rs => isabella/src/game/mod.rs +20 -1
@@ 14,11 14,14 @@ pub enum StartingPosition {
    Custom(#[serde(with = "chess_serde")] Chess),
}

#[derive(Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
#[derive(
    Copy, Clone, Debug, Default, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize,
)]
pub enum GameResult {
    WhiteWon,
    BlackWon,
    Drawn,
    #[default]
    Other,
}



@@ 39,4 42,20 @@ pub struct Game {
    #[serde(default, with = "vec_move_def")]
    pub moves: Vec<Move>,
    pub tags: Vec<(StringID, StringID)>,
    pub result: GameResult,
}

#[derive(Debug, Eq, PartialEq, Ord, PartialOrd)]
pub struct GameSortKey {
    pub result: GameResult,
    pub moves: Vec<String>,
}

impl Game {
    pub fn sort_key(&self) -> GameSortKey {
        let moves = self.moves.iter().take(5).map(|m| m.to_string()).collect();
        let result = self.result;

        GameSortKey { result, moves }
    }
}

M isabella/src/index/game_result.rs => isabella/src/index/game_result.rs +6 -23
@@ 26,31 26,14 @@ impl GameResultIndex {
    pub fn construct(db: &GameDB) -> GameResultIndex {
        let mut index = GameResultIndex::of_size(db.len());

        let strings = db.strings();

        let result_string_id = match strings.id_of("Result") {
            Some(id) => id,
            None => {
                tracing::info!("no games exist with tag \"Result\"");
                return index;
            }
        };

        for (idx, game) in db.games().iter().enumerate() {
            if let Some((_, val_id)) = game.tags.iter().find(|(key, _)| *key == result_string_id) {
                match GameResult::from(
                    strings
                        .by_id(*val_id)
                        .expect("result should be an interned string")
                        .as_ref(),
                ) {
                    GameResult::WhiteWon => index.white_won.set(idx),
                    GameResult::BlackWon => index.black_won.set(idx),
                    GameResult::Drawn => index.drawn.set(idx),
                    GameResult::Other => index.other.set(idx),
                }
                .expect("inserting into result bitmap should succeed");
            match game.result {
                GameResult::WhiteWon => index.white_won.set(idx),
                GameResult::BlackWon => index.black_won.set(idx),
                GameResult::Drawn => index.drawn.set(idx),
                GameResult::Other => index.other.set(idx),
            }
            .expect("setting result should work");

            if idx % 100_000 == 0 {
                tracing::debug!(idx, "construction in progress");

M isabella/src/web/templates.rs => isabella/src/web/templates.rs +6 -0
@@ 108,6 108,12 @@ impl<'a> BoardTemplate<'a> {
                    Fen::from_position(next_board, shakmaty::EnPassantMode::Legal).to_string();

                let stats = self.context.positions.get(&hash).map(|positions_bm| {
                    tracing::debug!(
                        "san: {:?}, bm: {:?}, result: {:?}",
                        san,
                        positions_bm,
                        self.context.results.white_won
                    );
                    let white_win_bm = positions_bm & &self.context.results.white_won;
                    let black_win_bm = positions_bm & &self.context.results.black_won;
                    let draw_bm = positions_bm & &self.context.results.drawn;