~ntietz/isabella-db

f948e3af3fc055c1eb3d3a02ee0a7c4af2ca1ca0 — Nicole Tietz-Sokolskaya 1 year, 1 month ago 08eebe8
Load shards and serve query results from all shards.

Implements: https://todo.sr.ht/~ntietz/isabella-db/14
A Makefile => Makefile +6 -0
@@ 0,0 1,6 @@
check:
	rustup update
	cargo build
	cargo test
	cargo fmt --check
	cargo clippy -- -Dwarnings

M isabella/src/bin/idb.rs => isabella/src/bin/idb.rs +3 -26
@@ 1,10 1,9 @@
use std::fs::{read_dir, File};
use std::io::{Read, Write};
use std::io::Read;

use clap::{Parser, Subcommand};
use isabella_db::db::{load_from_file, GameDB};
use isabella_db::index::{save::save_all, GameResultIndex, PositionIndex};
use pgn::load::PgnFile;
use rayon::prelude::*;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::EnvFilter;


@@ 20,9 19,8 @@ struct Args {

#[derive(Subcommand, Debug)]
enum Commands {
    Convert { pgn_filename: String },
    Index,
    ConvertBatch { pgn_directory: String },
    Convert { pgn_directory: String },
}

fn main() {


@@ 34,28 32,7 @@ fn main() {
    let args = Args::parse();

    match args.command {
        Commands::Convert { pgn_filename } => {
            let file = PgnFile::new(pgn_filename).expect("should open the file");
            let mut db = GameDB::from_pgn(file);
            db.sort();

            tracing::info!("constructed database");
            let buf = bincode::serialize(&db).expect("serializing should work");
            tracing::info!("serialized database");

            let mut outfile: File = File::options()
                .write(true)
                .create(true)
                .open(args.gamedb_filename)
                .expect("should open file to write");
            outfile.write_all(&buf).expect("should write to file");
            tracing::info!("wrote database");

            // To speed up exit times, we just forget about the memory without dropping it.
            std::mem::forget(buf);
            std::mem::forget(db);
        }
        Commands::ConvertBatch { pgn_directory } => {
        Commands::Convert { pgn_directory } => {
            // TODO: make directory, clean up if it exists already or warn?
            let output_dir = args.gamedb_filename;


M isabella/src/bin/preprocessor.rs => isabella/src/bin/preprocessor.rs +1 -0
@@ 24,6 24,7 @@ pub fn main() {

    for (idx, chunk) in parts[..].chunks(2).enumerate() {
        shards[idx % args.num_shards] += &chunk.join("\n\n");
        shards[idx % args.num_shards] += "\n\n";
    }

    for (idx, shard) in shards.iter().enumerate() {

M isabella/src/bin/web.rs => isabella/src/bin/web.rs +25 -7
@@ 1,13 1,15 @@
use std::fs;
use std::io::Read;
use std::{fs::File, sync::Arc};

use actix_web::{middleware::Logger, web, App, HttpServer};
use clap::Parser;
use isabella_db::web::context::Context;
use isabella_db::web::context::{Context, Shard};
use isabella_db::{
    index::{GameResultIndex, PositionIndex},
    web::routes::add_routes,
};
use rayon::prelude::*;
use serde::de::DeserializeOwned;
use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter};



@@ 28,14 30,30 @@ pub async fn main() -> std::io::Result<()> {

    tracing::debug!(args=?args, "Parsed args");

    let positions: PositionIndex =
        load_from_disk(&format!("{}.position.idx", args.gamedb_filename));
    tracing::debug!("Loaded position index");
    let base_paths: Vec<_> = fs::read_dir(args.gamedb_filename)
        .expect("data directory should exist and be readable")
        .map(|item| item.unwrap())
        .filter(|item| item.path().to_string_lossy().ends_with(".isa"))
        .collect();

    let results: GameResultIndex = load_from_disk(&format!("{}.result.idx", args.gamedb_filename));
    tracing::debug!("Loaded game result index");
    let shards: Vec<_> = base_paths
        .par_iter()
        .map(|item| {
            let positions: PositionIndex =
                load_from_disk(&format!("{}.position.idx", item.path().to_string_lossy()));
            tracing::debug!("Loaded position index");

    let context = Arc::new(Context { positions, results });
            let results: GameResultIndex =
                load_from_disk(&format!("{}.result.idx", item.path().to_string_lossy()));
            tracing::debug!("Loaded game result index");

            Arc::new(Shard { positions, results })
        })
        .collect();

    tracing::debug!(shards=?shards, "Loaded shards");

    let context = Arc::new(Context { shards });

    tracing::info!("Starting server");


M isabella/src/web/context.rs => isabella/src/web/context.rs +8 -0
@@ 1,6 1,14 @@
use std::sync::Arc;

use crate::index::{GameResultIndex, PositionIndex};

#[derive(Debug)]
pub struct Context {
    pub shards: Vec<Arc<Shard>>,
}

#[derive(Debug)]
pub struct Shard {
    pub positions: PositionIndex,
    pub results: GameResultIndex,
}

M isabella/src/web/templates.rs => isabella/src/web/templates.rs +28 -18
@@ 3,7 3,7 @@ use shakmaty::fen::Fen;
use shakmaty::san::SanPlus;
use shakmaty::{zobrist::ZobristHash, Chess, Color, Piece, Position, Role, Square};

use super::context::Context;
use super::context::{Context, Shard};

#[derive(Template)]
#[template(path = "home.html")]


@@ 110,23 110,18 @@ impl<'a> BoardTemplate<'a> {
                let fen =
                    Fen::from_position(next_board, shakmaty::EnPassantMode::Legal).to_string();

                let stats = self.context.positions.get(&hash).map(|positions_bm| {
                    tracing::debug!(
                        "san: {:?}, bm: {:?}, result: {:?}",
                        san,
                        positions_bm,
                        self.context.results.white_won
                    );
                    let white_win_bm = positions_bm & &self.context.results.white_won;
                    let black_win_bm = positions_bm & &self.context.results.black_won;
                    let draw_bm = positions_bm & &self.context.results.drawn;

                    GameResultStats::from_counts(
                        white_win_bm.len(),
                        black_win_bm.len(),
                        draw_bm.len(),
                    )
                });
                let counts = self
                    .context
                    .shards
                    .iter()
                    .flat_map(|shard| process_shard(shard, hash))
                    .fold((0, 0, 0), |a, b| (a.0 + b.0, a.1 + b.1, a.2 + b.2));
                let total = counts.0 + counts.1 + counts.2;
                let stats = if total > 0 {
                    Some(GameResultStats::from_counts(counts.0, counts.1, counts.2))
                } else {
                    None
                };

                available_moves.push(AvailableMove {
                    san: san.to_string(),


@@ 153,6 148,21 @@ impl<'a> BoardTemplate<'a> {
    }
}

pub fn process_shard(shard: &Shard, hash: u64) -> Option<(usize, usize, usize)> {
    shard.positions.get(&hash).map(|positions_bm| {
        tracing::debug!(
            "bm: {:?}, result: {:?}",
            positions_bm,
            shard.results.white_won
        );
        let white_win_bm = positions_bm & &shard.results.white_won;
        let black_win_bm = positions_bm & &shard.results.black_won;
        let draw_bm = positions_bm & &shard.results.drawn;

        (white_win_bm.len(), black_win_bm.len(), draw_bm.len())
    })
}

pub fn html_display_piece(piece: Option<Piece>) -> String {
    if let Some(piece) = piece {
        let s = match (piece.color, piece.role) {