From 1a6ea07ed98b47e58314205edfdfc26311bfeb72 Mon Sep 17 00:00:00 2001 From: Jan-Peter von Hunnius Date: Mon, 31 Mar 2025 12:15:01 +0200 Subject: [PATCH] Get rid of DB, parse into memory --- Cargo.toml | 1 - bestCarForTrack.sql | 8 - bestCars.sql | 48 ----- bestCarsPerTrack.sql | 51 ----- src/records.rs | 465 +++++++++++-------------------------------- 5 files changed, 118 insertions(+), 455 deletions(-) delete mode 100644 bestCarForTrack.sql delete mode 100644 bestCars.sql delete mode 100644 bestCarsPerTrack.sql diff --git a/Cargo.toml b/Cargo.toml index e2282c5..8853415 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ path = "src/records.rs" scraper = "0.23.1" reqwest = "0.12.15" tokio = { version = "1.44.1", features = ["full"] } -rusqlite = { version = "0.32.1", features = ["bundled", "backup"] } serde_json = "1.0.140" futures = "0.3.31" once_cell = "1.20.2" diff --git a/bestCarForTrack.sql b/bestCarForTrack.sql deleted file mode 100644 index 1bbbcc0..0000000 --- a/bestCarForTrack.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT rq, brand, onlyName, year, configuration, time -FROM car_track_times t -JOIN cars c ON t.car_rid = c.rid -WHERE track_id = 'Mountain Twisty Road' - AND track_type = 'Wet' - AND rq <= 90 -ORDER BY CAST(t.time AS DECIMAL) ASC -LIMIT 1; \ No newline at end of file diff --git a/bestCars.sql b/bestCars.sql deleted file mode 100644 index be78ad0..0000000 --- a/bestCars.sql +++ /dev/null @@ -1,48 +0,0 @@ -WITH RECURSIVE rq_range AS ( - SELECT DISTINCT rq from cars -), -track_combinations AS ( - SELECT DISTINCT track_id, track_type - FROM car_track_times -), -ranked_cars AS ( - SELECT - rq_range.rq AS max_rq, - t.track_id, - t.track_type, - c.rq, - c.brand, - c.onlyName, - c.year, - t.configuration, - t.time, - ROW_NUMBER() OVER ( - PARTITION BY rq_range.rq, t.track_id, t.track_type - ORDER BY t.time ASC - ) AS rank - FROM - rq_range - CROSS JOIN - track_combinations tc - JOIN - car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type - JOIN - cars c ON t.car_rid = c.rid - WHERE - t.time > 0 -- Exclude invalid times - AND t.time IS NOT NULL - AND c.rq <= rq_range.rq -) -SELECT - rq, - brand, - onlyName, - year, - Count(time) -FROM - ranked_cars -WHERE - rank = 1 -- Only the best car per combination -GROUP BY brand, onlyName, year -ORDER BY - rq, brand, onlyName, year; diff --git a/bestCarsPerTrack.sql b/bestCarsPerTrack.sql deleted file mode 100644 index e438f11..0000000 --- a/bestCarsPerTrack.sql +++ /dev/null @@ -1,51 +0,0 @@ -WITH RECURSIVE rq_range AS ( - SELECT DISTINCT rq from cars -), -track_combinations AS ( - SELECT DISTINCT track_id, track_type - FROM car_track_times -), -ranked_cars AS ( - SELECT - rq_range.rq AS max_rq, - t.track_id, - t.track_type, - c.rq, - c.brand, - c.onlyName, - c.year, - t.configuration, - t.time, - ROW_NUMBER() OVER ( - PARTITION BY rq_range.rq, t.track_id, t.track_type - ORDER BY t.time ASC - ) AS rank - FROM - rq_range - CROSS JOIN - track_combinations tc - JOIN - car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type - JOIN - cars c ON t.car_rid = c.rid - WHERE - t.time > 0 -- Exclude invalid times - AND t.time IS NOT NULL - AND c.rq <= rq_range.rq -) -SELECT - max_rq, - track_id, - track_type, - rq, - brand, - onlyName, - year, - configuration, - time AS best_time -FROM - ranked_cars -WHERE - rank = 1 -- Only the best car per combination -ORDER BY - max_rq, track_id, track_type; diff --git a/src/records.rs b/src/records.rs index 14fff61..8bf6b6c 100644 --- a/src/records.rs +++ b/src/records.rs @@ -1,14 +1,13 @@ -use std::collections::HashMap; +use futures::future::join_all; +use once_cell::sync::Lazy; use reqwest::get; -use rusqlite::backup::Backup; -use rusqlite::{params, CachedStatement, Connection}; use serde_json::Value; +use std::collections::HashMap; use std::env; use std::fs; -use once_cell::sync::Lazy; use std::sync::Mutex; -use futures::future::{join_all}; use tokio::sync::Semaphore; +use tokio::task; #[derive(Debug, Clone)] struct Car { @@ -17,20 +16,20 @@ struct Car { only_name: String, year: u16, rq: u8, - drive: String, - tyres: String, - top_speed: u32, - acel: f64, - hand: u32, } #[derive(Debug, Clone)] struct BestCarRecord { car_rid: String, - configuration: String, + brand: String, + only_name: String, + year: u16, + rq: u8, + config: String, time: f64, } type TripletKey = (String, u8, String); // (Track ID, RQ, Surface) -static BEST_CARS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); +static BEST_CARS: Lazy>> = + Lazy::new(|| Mutex::new(HashMap::new())); #[tokio::main] async fn main() { @@ -43,62 +42,24 @@ async fn main() { Some(argument) => { let command = argument.as_str(); match command { - "init" => { - let mut connection = - Connection::open_in_memory().expect("Issue opening in-memory DB!"); - - init_db(&mut connection); - parse_cars(&mut connection); - - let _ = fs::remove_file("top_drives.db"); - let mut file = - Connection::open("top_drives.db").expect("Issue opening DB file!"); - let backup = - Backup::new(&connection, &mut file).expect("Could not create backup!"); - backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once - drop(backup); - connection.close().unwrap(); - file.close().unwrap(); - } "fetch" => { get_cars_file().await; get_tracks_file().await; fetch_times().await; } - "parse" => { - let mut file = - Connection::open("top_drives.db").expect("Issue opening DB file!"); - let mut connection = - Connection::open_in_memory().expect("Issue opening in-memory DB!"); - - let backup = - Backup::new(&file, &mut connection).expect("Could not create backup!"); - backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once - drop(backup); - file.close().unwrap(); - - parse_all_times(&mut connection).await; - - let _ = fs::remove_file("top_drives.db"); - file = Connection::open("top_drives.db").expect("Issue opening DB file!"); - let backup = - Backup::new(&connection, &mut file).expect("Could not create backup!"); - backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once - drop(backup); - - connection.close().unwrap(); - file.close().unwrap(); - } "best" => { - list_best_cars().await; + parse_all_times().await; + print_best_cars(); } "details" => { - list_details().await; + parse_all_times().await; + print_best_times(); } "track" => { if args.get(4).is_none() { println!("Track name, condition and RQ limit required."); } else { + parse_all_times().await; let track = args.get(2).unwrap(); let condition = args.get(3).unwrap(); let rq = args.get(4).unwrap(); @@ -112,182 +73,72 @@ async fn main() { } } } +fn print_best_times() { + let best_cars = BEST_CARS.lock().unwrap(); -fn print_best_car(track: &String, condition: &String, rq: &String) { - let connection = Connection::open("top_drives.db").expect("Issue opening DB file!"); + // Collect keys and sort them by RQ and Track ID + let mut sorted_keys: Vec<_> = best_cars.keys().collect(); + sorted_keys.sort_by(|a, b| a.1.cmp(&b.1).then(a.0.cmp(&b.0))); // Sort by RQ, then Track ID - let mut stmt = connection - .prepare( - r#" - SELECT rq, brand, onlyName, year, configuration, time - FROM car_track_times t - JOIN cars c ON t.car_rid = c.rid - WHERE track_id = ? - AND track_type = ? - AND rq <= ? - ORDER BY CAST(t.time AS DECIMAL) ASC - LIMIT 1;"# - ).expect("Could not prepare query!"); - - let mut rows = stmt.query(params!(track, condition, rq)).expect("Could not execute query!"); - while let Ok(Some(row)) = rows.next() { - let rq :u8 = row.get(0).expect("Cannot extract RQ"); - let brand: String = row.get(1).expect("Cannot extract RQ"); - let only_name: String = row.get(2).expect("Cannot extract RQ"); - let year: u16 = row.get(3).expect("Cannot extract RQ"); - let configuration: String = row.get(4).expect("Cannot extract RQ"); - let time: f32 = row.get(5).expect("Cannot extract RQ"); - println!("{} {} {} {} ({}): {}", rq, brand, only_name, year, configuration, time ); - } -} - -async fn list_details() { - let file = Connection::open("top_drives.db").expect("Issue opening DB file!"); - let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!"); - - let backup = Backup::new(&file, &mut connection).expect("Could not create backup!"); - backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once - drop(backup); - drop(file); - - let stmt = connection.prepare( - r#" - WITH RECURSIVE rq_range AS ( - SELECT DISTINCT rq from cars - ), - track_combinations AS ( - SELECT DISTINCT track_id, track_type - FROM car_track_times - ), - ranked_cars AS ( - SELECT - rq_range.rq AS max_rq, - t.track_id, - t.track_type, - c.rq, - c.brand, - c.onlyName, - c.year, - t.configuration, - t.time, - ROW_NUMBER() OVER ( - PARTITION BY rq_range.rq, t.track_id, t.track_type - ORDER BY t.time ASC - ) AS rank - FROM rq_range - CROSS JOIN track_combinations tc - JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type - JOIN cars c ON t.car_rid = c.rid - WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq - ) - SELECT max_rq, track_id, track_type, rq, brand, onlyName, year, configuration, time AS best_time - FROM ranked_cars - WHERE rank = 1 - ORDER BY max_rq, track_id, track_type - "#, - ); - - let mut result = stmt.expect("Query failed!"); - let rows = result - .query_map([], |row| { - Ok(( - row.get::<_, i32>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i32>(3)?, - row.get::<_, String>(4)?, - row.get::<_, String>(5)?, - row.get::<_, i32>(6)?, - row.get::<_, String>(7)?, - row.get::<_, f64>(8)?, - )) - }) - .expect("Result retrieval failed!"); - - for row in rows { - let (max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time) = - row.expect("Row retrieval failed!"); + for key in sorted_keys { + let best_car = best_cars.get(key).unwrap(); println!( "{},{},{},{},{},{},{},{},{}", - max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time + key.1, + key.0, + key.2, + best_car.rq, + best_car.brand, + best_car.only_name, + best_car.year, + best_car.config, + best_car.time ); } - - drop(result); - connection.close().unwrap(); } +fn print_best_cars() { + let best_cars = BEST_CARS.lock().unwrap(); + let mut car_count: HashMap = HashMap::new(); -async fn list_best_cars() { - let file = Connection::open("top_drives.db").expect("Issue opening DB file!"); - let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!"); - - let backup = Backup::new(&file, &mut connection).expect("Could not create backup!"); - backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once - drop(backup); - drop(file); - - let stmt = connection.prepare( - r#" - WITH RECURSIVE rq_range AS ( - SELECT DISTINCT rq from cars - ), - track_combinations AS ( - SELECT DISTINCT track_id, track_type - FROM car_track_times - ), - ranked_cars AS ( - SELECT - rq_range.rq AS max_rq, - t.track_id, - t.track_type, - c.rq, - c.brand, - c.onlyName, - c.year, - t.configuration, - t.time, - ROW_NUMBER() OVER ( - PARTITION BY rq_range.rq, t.track_id, t.track_type - ORDER BY t.time ASC - ) AS rank - FROM rq_range - CROSS JOIN track_combinations tc - JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type - JOIN cars c ON t.car_rid = c.rid - WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq - ) - SELECT rq, brand, onlyName, year, COUNT(time) AS occurrences - FROM ranked_cars - WHERE rank = 1 - GROUP BY brand, onlyName, year - ORDER BY rq, brand, onlyName, year - "#, - ); - - let mut result = stmt.expect("Query failed!"); - let rows = result - .query_map([], |row| { - Ok(( - row.get::<_, i32>(0).expect("Could not assign value"), - row.get::<_, String>(1).expect("Could not assign value"), - row.get::<_, String>(2).expect("Could not assign value"), - row.get::<_, i32>(3).expect("Could not assign value"), - row.get::<_, i32>(4).expect("Could not assign value"), - )) - }) - .expect("Result retrieval failed!"); - - for row in rows { - let (rq, brand, only_name, year, occurrences) = row.expect("Row retrieval failed!"); - println!("{},{},{},{},{}", rq, brand, only_name, year, occurrences); + // Count occurrences and store RQ + for ((_, rq, _), best_car) in best_cars.iter() { + let entry = car_count + .entry(best_car.car_rid.clone()) + .or_insert((0, *rq, best_car.clone())); + entry.0 += 1; } - drop(result); - connection.close().unwrap(); + // Convert to vector and sort by RQ first, then frequency + let mut car_count_vec: Vec<_> = car_count.iter().collect(); + car_count_vec.sort_by(|a, b| a.1 .1.cmp(&b.1 .1).then(a.0.cmp(b.0))); + + for (_car_rid, (count, rq, car_record)) in car_count_vec { + println!( + "{},{},{},{},{}", + rq, car_record.brand, car_record.only_name, car_record.year, count + ); + } +} +fn print_best_car(track: &String, condition: &String, rq: &String) { + let best_cars = BEST_CARS.lock().unwrap(); + let key = (track.clone(), rq.parse::().unwrap_or(115), condition.clone()); + + if let Some(best_car) = best_cars.get(&key) { + println!( + "Best car for Track: {}, Condition: {}, RQ: {} -> {} {} ({}) with time {:.2}", + track, condition, rq, + best_car.brand, best_car.only_name, best_car.year, best_car.time + ); + } else { + println!( + "No best car found for Track: {}, Condition: {}, RQ: {}", + track, condition, rq + ); + } } -async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) { - let file_name = rid.to_owned() + ".json"; +async fn parse_car_times(car: Car) { + let file_name = car.rid.to_owned() + ".json"; let local_file_path = "times/".to_owned() + file_name.as_str(); let file_content = fs::read_to_string(local_file_path); @@ -317,86 +168,53 @@ async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) { // Insert into the database if time > 1.2 && no_down_votes { - let _ = statement.execute([ - rid.as_str(), + update_best_car( translate_track_name(cleaned_track_id), + car.rq, translate_track_type(track_type), - time.to_string().as_str(), - config, - ]); -//TODO: update_best_car(cleaned_track_id, translate_track_type(track_type), None) + BestCarRecord { + car_rid: car.rid.clone(), + brand: car.brand.clone(), + only_name: car.only_name.clone(), + year: car.year, + rq: car.rq, + config: config.clone(), + time, + }, + ); } } } } } } - println!("Track times imported for {}", rid); } -fn update_best_car(track_id: &str, rq: u8, surface: &str, new_record: BestCarRecord) { - let key = (track_id.to_string(), rq, surface.to_string()); +fn update_best_car(track_id: &str, car_rq: u8, surface: &str, new_record: BestCarRecord) { let mut best_cars = BEST_CARS.lock().unwrap(); - if let Some(existing) = best_cars.get(&key) { - if new_record.time < existing.time { - best_cars.insert(key, new_record); + for rq in car_rq..116 { + let key = (track_id.to_string(), rq, surface.to_string()); + + if let Some(existing) = best_cars.get(&key) { + if new_record.time < existing.time { + best_cars.insert(key, new_record.clone()); + } + } else { + best_cars.insert(key, new_record.clone()); } - } else { - best_cars.insert(key, new_record); } } -async fn parse_all_times(connection: &mut Connection) { - let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!"); +async fn parse_all_times() { + let mut handles = vec![]; - for car in get_cars_from_db() { - parse_car_times(car, &mut statement).await; + for car in get_car_list() { + let handle = task::spawn(async move { + parse_car_times(car).await; + }); + handles.push(handle); } - connection.flush_prepared_statement_cache(); - let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ()); -} -fn parse_cars(connection: &mut Connection) { - println!("Parsing cars into DB..."); - let cars_file = "cars_repo.json"; - let cars_data: Vec = - serde_json::from_str(&fs::read_to_string(cars_file).unwrap()).unwrap(); - - for car in cars_data { - if let Err(e) = connection.execute("INSERT OR IGNORE INTO cars - (class, rq, onlyName, brand, year, abs, tcs, clearance, country, topSpeed, acel, hand, - drive, tyres, mra, weight, name, rid, prize, bodyTypes, fuel, seats, engine, tags, guid, brake) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26);", - params![ - car["class"].as_str().unwrap(), - car["rq"].as_u64().unwrap(), - car["onlyName"].as_str().unwrap(), - car["brand"].as_str().unwrap(), - car["year"].as_u64().unwrap(), - car["abs"].as_bool().unwrap(), - car["tcs"].as_bool().unwrap(), - car["clearance"].as_str().unwrap(), - car["country"].as_str().unwrap(), - car["topSpeed"].as_u64().unwrap(), - car["acel"].as_f64().unwrap_or_default(), - car["hand"].as_u64().unwrap(), - car["drive"].as_str().unwrap(), - car["tyres"].as_str().unwrap(), - car["mra"].as_f64().unwrap_or_default(), - car["weight"].as_u64().unwrap(), - car["name"].as_str().unwrap(), - car["rid"].as_str().unwrap(), - car["prize"].as_bool().unwrap(), - Some(serde_json::to_string(&car["bodyTypes"]).unwrap().as_str()).unwrap_or_default(), - car["fuel"].as_str().unwrap(), - car["seats"].as_u64().unwrap_or_default(), - car["engine"].as_str().unwrap(), - Some(serde_json::to_string(&car["tags"]).unwrap().as_str()).unwrap_or_default(), - car["guid"].as_str().unwrap(), - car["brake"].as_str().unwrap(), - ]) { - eprintln!("Insertion error: {}", e); - }; - } + join_all(handles).await; } fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a str, &'a str)> { input.find(delimiter).map(|index| { @@ -406,13 +224,13 @@ fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a s }) } async fn fetch_times() { - let semaphore = std::sync::Arc::new(Semaphore::new(20)); // Adjust the number of concurrent requests + let semaphore = std::sync::Arc::new(Semaphore::new(20)); let mut handles = vec![]; - for car in get_cars_from_db() { + for car in get_car_list() { let permit = semaphore.clone().acquire_owned().await.unwrap(); handles.push(tokio::spawn(async move { - fetch_car(car).await; + fetch_car(car.rid).await; drop(permit); })); } @@ -426,80 +244,34 @@ async fn fetch_car(rid: String) { println!("Fetching {}.", url); get_file(url.as_str(), local_file_path.as_str()).await; } -fn get_cars_from_db() -> Vec { - let mut car_list = Vec::new(); +fn get_car_list() -> Vec { + let mut car_list: Vec = Vec::new(); - let connection = Connection::open("top_drives.db").expect("Issue opening DB file!"); + let cars_file = "cars_repo.json"; + let cars_data: Vec = + serde_json::from_str(&fs::read_to_string(cars_file).unwrap()).unwrap(); - let mut stmt = connection - .prepare("SELECT rid FROM cars") - .expect("Could not prepare query!"); - let mut rows = stmt.query([]).expect("Could not execute query!"); + for car in cars_data { + let new_car = Car { + rid: String::from(car["rid"].as_str().unwrap_or("No RID")), + brand: String::from(car["brand"].as_str().unwrap_or("No Brand")), + only_name: String::from(car["onlyName"].as_str().unwrap_or("No Name")), + year: car["year"].as_u64().unwrap_or(1900) as u16, + rq: car["rq"].as_u64().unwrap_or(200) as u8, + }; - while let Ok(Some(row)) = rows.next() { - car_list.push(row.get::(0).expect("Could not get column!")); + car_list.push(new_car); } car_list } -fn init_db(connection: &mut Connection) { - println!("Initializing DB..."); - - let _ = connection.execute( - "CREATE TABLE IF NOT EXISTS cars ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - class TEXT, - rq INTEGER, - onlyName TEXT, - brand TEXT, - year INTEGER, - abs BOOLEAN, - tcs BOOLEAN, - clearance TEXT, - country TEXT, - topSpeed INTEGER, - acel REAL, - hand INTEGER, - drive TEXT, - tyres TEXT, - mra REAL, - weight INTEGER, - name TEXT, - rid TEXT UNIQUE, - prize BOOLEAN, - bodyTypes TEXT, - fuel TEXT, - seats TEXT, - engine TEXT, - tags TEXT, - guid TEXT UNIQUE, - brake TEXT, - owned BOOLEAN - );", - [], - ); - - let _ = connection.execute( - "CREATE TABLE IF NOT EXISTS car_track_times ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - car_rid TEXT, - configuration TEXT, - track_id TEXT, - track_type INTEGER, - time DECIMAL, - FOREIGN KEY (car_rid) REFERENCES cars(rid), - UNIQUE(car_rid, track_id, track_type, configuration) - );", - [], - ); -} async fn get_tracks_file() { println!("Fetching tracks file..."); get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/tracks_repo.json","tracks_repo.json").await; } async fn get_cars_file() { println!("Fetching cars file..."); - get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/cars_final_PL25-fixed.json", "cars_repo.json").await; + get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/cars_final.json", "cars_repo.json").await; } async fn get_file(file_url: &str, local_file_path: &str) { let _ = fs::remove_file(local_file_path); @@ -627,7 +399,6 @@ fn translate_track_name(track_code: &str) -> &str { "miBeachDrag" => "Miami Beach Drag", "mnGforce" => "Monaco G-Force Test", "mnHairpin" => "Monaco Hairpin", -// "mnNarrowStreets" => "Monaco Narrow Streets", "mnCityNarrow" => "Monaco Narrow Streets", "mnCity" => "Monaco Streets", "mnCityLong" => "Monaco Streets Long",