jan/top_drives
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get rid of DB, parse into memory

Browse source
This commit is contained in:
Jan-Peter von Hunnius 2025-03-31 12:15:01 +02:00
parent 12a3868a46
commit 1a6ea07ed9
5 changed files with 118 additions and 455 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -15,7 +15,6 @@ path = "src/records.rs"
scraper = "0.23.1"
reqwest = "0.12.15"
tokio = { version = "1.44.1", features = ["full"] }
rusqlite = { version = "0.32.1", features = ["bundled", "backup"] }
serde_json = "1.0.140"
futures = "0.3.31"
once_cell = "1.20.2"

8
bestCarForTrack.sql
View file

@ -1,8 +0,0 @@
SELECT rq, brand, onlyName, year, configuration, time
FROM car_track_times t
JOIN cars c ON t.car_rid = c.rid
WHERE track_id = 'Mountain Twisty Road'
AND track_type = 'Wet'
AND rq <= 90
ORDER BY CAST(t.time AS DECIMAL) ASC
LIMIT 1;

48
bestCars.sql
View file

@ -1,48 +0,0 @@
WITH RECURSIVE rq_range AS (
SELECT DISTINCT rq from cars
),
track_combinations AS (
SELECT DISTINCT track_id, track_type
FROM car_track_times
),
ranked_cars AS (
SELECT
rq_range.rq AS max_rq,
t.track_id,
t.track_type,
c.rq,
c.brand,
c.onlyName,
c.year,
t.configuration,
t.time,
ROW_NUMBER() OVER (
PARTITION BY rq_range.rq, t.track_id, t.track_type
ORDER BY t.time ASC
) AS rank
FROM
rq_range
CROSS JOIN
track_combinations tc
JOIN
car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
JOIN
cars c ON t.car_rid = c.rid
WHERE
t.time > 0 -- Exclude invalid times
AND t.time IS NOT NULL
AND c.rq <= rq_range.rq
)
SELECT
rq,
brand,
onlyName,
year,
Count(time)
FROM
ranked_cars
WHERE
rank = 1 -- Only the best car per combination
GROUP BY brand, onlyName, year
ORDER BY
rq, brand, onlyName, year;

51
bestCarsPerTrack.sql
View file

@ -1,51 +0,0 @@
WITH RECURSIVE rq_range AS (
SELECT DISTINCT rq from cars
),
track_combinations AS (
SELECT DISTINCT track_id, track_type
FROM car_track_times
),
ranked_cars AS (
SELECT
rq_range.rq AS max_rq,
t.track_id,
t.track_type,
c.rq,
c.brand,
c.onlyName,
c.year,
t.configuration,
t.time,
ROW_NUMBER() OVER (
PARTITION BY rq_range.rq, t.track_id, t.track_type
ORDER BY t.time ASC
) AS rank
FROM
rq_range
CROSS JOIN
track_combinations tc
JOIN
car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
JOIN
cars c ON t.car_rid = c.rid
WHERE
t.time > 0 -- Exclude invalid times
AND t.time IS NOT NULL
AND c.rq <= rq_range.rq
)
SELECT
max_rq,
track_id,
track_type,
rq,
brand,
onlyName,
year,
configuration,
time AS best_time
FROM
ranked_cars
WHERE
rank = 1 -- Only the best car per combination
ORDER BY
max_rq, track_id, track_type;

459
src/records.rs
View file

@ -1,14 +1,13 @@
use std::collections::HashMap;
use futures::future::join_all;
use once_cell::sync::Lazy;
use reqwest::get;
use rusqlite::backup::Backup;
use rusqlite::{params, CachedStatement, Connection};
use serde_json::Value;
use std::collections::HashMap;
use std::env;
use std::fs;
use once_cell::sync::Lazy;
use std::sync::Mutex;
use futures::future::{join_all};
use tokio::sync::Semaphore;
use tokio::task;
#[derive(Debug, Clone)]
struct Car {
@ -17,20 +16,20 @@ struct Car {
only_name: String,
year: u16,
rq: u8,
drive: String,
tyres: String,
top_speed: u32,
acel: f64,
hand: u32,
}
#[derive(Debug, Clone)]
struct BestCarRecord {
car_rid: String,
configuration: String,
brand: String,
only_name: String,
year: u16,
rq: u8,
config: String,
time: f64,
}
type TripletKey = (String, u8, String); // (Track ID, RQ, Surface)
static BEST_CARS: Lazy<Mutex<HashMap<TripletKey, BestCarRecord>>> = Lazy::new(|| Mutex::new(HashMap::new()));
static BEST_CARS: Lazy<Mutex<HashMap<TripletKey, BestCarRecord>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
#[tokio::main]
async fn main() {
@ -43,62 +42,24 @@ async fn main() {
Some(argument) => {
let command = argument.as_str();
match command {
"init" => {
let mut connection =
Connection::open_in_memory().expect("Issue opening in-memory DB!");
init_db(&mut connection);
parse_cars(&mut connection);
let _ = fs::remove_file("top_drives.db");
let mut file =
Connection::open("top_drives.db").expect("Issue opening DB file!");
let backup =
Backup::new(&connection, &mut file).expect("Could not create backup!");
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
connection.close().unwrap();
file.close().unwrap();
}
"fetch" => {
get_cars_file().await;
get_tracks_file().await;
fetch_times().await;
}
"parse" => {
let mut file =
Connection::open("top_drives.db").expect("Issue opening DB file!");
let mut connection =
Connection::open_in_memory().expect("Issue opening in-memory DB!");
let backup =
Backup::new(&file, &mut connection).expect("Could not create backup!");
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
file.close().unwrap();
parse_all_times(&mut connection).await;
let _ = fs::remove_file("top_drives.db");
file = Connection::open("top_drives.db").expect("Issue opening DB file!");
let backup =
Backup::new(&connection, &mut file).expect("Could not create backup!");
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
connection.close().unwrap();
file.close().unwrap();
}
"best" => {
list_best_cars().await;
parse_all_times().await;
print_best_cars();
}
"details" => {
list_details().await;
parse_all_times().await;
print_best_times();
}
"track" => {
if args.get(4).is_none() {
println!("Track name, condition and RQ limit required.");
} else {
parse_all_times().await;
let track = args.get(2).unwrap();
let condition = args.get(3).unwrap();
let rq = args.get(4).unwrap();
@ -112,182 +73,72 @@ async fn main() {
}
}
}
fn print_best_times() {
let best_cars = BEST_CARS.lock().unwrap();
fn print_best_car(track: &String, condition: &String, rq: &String) {
let connection = Connection::open("top_drives.db").expect("Issue opening DB file!");
// Collect keys and sort them by RQ and Track ID
let mut sorted_keys: Vec<_> = best_cars.keys().collect();
sorted_keys.sort_by(|a, b| a.1.cmp(&b.1).then(a.0.cmp(&b.0))); // Sort by RQ, then Track ID
let mut stmt = connection
.prepare(
r#"
SELECT rq, brand, onlyName, year, configuration, time
FROM car_track_times t
JOIN cars c ON t.car_rid = c.rid
WHERE track_id = ?
AND track_type = ?
AND rq <= ?
ORDER BY CAST(t.time AS DECIMAL) ASC
LIMIT 1;"#
).expect("Could not prepare query!");
let mut rows = stmt.query(params!(track, condition, rq)).expect("Could not execute query!");
while let Ok(Some(row)) = rows.next() {
let rq :u8 = row.get(0).expect("Cannot extract RQ");
let brand: String = row.get(1).expect("Cannot extract RQ");
let only_name: String = row.get(2).expect("Cannot extract RQ");
let year: u16 = row.get(3).expect("Cannot extract RQ");
let configuration: String = row.get(4).expect("Cannot extract RQ");
let time: f32 = row.get(5).expect("Cannot extract RQ");
println!("{} {} {} {} ({}): {}", rq, brand, only_name, year, configuration, time );
}
}
async fn list_details() {
let file = Connection::open("top_drives.db").expect("Issue opening DB file!");
let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
let backup = Backup::new(&file, &mut connection).expect("Could not create backup!");
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
drop(file);
let stmt = connection.prepare(
r#"
WITH RECURSIVE rq_range AS (
SELECT DISTINCT rq from cars
),
track_combinations AS (
SELECT DISTINCT track_id, track_type
FROM car_track_times
),
ranked_cars AS (
SELECT
rq_range.rq AS max_rq,
t.track_id,
t.track_type,
c.rq,
c.brand,
c.onlyName,
c.year,
t.configuration,
t.time,
ROW_NUMBER() OVER (
PARTITION BY rq_range.rq, t.track_id, t.track_type
ORDER BY t.time ASC
) AS rank
FROM rq_range
CROSS JOIN track_combinations tc
JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
JOIN cars c ON t.car_rid = c.rid
WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq
)
SELECT max_rq, track_id, track_type, rq, brand, onlyName, year, configuration, time AS best_time
FROM ranked_cars
WHERE rank = 1
ORDER BY max_rq, track_id, track_type
"#,
);
let mut result = stmt.expect("Query failed!");
let rows = result
.query_map([], |row| {
Ok((
row.get::<_, i32>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, i32>(3)?,
row.get::<_, String>(4)?,
row.get::<_, String>(5)?,
row.get::<_, i32>(6)?,
row.get::<_, String>(7)?,
row.get::<_, f64>(8)?,
))
})
.expect("Result retrieval failed!");
for row in rows {
let (max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time) =
row.expect("Row retrieval failed!");
for key in sorted_keys {
let best_car = best_cars.get(key).unwrap();
println!(
"{},{},{},{},{},{},{},{},{}",
max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time
key.1,
key.0,
key.2,
best_car.rq,
best_car.brand,
best_car.only_name,
best_car.year,
best_car.config,
best_car.time
);
}
drop(result);
connection.close().unwrap();
}
fn print_best_cars() {
let best_cars = BEST_CARS.lock().unwrap();
let mut car_count: HashMap<String, (usize, u8, BestCarRecord)> = HashMap::new();
async fn list_best_cars() {
let file = Connection::open("top_drives.db").expect("Issue opening DB file!");
let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
let backup = Backup::new(&file, &mut connection).expect("Could not create backup!");
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
drop(file);
let stmt = connection.prepare(
r#"
WITH RECURSIVE rq_range AS (
SELECT DISTINCT rq from cars
),
track_combinations AS (
SELECT DISTINCT track_id, track_type
FROM car_track_times
),
ranked_cars AS (
SELECT
rq_range.rq AS max_rq,
t.track_id,
t.track_type,
c.rq,
c.brand,
c.onlyName,
c.year,
t.configuration,
t.time,
ROW_NUMBER() OVER (
PARTITION BY rq_range.rq, t.track_id, t.track_type
ORDER BY t.time ASC
) AS rank
FROM rq_range
CROSS JOIN track_combinations tc
JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
JOIN cars c ON t.car_rid = c.rid
WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq
)
SELECT rq, brand, onlyName, year, COUNT(time) AS occurrences
FROM ranked_cars
WHERE rank = 1
GROUP BY brand, onlyName, year
ORDER BY rq, brand, onlyName, year
"#,
);
let mut result = stmt.expect("Query failed!");
let rows = result
.query_map([], |row| {
Ok((
row.get::<_, i32>(0).expect("Could not assign value"),
row.get::<_, String>(1).expect("Could not assign value"),
row.get::<_, String>(2).expect("Could not assign value"),
row.get::<_, i32>(3).expect("Could not assign value"),
row.get::<_, i32>(4).expect("Could not assign value"),
))
})
.expect("Result retrieval failed!");
for row in rows {
let (rq, brand, only_name, year, occurrences) = row.expect("Row retrieval failed!");
println!("{},{},{},{},{}", rq, brand, only_name, year, occurrences);
// Count occurrences and store RQ
for ((_, rq, _), best_car) in best_cars.iter() {
let entry = car_count
.entry(best_car.car_rid.clone())
.or_insert((0, *rq, best_car.clone()));
entry.0 += 1;
}
drop(result);
connection.close().unwrap();
// Convert to vector and sort by RQ first, then frequency
let mut car_count_vec: Vec<_> = car_count.iter().collect();
car_count_vec.sort_by(|a, b| a.1 .1.cmp(&b.1 .1).then(a.0.cmp(b.0)));
for (_car_rid, (count, rq, car_record)) in car_count_vec {
println!(
"{},{},{},{},{}",
rq, car_record.brand, car_record.only_name, car_record.year, count
);
}
}
fn print_best_car(track: &String, condition: &String, rq: &String) {
let best_cars = BEST_CARS.lock().unwrap();
let key = (track.clone(), rq.parse::<u8>().unwrap_or(115), condition.clone());
if let Some(best_car) = best_cars.get(&key) {
println!(
"Best car for Track: {}, Condition: {}, RQ: {} -> {} {} ({}) with time {:.2}",
track, condition, rq,
best_car.brand, best_car.only_name, best_car.year, best_car.time
);
} else {
println!(
"No best car found for Track: {}, Condition: {}, RQ: {}",
track, condition, rq
);
}
}
async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
let file_name = rid.to_owned() + ".json";
async fn parse_car_times(car: Car) {
let file_name = car.rid.to_owned() + ".json";
let local_file_path = "times/".to_owned() + file_name.as_str();
let file_content = fs::read_to_string(local_file_path);
@ -317,86 +168,53 @@ async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
// Insert into the database
if time > 1.2 && no_down_votes {
let _ = statement.execute([
rid.as_str(),
update_best_car(
translate_track_name(cleaned_track_id),
car.rq,
translate_track_type(track_type),
time.to_string().as_str(),
config,
]);
//TODO: update_best_car(cleaned_track_id, translate_track_type(track_type), None)
BestCarRecord {
car_rid: car.rid.clone(),
brand: car.brand.clone(),
only_name: car.only_name.clone(),
year: car.year,
rq: car.rq,
config: config.clone(),
time,
},
);
}
}
}
}
}
}
println!("Track times imported for {}", rid);
}
fn update_best_car(track_id: &str, rq: u8, surface: &str, new_record: BestCarRecord) {
let key = (track_id.to_string(), rq, surface.to_string());
fn update_best_car(track_id: &str, car_rq: u8, surface: &str, new_record: BestCarRecord) {
let mut best_cars = BEST_CARS.lock().unwrap();
for rq in car_rq..116 {
let key = (track_id.to_string(), rq, surface.to_string());
if let Some(existing) = best_cars.get(&key) {
if new_record.time < existing.time {
best_cars.insert(key, new_record);
best_cars.insert(key, new_record.clone());
}
} else {
best_cars.insert(key, new_record);
best_cars.insert(key, new_record.clone());
}
}
}
async fn parse_all_times(connection: &mut Connection) {
let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!");
async fn parse_all_times() {
let mut handles = vec![];
for car in get_cars_from_db() {
parse_car_times(car, &mut statement).await;
for car in get_car_list() {
let handle = task::spawn(async move {
parse_car_times(car).await;
});
handles.push(handle);
}
connection.flush_prepared_statement_cache();
let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ());
}
fn parse_cars(connection: &mut Connection) {
println!("Parsing cars into DB...");
let cars_file = "cars_repo.json";
let cars_data: Vec<Value> =
serde_json::from_str(&fs::read_to_string(cars_file).unwrap()).unwrap();
for car in cars_data {
if let Err(e) = connection.execute("INSERT OR IGNORE INTO cars
(class, rq, onlyName, brand, year, abs, tcs, clearance, country, topSpeed, acel, hand,
drive, tyres, mra, weight, name, rid, prize, bodyTypes, fuel, seats, engine, tags, guid, brake)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26);",
params![
car["class"].as_str().unwrap(),
car["rq"].as_u64().unwrap(),
car["onlyName"].as_str().unwrap(),
car["brand"].as_str().unwrap(),
car["year"].as_u64().unwrap(),
car["abs"].as_bool().unwrap(),
car["tcs"].as_bool().unwrap(),
car["clearance"].as_str().unwrap(),
car["country"].as_str().unwrap(),
car["topSpeed"].as_u64().unwrap(),
car["acel"].as_f64().unwrap_or_default(),
car["hand"].as_u64().unwrap(),
car["drive"].as_str().unwrap(),
car["tyres"].as_str().unwrap(),
car["mra"].as_f64().unwrap_or_default(),
car["weight"].as_u64().unwrap(),
car["name"].as_str().unwrap(),
car["rid"].as_str().unwrap(),
car["prize"].as_bool().unwrap(),
Some(serde_json::to_string(&car["bodyTypes"]).unwrap().as_str()).unwrap_or_default(),
car["fuel"].as_str().unwrap(),
car["seats"].as_u64().unwrap_or_default(),
car["engine"].as_str().unwrap(),
Some(serde_json::to_string(&car["tags"]).unwrap().as_str()).unwrap_or_default(),
car["guid"].as_str().unwrap(),
car["brake"].as_str().unwrap(),
]) {
eprintln!("Insertion error: {}", e);
};
}
join_all(handles).await;
}
fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a str, &'a str)> {
input.find(delimiter).map(|index| {
@ -406,13 +224,13 @@ fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a s
})
}
async fn fetch_times() {
let semaphore = std::sync::Arc::new(Semaphore::new(20)); // Adjust the number of concurrent requests
let semaphore = std::sync::Arc::new(Semaphore::new(20));
let mut handles = vec![];
for car in get_cars_from_db() {
for car in get_car_list() {
let permit = semaphore.clone().acquire_owned().await.unwrap();
handles.push(tokio::spawn(async move {
fetch_car(car).await;
fetch_car(car.rid).await;
drop(permit);
}));
}
@ -426,80 +244,34 @@ async fn fetch_car(rid: String) {
println!("Fetching {}.", url);
get_file(url.as_str(), local_file_path.as_str()).await;
}
fn get_cars_from_db() -> Vec<String> {
let mut car_list = Vec::new();
fn get_car_list() -> Vec<Car> {
let mut car_list: Vec<Car> = Vec::new();
let connection = Connection::open("top_drives.db").expect("Issue opening DB file!");
let cars_file = "cars_repo.json";
let cars_data: Vec<Value> =
serde_json::from_str(&fs::read_to_string(cars_file).unwrap()).unwrap();
let mut stmt = connection
.prepare("SELECT rid FROM cars")
.expect("Could not prepare query!");
let mut rows = stmt.query([]).expect("Could not execute query!");
for car in cars_data {
let new_car = Car {
rid: String::from(car["rid"].as_str().unwrap_or("No RID")),
brand: String::from(car["brand"].as_str().unwrap_or("No Brand")),
only_name: String::from(car["onlyName"].as_str().unwrap_or("No Name")),
year: car["year"].as_u64().unwrap_or(1900) as u16,
rq: car["rq"].as_u64().unwrap_or(200) as u8,
};
while let Ok(Some(row)) = rows.next() {
car_list.push(row.get::<usize, String>(0).expect("Could not get column!"));
car_list.push(new_car);
}
car_list
}
fn init_db(connection: &mut Connection) {
println!("Initializing DB...");
let _ = connection.execute(
"CREATE TABLE IF NOT EXISTS cars (
id INTEGER PRIMARY KEY AUTOINCREMENT,
class TEXT,
rq INTEGER,
onlyName TEXT,
brand TEXT,
year INTEGER,
abs BOOLEAN,
tcs BOOLEAN,
clearance TEXT,
country TEXT,
topSpeed INTEGER,
acel REAL,
hand INTEGER,
drive TEXT,
tyres TEXT,
mra REAL,
weight INTEGER,
name TEXT,
rid TEXT UNIQUE,
prize BOOLEAN,
bodyTypes TEXT,
fuel TEXT,
seats TEXT,
engine TEXT,
tags TEXT,
guid TEXT UNIQUE,
brake TEXT,
owned BOOLEAN
);",
[],
);
let _ = connection.execute(
"CREATE TABLE IF NOT EXISTS car_track_times (
id INTEGER PRIMARY KEY AUTOINCREMENT,
car_rid TEXT,
configuration TEXT,
track_id TEXT,
track_type INTEGER,
time DECIMAL,
FOREIGN KEY (car_rid) REFERENCES cars(rid),
UNIQUE(car_rid, track_id, track_type, configuration)
);",
[],
);
}
async fn get_tracks_file() {
println!("Fetching tracks file...");
get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/tracks_repo.json","tracks_repo.json").await;
}
async fn get_cars_file() {
println!("Fetching cars file...");
get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/cars_final_PL25-fixed.json", "cars_repo.json").await;
get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/cars_final.json", "cars_repo.json").await;
}
async fn get_file(file_url: &str, local_file_path: &str) {
let _ = fs::remove_file(local_file_path);
@ -627,7 +399,6 @@ fn translate_track_name(track_code: &str) -> &str {
"miBeachDrag" => "Miami Beach Drag",
"mnGforce" => "Monaco G-Force Test",
"mnHairpin" => "Monaco Hairpin",
// "mnNarrowStreets" => "Monaco Narrow Streets",
"mnCityNarrow" => "Monaco Narrow Streets",
"mnCity" => "Monaco Streets",
"mnCityLong" => "Monaco Streets Long",