Perform parsing in-memory

Cargo.toml

@@ -12,8 +12,8 @@ name = "records"
 path = "src/records.rs"
 
 [dependencies]
-scraper = "0.20.0"
+scraper = "0.23.1"
-reqwest = "0.12.7"
+reqwest = "0.12.15"
-tokio = { version = "1.40.0", features = ["full"] }
+tokio = { version = "1.44.1", features = ["full"] }
-rusqlite = { version = "0.32.0", features = ["bundled"] }
+rusqlite = { version = "0.32.1", features = ["bundled", "backup"] }
-serde_json = "1.0.133"
+serde_json = "1.0.140"

src/records.rs

@@ -1,8 +1,10 @@
 use reqwest::get;
 use rusqlite::{params, CachedStatement, Connection};
+use rusqlite::backup::Backup;
 use serde_json::Value;
 use std::env;
 use std::fs;
+
 #[tokio::main]
 async fn main() {
     let args: Vec<_> = env::args().collect();
@@ -17,18 +19,58 @@ async fn main() {
                 "init" => {
                     get_cars_file().await;
                     get_tracks_file().await;
-                    init_db();
+
-                    parse_cars();
+                    let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
+
+                    init_db(&mut connection);
+                    parse_cars(&mut connection);
+
+                    let _ = fs::remove_file("top_drives.db");
+                    let mut file = Connection::open("top_drives.db").expect("Issue opening DB file!");
+                    let backup = Backup::new(&connection, &mut file).expect("Could not create backup!");
+                    backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
+                    drop(backup);
+                    connection.close().unwrap();
+                    file.close().unwrap();
                 }
                 "fetch" => {
                     fetch_times().await;
                 }
                 "parse" => {
-                    parse_all_times().await;
+                    let mut file = Connection::open("top_drives.db").expect("Issue opening DB file!");
+                    let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
+
+                    let backup = Backup::new(&file, &mut connection).expect("Could not create backup!");
+                    backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
+                    drop(backup);
+                    file.close().unwrap();
+
+                    parse_all_times(&mut connection).await;
+
+                    let _ = fs::remove_file("top_drives.db");
+                    file = Connection::open("top_drives.db").expect("Issue opening DB file!");
+                    let backup = Backup::new(&connection, &mut file).expect("Could not create backup!");
+                    backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
+                    drop(backup);
+
+                    connection.close().unwrap();
+                    file.close().unwrap();
                 }
-                "analyze" => {
+                "best" => {
                     list_best_cars().await;
                 }
+                "details" => {
+                    list_details().await;
+                }
+                "track" => {
+                    if args.get(3).is_none() {
+                        println!("Track name and RQ limit required.");
+                    } else {
+                        let track = args.get(2).unwrap();
+                        let rq = args.get(3).unwrap();
+                        //TODO: Get best car for track and RQ limit
+                    }
+                }
                 &_ => {
                     println!("Unknown command '{}'.", argument)
                 }
@@ -36,9 +78,147 @@ async fn main() {
         }
     }
 }
-async fn list_best_cars() {
+async fn list_details() {
-    todo!()
+    let file = Connection::open("top_drives.db").expect("Issue opening DB file!");
+    let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
+
+    let backup = Backup::new(&file, &mut connection).expect("Could not create backup!");
+    backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
+    drop(backup);
+    drop (file);
+    let stmt = connection.prepare(
+        r#"
+        WITH RECURSIVE rq_range AS (
+            SELECT DISTINCT rq from cars
+        ),
+        track_combinations AS (
+            SELECT DISTINCT track_id, track_type
+            FROM car_track_times
+        ),
+        ranked_cars AS (
+            SELECT
+                rq_range.rq AS max_rq,
+                t.track_id,
+                t.track_type,
+                c.rq,
+                c.brand,
+                c.onlyName,
+                c.year,
+                t.configuration,
+                t.time,
+                ROW_NUMBER() OVER (
+                    PARTITION BY rq_range.rq, t.track_id, t.track_type
+                    ORDER BY t.time ASC
+                ) AS rank
+            FROM rq_range
+            CROSS JOIN track_combinations tc
+            JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
+            JOIN cars c ON t.car_rid = c.rid
+            WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq
+        )
+        SELECT max_rq, track_id, track_type, rq, brand, onlyName, year, configuration, time AS best_time
+        FROM ranked_cars
+        WHERE rank = 1
+        ORDER BY max_rq, track_id, track_type
+        "#,
+    );
+
+    let mut result = stmt.expect("Query failed!");
+    let rows = result.query_map([], |row| {
+            Ok((
+                row.get::<_, i32>(0)?,
+                row.get::<_, String>(1)?,
+                row.get::<_, String>(2)?,
+                row.get::<_, i32>(3)?,
+                row.get::<_, String>(4)?,
+                row.get::<_, String>(5)?,
+                row.get::<_, i32>(6)?,
+                row.get::<_, String>(7)?,
+                row.get::<_, f64>(8)?,
+            ))
+        })
+        .expect("Result retrieval failed!");
+
+    for row in rows {
+        let (max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time) =
+            row.expect("Row retrieval failed!");
+        println!(
+            "{},{},{},{},{},{},{},{},{}",
+            max_rq, track_id, track_type, rq, brand, only_name, year, configuration, best_time
+        );
     }
+
+    drop(result);
+    connection.close().unwrap();
+}
+
+async fn list_best_cars() {
+    let file = Connection::open("top_drives.db").expect("Issue opening DB file!");
+    let mut connection = Connection::open_in_memory().expect("Issue opening in-memory DB!");
+
+    let backup = Backup::new(&file, &mut connection).expect("Could not create backup!");
+    backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
+    drop(backup);
+    drop (file);
+    let stmt = connection.prepare(
+        r#"
+        WITH RECURSIVE rq_range AS (
+            SELECT DISTINCT rq from cars
+        ),
+        track_combinations AS (
+            SELECT DISTINCT track_id, track_type
+            FROM car_track_times
+        ),
+        ranked_cars AS (
+            SELECT
+                rq_range.rq AS max_rq,
+                t.track_id,
+                t.track_type,
+                c.rq,
+                c.brand,
+                c.onlyName,
+                c.year,
+                t.configuration,
+                t.time,
+                ROW_NUMBER() OVER (
+                    PARTITION BY rq_range.rq, t.track_id, t.track_type
+                    ORDER BY t.time ASC
+                ) AS rank
+            FROM rq_range
+            CROSS JOIN track_combinations tc
+            JOIN car_track_times t ON tc.track_id = t.track_id AND tc.track_type = t.track_type
+            JOIN cars c ON t.car_rid = c.rid
+            WHERE t.time > 0 AND t.time IS NOT NULL AND c.rq <= rq_range.rq
+        )
+        SELECT rq, brand, onlyName, year, COUNT(time) AS occurrences
+        FROM ranked_cars
+        WHERE rank = 1
+        GROUP BY brand, onlyName, year
+        ORDER BY rq, brand, onlyName, year
+        "#,
+    );
+
+    let mut result = stmt.expect("Query failed!");
+    let rows = result.query_map([], |row| {
+            Ok((
+                row.get::<_, i32>(0).expect("Could not assign value"),
+                row.get::<_, String>(1).expect("Could not assign value"),
+                row.get::<_, String>(2).expect("Could not assign value"),
+                row.get::<_, i32>(3).expect("Could not assign value"),
+                row.get::<_, i32>(4).expect("Could not assign value"),
+            ))
+        })
+        .expect("Result retrieval failed!");
+
+    for row in rows {
+        let (rq, brand, only_name, year, occurrences) = row.expect("Row retrieval failed!");
+        println!("{},{},{},{},{}", rq, brand, only_name, year, occurrences);
+    }
+
+    drop(result);
+    connection.close().unwrap();
+}
+
 async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
     let file_name = rid.to_owned() + ".json";
     let local_file_path = "times/".to_owned() + file_name.as_str();
@@ -85,8 +265,7 @@ async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
     }
     println!("Track times imported for {}", rid);
 }
-async fn parse_all_times() {
+async fn parse_all_times(connection: &mut Connection) {
-    let connection = Connection::open("top_drives.db").expect("Issue opening DB file!");
     let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!");
     for car in get_cars_from_db() {
         parse_car_times(car, &mut statement).await;
@@ -95,9 +274,8 @@ async fn parse_all_times() {
 
     let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ());
 }
-fn parse_cars() {
+fn parse_cars(connection: &mut Connection) {
-    let connection = Connection::open("top_drives.db").expect("Issue opening DB file!");
+    println!("Parsing cars into DB...");
-
     let cars_file = "cars_repo.json";
     let cars_data: Vec<Value> =
         serde_json::from_str(&fs::read_to_string(cars_file).unwrap()).unwrap();
@@ -174,9 +352,8 @@ fn get_cars_from_db() -> Vec<String> {
 
     car_list
 }
-fn init_db() {
+fn init_db(connection: &mut Connection) {
-    let _ = fs::remove_file("top_drives.db");
+    println!("Initializing DB...");
-    let connection = Connection::open("top_drives.db").expect("Issue opening DB file!");
 
     let _ = connection.execute(
         "CREATE TABLE IF NOT EXISTS cars (
@@ -227,9 +404,11 @@ fn init_db() {
     );
 }
 async fn get_tracks_file() {
+    println!("Fetching tracks file...");
     get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/tracks_repo.json","tracks_repo.json").await;
 }
 async fn get_cars_file() {
+    println!("Fetching cars file...");
     get_file("https://github.com/TiagoXavi/topdrives-records/raw/refs/heads/main/src/database/cars_final_PL25-fixed.json", "cars_repo.json").await;
 }
 async fn get_file(file_url: &str, local_file_path: &str) {