jan/top_drives
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Multi-threaded fetch

Browse source
This commit is contained in:
Jan-Peter von Hunnius 2025-03-30 16:33:04 +02:00
parent 431ea2814b
commit 02ff6eddb2
2 changed files with 20 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -17,3 +17,4 @@ reqwest = "0.12.15"
tokio = { version = "1.44.1", features = ["full"] } tokio = { version = "1.44.1", features = ["full"] }
rusqlite = { version = "0.32.1", features = ["bundled", "backup"] } rusqlite = { version = "0.32.1", features = ["bundled", "backup"] }
serde_json = "1.0.140" serde_json = "1.0.140"
futures = "0.3.31"

21
src/records.rs
View file

@ -4,6 +4,8 @@ use rusqlite::{params, CachedStatement, Connection};
use serde_json::Value; use serde_json::Value;
use std::env; use std::env;
use std::fs; use std::fs;
use tokio::sync::Semaphore;
use futures::future::join_all;
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
@ -123,6 +125,7 @@ async fn list_details() {
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup); drop(backup);
drop(file); drop(file);
let stmt = connection.prepare( let stmt = connection.prepare(
r#" r#"
WITH RECURSIVE rq_range AS ( WITH RECURSIVE rq_range AS (
@ -198,6 +201,7 @@ async fn list_best_cars() {
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup); drop(backup);
drop(file); drop(file);
let stmt = connection.prepare( let stmt = connection.prepare(
r#" r#"
WITH RECURSIVE rq_range AS ( WITH RECURSIVE rq_range AS (
@ -306,11 +310,12 @@ async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
} }
async fn parse_all_times(connection: &mut Connection) { async fn parse_all_times(connection: &mut Connection) {
let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!"); let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!");
for car in get_cars_from_db() { for car in get_cars_from_db() {
parse_car_times(car, &mut statement).await; parse_car_times(car, &mut statement).await;
connection.flush_prepared_statement_cache();
} }
connection.flush_prepared_statement_cache();
let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ()); let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ());
} }
fn parse_cars(connection: &mut Connection) { fn parse_cars(connection: &mut Connection) {
@ -364,9 +369,18 @@ fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a s
}) })
} }
async fn fetch_times() { async fn fetch_times() {
let semaphore = std::sync::Arc::new(Semaphore::new(20)); // Adjust the number of concurrent requests
let mut handles = vec![];
for car in get_cars_from_db() { for car in get_cars_from_db() {
let permit = semaphore.clone().acquire_owned().await.unwrap();
handles.push(tokio::spawn(async move {
fetch_car(car).await; fetch_car(car).await;
drop(permit); // Release semaphore
}));
} }
// Wait for all fetch tasks to complete
join_all(handles).await;
} }
async fn fetch_car(rid: String) { async fn fetch_car(rid: String) {
let file_name = rid.to_owned() + ".json"; let file_name = rid.to_owned() + ".json";
@ -508,6 +522,7 @@ fn translate_track_name(track_code: &str) -> &str {
"drag100150" => "100-150mph", "drag100150" => "100-150mph",
"drag30130" => "30-130mph", "drag30130" => "30-130mph",
"drag50150" => "50-150mph", "drag50150" => "50-150mph",
"drag75125" => "75-125mph",
"airplaneHangars" => "Airplane Hangars", "airplaneHangars" => "Airplane Hangars",
"airplaneSlalom" => "Airplane Slalom", "airplaneSlalom" => "Airplane Slalom",
"smallRoad" => "Back Road", "smallRoad" => "Back Road",
@ -575,11 +590,11 @@ fn translate_track_name(track_code: &str) -> &str {
"miBeachDrag" => "Miami Beach Drag", "miBeachDrag" => "Miami Beach Drag",
"mnGforce" => "Monaco G-Force Test", "mnGforce" => "Monaco G-Force Test",
"mnHairpin" => "Monaco Hairpin", "mnHairpin" => "Monaco Hairpin",
"mnNarrowStreets" => "Monaco Narrow Streets", // "mnNarrowStreets" => "Monaco Narrow Streets",
"mnCityNarrow" => "Monaco Narrow Streets",
"mnCity" => "Monaco Streets", "mnCity" => "Monaco Streets",
"mnCityLong" => "Monaco Streets Long", "mnCityLong" => "Monaco Streets Long",
"moto" => "Motocross Track", "moto" => "Motocross Track",
"mtButte" => "mtButte",
"mtHairpin" => "Mountain Hairpin", "mtHairpin" => "Mountain Hairpin",
"mtHill" => "Mountain Hill Climb", "mtHill" => "Mountain Hill Climb",
"mtIncline" => "Mountain Incline Road", "mtIncline" => "Mountain Incline Road",