jan/top_drives
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Multi-threaded fetch

Browse source
This commit is contained in:
Jan-Peter von Hunnius 2025-03-30 16:33:04 +02:00
parent 431ea2814b
commit 02ff6eddb2
2 changed files with 20 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/records.rs
View file

@ -4,6 +4,8 @@ use rusqlite::{params, CachedStatement, Connection};
use serde_json::Value;
use std::env;
use std::fs;
use tokio::sync::Semaphore;
use futures::future::join_all;
#[tokio::main]
async fn main() {
@ -123,6 +125,7 @@ async fn list_details() {
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
drop(file);
let stmt = connection.prepare(
r#"
WITH RECURSIVE rq_range AS (
@ -198,6 +201,7 @@ async fn list_best_cars() {
backup.step(-1).expect("Could not create backup!"); // -1 copies all pages at once
drop(backup);
drop(file);
let stmt = connection.prepare(
r#"
WITH RECURSIVE rq_range AS (
@ -306,11 +310,12 @@ async fn parse_car_times(rid: String, statement: &mut CachedStatement<'_>) {
}
async fn parse_all_times(connection: &mut Connection) {
let mut statement = connection.prepare_cached("INSERT INTO car_track_times (car_rid, track_id, track_type, time, configuration) VALUES (?1, ?2, ?3, ?4, ?5)").expect("Could not prepare statement!");
for car in get_cars_from_db() {
parse_car_times(car, &mut statement).await;
connection.flush_prepared_statement_cache();
}
connection.flush_prepared_statement_cache();
let _ = connection.execute("DELETE FROM car_track_times WHERE time < 1.2;", ());
}
fn parse_cars(connection: &mut Connection) {
@ -364,9 +369,18 @@ fn split_prefix_postfix<'a>(input: &'a str, delimiter: &'a str) -> Option<(&'a s
})
}
async fn fetch_times() {
let semaphore = std::sync::Arc::new(Semaphore::new(20)); // Adjust the number of concurrent requests
let mut handles = vec![];
for car in get_cars_from_db() {
fetch_car(car).await;
let permit = semaphore.clone().acquire_owned().await.unwrap();
handles.push(tokio::spawn(async move {
fetch_car(car).await;
drop(permit); // Release semaphore
}));
}
// Wait for all fetch tasks to complete
join_all(handles).await;
}
async fn fetch_car(rid: String) {
let file_name = rid.to_owned() + ".json";
@ -508,6 +522,7 @@ fn translate_track_name(track_code: &str) -> &str {
"drag100150" => "100-150mph",
"drag30130" => "30-130mph",
"drag50150" => "50-150mph",
"drag75125" => "75-125mph",
"airplaneHangars" => "Airplane Hangars",
"airplaneSlalom" => "Airplane Slalom",
"smallRoad" => "Back Road",
@ -575,11 +590,11 @@ fn translate_track_name(track_code: &str) -> &str {
"miBeachDrag" => "Miami Beach Drag",
"mnGforce" => "Monaco G-Force Test",
"mnHairpin" => "Monaco Hairpin",
"mnNarrowStreets" => "Monaco Narrow Streets",
// "mnNarrowStreets" => "Monaco Narrow Streets",
"mnCityNarrow" => "Monaco Narrow Streets",
"mnCity" => "Monaco Streets",
"mnCityLong" => "Monaco Streets Long",
"moto" => "Motocross Track",
"mtButte" => "mtButte",
"mtHairpin" => "Mountain Hairpin",
"mtHill" => "Mountain Hill Climb",
"mtIncline" => "Mountain Incline Road",