jan/top_drives
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fetch car times sequentially to respect the API's provisioned throughput
All checks were successful
build / build-test (push) Successful in 1m17s
Details

Browse source
This commit is contained in:
Jan-Peter von Hunnius 2026-09-29 10:37:06 +02:00
parent 0b1a710ffa
commit 119a59f145
1 changed files with 16 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/parsing/mod.rs
View file

@ -362,11 +362,15 @@ pub async fn parse_all_times(
/// Remote URL for the per-car timing API. /// Remote URL for the per-car timing API.
const CAR_TIMING_API_BASE: &str = "https://api.topdrivesrecords.com/car/"; const CAR_TIMING_API_BASE: &str = "https://api.topdrivesrecords.com/car/";
/// Fetches timing data files for all cars using controlled concurrency. /// Fetches timing data files for all cars sequentially.
///
/// The records API is a hobbyist backend backed by a provisioned-throughput
/// database; concurrent fetching outruns its read capacity and produces HTTP
/// 400 "provisioned throughput exceeded" errors, so downloads are issued one
/// at a time to stay a polite client.
/// ///
/// Displays a single-line progress counter (completed/total cars) that stays /// Displays a single-line progress counter (completed/total cars) that stays
/// on the bottom line while downloads run. The counter tracks completions, /// on the bottom line while downloads run.
/// not cars seen, since downloads run concurrently under the semaphore.
/// ///
/// Returns the first error encountered, or `Ok(())` if all downloads succeeded. /// Returns the first error encountered, or `Ok(())` if all downloads succeeded.
pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppError> { pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppError> {
@ -379,47 +383,19 @@ pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppE
); );
progress.set_message("Downloading track times for car"); progress.set_message("Downloading track times for car");
let semaphore = Arc::new(Semaphore::new(CONCURRENCY_LIMIT));
let mut handles = vec![];
for car in car_list {
let permit = semaphore
.clone()
.acquire_owned()
.await
.map_err(|e| AppError::Network(format!("Semaphore closed: {e}")))?;
let times_dir = times_dir.to_string();
let progress = progress.clone();
handles.push(tokio::spawn(async move {
let _permit = permit; // held until end of block
let result = fetch_car(&car.rid, &times_dir).await;
progress.inc(1);
result
}));
}
let results = join_all(handles).await;
// Clear the counter before error logging so warnings don't garble the line.
progress.finish_and_clear();
let mut first_error: Option<AppError> = None; let mut first_error: Option<AppError> = None;
for result in results { for car in car_list {
match result { if let Err(e) = fetch_car(&car.rid, times_dir).await {
Ok(Ok(())) => {} log::warn!("{e}");
Ok(Err(e)) => { if first_error.is_none() {
log::warn!("{e}"); first_error = Some(e);
if first_error.is_none() {
first_error = Some(e);
}
}
Err(e) => {
let msg = AppError::Network(format!("Task join error: {e}"));
log::warn!("{msg}");
if first_error.is_none() {
first_error = Some(msg);
}
} }
} }
progress.inc(1);
} }
// Clear the counter before error logging so warnings don't garble the line.
progress.finish_and_clear();
first_error.map_or(Ok(()), Err) first_error.map_or(Ok(()), Err)
} }