Fetch car times sequentially to respect the API's provisioned throughput
All checks were successful
build / build-test (push) Successful in 1m17s
All checks were successful
build / build-test (push) Successful in 1m17s
This commit is contained in:
parent
0b1a710ffa
commit
119a59f145
1 changed files with 16 additions and 40 deletions
|
|
@ -362,11 +362,15 @@ pub async fn parse_all_times(
|
||||||
/// Remote URL for the per-car timing API.
|
/// Remote URL for the per-car timing API.
|
||||||
const CAR_TIMING_API_BASE: &str = "https://api.topdrivesrecords.com/car/";
|
const CAR_TIMING_API_BASE: &str = "https://api.topdrivesrecords.com/car/";
|
||||||
|
|
||||||
/// Fetches timing data files for all cars using controlled concurrency.
|
/// Fetches timing data files for all cars sequentially.
|
||||||
|
///
|
||||||
|
/// The records API is a hobbyist backend backed by a provisioned-throughput
|
||||||
|
/// database; concurrent fetching outruns its read capacity and produces HTTP
|
||||||
|
/// 400 "provisioned throughput exceeded" errors, so downloads are issued one
|
||||||
|
/// at a time to stay a polite client.
|
||||||
///
|
///
|
||||||
/// Displays a single-line progress counter (completed/total cars) that stays
|
/// Displays a single-line progress counter (completed/total cars) that stays
|
||||||
/// on the bottom line while downloads run. The counter tracks completions,
|
/// on the bottom line while downloads run.
|
||||||
/// not cars seen, since downloads run concurrently under the semaphore.
|
|
||||||
///
|
///
|
||||||
/// Returns the first error encountered, or `Ok(())` if all downloads succeeded.
|
/// Returns the first error encountered, or `Ok(())` if all downloads succeeded.
|
||||||
pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppError> {
|
pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppError> {
|
||||||
|
|
@ -379,47 +383,19 @@ pub async fn fetch_times(cars_db_path: &str, times_dir: &str) -> Result<(), AppE
|
||||||
);
|
);
|
||||||
progress.set_message("Downloading track times for car");
|
progress.set_message("Downloading track times for car");
|
||||||
|
|
||||||
let semaphore = Arc::new(Semaphore::new(CONCURRENCY_LIMIT));
|
|
||||||
let mut handles = vec![];
|
|
||||||
|
|
||||||
for car in car_list {
|
|
||||||
let permit = semaphore
|
|
||||||
.clone()
|
|
||||||
.acquire_owned()
|
|
||||||
.await
|
|
||||||
.map_err(|e| AppError::Network(format!("Semaphore closed: {e}")))?;
|
|
||||||
let times_dir = times_dir.to_string();
|
|
||||||
let progress = progress.clone();
|
|
||||||
handles.push(tokio::spawn(async move {
|
|
||||||
let _permit = permit; // held until end of block
|
|
||||||
let result = fetch_car(&car.rid, ×_dir).await;
|
|
||||||
progress.inc(1);
|
|
||||||
result
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
let results = join_all(handles).await;
|
|
||||||
// Clear the counter before error logging so warnings don't garble the line.
|
|
||||||
progress.finish_and_clear();
|
|
||||||
let mut first_error: Option<AppError> = None;
|
let mut first_error: Option<AppError> = None;
|
||||||
for result in results {
|
for car in car_list {
|
||||||
match result {
|
if let Err(e) = fetch_car(&car.rid, times_dir).await {
|
||||||
Ok(Ok(())) => {}
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
log::warn!("{e}");
|
log::warn!("{e}");
|
||||||
if first_error.is_none() {
|
if first_error.is_none() {
|
||||||
first_error = Some(e);
|
first_error = Some(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
progress.inc(1);
|
||||||
let msg = AppError::Network(format!("Task join error: {e}"));
|
|
||||||
log::warn!("{msg}");
|
|
||||||
if first_error.is_none() {
|
|
||||||
first_error = Some(msg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clear the counter before error logging so warnings don't garble the line.
|
||||||
|
progress.finish_and_clear();
|
||||||
first_error.map_or(Ok(()), Err)
|
first_error.map_or(Ok(()), Err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue