Capture ephemeral data.
Lose nothing.

1use std::{time::Duration, env};2 3use meathook::{4    FlushPolicy, HfSink, JsonEncoder, JsonlStore, Meathook, MemStore,5    Pipeline, SatayCollector, SinkStack,6};7use satay_reqwest::ReqwestActionExt as _;8use reqwest::Client;9 10#[tokio::main]11async fn main() -> Result<(), meathook::runtime::RuntimeError> {12    let client = Client::new();13    let token = env::var("HF_TOKEN").expect("HF_TOKEN must be set");14 15    Meathook::builder()16        .pipeline(move || {17            let api = nea_rs::Api::new();18            let collector = SatayCollector::new(19                "air_temperature",20                client.clone(),21                move |client| {22                    let api = api.clone();23                    async move { api.air_temperature().send_with(&client).await }24                },25                |response| flatten(response),26            );27 28            // Declaration order is record flow: memory batches for five minutes29            // or 10k records, JSONL fsyncs each batch and forwards hourly, then30            // the terminal uploads it. Disk is reached only when memory fires.31            let sink = SinkStack::new()32                .tier(33                    MemStore::new(),34                    FlushPolicy::new(Duration::from_secs(300), 10_000),35                )36                .tier(37                    JsonlStore::new("/var/lib/meathook/spool/air_temperature"),38                    FlushPolicy::hourly(),39                )40                .terminal(41                    HfSink::new(client.clone(), "you/your-dataset", token.clone())42                        .encoder(JsonEncoder),43                );44 45            Pipeline::new(collector, sink, Duration::from_secs(60))46                .with_key_fn(|r: &MyRecord| (r.station_id.clone(), r.timestamp.clone()))47        })48        .run()49        .await50}