feat: add explicit source synchronization command

Author: bernard-ng

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "basango"
-version = "0.2.2"
+version = "0.2.3"
 edition = "2024"
 rust-version = "1.85"
 description = "A Rust-native Basango news crawler with HTML and WordPress adapters"

Cargo.toml

@@ -47,6 +47,17 @@ sudo journalctl -fu basango-crawler-worker.service
 
 The worker starts automatically at boot. Stopping it gracefully drains active work and leaves incomplete runs open so the next worker process can resume them. Repeated Redis connection failures make the process exit, allowing systemd to restart it without closing the affected runs.
 
+### Synchronize sources
+
+Run source registration and archive-size estimation once after installation, or again whenever the configured source list changes:
+
+```bash
+cd /opt/crawler
+sudo -u basango ./crawler source
+```
+
+Normal `crawl`, `schedule`, and `worker` runs do not synchronize or estimate sources.
+
 ### Schedule crawls (Initial)
 
 Schedule every registered source that does not require an indexed category:

README.md

@@ -44,10 +44,12 @@ curl -fsSL https://raw.githubusercontent.com/bernard-ng/basango-crawler/refs/hea
 
 From a local checkout, run `sudo ./deploy/uninstall.sh`. The script stops and disables the worker, removes its systemd unit, `/opt/crawler`, `/var/lib/crawler`, and the installer-created `basango` system account and group. It asks for confirmation first; pass `--yes` for unattended removal.
 
-Pushing a `v*` Git tag runs the release workflow, which publishes native `aarch64` (Raspberry Pi) and `x86_64` Linux archives to the GitHub release.
+Pushing a release tag runs the release workflow, which publishes native `aarch64` (Raspberry Pi) and `x86_64` Linux archives to the GitHub release.
 
 Each agent ID prefixes its discovery, article, and delivery queue names, so multiple Pis can safely share Redis. The worker consumes all three concurrently and reconciles SQLite delivery records after a restart. The installer does not schedule crawls. Run `crawler schedule` yourself or configure cron later with the sources and cadence assigned to that device.
 
+After installation, register the configured sources and estimate their archive sizes once with `sudo -u basango /opt/crawler/crawler source`. Run it again only when the configured sources change; normal crawler and worker runs do not perform source synchronization.
+
 To reset a Pi, stop its worker before clearing its scoped queues and SQLite outbox:
 
 ```bash

deploy/README.md

@@ -33,6 +33,8 @@ struct Cli {
 
 #[derive(Debug, Subcommand)]
 enum Command {
+    /// Register configured sources and estimate their archive sizes.
+    Source,
     Crawl(CrawlArgs),
     Schedule(ScheduleArgs),
     Worker(WorkerArgs),
@@ -119,6 +121,10 @@ pub async fn run() -> anyhow::Result<()> {
     .context("could not initialize crawler")?;
 
     match cli.command {
+        Command::Source => {
+            crawler.synchronize_sources().await?;
+            tracing::info!("source synchronization and estimation completed");
+        }
         Command::Crawl(arguments) => {
             let report = crawler.crawl(arguments.into()).await?;
             tracing::info!(?report, "crawl completed");

src/cli.rs

@@ -99,13 +99,11 @@ impl Crawler {
 
     /// Crawl now, streaming collected drafts into the durable outbox.
     pub async fn crawl(&self, request: CrawlRequest) -> Result<CrawlReport> {
-        self.runtime.synchronize_sources().await?;
         crawl_now(&self.runtime, request).await
     }
 
     /// Schedule source discovery in BullMQ.
     pub async fn schedule(&self, mut request: CrawlRequest) -> Result<String> {
-        self.runtime.synchronize_sources().await?;
         self.runtime.config.prepare_request(&mut request)?;
         let reporter = RunReporter::new(
             &self.runtime.config.ingestion,
@@ -159,10 +157,14 @@ impl Crawler {
 
     /// Run BullMQ consumers until the process receives Ctrl-C.
     pub async fn work(&self, queues: Vec<String>, concurrency: usize) -> Result<()> {
-        self.runtime.synchronize_sources().await?;
         run_worker(self.runtime.clone(), queues, concurrency).await
     }
 
+    /// Register configured sources and update their archive-size estimates.
+    pub async fn synchronize_sources(&self) -> Result<()> {
+        self.runtime.synchronize_sources().await
+    }
+
     /// Read the current agent's local outbox and Redis queue state.
     pub async fn status(&self) -> CrawlerStatus {
         let sqlite_path = self.runtime.config.sqlite_path();

src/crawler.rs

@@ -18,10 +18,6 @@ pub(crate) use worker::run_worker;
 
 use std::sync::Arc;
 
-use chrono::{DateTime, Utc};
-use serde::Deserialize;
-use tokio::sync::OnceCell;
-
 use crate::{
     articles::endpoint_url,
     config::CrawlerConfig,
@@ -30,6 +26,8 @@ use crate::{
     http::HttpClient,
     telemetry::agent_id,
 };
+use chrono::{DateTime, Utc};
+use serde::Deserialize;
 
 /// Shared, immutable dependencies are placed in `Arc` so queued jobs can own a
 /// cheap reference while running concurrently.
@@ -38,7 +36,6 @@ pub(crate) struct Runtime {
     pub config: Arc<CrawlerConfig>,
     pub http: HttpClient,
     pub agent_id: String,
-    source_sync: Arc<OnceCell<()>>,
 }
 
 impl Runtime {
@@ -50,15 +47,11 @@ impl Runtime {
             config: Arc::new(config),
             http,
             agent_id,
-            source_sync: Arc::new(OnceCell::new()),
         })
     }
 
     pub async fn synchronize_sources(&self) -> Result<()> {
-        self.source_sync
-            .get_or_try_init(|| async { source_sync::synchronize(self).await })
-            .await
-            .map(|_| ())
+        source_sync::synchronize(self).await
     }
 
     /// Ask the ingestion API for the last known article boundary when the caller did

src/execution.rs

@@ -30,6 +30,12 @@ fn status_is_a_standalone_command() {
     assert!(matches!(cli.command, Command::Status));
 }
 
+#[test]
+fn source_is_a_standalone_command() {
+    let cli = Cli::try_parse_from(["crawler", "source"]).unwrap();
+    assert!(matches!(cli.command, Command::Source));
+}
+
 #[test]
 fn schedule_requires_an_explicit_source() {
     assert!(Cli::try_parse_from(["crawler", "schedule"]).is_err());