test: US-08 DWN sync tests pass 50/50 — fix sync performance
- Make dwn.sync endpoint async: spawns background task, returns immediately - Add 90s overall timeout to sync_with_peers via tokio::time::timeout - Deduplicate peer onion addresses before syncing - Batch message pushes (50 per request) instead of one-at-a-time over Tor - Add 15s connect_timeout to Tor SOCKS5 client - Cap local message query to 200 messages per sync - Fix DWN HTTP handler to process ALL messages in batch (was only first) - Add recordId deduplication in handler to prevent duplicate imports - Update test script to poll dwn.status for sync completion Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dorian
@@ -652,6 +652,7 @@ impl ApiHandler {
|
||||
}
|
||||
|
||||
/// DWN message processing endpoint — handles RecordsWrite, RecordsQuery, RecordsRead, RecordsDelete.
|
||||
/// Supports batch processing: all messages in the array are processed.
|
||||
async fn handle_dwn_message(
|
||||
body: hyper::body::Bytes,
|
||||
config: &Config,
|
||||
@@ -668,100 +669,145 @@ impl ApiHandler {
|
||||
}
|
||||
};
|
||||
|
||||
// Support both formats: {"message": {...}} and {"messages": [{...}]}
|
||||
let message = if request.get("message").is_some() {
|
||||
request["message"].clone()
|
||||
// Collect all messages to process
|
||||
let messages: Vec<serde_json::Value> = if request.get("message").is_some() {
|
||||
vec![request["message"].clone()]
|
||||
} else if let Some(msgs) = request["messages"].as_array() {
|
||||
msgs.first().cloned().unwrap_or_default()
|
||||
msgs.clone()
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
vec![serde_json::Value::Null]
|
||||
};
|
||||
|
||||
let interface = message["descriptor"]["interface"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
let method = message["descriptor"]["method"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
|
||||
let store = DwnStore::new(&config.data_dir).await?;
|
||||
let mut results = Vec::new();
|
||||
|
||||
let result = match (interface, method) {
|
||||
("Records", "Write") => {
|
||||
let author = message["author"].as_str().unwrap_or("unknown");
|
||||
let protocol = message["descriptor"]["protocol"].as_str();
|
||||
let schema = message["descriptor"]["schema"].as_str();
|
||||
let data_format = message["descriptor"]["dataFormat"].as_str();
|
||||
let data = message.get("data").cloned();
|
||||
match store.write_message(author, protocol, schema, data_format, data).await {
|
||||
Ok(msg) => serde_json::json!({"status": {"code": 202}, "entry": msg}),
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
("Records", "Query") => {
|
||||
let query = crate::network::dwn_store::MessageQuery {
|
||||
protocol: message["descriptor"]["filter"]["protocol"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
schema: message["descriptor"]["filter"]["schema"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
author: message["descriptor"]["filter"]["author"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
date_from: message["descriptor"]["filter"]["dateFrom"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
date_to: message["descriptor"]["filter"]["dateTo"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
limit: message["descriptor"]["filter"]["limit"]
|
||||
.as_u64()
|
||||
.map(|n| n as usize),
|
||||
};
|
||||
match store.query_messages(&query).await {
|
||||
Ok(messages) => serde_json::json!({"status": {"code": 200}, "entries": messages}),
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
("Records", "Read") => {
|
||||
let record_id = message["descriptor"]["recordId"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
match store.read_message(record_id).await {
|
||||
Ok(Some(msg)) => serde_json::json!({"status": {"code": 200}, "entry": msg}),
|
||||
Ok(None) => serde_json::json!({"status": {"code": 404, "detail": "Record not found"}}),
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
("Records", "Delete") => {
|
||||
let record_id = message["descriptor"]["recordId"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
match store.delete_message(record_id).await {
|
||||
Ok(true) => serde_json::json!({"status": {"code": 200}}),
|
||||
Ok(false) => serde_json::json!({"status": {"code": 404, "detail": "Record not found"}}),
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
serde_json::json!({"status": {"code": 400, "detail": format!("Unknown method: {}.{}", interface, method)}})
|
||||
}
|
||||
};
|
||||
for message in &messages {
|
||||
let interface = message["descriptor"]["interface"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
let method = message["descriptor"]["method"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
|
||||
let status_code = result["status"]["code"].as_u64().unwrap_or(200);
|
||||
let http_status = match status_code {
|
||||
202 => StatusCode::ACCEPTED,
|
||||
400 => StatusCode::BAD_REQUEST,
|
||||
404 => StatusCode::NOT_FOUND,
|
||||
500 => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
_ => StatusCode::OK,
|
||||
let result = match (interface, method) {
|
||||
("Records", "Write") => {
|
||||
let author = message["author"].as_str().unwrap_or("unknown");
|
||||
let protocol = message["descriptor"]["protocol"].as_str();
|
||||
let schema = message["descriptor"]["schema"].as_str();
|
||||
let data_format = message["descriptor"]["dataFormat"].as_str();
|
||||
let data = message.get("data").cloned();
|
||||
// Deduplicate: check if recordId already exists
|
||||
if let Some(record_id) = message["recordId"].as_str() {
|
||||
if store.read_message(record_id).await.ok().flatten().is_some() {
|
||||
serde_json::json!({"status": {"code": 200, "detail": "Already exists"}})
|
||||
} else {
|
||||
match store
|
||||
.write_message(author, protocol, schema, data_format, data)
|
||||
.await
|
||||
{
|
||||
Ok(msg) => {
|
||||
serde_json::json!({"status": {"code": 202}, "entry": msg})
|
||||
}
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match store
|
||||
.write_message(author, protocol, schema, data_format, data)
|
||||
.await
|
||||
{
|
||||
Ok(msg) => serde_json::json!({"status": {"code": 202}, "entry": msg}),
|
||||
Err(e) => serde_json::json!({"status": {"code": 500, "detail": e.to_string()}}),
|
||||
}
|
||||
}
|
||||
}
|
||||
("Records", "Query") => {
|
||||
let query = crate::network::dwn_store::MessageQuery {
|
||||
protocol: message["descriptor"]["filter"]["protocol"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
schema: message["descriptor"]["filter"]["schema"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
author: message["descriptor"]["filter"]["author"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
date_from: message["descriptor"]["filter"]["dateFrom"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
date_to: message["descriptor"]["filter"]["dateTo"]
|
||||
.as_str()
|
||||
.map(|s| s.to_string()),
|
||||
limit: message["descriptor"]["filter"]["limit"]
|
||||
.as_u64()
|
||||
.map(|n| n as usize),
|
||||
};
|
||||
match store.query_messages(&query).await {
|
||||
Ok(messages) => {
|
||||
serde_json::json!({"status": {"code": 200}, "entries": messages})
|
||||
}
|
||||
Err(e) => {
|
||||
serde_json::json!({"status": {"code": 500, "detail": e.to_string()}})
|
||||
}
|
||||
}
|
||||
}
|
||||
("Records", "Read") => {
|
||||
let record_id = message["descriptor"]["recordId"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
match store.read_message(record_id).await {
|
||||
Ok(Some(msg)) => {
|
||||
serde_json::json!({"status": {"code": 200}, "entry": msg})
|
||||
}
|
||||
Ok(None) => serde_json::json!({"status": {"code": 404, "detail": "Record not found"}}),
|
||||
Err(e) => {
|
||||
serde_json::json!({"status": {"code": 500, "detail": e.to_string()}})
|
||||
}
|
||||
}
|
||||
}
|
||||
("Records", "Delete") => {
|
||||
let record_id = message["descriptor"]["recordId"]
|
||||
.as_str()
|
||||
.unwrap_or("");
|
||||
match store.delete_message(record_id).await {
|
||||
Ok(true) => serde_json::json!({"status": {"code": 200}}),
|
||||
Ok(false) => serde_json::json!({"status": {"code": 404, "detail": "Record not found"}}),
|
||||
Err(e) => {
|
||||
serde_json::json!({"status": {"code": 500, "detail": e.to_string()}})
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
serde_json::json!({"status": {"code": 400, "detail": format!("Unknown method: {}.{}", interface, method)}})
|
||||
}
|
||||
};
|
||||
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
// Return single result for single message, array for batch
|
||||
let (response_body, http_status) = if results.len() == 1 {
|
||||
let result = &results[0];
|
||||
let status_code = result["status"]["code"].as_u64().unwrap_or(200);
|
||||
let http_status = match status_code {
|
||||
202 => StatusCode::ACCEPTED,
|
||||
400 => StatusCode::BAD_REQUEST,
|
||||
404 => StatusCode::NOT_FOUND,
|
||||
500 => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
_ => StatusCode::OK,
|
||||
};
|
||||
(result.to_string(), http_status)
|
||||
} else {
|
||||
(
|
||||
serde_json::json!({"replies": results}).to_string(),
|
||||
StatusCode::OK,
|
||||
)
|
||||
};
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(http_status)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(hyper::Body::from(result.to_string()))
|
||||
.body(hyper::Body::from(response_body))
|
||||
.unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,18 @@ impl RpcHandler {
|
||||
}
|
||||
|
||||
/// Trigger DWN sync with connected peers.
|
||||
/// Spawns sync as a background task and returns immediately.
|
||||
pub(super) async fn handle_dwn_sync(&self) -> Result<serde_json::Value> {
|
||||
// Check if already syncing
|
||||
let current_state = dwn_sync::load_sync_state(&self.config.data_dir).await?;
|
||||
if matches!(current_state.status, dwn_sync::SyncStatus::Syncing) {
|
||||
return Ok(serde_json::json!({
|
||||
"sync_status": "syncing",
|
||||
"last_sync": current_state.last_sync,
|
||||
"messages_synced": current_state.messages_synced,
|
||||
}));
|
||||
}
|
||||
|
||||
let nodes = federation::load_nodes(&self.config.data_dir).await?;
|
||||
let onions: Vec<String> = nodes
|
||||
.iter()
|
||||
@@ -39,12 +50,19 @@ impl RpcHandler {
|
||||
.map(|n| n.onion.clone())
|
||||
.collect();
|
||||
|
||||
let state = dwn_sync::sync_with_peers(&self.config.data_dir, &onions).await?;
|
||||
// Spawn sync in background so we don't block the RPC response
|
||||
let data_dir = self.config.data_dir.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = dwn_sync::sync_with_peers(&data_dir, &onions).await {
|
||||
tracing::warn!(error = %e, "DWN background sync failed");
|
||||
}
|
||||
});
|
||||
|
||||
// Return immediately with "syncing" status
|
||||
Ok(serde_json::json!({
|
||||
"sync_status": state.status,
|
||||
"last_sync": state.last_sync,
|
||||
"messages_synced": state.messages_synced,
|
||||
"sync_status": "syncing",
|
||||
"last_sync": current_state.last_sync,
|
||||
"messages_synced": current_state.messages_synced,
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user