Improve Orchestrator error logs

orchestrator/Cargo.toml

@@ -2,7 +2,7 @@
 # Also, the name was changed from orchestrator to gravity_bridge.
 [package]
 name = "gravity_bridge"
-version = "4.0.0"
+version = "4.0.1"
 authors = ["PeggyJV"]
 license = "Apache-2.0"
 edition = "2018"

orchestrator/cosmos_gravity/src/query.rs

@@ -66,7 +66,10 @@ pub async fn get_all_valset_confirms(
     let mut parsed_confirms = Vec::new();
     for item in confirms {
         if let Err(e) = Signature::from_bytes(&item.signature)?.error_check() {
-            warn!("ignoring valset confirmation with invalid signature from {}", item.ethereum_signer);
+            warn!(
+                "ignoring valset confirmation with invalid signature from {}",
+                item.ethereum_signer
+            );
             debug!("reason given for invalid signature: {e:?}");
 
             continue;
@@ -139,7 +142,10 @@ pub async fn get_transaction_batch_signatures(
     let mut out = Vec::new();
     for confirm in batch_confirms {
         if let Err(e) = Signature::from_bytes(&confirm.signature)?.error_check() {
-            warn!("ignoring batch confirmation with invalid signature from {}", confirm.ethereum_signer);
+            warn!(
+                "ignoring batch confirmation with invalid signature from {}",
+                confirm.ethereum_signer
+            );
             debug!("reason given for invalid signature: {e:?}");
 
             continue;
@@ -201,7 +207,10 @@ pub async fn get_logic_call_signatures(
     let mut out = Vec::new();
     for confirm in call_confirms {
         if let Err(e) = Signature::from_bytes(&confirm.signature)?.error_check() {
-            warn!("ignoring logic call confirmation with invalid signature from {}", confirm.ethereum_signer);
+            warn!(
+                "ignoring logic call confirmation with invalid signature from {}",
+                confirm.ethereum_signer
+            );
             debug!("reason given for invalid signature: {e:?}");
 
             continue;

orchestrator/orchestrator/src/get_with_retry.rs

@@ -14,7 +14,10 @@ pub const RETRY_TIME: Duration = Duration::from_secs(5);
 pub async fn get_block_number_with_retry(eth_client: EthClient) -> U64 {
     let mut res = eth_client.get_block_number().await;
     while res.is_err() {
-        error!("Failed to get latest block! Is your Eth node working?");
+        error!(
+            "Failed to get latest block! (Is your Eth node working?): {:?}",
+            res
+        );
         delay_for(RETRY_TIME).await;
         res = eth_client.get_block_number().await;
     }
@@ -29,7 +32,7 @@ pub async fn get_last_event_nonce_with_retry(
     let mut res = get_last_event_nonce(client, our_cosmos_address).await;
     while res.is_err() {
         error!(
-            "Failed to get last event nonce, is the Cosmos GRPC working? {:?}",
+            "Failed to get last event nonce, is the Cosmos GRPC working? (may be transient): {:?}",
             res
         );
         delay_for(RETRY_TIME).await;
@@ -42,7 +45,10 @@ pub async fn get_last_event_nonce_with_retry(
 pub async fn get_chain_id_with_retry(eth_client: EthClient) -> U256 {
     let mut res = eth_client.get_chainid().await;
     while res.is_err() {
-        error!("Failed to get chain ID! Is your Eth node working?");
+        error!(
+            "Failed to get chain ID! (Is your Eth node working?): {:?}",
+            res
+        );
         delay_for(RETRY_TIME).await;
         res = eth_client.get_chainid().await;
     }

orchestrator/orchestrator/src/main_loop.rs

@@ -257,7 +257,10 @@ pub async fn eth_signer_main_loop(
 
     let gravity_id = get_gravity_id(contract_address, eth_client.clone()).await;
     if gravity_id.is_err() {
-        error!("Failed to get GravityID, check your Eth node");
+        error!(
+            "Failed to get GravityID, check your Eth node: {:?}",
+            gravity_id
+        );
         return;
     }
     let gravity_id = gravity_id.unwrap();

orchestrator/orchestrator/src/oracle_resync.rs

@@ -77,32 +77,65 @@ pub async fn get_last_checked_block(
         transaction_batch_filter = transaction_batch_filter.select(search_range.clone());
         valset_updated_filter = valset_updated_filter.select(search_range.clone());
 
-        let erc20_deployed_events = eth_client.get_logs(&erc20_deployed_filter).await;
-        let logic_call_events = eth_client.get_logs(&logic_call_filter).await;
-        let send_to_cosmos_events = eth_client.get_logs(&send_to_cosmos_filter).await;
-        let transaction_batch_events = eth_client.get_logs(&transaction_batch_filter).await;
+        let erc20_deployed_events = match eth_client.get_logs(&erc20_deployed_filter).await {
+            Ok(events) => events,
+            Err(e) => {
+                error!(
+                    "Failed to get ERC20 deployed events (may be transient): {:?}",
+                    e
+                );
+                delay_for(RETRY_TIME).await;
+                continue;
+            }
+        };
+        let logic_call_events = match eth_client.get_logs(&logic_call_filter).await {
+            Ok(events) => events,
+            Err(e) => {
+                error!(
+                    "Failed to get logic call events (may be transient): {:?}",
+                    e
+                );
+                delay_for(RETRY_TIME).await;
+                continue;
+            }
+        };
+        let send_to_cosmos_events = match eth_client.get_logs(&send_to_cosmos_filter).await {
+            Ok(events) => events,
+            Err(e) => {
+                error!(
+                    "Failed to get send to cosmos events (may be transient): {:?}",
+                    e
+                );
+                delay_for(RETRY_TIME).await;
+                continue;
+            }
+        };
+        let transaction_batch_events = match eth_client.get_logs(&transaction_batch_filter).await {
+            Ok(events) => events,
+            Err(e) => {
+                error!(
+                    "Failed to get transaction batch events (may be transient): {:?}",
+                    e
+                );
+                delay_for(RETRY_TIME).await;
+                continue;
+            }
+        };
         // valset update events have one special property that is useful to us in this handler:
         // a valset update event for nonce 0 is emitted in the contract constructor meaning once you
         // find that event you can exit the search with confidence that you have not missed any events
         // without searching the entire blockchain history
-        let valset_updated_events = eth_client.get_logs(&valset_updated_filter).await;
-
-        if erc20_deployed_events.is_err()
-            || logic_call_events.is_err()
-            || send_to_cosmos_events.is_err()
-            || transaction_batch_events.is_err()
-            || valset_updated_events.is_err()
-        {
-            error!("Failed to get blockchain events while resyncing, is your Eth node working? If you see only one of these it's fine");
-            delay_for(RETRY_TIME).await;
-            continue;
-        }
-
-        let erc20_deployed_events = erc20_deployed_events.unwrap();
-        let logic_call_events = logic_call_events.unwrap();
-        let send_to_cosmos_events = send_to_cosmos_events.unwrap();
-        let transaction_batch_events = transaction_batch_events.unwrap();
-        let mut valset_updated_events = valset_updated_events.unwrap();
+        let mut valset_updated_events = match eth_client.get_logs(&valset_updated_filter).await {
+            Ok(events) => events,
+            Err(e) => {
+                error!(
+                    "Failed to get valset updated events (may be transient): {:?}",
+                    e
+                );
+                delay_for(RETRY_TIME).await;
+                continue;
+            }
+        };
 
         // look for and return the block number of the event last seen on the Cosmos chain
         // then we will play events from that block (including that block, just in case

orchestrator/relayer/src/batch_relaying.rs

@@ -106,7 +106,7 @@ async fn get_batches_and_signatures(
             );
         }
     }
-    
+
     possible_batches
 }
 

orchestrator/relayer/src/logic_call_relaying.rs

@@ -169,7 +169,7 @@ pub async fn relay_logic_calls(
         .await;
 
         if res.is_err() {
-            warn!("LogicCall submission failed");
+            warn!("LogicCall submission failed with {:?}", res);
             let should_permanently_skip = handle_contract_error(res.unwrap_err());
             if should_permanently_skip {
                 logic_call_skips.skip_permanently(&oldest_signed_call);

orchestrator/relayer/src/main_loop.rs

@@ -24,7 +24,10 @@ pub async fn relayer_main_loop(
     let mut grpc_client = grpc_client;
     let gravity_id = get_gravity_id(gravity_contract_address, eth_client.clone()).await;
     if gravity_id.is_err() {
-        error!("Failed to get GravityID, check your Eth node");
+        error!(
+            "Failed to get GravityID, check your Eth node: {:?}",
+            gravity_id
+        );
         return;
     }
     let gravity_id = gravity_id.unwrap();