From 1f8250058b2d0a76e892b8ab649f9f934821a0fd Mon Sep 17 00:00:00 2001 From: spalen0 Date: Mon, 29 Jun 2026 18:58:39 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat(pegs):=20L2=20oracle=20health=20monito?= =?UTF-8?q?r=20=E2=80=94=20stables/oracles.py=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer 2 of peg monitoring: an hourly script that watches the on-chain oracles our lending markets actually liquidate on, driven by the shared PeggedAsset registry. Stacks on the foundation branch (#299 / #305). Per Chainlink-backed asset (USDC, USDT, USDS, USDe, cbBTC, LBTC): - staleness (now - updatedAt > heartbeat + buffer), - round sanity / monotonicity (positive answer, completed round, answeredInRound >= roundId, non-decreasing roundId vs cached run), - deviation from peg (oracle price vs PegTarget), - oracle <-> market divergence (Chainlink vs DeFiLlama) — the liquidation signal. Per-check logic is pure (Alert | None) so forced-stale / forced-divergence / off-peg / round-malfunction cases are unit-tested without a chain. Rate / fundamental oracles: generic monotonicity + delta-vs-cached handler (apyusd approach); a monotonic/capped decrease is CRITICAL (per #196). LBTC Redstone is already covered by a Tenderly alert, so it is documented in TENDERLY_COVERED and not double-polled; gap-analysis notes included. Foundation: add ChainlinkFeed.quote (PegTarget) so BTC-denominated feeds (LBTC/BTC) convert to a USD basis for divergence; LBTC feed marked quote=BTC. Wired into the hourly profile in automation/jobs.yaml (renders; 24 tasks). Validation: ruff + mypy clean (new files); 52 peg tests + 13 automation tests pass; live mainnet smoke run reads all 6 feeds and computes peg/market deviations correctly (LBTC $60,405 = 1.0043 BTC x $60,146). Co-Authored-By: Claude Opus 4.8 (1M context) --- automation/jobs.yaml | 1 + protocols/stables/oracles.py | 341 ++++++++++++++++++++++++++++++++++ tests/test_stables_oracles.py | 180 ++++++++++++++++++ utils/pegged_assets.py | 12 +- 4 files changed, 532 insertions(+), 2 deletions(-) create mode 100644 protocols/stables/oracles.py create mode 100644 tests/test_stables_oracles.py diff --git a/automation/jobs.yaml b/automation/jobs.yaml index aafaff1..892b6b1 100644 --- a/automation/jobs.yaml +++ b/automation/jobs.yaml @@ -37,6 +37,7 @@ profiles: - { name: "usdai", script: protocols/usdai/main.py } - { name: "usdai-large-mints", script: protocols/usdai/large_mints.py } - { name: "stables-dune-large-transfers", script: protocols/stables/dune_large_transfers.py } + - { name: "stables-oracles", script: protocols/stables/oracles.py } - { name: "yearn-alert-large-flows", script: protocols/yearn/alert_large_flows.py } # Cache: tks-trigger-cache.json under $CACHE_DIR (check_stuck_triggers.DEFAULT_CACHE_FILE). - { name: "yearn-check-stuck-triggers", script: protocols/yearn/check_stuck_triggers.py, enabled: false } diff --git a/protocols/stables/oracles.py b/protocols/stables/oracles.py new file mode 100644 index 0000000..f71ba77 --- /dev/null +++ b/protocols/stables/oracles.py @@ -0,0 +1,341 @@ +"""Layer 2 peg monitoring — on-chain oracle health for pegged assets (hourly). + +Where ``protocols/stables/main.py`` watches *market* price (DeFiLlama), this watches +the **on-chain oracles our lending markets actually liquidate on**. Driven by the +shared :data:`PeggedAsset` registry, for every Chainlink-backed asset it checks: + +* **staleness** — ``now − updatedAt > heartbeat + buffer``; +* **round sanity / monotonicity** — positive answer, completed round, + ``answeredInRound ≥ roundId``, and a non-decreasing ``roundId`` vs the cached run; +* **deviation from peg** — oracle price vs the asset's :class:`PegTarget`; +* **oracle ↔ market divergence** — Chainlink vs DeFiLlama, the actual + liquidation-risk signal (markets liquidate on the oracle, not market price). + +For **rate / fundamental oracles** (vault-rate, capped Redstone feeds) it checks +monotonicity + delta-vs-cached (the ``protocols/apyusd/main.py`` approach); any +fundamental-oracle depeg is ``CRITICAL`` (per #196). Fundamental oracles already +covered by a Tenderly alert are listed in :data:`TENDERLY_COVERED` and skipped +here to avoid duplicate alerting. + +Runs hourly via ``automation/jobs.yaml``. +""" + +from dataclasses import dataclass +from decimal import Decimal + +from utils.alert import Alert, AlertSeverity, send_alert +from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.chainlink import FeedReading, read_feeds, round_issues +from utils.chains import Chain +from utils.config import Config +from utils.defillama import fetch_prices +from utils.logger import get_logger +from utils.pegged_assets import PEGGED_ASSETS, PeggedAsset, PegTarget, price_deviation, resolve_peg_prices +from utils.web3_wrapper import ChainManager, Web3Client + +PROTOCOL = "pegs" +logger = get_logger("stables-oracles") + +# Tunables (env-overridable). +STALENESS_BUFFER = Config.get_env_int("PEG_ORACLE_STALENESS_BUFFER", 600) # 10 min grace on heartbeat +DIVERGENCE_THRESHOLD = Decimal(str(Config.get_env_float("PEG_ORACLE_DIVERGENCE_THRESHOLD", 0.01))) # 1% +RATE_DELTA_THRESHOLD = Decimal(str(Config.get_env_float("PEG_ORACLE_RATE_DELTA_THRESHOLD", 0.05))) # 5% + +CACHE_FILE = cache_filename + + +def _round_cache_key(address: str) -> str: + return f"peg_oracle_round_{address.lower()}" + + +def _rate_cache_key(address: str) -> str: + return f"peg_oracle_rate_{address.lower()}" + + +# Fundamental oracles already covered by an existing Tenderly alert — NOT polled +# here to avoid duplicate alerting. See protocols/lrt-pegs/README.md. +# +# Gap analysis (per #196 step 6): the registry currently exposes no *uncovered* +# fundamental oracle. Adding active polling for a new one needs the oracle +# contract address, its read function + precision, and whether it is monotonic +# (capped) — wire it as a ``rate_oracle`` on the relevant ``PeggedAsset``. +TENDERLY_COVERED: dict[str, str] = { + # LBTC Redstone fundamental oracle, upper-capped at 1 (healthy == 1). + "LBTC Redstone (0xb415eAA355D8440ac7eCB602D3fb67ccC1f0bc81)": ( + "https://dashboard.tenderly.co/yearn/sam/alerts/rules/eca272ef-979a-47b3-a7f0-2e67172889bb" + ), +} + + +# --------------------------------------------------------------------------- +# Observation + pure per-check helpers (unit tested without a chain) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class OracleObservation: + """Everything needed to evaluate one Chainlink-backed asset for one run.""" + + asset: PeggedAsset + reading: FeedReading + peg_price_usd: Decimal # USD price of the asset's peg target + quote_price_usd: Decimal # USD price of the feed's quote unit + now: int + market_price_usd: Decimal | None = None # DeFiLlama; None when unavailable + prev_round_id: int | None = None # cached roundId from the previous run + + @property + def oracle_price_usd(self) -> Decimal: + """Chainlink answer expressed in USD (answer × quote price).""" + return self.reading.price * self.quote_price_usd + + +def check_staleness(obs: OracleObservation, buffer: int = STALENESS_BUFFER) -> Alert | None: + """Alert (HIGH) if the feed has not updated within heartbeat + buffer.""" + feed = obs.asset.chainlink_feed + assert feed is not None + updated_at = obs.reading.round_data.updated_at + age = obs.now - updated_at + if updated_at <= 0 or age > feed.heartbeat + buffer: + return Alert( + AlertSeverity.HIGH, + f"*{obs.asset.name} oracle stale* ({feed.description})\n" + f"Age: {age}s — heartbeat {feed.heartbeat}s + buffer {buffer}s\n" + f"updatedAt: {updated_at}\n" + f"Feed: `{feed.address}`", + PROTOCOL, + channel=obs.asset.channel, + ) + return None + + +def check_round_health(obs: OracleObservation) -> Alert | None: + """Alert if round sanity checks fail or ``roundId`` moved backwards. + + A non-positive answer, an incomplete round, or a backwards ``roundId`` is a + feed malfunction (``CRITICAL``); a stale ``answeredInRound`` is ``HIGH``. + """ + feed = obs.asset.chainlink_feed + assert feed is not None + round_data = obs.reading.round_data + issues = round_issues(round_data) + + if obs.prev_round_id is not None and round_data.round_id < obs.prev_round_id: + issues.append(f"roundId went backwards ({obs.prev_round_id} -> {round_data.round_id})") + + if not issues: + return None + + # Anything beyond a lagging answeredInRound means the feed is broken. + critical = any("answeredInRound" not in issue for issue in issues) + severity = AlertSeverity.CRITICAL if critical else AlertSeverity.HIGH + return Alert( + severity, + f"*{obs.asset.name} oracle round unhealthy* ({feed.description})\n" + + "\n".join(f"- {issue}" for issue in issues) + + f"\nFeed: `{feed.address}`", + PROTOCOL, + channel=obs.asset.channel, + ) + + +def check_peg_deviation(obs: OracleObservation) -> Alert | None: + """Alert (HIGH) if the oracle price deviates from the peg beyond ``depeg_pct``.""" + if obs.peg_price_usd <= 0: + return None + if not obs.asset.is_depegged(obs.oracle_price_usd, obs.peg_price_usd): + return None + dev = obs.asset.deviation(obs.oracle_price_usd, obs.peg_price_usd) + return Alert( + AlertSeverity.HIGH, + f"*{obs.asset.name} oracle off peg* ({obs.asset.peg.value})\n" + f"Oracle: ${obs.oracle_price_usd:.6f}\n" + f"Peg: ${obs.peg_price_usd:.6f}\n" + f"Deviation: {dev:+.2%} (tolerance {obs.asset.depeg_pct:.2%})", + PROTOCOL, + channel=obs.asset.channel, + ) + + +def check_market_divergence(obs: OracleObservation, threshold: Decimal = DIVERGENCE_THRESHOLD) -> Alert | None: + """Alert (HIGH) if the oracle and DeFiLlama market price diverge beyond ``threshold``.""" + if obs.market_price_usd is None or obs.market_price_usd <= 0: + return None + dev = price_deviation(obs.oracle_price_usd, obs.market_price_usd) + if abs(dev) < threshold: + return None + return Alert( + AlertSeverity.HIGH, + f"*{obs.asset.name} oracle ↔ market divergence*\n" + f"Oracle: ${obs.oracle_price_usd:.6f}\n" + f"Market (DeFiLlama): ${obs.market_price_usd:.6f}\n" + f"Divergence: {dev:+.2%} (threshold {threshold:.2%})", + PROTOCOL, + channel=obs.asset.channel, + ) + + +def evaluate_chainlink_asset( + obs: OracleObservation, + *, + buffer: int = STALENESS_BUFFER, + divergence_threshold: Decimal = DIVERGENCE_THRESHOLD, +) -> list[Alert]: + """Run all Chainlink-asset checks, returning the alerts that fired.""" + candidates = [ + check_staleness(obs, buffer), + check_round_health(obs), + check_peg_deviation(obs), + check_market_divergence(obs, divergence_threshold), + ] + return [alert for alert in candidates if alert is not None] + + +def check_rate_oracle( + asset: PeggedAsset, + current_rate: int, + prev_rate: int | None, + threshold: Decimal = RATE_DELTA_THRESHOLD, +) -> list[Alert]: + """Monotonicity + delta-vs-cached checks for a fundamental / rate oracle. + + A decrease in a monotonic (capped) oracle is a fundamental depeg + (``CRITICAL``); any delta beyond ``threshold`` is ``HIGH``. + """ + oracle = asset.rate_oracle + assert oracle is not None + if prev_rate is None or prev_rate <= 0: + return [] + + alerts: list[Alert] = [] + delta = Decimal(current_rate - prev_rate) / Decimal(prev_rate) + + if oracle.monotonic and current_rate < prev_rate: + alerts.append( + Alert( + AlertSeverity.CRITICAL, + f"*{asset.name} fundamental oracle DECREASED* (monotonic/capped)\n" + f"Previous: {prev_rate}\nCurrent: {current_rate}\nDelta: {delta:+.4%}\n" + f"Oracle: `{oracle.address}`", + PROTOCOL, + channel=asset.channel, + ) + ) + elif abs(delta) >= threshold: + alerts.append( + Alert( + AlertSeverity.HIGH, + f"*{asset.name} fundamental oracle delta* {delta:+.4%} (threshold {threshold:.2%})\n" + f"Previous: {prev_rate}\nCurrent: {current_rate}\nOracle: `{oracle.address}`", + PROTOCOL, + channel=asset.channel, + ) + ) + return alerts + + +# --------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------- + + +def _build_rate_oracle_abi(function: str) -> list[dict]: + return [ + { + "inputs": [], + "name": function, + "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], + "stateMutability": "view", + "type": "function", + } + ] + + +def _monitor_chainlink_assets(client: Web3Client) -> None: + """Check every registry asset that has a Chainlink feed.""" + assets = [a for a in PEGGED_ASSETS if a.chainlink_feed is not None] + if not assets: + return + + needed_targets: set[PegTarget] = set() + for asset in assets: + assert asset.chainlink_feed is not None + needed_targets.add(asset.peg) + needed_targets.add(asset.chainlink_feed.quote) + peg_prices = resolve_peg_prices(needed_targets) + + market_prices = fetch_prices([a.defillama_key for a in assets]) + readings = read_feeds(client, [a.chainlink_feed.address for a in assets]) # type: ignore[union-attr] + now = int(client.eth.get_block("latest")["timestamp"]) + + for asset in assets: + feed = asset.chainlink_feed + assert feed is not None + reading = readings[feed.address] + + prev_round_raw = get_last_value_for_key_from_file(CACHE_FILE, _round_cache_key(feed.address)) + try: + prev_round_id: int | None = int(str(prev_round_raw)) if str(prev_round_raw) != "0" else None + except ValueError: + prev_round_id = None + + obs = OracleObservation( + asset=asset, + reading=reading, + peg_price_usd=peg_prices[asset.peg], + quote_price_usd=peg_prices[feed.quote], + now=now, + market_price_usd=market_prices.get(asset.defillama_key), + prev_round_id=prev_round_id, + ) + + alerts = evaluate_chainlink_asset(obs) + logger.info( + "%s oracle: $%.6f (peg $%.6f, market %s) — %d alert(s)", + asset.name, + obs.oracle_price_usd, + obs.peg_price_usd, + f"${obs.market_price_usd:.6f}" if obs.market_price_usd is not None else "n/a", + len(alerts), + ) + for alert in alerts: + send_alert(alert) + + write_last_value_to_file(CACHE_FILE, _round_cache_key(feed.address), reading.round_data.round_id) + + +def _monitor_rate_oracles(client: Web3Client) -> None: + """Check every registry asset that has a fundamental / rate oracle.""" + assets = [a for a in PEGGED_ASSETS if a.rate_oracle is not None] + for asset in assets: + oracle = asset.rate_oracle + assert oracle is not None + contract = client.get_contract(oracle.address, _build_rate_oracle_abi(oracle.function)) + current_rate = int(contract.functions[oracle.function]().call()) + + prev_raw = get_last_value_for_key_from_file(CACHE_FILE, _rate_cache_key(oracle.address)) + try: + prev_rate: int | None = int(str(prev_raw)) if str(prev_raw) != "0" else None + except ValueError: + prev_rate = None + + alerts = check_rate_oracle(asset, current_rate, prev_rate) + logger.info("%s rate oracle %s: %d (%d alert(s))", asset.name, oracle.address, current_rate, len(alerts)) + for alert in alerts: + send_alert(alert) + + write_last_value_to_file(CACHE_FILE, _rate_cache_key(oracle.address), current_rate) + + +def main() -> None: + """Run all L2 oracle-health checks driven by the pegged-asset registry.""" + client = ChainManager.get_client(Chain.MAINNET) + _monitor_chainlink_assets(client) + _monitor_rate_oracles(client) + logger.info("L2 oracle health check complete (%d Tenderly-covered oracle(s) skipped)", len(TENDERLY_COVERED)) + + +if __name__ == "__main__": + from utils.runner import run_with_alert + + run_with_alert(main, PROTOCOL) diff --git a/tests/test_stables_oracles.py b/tests/test_stables_oracles.py new file mode 100644 index 0000000..41e9654 --- /dev/null +++ b/tests/test_stables_oracles.py @@ -0,0 +1,180 @@ +import unittest +from decimal import Decimal + +from protocols.stables.oracles import ( + OracleObservation, + check_market_divergence, + check_peg_deviation, + check_rate_oracle, + check_round_health, + check_staleness, + evaluate_chainlink_asset, +) +from utils.alert import AlertSeverity +from utils.chainlink import FeedReading, RoundData +from utils.pegged_assets import PeggedAsset, PegTarget, RateOracle, get_asset + +NOW = 2_000_000 +HEARTBEAT = 86_400 # matches registry _STABLE_HEARTBEAT + + +def _reading( + address: str, + answer: int, + *, + decimals: int = 8, + round_id: int = 100, + updated_at: int = NOW - 100, + answered_in_round: int = 100, +) -> FeedReading: + rd = RoundData( + round_id=round_id, + answer=answer, + started_at=updated_at, + updated_at=updated_at, + answered_in_round=answered_in_round, + ) + return FeedReading(address=address, round_data=rd, decimals=decimals) + + +def _cbbtc_obs(**overrides) -> OracleObservation: + """A healthy cbBTC (USD-quoted feed, BTC peg) observation; override per test.""" + asset = get_asset("cbBTC") + defaults = dict( + asset=asset, + reading=_reading(asset.chainlink_feed.address, 60_100 * 10**8), # $60,100 + peg_price_usd=Decimal("60000"), + quote_price_usd=Decimal("1"), # USD-quoted feed + now=NOW, + market_price_usd=Decimal("60100"), + prev_round_id=99, + ) + defaults.update(overrides) + return OracleObservation(**defaults) + + +class TestHealthyAsset(unittest.TestCase): + def test_no_alerts_when_healthy(self): + self.assertEqual(evaluate_chainlink_asset(_cbbtc_obs()), []) + + +class TestStaleness(unittest.TestCase): + def test_fresh_feed_ok(self): + self.assertIsNone(check_staleness(_cbbtc_obs(), buffer=600)) + + def test_forced_stale_fires(self): + stale = _cbbtc_obs( + reading=_reading( + get_asset("cbBTC").chainlink_feed.address, 60_100 * 10**8, updated_at=NOW - (HEARTBEAT + 1000) + ) + ) + alert = check_staleness(stale, buffer=600) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.HIGH) + self.assertEqual(alert.channel, "pegs") + + def test_zero_updated_at_is_stale(self): + obs = _cbbtc_obs(reading=_reading(get_asset("cbBTC").chainlink_feed.address, 60_100 * 10**8, updated_at=0)) + self.assertIsNotNone(check_staleness(obs)) + + +class TestRoundHealth(unittest.TestCase): + def test_healthy_round(self): + self.assertIsNone(check_round_health(_cbbtc_obs())) + + def test_non_positive_answer_is_critical(self): + obs = _cbbtc_obs(reading=_reading(get_asset("cbBTC").chainlink_feed.address, 0)) + alert = check_round_health(obs) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.CRITICAL) + + def test_lagging_answered_in_round_is_high(self): + addr = get_asset("cbBTC").chainlink_feed.address + obs = _cbbtc_obs(reading=_reading(addr, 60_100 * 10**8, round_id=100, answered_in_round=99)) + alert = check_round_health(obs) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.HIGH) + + def test_roundid_backwards_is_critical(self): + obs = _cbbtc_obs(prev_round_id=200) # current round_id is 100 + alert = check_round_health(obs) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.CRITICAL) + + +class TestPegDeviation(unittest.TestCase): + def test_within_tolerance_ok(self): + self.assertIsNone(check_peg_deviation(_cbbtc_obs())) + + def test_off_peg_fires(self): + # oracle $63,000 vs $60,000 peg = +5% > cbBTC 2% tolerance + obs = _cbbtc_obs(reading=_reading(get_asset("cbBTC").chainlink_feed.address, 63_000 * 10**8)) + alert = check_peg_deviation(obs) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.HIGH) + + +class TestMarketDivergence(unittest.TestCase): + def test_aligned_ok(self): + self.assertIsNone(check_market_divergence(_cbbtc_obs(), threshold=Decimal("0.01"))) + + def test_forced_divergence_fires(self): + # oracle $60,100 vs market $50,000 ~ +20% + obs = _cbbtc_obs(market_price_usd=Decimal("50000")) + alert = check_market_divergence(obs, threshold=Decimal("0.01")) + self.assertIsNotNone(alert) + self.assertEqual(alert.severity, AlertSeverity.HIGH) + + def test_missing_market_price_skips(self): + self.assertIsNone(check_market_divergence(_cbbtc_obs(market_price_usd=None))) + + +class TestQuoteConversion(unittest.TestCase): + def test_btc_quoted_feed_scales_to_usd(self): + # LBTC/BTC feed answer 1.004 BTC, BTC at $60,000 -> oracle $60,240 + lbtc = get_asset("LBTC") + obs = OracleObservation( + asset=lbtc, + reading=_reading(lbtc.chainlink_feed.address, 100_400_000), # 1.004 * 1e8 + peg_price_usd=Decimal("60000"), + quote_price_usd=Decimal("60000"), # feed quotes in BTC + now=NOW, + market_price_usd=Decimal("60240"), + ) + self.assertEqual(obs.oracle_price_usd, Decimal("60240.000")) + self.assertEqual(evaluate_chainlink_asset(obs), []) + + +class TestRateOracle(unittest.TestCase): + def _asset(self, monotonic: bool = True) -> PeggedAsset: + return PeggedAsset( + name="fakeRate", + defillama_key="ethereum:0x0000000000000000000000000000000000000001", + channel="pegs", + peg=PegTarget.USD, + depeg_pct=Decimal("0.02"), + rate_oracle=RateOracle(address="0xRate", monotonic=monotonic), + ) + + def test_no_previous_rate_no_alert(self): + self.assertEqual(check_rate_oracle(self._asset(), current_rate=10**18, prev_rate=None), []) + + def test_monotonic_decrease_is_critical(self): + alerts = check_rate_oracle(self._asset(monotonic=True), current_rate=9 * 10**17, prev_rate=10**18) + self.assertEqual(len(alerts), 1) + self.assertEqual(alerts[0].severity, AlertSeverity.CRITICAL) + + def test_large_increase_is_high(self): + alerts = check_rate_oracle(self._asset(), current_rate=12 * 10**17, prev_rate=10**18, threshold=Decimal("0.05")) + self.assertEqual(len(alerts), 1) + self.assertEqual(alerts[0].severity, AlertSeverity.HIGH) + + def test_small_change_no_alert(self): + self.assertEqual( + check_rate_oracle(self._asset(), current_rate=101 * 10**16, prev_rate=10**18, threshold=Decimal("0.05")), + [], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/pegged_assets.py b/utils/pegged_assets.py index a5b7742..ceda2a7 100644 --- a/utils/pegged_assets.py +++ b/utils/pegged_assets.py @@ -41,11 +41,17 @@ class PegTarget(Enum): @dataclass(frozen=True) class ChainlinkFeed: - """A Chainlink aggregator backing an asset's price (consumed by L2).""" + """A Chainlink aggregator backing an asset's price (consumed by L2). + + ``quote`` is the unit the feed is denominated in (e.g. a ``LBTC/BTC`` feed + quotes in ``BTC``, a ``cbBTC/USD`` feed in ``USD``); L2 multiplies the raw + answer by the quote's live price to compare oracle and market on a USD basis. + """ address: str heartbeat: int # max expected seconds between updates (Chainlink mainnet default) description: str = "" + quote: PegTarget = PegTarget.USD @dataclass(frozen=True) @@ -233,7 +239,9 @@ def get_asset(name: str) -> PeggedAsset: peg=PegTarget.BTC, depeg_pct=Decimal("0.03"), # LBTC/BTC market-rate feed (8 decimals); price ~1 BTC per LBTC. - chainlink_feed=ChainlinkFeed("0x5c29868C58b6e15e2b962943278969Ab6a7D3212", _STABLE_HEARTBEAT, "LBTC/BTC"), + chainlink_feed=ChainlinkFeed( + "0x5c29868C58b6e15e2b962943278969Ab6a7D3212", _STABLE_HEARTBEAT, "LBTC/BTC", quote=PegTarget.BTC + ), ), ] From 210658867e0e2a4f754d8dcb44abd9e73f120c77 Mon Sep 17 00:00:00 2001 From: spalen0 Date: Mon, 29 Jun 2026 20:20:38 +0000 Subject: [PATCH 2/3] fix(pegs): logical protocol for dispatch + non-poisoning round cache (#301) Address review on stables/oracles.py: 1. Emergency dispatch (utils/dispatch.dispatch_emergency_withdrawal) keys off Alert.protocol and skips anything outside DISPATCHABLE_PROTOCOLS. Alerts were hardcoding protocol="pegs" and putting the owner only in channel, so an USDe oracle failure routed to Telegram but never triggered ethena's emergency withdrawal. Rename PeggedAsset.channel -> protocol (the logical owner) and add an optional channel override (empty falls back to protocol routing). Alerts now emit protocol=asset.protocol, channel=asset.channel, so dispatch fires for ethena/cap/infinifi while Telegram routing is unchanged. 2. The round cache was overwritten with the current roundId unconditionally, so a backwards roundId became the new baseline and the regression was caught only once. Add pure next_cached_round(): a health-gated high-water mark that never lowers the cached round or stores a malfunctioning one. Tests: add dispatch-routing coverage (alert.protocol in DISPATCHABLE_PROTOCOLS) and next_cached_round cases. Full suite: 637 passed, 6 skipped. ruff + mypy clean (new files). Live no-send smoke reads all six feeds, 0 alerts under current conditions. Co-Authored-By: Claude Opus 4.8 (1M context) --- protocols/stables/oracles.py | 34 +++++++++++++++++++------ tests/test_stables_oracles.py | 47 +++++++++++++++++++++++++++++++++-- utils/pegged_assets.py | 21 ++++++++-------- 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/protocols/stables/oracles.py b/protocols/stables/oracles.py index f71ba77..5345935 100644 --- a/protocols/stables/oracles.py +++ b/protocols/stables/oracles.py @@ -25,7 +25,7 @@ from utils.alert import Alert, AlertSeverity, send_alert from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file -from utils.chainlink import FeedReading, read_feeds, round_issues +from utils.chainlink import FeedReading, RoundData, is_round_healthy, read_feeds, round_issues from utils.chains import Chain from utils.config import Config from utils.defillama import fetch_prices @@ -103,7 +103,7 @@ def check_staleness(obs: OracleObservation, buffer: int = STALENESS_BUFFER) -> A f"Age: {age}s — heartbeat {feed.heartbeat}s + buffer {buffer}s\n" f"updatedAt: {updated_at}\n" f"Feed: `{feed.address}`", - PROTOCOL, + obs.asset.protocol, channel=obs.asset.channel, ) return None @@ -134,7 +134,7 @@ def check_round_health(obs: OracleObservation) -> Alert | None: f"*{obs.asset.name} oracle round unhealthy* ({feed.description})\n" + "\n".join(f"- {issue}" for issue in issues) + f"\nFeed: `{feed.address}`", - PROTOCOL, + obs.asset.protocol, channel=obs.asset.channel, ) @@ -152,7 +152,7 @@ def check_peg_deviation(obs: OracleObservation) -> Alert | None: f"Oracle: ${obs.oracle_price_usd:.6f}\n" f"Peg: ${obs.peg_price_usd:.6f}\n" f"Deviation: {dev:+.2%} (tolerance {obs.asset.depeg_pct:.2%})", - PROTOCOL, + obs.asset.protocol, channel=obs.asset.channel, ) @@ -170,7 +170,7 @@ def check_market_divergence(obs: OracleObservation, threshold: Decimal = DIVERGE f"Oracle: ${obs.oracle_price_usd:.6f}\n" f"Market (DeFiLlama): ${obs.market_price_usd:.6f}\n" f"Divergence: {dev:+.2%} (threshold {threshold:.2%})", - PROTOCOL, + obs.asset.protocol, channel=obs.asset.channel, ) @@ -191,6 +191,22 @@ def evaluate_chainlink_asset( return [alert for alert in candidates if alert is not None] +def next_cached_round(prev_round_id: int | None, round_data: RoundData) -> int: + """High-water-mark ``roundId`` to persist so a regression never poisons the cache. + + The cached ``roundId`` is the baseline the next run compares against for + monotonicity. Writing a lower or malfunctioning round would make the + regression the new baseline, so it is flagged only once and a feed stuck at + (or crawling below) the regressed round looks "monotonic" forever. Only a + healthy, non-decreasing round advances the mark. + """ + if not is_round_healthy(round_data): + return prev_round_id or 0 + if prev_round_id is None: + return round_data.round_id + return max(prev_round_id, round_data.round_id) + + def check_rate_oracle( asset: PeggedAsset, current_rate: int, @@ -217,7 +233,7 @@ def check_rate_oracle( f"*{asset.name} fundamental oracle DECREASED* (monotonic/capped)\n" f"Previous: {prev_rate}\nCurrent: {current_rate}\nDelta: {delta:+.4%}\n" f"Oracle: `{oracle.address}`", - PROTOCOL, + asset.protocol, channel=asset.channel, ) ) @@ -227,7 +243,7 @@ def check_rate_oracle( AlertSeverity.HIGH, f"*{asset.name} fundamental oracle delta* {delta:+.4%} (threshold {threshold:.2%})\n" f"Previous: {prev_rate}\nCurrent: {current_rate}\nOracle: `{oracle.address}`", - PROTOCOL, + asset.protocol, channel=asset.channel, ) ) @@ -301,7 +317,9 @@ def _monitor_chainlink_assets(client: Web3Client) -> None: for alert in alerts: send_alert(alert) - write_last_value_to_file(CACHE_FILE, _round_cache_key(feed.address), reading.round_data.round_id) + write_last_value_to_file( + CACHE_FILE, _round_cache_key(feed.address), next_cached_round(prev_round_id, reading.round_data) + ) def _monitor_rate_oracles(client: Web3Client) -> None: diff --git a/tests/test_stables_oracles.py b/tests/test_stables_oracles.py index 41e9654..8a7283d 100644 --- a/tests/test_stables_oracles.py +++ b/tests/test_stables_oracles.py @@ -9,9 +9,11 @@ check_round_health, check_staleness, evaluate_chainlink_asset, + next_cached_round, ) from utils.alert import AlertSeverity from utils.chainlink import FeedReading, RoundData +from utils.dispatch import DISPATCHABLE_PROTOCOLS from utils.pegged_assets import PeggedAsset, PegTarget, RateOracle, get_asset NOW = 2_000_000 @@ -71,7 +73,9 @@ def test_forced_stale_fires(self): alert = check_staleness(stale, buffer=600) self.assertIsNotNone(alert) self.assertEqual(alert.severity, AlertSeverity.HIGH) - self.assertEqual(alert.channel, "pegs") + # cbBTC has no dispatchable owner: protocol is "pegs", channel override empty. + self.assertEqual(alert.protocol, "pegs") + self.assertEqual(alert.channel, "") def test_zero_updated_at_is_stale(self): obs = _cbbtc_obs(reading=_reading(get_asset("cbBTC").chainlink_feed.address, 60_100 * 10**8, updated_at=0)) @@ -150,7 +154,7 @@ def _asset(self, monotonic: bool = True) -> PeggedAsset: return PeggedAsset( name="fakeRate", defillama_key="ethereum:0x0000000000000000000000000000000000000001", - channel="pegs", + protocol="pegs", peg=PegTarget.USD, depeg_pct=Decimal("0.02"), rate_oracle=RateOracle(address="0xRate", monotonic=monotonic), @@ -176,5 +180,44 @@ def test_small_change_no_alert(self): ) +class TestDispatchRouting(unittest.TestCase): + """Alerts must carry the asset's logical protocol so emergency dispatch can fire.""" + + def test_owned_asset_uses_dispatchable_protocol(self): + usde = get_asset("USDe") # owner "ethena", peg USD, 3% tolerance + obs = OracleObservation( + asset=usde, + reading=_reading(usde.chainlink_feed.address, 90 * 10**6), # $0.90 -> off peg + peg_price_usd=Decimal("1"), + quote_price_usd=Decimal("1"), + now=NOW, + market_price_usd=Decimal("0.90"), + ) + alert = check_peg_deviation(obs) + self.assertIsNotNone(alert) + # protocol (not channel) carries the owner; dispatch keys off alert.protocol. + self.assertEqual(alert.protocol, "ethena") + self.assertIn(alert.protocol, DISPATCHABLE_PROTOCOLS) + + +class TestNextCachedRound(unittest.TestCase): + def _rd(self, round_id: int, answer: int = 60_100 * 10**8, updated_at: int = NOW - 100) -> RoundData: + return RoundData(round_id, answer, updated_at, updated_at, round_id) + + def test_first_run_caches_current(self): + self.assertEqual(next_cached_round(None, self._rd(100)), 100) + + def test_advances_on_increase(self): + self.assertEqual(next_cached_round(100, self._rd(101)), 101) + + def test_keeps_high_water_mark_on_regression(self): + # Backwards round must NOT lower the cached baseline (no poisoning). + self.assertEqual(next_cached_round(100, self._rd(99)), 100) + + def test_broken_round_does_not_poison_even_if_higher(self): + # answer == 0 -> unhealthy; keep last-good rather than caching a broken round. + self.assertEqual(next_cached_round(100, self._rd(200, answer=0)), 100) + + if __name__ == "__main__": unittest.main() diff --git a/utils/pegged_assets.py b/utils/pegged_assets.py index ceda2a7..6869f67 100644 --- a/utils/pegged_assets.py +++ b/utils/pegged_assets.py @@ -79,11 +79,12 @@ class PeggedAsset: name: str defillama_key: str # "chain:address" - channel: str # Telegram routing channel + protocol: str # logical owner — used as Alert.protocol so emergency dispatch can key off it peg: PegTarget depeg_pct: Decimal # deviation tolerance from the peg (e.g. Decimal("0.02") = 2%) chainlink_feed: ChainlinkFeed | None = None rate_oracle: RateOracle | None = None + channel: str = "" # Telegram channel override; empty falls back to ``protocol`` routing @property def address(self) -> str: @@ -171,7 +172,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="USDC", defillama_key="ethereum:0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", - channel="pegs", + protocol="pegs", peg=PegTarget.USD, depeg_pct=Decimal("0.02"), chainlink_feed=ChainlinkFeed("0x8fFfFfd4AfB6115b954Bd326cbe7B4BA576818f6", _STABLE_HEARTBEAT, "USDC/USD"), @@ -179,7 +180,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="USDT", defillama_key="ethereum:0xdAC17F958D2ee523a2206206994597C13D831ec7", - channel="pegs", + protocol="pegs", peg=PegTarget.USD, depeg_pct=Decimal("0.02"), chainlink_feed=ChainlinkFeed("0x3E7d1eAB13ad0104d2750B8863b489D65364e32D", _STABLE_HEARTBEAT, "USDT/USD"), @@ -187,7 +188,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="USDS", defillama_key="ethereum:0xdC035D45d973E3EC169d2276DDab16f1e407384F", - channel="pegs", + protocol="pegs", peg=PegTarget.USD, depeg_pct=Decimal("0.02"), chainlink_feed=ChainlinkFeed("0xfF30586cD0F29eD462364C7e81375FC0C71219b1", _STABLE_HEARTBEAT, "USDS/USD"), @@ -196,7 +197,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="USDe", defillama_key="ethereum:0x4c9EDD5852cd905f086C759E8383e09bff1E68B3", - channel="ethena", + protocol="ethena", peg=PegTarget.USD, depeg_pct=Decimal("0.03"), chainlink_feed=ChainlinkFeed("0xa569d910839Ae8865Da8F8e70FfFb0cBA869F961", _STABLE_HEARTBEAT, "USDe/USD"), @@ -204,14 +205,14 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="cUSD", defillama_key="ethereum:0xcccc62962d17b8914c62d74ffb843d73b2a3cccc", - channel="cap", + protocol="cap", peg=PegTarget.USD, depeg_pct=Decimal("0.05"), # cap cUSD price is more volatile ), PeggedAsset( name="iUSD", defillama_key="ethereum:0x48f9e38f3070AD8945DFEae3FA70987722E3D89c", - channel="infinifi", + protocol="infinifi", peg=PegTarget.USD, depeg_pct=Decimal("0.03"), ), @@ -219,7 +220,7 @@ def get_asset(name: str) -> PeggedAsset: # TODO: siUSD token address not yet verified on-chain — placeholder. name="siUSD", defillama_key=f"ethereum:{PLACEHOLDER_ADDRESS}", - channel="pegs", + protocol="pegs", peg=PegTarget.USD, depeg_pct=Decimal("0.05"), ), @@ -227,7 +228,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="cbBTC", defillama_key="ethereum:0xcbB7C0000aB88B473b1f5aFd9ef808440eed33Bf", - channel="pegs", + protocol="pegs", peg=PegTarget.BTC, depeg_pct=Decimal("0.02"), chainlink_feed=ChainlinkFeed("0x2665701293fCbEB223D11A08D826563EDcCE423A", _STABLE_HEARTBEAT, "cbBTC/USD"), @@ -235,7 +236,7 @@ def get_asset(name: str) -> PeggedAsset: PeggedAsset( name="LBTC", defillama_key="ethereum:0x8236a87084f8B84306f72007F36F2618A5634494", - channel="pegs", + protocol="pegs", peg=PegTarget.BTC, depeg_pct=Decimal("0.03"), # LBTC/BTC market-rate feed (8 decimals); price ~1 BTC per LBTC. From 477f4c86374df5406bf1c7687b1a67c5a5aaa26b Mon Sep 17 00:00:00 2001 From: spalen0 Date: Tue, 30 Jun 2026 17:51:31 +0200 Subject: [PATCH 3/3] chore: remove lbtc info --- protocols/stables/oracles.py | 21 ++------------------- utils/dispatch.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/protocols/stables/oracles.py b/protocols/stables/oracles.py index 319564e..f58dea1 100644 --- a/protocols/stables/oracles.py +++ b/protocols/stables/oracles.py @@ -17,9 +17,7 @@ For **rate / fundamental oracles** (vault-rate, capped Redstone feeds) it checks monotonicity + delta-vs-cached (the ``protocols/apyusd/main.py`` approach); any -fundamental-oracle depeg is ``CRITICAL`` (per #196). Fundamental oracles already -covered by a Tenderly alert are listed in :data:`TENDERLY_COVERED` and skipped -here to avoid duplicate alerting. +fundamental-oracle depeg is ``CRITICAL`` (per #196). Runs hourly via ``automation/jobs.yaml``. """ @@ -56,21 +54,6 @@ def _rate_cache_key(address: str) -> str: return f"peg_oracle_rate_{address.lower()}" -# Fundamental oracles already covered by an existing Tenderly alert — NOT polled -# here to avoid duplicate alerting. See protocols/lrt-pegs/README.md. -# -# Gap analysis (per #196 step 6): the registry currently exposes no *uncovered* -# fundamental oracle. Adding active polling for a new one needs the oracle -# contract address, its read function + precision, and whether it is monotonic -# (capped) — wire it as a ``rate_oracle`` on the relevant ``PeggedAsset``. -TENDERLY_COVERED: dict[str, str] = { - # LBTC Redstone fundamental oracle, upper-capped at 1 (healthy == 1). - "LBTC Redstone (0xb415eAA355D8440ac7eCB602D3fb67ccC1f0bc81)": ( - "https://dashboard.tenderly.co/yearn/sam/alerts/rules/eca272ef-979a-47b3-a7f0-2e67172889bb" - ), -} - - # --------------------------------------------------------------------------- # Observation + pure per-check helpers (unit tested without a chain) # --------------------------------------------------------------------------- @@ -388,7 +371,7 @@ def main() -> None: client = ChainManager.get_client(Chain.MAINNET) _monitor_chainlink_assets(client) _monitor_rate_oracles(client) - logger.info("L2 oracle health check complete (%d Tenderly-covered oracle(s) skipped)", len(TENDERLY_COVERED)) + logger.info("L2 oracle health check complete") if __name__ == "__main__": diff --git a/utils/dispatch.py b/utils/dispatch.py index e68680a..72930f7 100644 --- a/utils/dispatch.py +++ b/utils/dispatch.py @@ -29,7 +29,22 @@ # Protocols that have emergency withdrawal config in liquidity-monitoring. # Only these protocols will trigger a dispatch. -DISPATCHABLE_PROTOCOLS = {"infinifi", "cap", "ethena", "ethplus", "usdai", "origin", "maple", "3jane", "wbtc", "coinbase", "lombard", "tether", "circle", "maker"} +DISPATCHABLE_PROTOCOLS = { + "infinifi", + "cap", + "ethena", + "ethplus", + "usdai", + "origin", + "maple", + "3jane", + "wbtc", + "coinbase", + "lombard", + "tether", + "circle", + "maker", +} def _is_on_cooldown(protocol: str, cooldown_seconds: int = DEFAULT_COOLDOWN_SECONDS) -> bool: