diff --git a/docker/prometheus/rules/uptime.yaml b/docker/prometheus/rules/uptime.yaml index c4125cb..bff637c 100644 --- a/docker/prometheus/rules/uptime.yaml +++ b/docker/prometheus/rules/uptime.yaml @@ -1,6 +1,13 @@ groups: - name: UptimePackage rules: + # - alert: UptimeNoCCVUpdate + # expr: | + # cvms_block_height - on(chain_id) cvms_uptime_last_ccv_update > 5000 + # labels: + # severity: critical + # annotations: + # summary: The chain has not received a validator set update in the last 5000 blocks. # - alert: IncreasingMissCounterOver30%During1h # expr: | # # 30mins ago, the validator is unjailed. but got jailed diff --git a/internal/packages/consensus/uptime/api/api.go b/internal/packages/consensus/uptime/api/api.go index 9b13ad5..2b2c379 100644 --- a/internal/packages/consensus/uptime/api/api.go +++ b/internal/packages/consensus/uptime/api/api.go @@ -85,16 +85,24 @@ func GetConsumserUptimeStatus(exporter *common.Exporter, chainID string) (types. return types.CommonUptimeStatus{}, errors.Cause(err) } exporter.Debugf("got total consumer validator uptime: %d", len(validatorUptimeStatus)) - // 5. get on-chain slashing parameter signedBlocksWindow, minSignedPerWindow, err := getUptimeParams(consumerClient, exporter.ChainName) if err != nil { return types.CommonUptimeStatus{}, errors.Cause(err) } + // 6. get consumer channel uptime status + consumerUptimeStatus := &types.ConsumerUptimeStatus{} + channelStatus, err := getLastCCVUpdate(consumerClient) + if err != nil { + return types.CommonUptimeStatus{}, errors.Wrap(err, "failed to get CCV channel status") + } + consumerUptimeStatus.LastCCVUpdate = channelStatus + return types.CommonUptimeStatus{ - SignedBlocksWindow: signedBlocksWindow, - MinSignedPerWindow: minSignedPerWindow, - Validators: validatorUptimeStatus, + SignedBlocksWindow: signedBlocksWindow, + MinSignedPerWindow: minSignedPerWindow, + Validators: validatorUptimeStatus, + ConsumerUptimeStatus: consumerUptimeStatus, }, nil } diff --git a/internal/packages/consensus/uptime/api/uptime.go b/internal/packages/consensus/uptime/api/uptime.go index 50e49cb..7f07fae 100644 --- a/internal/packages/consensus/uptime/api/uptime.go +++ b/internal/packages/consensus/uptime/api/uptime.go @@ -3,7 +3,10 @@ package api import ( "context" "encoding/hex" + "encoding/json" + "fmt" "net/http" + "net/url" "sort" "strconv" "sync" @@ -300,3 +303,46 @@ func sliceStakingValidatorByVP(stakingValidators []commontypes.CosmosStakingVali }) return stakingValidators[:totalConsensusValidators] } + +func getLastCCVUpdate(c common.CommonClient) (uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), common.Timeout) + defer cancel() + + requester := c.APIClient.R().SetContext(ctx) + queryParams := url.Values{} + queryParams.Add("query", "ccv_packet.valset_update_id>1") + queryParams.Add("order_by", "ORDER_BY_DESC") + queryParams.Add("page", "1") + queryParams.Add("limit", "1") + endpoint := "/cosmos/tx/v1beta1/txs?" + queryParams.Encode() + c.Infof("endpoint: %s", endpoint) + resp, err := requester.Get(endpoint) + if err != nil { + return 0, errors.Cause(err) + } + if resp.StatusCode() != http.StatusOK { + return 0, errors.Errorf("api error: got %d code from %s", resp.StatusCode(), resp.Request.URL) + } + + var result struct { + TxResponses []struct { + Height string `json:"height"` + } `json:"tx_responses"` + } + + if err := json.Unmarshal(resp.Body(), &result); err != nil { + return 0, errors.Cause(err) + } + + if len(result.TxResponses) == 0 { + c.Warnf("No CCV update found in consumer chain") + return 0, nil + } + + height, err := strconv.ParseUint(result.TxResponses[0].Height, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse height from tx (%+v): %w", result, err) + } + + return height, nil +} diff --git a/internal/packages/consensus/uptime/collector/collector.go b/internal/packages/consensus/uptime/collector/collector.go index 997dcd9..df12507 100644 --- a/internal/packages/consensus/uptime/collector/collector.go +++ b/internal/packages/consensus/uptime/collector/collector.go @@ -28,6 +28,7 @@ const ( JailedMetricName = "jailed" SignedBlocksWindowMetricName = "signed_blocks_window" MinSignedPerWindowMetricName = "min_signed_per_window" + LastCCVUpdateMetricName = "last_ccv_update" ) func Start(p common.Packager) error { @@ -102,6 +103,12 @@ func loop(exporter *common.Exporter, p common.Packager) { Name: MinSignedPerWindowMetricName, ConstLabels: packageLabels, }) + lastCCVUpdateMetric := p.Factory.NewGauge(prometheus.GaugeOpts{ + Namespace: common.Namespace, + Subsystem: Subsystem, + Name: LastCCVUpdateMetricName, + ConstLabels: packageLabels, + }) isUnhealth := false for { @@ -189,6 +196,11 @@ func loop(exporter *common.Exporter, p common.Packager) { signedBlocksWindowMetric.Set(status.SignedBlocksWindow) minSignedPerWindowMetric.Set(status.MinSignedPerWindow) + // Update consumer metrics + if status.ConsumerUptimeStatus != nil { + lastCCVUpdateMetric.Set(float64(status.ConsumerUptimeStatus.LastCCVUpdate)) + } + exporter.Infof("updated metrics successfully and going to sleep %s ...", SubsystemSleep.String()) // update health and ops diff --git a/internal/packages/consensus/uptime/types/types_common.go b/internal/packages/consensus/uptime/types/types_common.go index d2577e8..1fec15e 100644 --- a/internal/packages/consensus/uptime/types/types_common.go +++ b/internal/packages/consensus/uptime/types/types_common.go @@ -7,9 +7,15 @@ var ( // common type CommonUptimeStatus struct { - MinSignedPerWindow float64 `json:"slash_winodw"` - SignedBlocksWindow float64 `json:"vote_period"` - Validators []ValidatorUptimeStatus `json:"validators"` + MinSignedPerWindow float64 `json:"slash_winodw"` + SignedBlocksWindow float64 `json:"vote_period"` + Validators []ValidatorUptimeStatus `json:"validators"` + ConsumerUptimeStatus *ConsumerUptimeStatus `json:"consumer_uptime_status"` +} + +// consumer uptime status +type ConsumerUptimeStatus struct { + LastCCVUpdate uint64 `json:"last_ccv_update"` } // cosmos uptime status