diff --git a/DESCRIPTION b/DESCRIPTION index b3372588..78173fb9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -15,7 +15,9 @@ Imports: Suggests: testthat (>= 3.0.0), knitr (>= 1.50.0), - rmarkdown (>= 2.29.0) + rmarkdown (>= 2.29.0), + ggplot2 (>= 3.4.0), + scales (>= 1.2.0) Config/testthat/edition: 3 VignetteBuilder: knitr URL: https://rentosaijo.github.io/nhlscraper/, https://github.com/RentoSaijo/nhlscraper diff --git a/R/Calibration.R b/R/Calibration.R new file mode 100644 index 00000000..6a15ad54 --- /dev/null +++ b/R/Calibration.R @@ -0,0 +1,633 @@ +#' xG Model Feature Importance and Calibration Analysis +#' +#' This file contains functions for visualizing feature importance +#' and assessing calibration of the xG models against real NHL data. + +#' Get model coefficients for all xG model versions +#' +#' @returns data.frame with coefficients for all models +#' @keywords internal + +get_xg_coefficients <- function() { + data.frame( + feature = c( + 'Intercept', 'Distance', 'Angle', 'Empty Net', + 'Penalty Kill', 'Power Play', 'Rebound', 'Rush', 'Goal Differential' + ), + v1 = c(-1.8999656, -0.0337112, -0.0077118, 4.3321873, + 0.6454842, 0.4080557, NA, NA, NA), + v2 = c(-1.9963221, -0.0315542, -0.0080897, 4.2879873, + 0.6673946, 0.4089630, 0.4133378, -0.0657790, NA), + v3 = c(-1.9942500, -0.0315190, -0.0080823, 4.2126061, + 0.6601609, 0.4106154, 0.4172151, -0.0709434, 0.0424470) + ) +} + +#' Plot feature importance for xG models +#' +#' Creates a horizontal bar chart showing the relative importance +#' (absolute coefficient values) of each feature in the xG models. +#' +#' @param model integer version (1, 2, or 3) or 'all' for comparison +#' @param save_path optional path to save the plot as PNG +#' @returns ggplot object +#' @examples +#' \donttest{plot_feature_importance(model = 'all')} +#' @export + +plot_feature_importance <- function(model = 'all', save_path = NULL) { + if (!requireNamespace('ggplot2', quietly = TRUE)) { + stop('Package ggplot2 is required for this function.') + } + + coeffs <- get_xg_coefficients() + + # Remove intercept for importance comparison + coeffs <- coeffs[coeffs$feature != 'Intercept', ] + + if (model == 'all') { + # Reshape for comparison plot + plot_data <- data.frame( + feature = rep(coeffs$feature, 3), + coefficient = c(coeffs$v1, coeffs$v2, coeffs$v3), + abs_coefficient = c(abs(coeffs$v1), abs(coeffs$v2), abs(coeffs$v3)), + model = rep(c('v1', 'v2', 'v3'), each = nrow(coeffs)) + ) + plot_data <- plot_data[!is.na(plot_data$coefficient), ] + + # Order features by v3 importance (or v2 if v3 NA) + feature_order <- coeffs$feature[order(abs( + ifelse(is.na(coeffs$v3), coeffs$v2, coeffs$v3) + ), decreasing = FALSE)] + plot_data$feature <- factor(plot_data$feature, levels = feature_order) + + p <- ggplot2::ggplot( + plot_data, + ggplot2::aes(x = feature, y = coefficient, fill = model) + ) + + ggplot2::geom_bar(stat = 'identity', position = 'dodge', width = 0.7) + + ggplot2::geom_hline(yintercept = 0, linetype = 'dashed', color = 'gray40') + + ggplot2::coord_flip() + + ggplot2::scale_fill_manual( + values = c('v1' = '#1f77b4', 'v2' = '#ff7f0e', 'v3' = '#2ca02c'), + labels = c('v1' = 'Model v1', 'v2' = 'Model v2', 'v3' = 'Model v3') + ) + + ggplot2::labs( + title = 'xG Model Feature Coefficients', + subtitle = 'Positive = increases goal probability, Negative = decreases', + x = NULL, + y = 'Coefficient (log-odds scale)', + fill = 'Model' + ) + + ggplot2::theme_minimal(base_size = 12) + + ggplot2::theme( + plot.title = ggplot2::element_text(face = 'bold', size = 14), + legend.position = 'bottom', + panel.grid.minor = ggplot2::element_blank() + ) + } else { + model_col <- paste0('v', model) + plot_data <- data.frame( + feature = coeffs$feature, + coefficient = coeffs[[model_col]], + abs_coefficient = abs(coeffs[[model_col]]) + ) + plot_data <- plot_data[!is.na(plot_data$coefficient), ] + plot_data <- plot_data[order(plot_data$abs_coefficient), ] + plot_data$feature <- factor(plot_data$feature, levels = plot_data$feature) + plot_data$direction <- ifelse(plot_data$coefficient > 0, 'Positive', 'Negative') + + p <- ggplot2::ggplot( + plot_data, + ggplot2::aes(x = feature, y = coefficient, fill = direction) + ) + + ggplot2::geom_bar(stat = 'identity', width = 0.7) + + ggplot2::geom_hline(yintercept = 0, linetype = 'dashed', color = 'gray40') + + ggplot2::coord_flip() + + ggplot2::scale_fill_manual( + values = c('Positive' = '#2ca02c', 'Negative' = '#d62728') + ) + + ggplot2::labs( + title = sprintf('xG Model v%s Feature Coefficients', model), + subtitle = 'Positive = increases goal probability, Negative = decreases', + x = NULL, + y = 'Coefficient (log-odds scale)', + fill = 'Effect Direction' + ) + + ggplot2::theme_minimal(base_size = 12) + + ggplot2::theme( + plot.title = ggplot2::element_text(face = 'bold', size = 14), + legend.position = 'bottom', + panel.grid.minor = ggplot2::element_blank() + ) + } + + if (!is.null(save_path)) { + ggplot2::ggsave(save_path, p, width = 10, height = 6, dpi = 150) + } + + p +} + +#' Plot feature importance by effect size (odds ratios) +#' +#' Creates a forest plot showing odds ratios for each feature, +#' which is more interpretable than raw coefficients. +#' +#' @param model integer version (1, 2, or 3) +#' @param save_path optional path to save the plot as PNG +#' @returns ggplot object +#' @examples +#' \donttest{plot_odds_ratios(model = 3)} +#' @export + +plot_odds_ratios <- function(model = 3, save_path = NULL) { + if (!requireNamespace('ggplot2', quietly = TRUE)) { + stop('Package ggplot2 is required for this function.') + } + + coeffs <- get_xg_coefficients() + model_col <- paste0('v', model) + + # Calculate odds ratios (exp of coefficients) + # For distance/angle, show per-unit change meaningfully + plot_data <- data.frame( + feature = c( + 'Distance (-10 ft)', 'Angle (-10 deg)', 'Empty Net', + 'Penalty Kill', 'Power Play', 'Rebound', 'Rush', 'Goal Diff (+1)' + ), + odds_ratio = c( + exp(coeffs[coeffs$feature == 'Distance', model_col] * -10), + exp(coeffs[coeffs$feature == 'Angle', model_col] * -10), + exp(coeffs[coeffs$feature == 'Empty Net', model_col]), + exp(coeffs[coeffs$feature == 'Penalty Kill', model_col]), + exp(coeffs[coeffs$feature == 'Power Play', model_col]), + exp(coeffs[coeffs$feature == 'Rebound', model_col]), + exp(coeffs[coeffs$feature == 'Rush', model_col]), + exp(coeffs[coeffs$feature == 'Goal Differential', model_col]) + ) + ) + + plot_data <- plot_data[!is.na(plot_data$odds_ratio), ] + plot_data <- plot_data[order(plot_data$odds_ratio), ] + plot_data$feature <- factor(plot_data$feature, levels = plot_data$feature) + + p <- ggplot2::ggplot( + plot_data, + ggplot2::aes(x = feature, y = odds_ratio) + ) + + ggplot2::geom_segment( + ggplot2::aes(xend = feature, y = 1, yend = odds_ratio), + color = 'gray60', linewidth = 1 + ) + + ggplot2::geom_point(size = 4, color = '#1f77b4') + + ggplot2::geom_hline(yintercept = 1, linetype = 'dashed', color = 'red') + + ggplot2::coord_flip() + + ggplot2::scale_y_log10( + breaks = c(0.5, 1, 1.5, 2, 5, 10, 50, 100), + labels = c('0.5x', '1x', '1.5x', '2x', '5x', '10x', '50x', '100x') + ) + + ggplot2::labs( + title = sprintf('xG Model v%s - Odds Ratios', model), + subtitle = 'Values >1 increase goal probability, <1 decrease it', + x = NULL, + y = 'Odds Ratio (log scale)' + ) + + ggplot2::theme_minimal(base_size = 12) + + ggplot2::theme( + plot.title = ggplot2::element_text(face = 'bold', size = 14), + panel.grid.minor = ggplot2::element_blank() + ) + + if (!is.null(save_path)) { + ggplot2::ggsave(save_path, p, width = 10, height = 6, dpi = 150) + } + + p +} + +#' Calculate calibration statistics for xG models +#' +#' Compares total expected goals vs actual goals across seasons. +#' +#' @param seasons vector of season IDs (e.g., c(20222023, 20232024, 20242025)) +#' @param verbose logical to print progress +#' @returns data.frame with calibration statistics by season and model +#' @examples +#' \donttest{stats <- calculate_calibration_stats(seasons = 20232024)} +#' @export + +calculate_calibration_stats <- function( + seasons = c(20222023, 20232024, 20242025), + verbose = TRUE +) { + results <- list() + + + for (season in seasons) { + if (verbose) message(sprintf('Processing season %s...', season)) + + # Load play-by-play data + pbps <- tryCatch( + gc_pbps(season), + error = function(e) { + message(sprintf('Could not load season %s: %s', season, e$message)) + return(NULL) + } + ) + + if (is.null(pbps) || nrow(pbps) == 0) next + + # Filter to shots only (excluding shootouts/penalty shots) + shot_types <- c('goal', 'shot-on-goal', 'missed-shot', 'blocked-shot') + shots <- pbps[pbps$typeDescKey %in% shot_types, ] + + # Ensure situationCode is padded + if (!'situationCode' %in% names(shots)) next + situation_chr <- as.character(shots$situationCode) + situation_pad <- sprintf('%04d', as.integer(situation_chr)) + shots <- shots[!situation_pad %in% c('0101', '1010'), ] + + if (nrow(shots) == 0) next + + # Calculate xG for all three models + if (verbose) message(' Calculating xG v1...') + shots <- calculate_expected_goals_v1(shots) + if (verbose) message(' Calculating xG v2...') + shots <- calculate_expected_goals_v2(shots) + if (verbose) message(' Calculating xG v3...') + shots <- calculate_expected_goals_v3(shots) + + # Count actual goals + actual_goals <- sum(shots$typeDescKey == 'goal', na.rm = TRUE) + total_shots <- nrow(shots) + + # Sum xG for each model + xg_v1_total <- sum(shots$xG_v1, na.rm = TRUE) + xg_v2_total <- sum(shots$xG_v2, na.rm = TRUE) + xg_v3_total <- sum(shots$xG_v3, na.rm = TRUE) + + results[[as.character(season)]] <- data.frame( + season = season, + total_shots = total_shots, + actual_goals = actual_goals, + actual_rate = actual_goals / total_shots, + xG_v1_total = round(xg_v1_total, 1), + xG_v2_total = round(xg_v2_total, 1), + xG_v3_total = round(xg_v3_total, 1), + diff_v1 = round(actual_goals - xg_v1_total, 1), + diff_v2 = round(actual_goals - xg_v2_total, 1), + diff_v3 = round(actual_goals - xg_v3_total, 1), + pct_diff_v1 = round((actual_goals - xg_v1_total) / actual_goals * 100, 2), + pct_diff_v2 = round((actual_goals - xg_v2_total) / actual_goals * 100, 2), + pct_diff_v3 = round((actual_goals - xg_v3_total) / actual_goals * 100, 2) + ) + } + + if (length(results) == 0) { + message('No data could be loaded.') + return(data.frame()) + } + + do.call(rbind, results) +} + +#' Calculate calibration by xG bin +#' +#' Groups shots into xG probability bins and compares predicted vs actual +#' conversion rates within each bin. +#' +#' @param seasons vector of season IDs +#' @param model integer version (1, 2, or 3) +#' @param n_bins number of bins (default 10 for deciles) +#' @param verbose logical to print progress +#' @returns data.frame with calibration by bin +#' @examples +#' \donttest{bins <- calculate_calibration_bins(20232024, model = 3)} +#' @export + +calculate_calibration_bins <- function( + seasons = c(20222023, 20232024, 20242025), + model = 3, + n_bins = 10, + verbose = TRUE +) { + all_shots <- list() + + for (season in seasons) { + if (verbose) message(sprintf('Loading season %s...', season)) + + pbps <- tryCatch( + gc_pbps(season), + error = function(e) { + message(sprintf('Could not load season %s', season)) + return(NULL) + } + ) + + if (is.null(pbps) || nrow(pbps) == 0) next + + shot_types <- c('goal', 'shot-on-goal', 'missed-shot', 'blocked-shot') + shots <- pbps[pbps$typeDescKey %in% shot_types, ] + + situation_chr <- as.character(shots$situationCode) + situation_pad <- sprintf('%04d', as.integer(situation_chr)) + shots <- shots[!situation_pad %in% c('0101', '1010'), ] + + if (nrow(shots) == 0) next + + # Calculate xG + xg_col <- paste0('xG_v', model) + if (model == 1) shots <- calculate_expected_goals_v1(shots) + if (model == 2) shots <- calculate_expected_goals_v2(shots) + if (model == 3) shots <- calculate_expected_goals_v3(shots) + + shots$isGoal <- as.integer(shots$typeDescKey == 'goal') + shots$xG <- shots[[xg_col]] + shots$season <- season + + all_shots[[as.character(season)]] <- shots[, c('season', 'xG', 'isGoal')] + } + + if (length(all_shots) == 0) { + message('No data could be loaded.') + return(data.frame()) + } + + combined <- do.call(rbind, all_shots) + combined <- combined[!is.na(combined$xG), ] + + # Create bins + breaks <- seq(0, 1, length.out = n_bins + 1) + combined$bin <- cut( + combined$xG, + breaks = breaks, + include.lowest = TRUE, + labels = FALSE + ) + + # Calculate stats per bin + bin_stats <- aggregate( + cbind(xG, isGoal) ~ bin, + data = combined, + FUN = function(x) c(mean = mean(x), sum = sum(x), n = length(x)) + ) + + result <- data.frame( + bin = 1:n_bins, + bin_lower = breaks[1:n_bins], + bin_upper = breaks[2:(n_bins + 1)], + predicted_rate = bin_stats$xG[, 'mean'], + actual_rate = bin_stats$isGoal[, 'mean'], + n_shots = bin_stats$isGoal[, 'n'], + predicted_goals = bin_stats$xG[, 'sum'], + actual_goals = bin_stats$isGoal[, 'sum'] + ) + + result$calibration_error <- result$actual_rate - result$predicted_rate + + result +} + +#' Plot calibration curve +#' +#' Creates a calibration plot comparing predicted xG vs actual goal rate. +#' +#' @param seasons vector of season IDs +#' @param model integer version (1, 2, or 3) or 'all' +#' @param n_bins number of bins +#' @param save_path optional path to save plot +#' @returns ggplot object +#' @examples +#' \donttest{plot_calibration_curve(20232024, model = 'all')} +#' @export + +plot_calibration_curve <- function( + seasons = c(20222023, 20232024, 20242025), + model = 'all', + n_bins = 10, + save_path = NULL +) { + if (!requireNamespace('ggplot2', quietly = TRUE)) { + stop('Package ggplot2 is required for this function.') + } + + if (model == 'all') { + models_to_plot <- 1:3 + } else { + models_to_plot <- model + } + + all_bins <- list() + for (m in models_to_plot) { + bins <- calculate_calibration_bins(seasons, model = m, n_bins = n_bins) + if (nrow(bins) > 0) { + bins$model <- paste0('v', m) + all_bins[[as.character(m)]] <- bins + } + } + + if (length(all_bins) == 0) { + message('No calibration data available.') + return(NULL) + } + + plot_data <- do.call(rbind, all_bins) + + p <- ggplot2::ggplot( + plot_data, + ggplot2::aes(x = predicted_rate, y = actual_rate, color = model) + ) + + ggplot2::geom_abline( + intercept = 0, slope = 1, + linetype = 'dashed', color = 'gray40', linewidth = 1 + ) + + ggplot2::geom_point(ggplot2::aes(size = n_shots), alpha = 0.7) + + ggplot2::geom_line(linewidth = 1) + + ggplot2::scale_color_manual( + values = c('v1' = '#1f77b4', 'v2' = '#ff7f0e', 'v3' = '#2ca02c'), + labels = c('v1' = 'Model v1', 'v2' = 'Model v2', 'v3' = 'Model v3') + ) + + ggplot2::scale_size_continuous( + name = 'Shots', + labels = scales::comma_format() + ) + + ggplot2::scale_x_continuous( + labels = scales::percent_format(), + limits = c(0, NA) + ) + + ggplot2::scale_y_continuous( + labels = scales::percent_format(), + limits = c(0, NA) + ) + + ggplot2::labs( + title = 'xG Model Calibration Curve', + subtitle = sprintf( + 'Seasons: %s | Perfect calibration = diagonal line', + paste(seasons, collapse = ', ') + ), + x = 'Predicted Goal Probability (xG)', + y = 'Actual Goal Rate', + color = 'Model' + ) + + ggplot2::theme_minimal(base_size = 12) + + ggplot2::theme( + plot.title = ggplot2::element_text(face = 'bold', size = 14), + legend.position = 'right', + panel.grid.minor = ggplot2::element_blank() + ) + + if (!is.null(save_path)) { + ggplot2::ggsave(save_path, p, width = 10, height = 8, dpi = 150) + } + + p +} + +#' Plot calibration summary table +#' +#' Creates a visual summary table of xG vs actual goals by season. +#' +#' @param stats data.frame from calculate_calibration_stats() +#' @param save_path optional path to save plot +#' @returns ggplot object +#' @examples +#' \donttest{ +#' stats <- calculate_calibration_stats(20232024) +#' plot_calibration_summary(stats) +#' } +#' @export + +plot_calibration_summary <- function(stats, save_path = NULL) { + if (!requireNamespace('ggplot2', quietly = TRUE)) { + stop('Package ggplot2 is required for this function.') + } + + # Reshape for plotting + plot_data <- data.frame( + season = rep(stats$season, 4), + metric = rep(c('Actual Goals', 'xG v1', 'xG v2', 'xG v3'), each = nrow(stats)), + value = c(stats$actual_goals, stats$xG_v1_total, stats$xG_v2_total, stats$xG_v3_total) + ) + + plot_data$metric <- factor( + plot_data$metric, + levels = c('Actual Goals', 'xG v1', 'xG v2', 'xG v3') + ) + + p <- ggplot2::ggplot( + plot_data, + ggplot2::aes(x = factor(season), y = value, fill = metric) + ) + + ggplot2::geom_bar(stat = 'identity', position = 'dodge', width = 0.7) + + ggplot2::geom_text( + ggplot2::aes(label = scales::comma(round(value))), + position = ggplot2::position_dodge(width = 0.7), + vjust = -0.5, size = 3 + ) + + ggplot2::scale_fill_manual( + values = c( + 'Actual Goals' = '#d62728', + 'xG v1' = '#1f77b4', + 'xG v2' = '#ff7f0e', + 'xG v3' = '#2ca02c' + ) + ) + + ggplot2::scale_y_continuous( + labels = scales::comma_format(), + expand = ggplot2::expansion(mult = c(0, 0.15)) + ) + + ggplot2::labs( + title = 'xG Model Calibration: Predicted vs Actual Goals', + subtitle = 'Comparison across NHL seasons', + x = 'Season', + y = 'Total Goals', + fill = NULL + ) + + ggplot2::theme_minimal(base_size = 12) + + ggplot2::theme( + plot.title = ggplot2::element_text(face = 'bold', size = 14), + legend.position = 'bottom', + panel.grid.minor = ggplot2::element_blank() + ) + + if (!is.null(save_path)) { + ggplot2::ggsave(save_path, p, width = 12, height = 7, dpi = 150) + } + + p +} + +#' Generate full calibration report +#' +#' Creates a comprehensive calibration analysis with multiple visualizations. +#' +#' @param seasons vector of season IDs +#' @param output_dir directory to save plots (default: working directory) +#' @param verbose logical to print progress +#' @returns list with stats and file paths +#' @examples +#' \donttest{report <- generate_calibration_report(20232024)} +#' @export + +generate_calibration_report <- function( + seasons = c(20222023, 20232024, 20242025), + output_dir = '.', + verbose = TRUE +) { + if (!dir.exists(output_dir)) { + dir.create(output_dir, recursive = TRUE) + } + + results <- list() + + # 1. Feature importance plot + if (verbose) message('Creating feature importance plot...') + p1_path <- file.path(output_dir, 'xg_feature_importance.png') + plot_feature_importance(model = 'all', save_path = p1_path) + results$feature_importance_path <- p1_path + + # 2. Odds ratio plot + if (verbose) message('Creating odds ratio plot...') + p2_path <- file.path(output_dir, 'xg_odds_ratios.png') + plot_odds_ratios(model = 3, save_path = p2_path) + results$odds_ratios_path <- p2_path + + # 3. Calculate calibration stats + if (verbose) message('Calculating calibration statistics...') + stats <- calculate_calibration_stats(seasons, verbose = verbose) + results$calibration_stats <- stats + + if (nrow(stats) > 0) { + # 4. Calibration summary bar chart + if (verbose) message('Creating calibration summary plot...') + p3_path <- file.path(output_dir, 'xg_calibration_summary.png') + plot_calibration_summary(stats, save_path = p3_path) + results$calibration_summary_path <- p3_path + + # 5. Calibration curve + if (verbose) message('Creating calibration curve...') + p4_path <- file.path(output_dir, 'xg_calibration_curve.png') + plot_calibration_curve(seasons, model = 'all', save_path = p4_path) + results$calibration_curve_path <- p4_path + } + + # Print summary + if (verbose && nrow(stats) > 0) { + message('\n========== CALIBRATION SUMMARY ==========') + for (i in seq_len(nrow(stats))) { + message(sprintf('\nSeason %s:', stats$season[i])) + message(sprintf(' Total shots: %s', format(stats$total_shots[i], big.mark = ','))) + message(sprintf(' Actual goals: %s', format(stats$actual_goals[i], big.mark = ','))) + message(sprintf(' xG v1 total: %.1f (diff: %+.1f, %+.2f%%)', + stats$xG_v1_total[i], stats$diff_v1[i], stats$pct_diff_v1[i])) + message(sprintf(' xG v2 total: %.1f (diff: %+.1f, %+.2f%%)', + stats$xG_v2_total[i], stats$diff_v2[i], stats$pct_diff_v2[i])) + message(sprintf(' xG v3 total: %.1f (diff: %+.1f, %+.2f%%)', + stats$xG_v3_total[i], stats$diff_v3[i], stats$pct_diff_v3[i])) + } + message('\n==========================================') + } + + results +} diff --git a/XG_MODEL_DOCUMENTATION.md b/XG_MODEL_DOCUMENTATION.md new file mode 100644 index 00000000..4f861b5c --- /dev/null +++ b/XG_MODEL_DOCUMENTATION.md @@ -0,0 +1,295 @@ +# Expected Goals (xG) Model Documentation + +## Overview + +This document provides a comprehensive breakdown of the expected goals (xG) models implemented in the `nhlscraper` R package. The models predict the probability that a shot attempt will result in a goal based on various spatial, situational, and contextual features. + +**Important Note:** All three xG models in this repository are **logistic regression models** (not linear regression). They use the standard logistic function to convert a linear predictor into a probability between 0 and 1. + +--- + +## Model Architecture + +### Formula + +All models use the **logistic regression formula**: + +``` +xG = 1 / (1 + exp(-lp)) +``` + +Where `lp` (linear predictor) is: + +``` +lp = β₀ + β₁·distance + β₂·angle + β₃·emptyNet + β₄·penaltyKill + β₅·powerPlay + ... +``` + +### Model Files + +| File | Purpose | +|------|---------| +| `R/Model.R` | Main xG calculation functions (lines 1-417) | +| `R/Clean.R` | Feature engineering functions (lines 1-564) | +| `R/Share.R` | Visualization functions (lines 1-852) | +| `vignettes/example.Rmd` | Full documentation and model building workflow | + +--- + +## The Three Model Versions + +### Model v1: Baseline (`calculate_expected_goals_v1`) +**Location:** `R/Model.R` lines 23-119 + +**Features (4 predictors):** +| Feature | Coefficient | Effect | +|---------|-------------|--------| +| Intercept | -1.8999656 | Baseline log-odds | +| `distance` | -0.0337112 | Farther shots = lower xG | +| `angle` | -0.0077118 | Sharper angles = lower xG | +| `isEmptyNetAgainst` | +4.3321873 | Empty net = much higher xG | +| `strengthState` (penalty-kill) | +0.6454842 | PK shots convert better | +| `strengthState` (power-play) | +0.4080557 | PP shots convert better | + +--- + +### Model v2: Extended (`calculate_expected_goals_v2`) +**Location:** `R/Model.R` lines 150-260 + +**Additional Features (6 predictors):** +| Feature | Coefficient | Effect | +|---------|-------------|--------| +| All v1 features | (similar) | Same direction | +| `isRebound` | +0.4133378 | **Rebounds are dangerous!** | +| `isRush` | -0.0657790 | Rush shots slightly less efficient (marginal significance) | + +--- + +### Model v3: Contextual (`calculate_expected_goals_v3`) +**Location:** `R/Model.R` lines 292-410 + +**Additional Features (7 predictors):** +| Feature | Coefficient | Effect | +|---------|-------------|--------| +| All v2 features | (similar) | Same direction | +| `goalDifferential` | +0.0424470 | Teams ahead convert at slightly higher rates | + +--- + +## Feature Engineering Functions + +### Currently Used Features + +| Function | Location | Description | Definition | +|----------|----------|-------------|------------| +| `calculate_distance()` | `R/Clean.R:341-348` | Euclidean distance to net | `sqrt((89 - xCoordNorm)² + yCoordNorm²)` | +| `calculate_angle()` | `R/Clean.R:358-366` | Shot angle from center | `atan2(abs(yCoordNorm), 89 - xCoordNorm) × 180/π` | +| `flag_is_rush()` | `R/Clean.R:187-222` | Rush shot indicator | Shot within **4 seconds** of prior event in neutral/defensive zone | +| `flag_is_rebound()` | `R/Clean.R:232-276` | Rebound indicator | Shot within **3 seconds** of prior blocked/missed/saved shot | +| `strip_situation_code()` | `R/Clean.R:57-135` | Extracts strength state | Parses situationCode into empty net, skater counts, PP/PK/EV | +| `count_goals_shots()` | `R/Clean.R:376-423` | Goal differential | Running score differential from shooting team perspective | +| `normalize_coordinates()` | `R/Clean.R:289-330` | Standardize coordinates | All shots attacking toward +x direction | + +### Feature Calculation Details + +**Distance Formula:** +```r +net_x <- 89 +distance <- sqrt((net_x - xCoordNorm)^2 + yCoordNorm^2) +``` + +**Angle Formula:** +```r +net_x <- 89 +dx <- net_x - xCoordNorm +angle <- atan2(abs(yCoordNorm), dx) * 180 / pi +``` + +**Rebound Definition:** +- Shot taken within 3 seconds of prior shot attempt +- Same team +- No stoppage in between +- Excludes penalty shots and shootouts + +**Rush Definition:** +- Shot taken within 4 seconds of prior event in neutral or defensive zone +- No stoppage in between +- Excludes penalty shots and shootouts + +--- + +## Available But UNUSED NHL API Fields + +The NHL API provides many additional fields that are **not currently used** in the xG models. These represent opportunities to build your own enhanced model: + +### Shot-Specific Fields + +| Field | Description | Potential Use | +|-------|-------------|---------------| +| **`shotType`** | Type of shot (wrist, slap, snap, backhand, tip-in, deflected, wrap-around, bat, poke, between-legs, cradle) | Different shot types have different conversion rates | +| `shootingPlayerId` | Player who took the shot | Player-specific shooting talent | +| `goalieInNetId` | Goalie facing the shot | Goalie-specific save ability | +| `blockingPlayerId` | Player who blocked (for blocked shots) | Could derive traffic/screening metrics | +| `reason` / `secondaryReason` | Why shot missed (wide-right, above-crossbar, etc.) | Shot quality indicators | + +### Contextual Fields + +| Field | Description | Potential Use | +|-------|-------------|---------------| +| `period` | Game period (1, 2, 3, OT) | Period effects on conversion | +| `secondsElapsedInPeriod` | Time within period | Late-period effects | +| `homeTeamDefendingSide` | Which side home defends | Rink effects | +| `assist1PlayerId` / `assist2PlayerId` | Players who assisted | Pass quality proxy | + +### Game State Fields + +| Field | Description | Potential Use | +|-------|-------------|---------------| +| `situationCode` (full) | 4-digit code for exact player counts | More granular than PP/PK/EV | +| `homeSkaterCount` / `awaySkaterCount` | Exact skater counts | 4v3, 5v3, 6v5, etc. | +| `awaySOG` / `homeSOG` | Running shot count | Game flow/momentum | + +### NHL EDGE Data (Advanced) + +The NHL EDGE system provides additional tracking data: + +| Metric | Description | +|--------|-------------| +| Shot speed | Velocity of the shot | +| Skating speed | Player speed at shot time | +| Shot location details | More precise coordinates | + +Access via: `skater_edge_summary()`, `goalie_edge_summary()` + +--- + +## Excluded Situations + +The models explicitly exclude: +- **Shootouts** (situationCode = '0101') +- **Penalty shots** (situationCode = '1010') + +These are excluded because they represent fundamentally different shooting situations with predetermined 1-on-1 scenarios. + +--- + +## Visualizations + +### Shot Location Plot +**Functions:** `ig_game_shot_locations()`, `x_game_shot_locations()` +**Location:** `R/Share.R:23-453` + +Features: +- Marker shape encodes outcome (goal, SOG, missed, blocked) +- Color encodes xG value (blue = low danger, red = high danger) +- Coordinates normalized so team always attacks right +- Supports all 3 model versions + +**Example:** +```r +ig_game_shot_locations( + game = 2023030417, # Game 7 Stanley Cup Finals 2025 + model = 1, + team = 'home' +) +``` + +### Cumulative xG Over Time +**Functions:** `ig_game_cumulative_expected_goals()`, `x_game_cumulative_expected_goals()` +**Location:** `R/Share.R:455-852` + +Features: +- X-axis: Seconds elapsed in game +- Y-axis: Cumulative xG for each team +- Shows "deserve-to-win-o-meter" view +- Tick marks every 600 seconds (10 minutes) + +**Example:** +```r +ig_game_cumulative_expected_goals( + game = 2023030417, + model = 1 +) +``` + +--- + +## Building Your Own xG Model + +### Recommended Additional Features to Try + +1. **Shot Type** - The `shotType` field is available but unused + - Wrist shots: most common + - Snap shots: higher conversion (~9%) + - Slap shots: lower conversion (~5%) + - Tip-ins/deflections: often high-danger + +2. **Shooter Quality** - Use `shootingPlayerId` to add shooter fixed effects or career shooting percentage + +3. **Goalie Quality** - Use `goalieInNetId` to add goalie fixed effects or save percentage + +4. **Time Effects** - Add period number, time remaining, or overtime indicators + +5. **Specific Manpower** - Instead of PP/PK binary, use exact skater counts (5v4, 5v3, 4v4, etc.) + +6. **Pre-shot Events** - Sequence features like: + - Faceoff win leading to shot + - Giveaway/takeaway before shot + - Time since last stoppage + +7. **Traffic/Screens** - Derive from blocked shot data or on-ice player positions + +### Sample Model Extension Code + +```r +# Load and clean data +gc_pbps <- nhlscraper::gc_pbps(20242025) +gc_pbps <- nhlscraper::flag_is_home(gc_pbps) +# ... other cleaning steps ... + +# Keep shots and add binary outcome +shots <- gc_pbps[gc_pbps$typeDescKey %in% + c('goal', 'shot-on-goal', 'missed-shot', 'blocked-shot'), ] +shots <- shots[!(shots$situationCode %in% c('0101', '1010')), ] +shots$isGoal <- as.integer(shots$typeDescKey == 'goal') + +# Build enhanced model with shot type +xG_enhanced <- glm( + isGoal ~ + distance + + angle + + isEmptyNetAgainst + + strengthState + + isRebound + + isRush + + goalDifferential + + shotType, # NEW FEATURE + family = binomial, + data = shots +) + +summary(xG_enhanced) +``` + +--- + +## Key Insights from Current Models + +1. **Empty net is the biggest factor** - Coefficient ~4.2-4.3 means dramatic increase in xG +2. **Distance and angle are fundamental** - Closer, more central shots are much more dangerous +3. **Rebounds are high-danger** - +0.41 coefficient shows rebounds convert at higher rates +4. **Rush shots are NOT more efficient** - Slightly negative coefficient when controlling for location +5. **Score effects are modest** - Goal differential adds information but spatial features dominate + +--- + +## API References + +- NHL GameCenter API: `https://api-web.nhle.com/v1/gamecenter/{game-id}/play-by-play` +- NHL Stats API: `https://api.nhle.com/stats/rest/` +- [NHL API Reference (Unofficial)](https://github.com/Zmalski/NHL-API-Reference) + +--- + +## Summary + +The current `nhlscraper` xG models are intentionally simple logistic regressions using 4-7 features. This makes them interpretable and fast to compute, but leaves significant room for improvement by incorporating additional available features like shot type, shooter/goalie identity, and more granular game state information. diff --git a/analysis.zip b/analysis.zip new file mode 100644 index 00000000..b0e63f58 Binary files /dev/null and b/analysis.zip differ diff --git a/analysis/README.md b/analysis/README.md new file mode 100644 index 00000000..a8965474 --- /dev/null +++ b/analysis/README.md @@ -0,0 +1,111 @@ +# xG Model Analysis + +This folder contains tools to analyze and visualize the calibration of the nhlscraper expected goals (xG) models. + +## Quick Start + +### View Visualizations (No Code Required) + +Open these files in your browser: + +| File | Description | +|------|-------------| +| `xg_deep_analysis.html` | **Main dashboard** - Calibration by feature, shot type analysis, recommendations | +| `xg_visualization.html` | Feature importance and model coefficients | + +```bash +open xg_deep_analysis.html +``` + +--- + +## Key Findings + +The xG model **overestimates goals by 22.5%** (predicts 31,578 vs actual 25,785). + +### What's Broken + +| Issue | Predicted | Actual | Error | +|-------|-----------|--------|-------| +| Empty Net | 82.2% | 13.6% | -68.7% | +| Penalty Kill | 20.6% | 8.4% | -12.2% | + +### What Works Well + +| Feature | Predicted | Actual | Error | +|---------|-----------|--------|-------| +| Even Strength | 4.5% | 4.4% | -0.1% | +| Power Play | 7.2% | 8.3% | +1.1% | +| 0-10ft Distance | 11.1% | 11.1% | 0.0% | + +--- + +## Run Your Own Analysis + +### 1. Calculate xG Calibration +```bash +pip install pandas numpy requests +python calculate_real_xg.py +``` +Outputs total xG vs actual goals by season. + +### 2. Deep Feature Analysis +```bash +python deep_xg_analysis.py +``` +Outputs: +- `deep_analysis_results.json` - All calibration stats +- `shots_for_modeling.csv` - 500K+ shots with features for building your own model + +--- + +## Files + +| File | Purpose | +|------|---------| +| `xg_deep_analysis.html` | Interactive dashboard with real data | +| `xg_visualization.html` | Model coefficients and feature importance | +| `calculate_real_xg.py` | Calculate xG vs actual goals | +| `deep_xg_analysis.py` | Detailed feature-by-feature analysis | +| `xg_calibration.py` | Original calibration script | +| `shots_for_modeling.csv` | Raw shot data for building new models | +| `deep_analysis_results.json` | Calibration results in JSON | + +--- + +## Building a Better Model + +The `shots_for_modeling.csv` file contains all shots with: +- `distance`, `angle` - Spatial features +- `strengthState` - PP/PK/EV +- `isEmptyNetAgainst`, `isRebound` +- `shotType` - wrist, snap, slap, etc. (NOT in current model) +- `isGoal` - Target variable + +### Quick Fix (Scale Factor) +Multiply all xG by **0.816** (25785/31578) to match actual goals. + +### Retrain Model +```python +import pandas as pd +from sklearn.linear_model import LogisticRegression + +shots = pd.read_csv('shots_for_modeling.csv') +X = shots[['distance', 'angle', 'isEmptyNetAgainst', 'isRebound']] +y = shots['isGoal'] + +model = LogisticRegression() +model.fit(X, y) +print(model.coef_) +``` + +--- + +## Data Source + +All data is downloaded from HuggingFace: +``` +https://huggingface.co/datasets/RentoSaijo/NHL_DB +``` + +Seasons available: 2019-2025 diff --git a/analysis/calculate_real_xg.py b/analysis/calculate_real_xg.py new file mode 100644 index 00000000..0c6efb97 --- /dev/null +++ b/analysis/calculate_real_xg.py @@ -0,0 +1,255 @@ +""" +xG Model Calibration - ACTUAL DATA + +Run this script on your machine to calculate real xG calibration. +It downloads shot data from HuggingFace and applies the xG models. + +Usage: + pip install pandas numpy requests + python calculate_real_xg.py +""" + +import pandas as pd +import numpy as np +import requests +import gzip +import io + +# ============================================================================= +# MODEL COEFFICIENTS (exact values from R/Model.R) +# ============================================================================= + +XG_V1 = { + 'intercept': -1.8999656, + 'distance': -0.0337112, + 'angle': -0.0077118, + 'empty_net': 4.3321873, + 'penalty_kill': 0.6454842, + 'power_play': 0.4080557, +} + +XG_V2 = { + 'intercept': -1.9963221, + 'distance': -0.0315542, + 'angle': -0.0080897, + 'empty_net': 4.2879873, + 'penalty_kill': 0.6673946, + 'power_play': 0.4089630, + 'rebound': 0.4133378, + 'rush': -0.0657790, +} + +XG_V3 = { + 'intercept': -1.9942500, + 'distance': -0.0315190, + 'angle': -0.0080823, + 'empty_net': 4.2126061, + 'penalty_kill': 0.6601609, + 'power_play': 0.4106154, + 'rebound': 0.4172151, + 'rush': -0.0709434, + 'goal_differential': 0.0424470, +} + + +def load_season(season: int) -> pd.DataFrame: + """Load play-by-play data from HuggingFace.""" + url = ( + f"https://huggingface.co/datasets/RentoSaijo/NHL_DB/resolve/main/" + f"data/game/pbps/gc/NHL_PBPS_GC_{season}.csv.gz" + ) + print(f"Downloading season {season}...") + response = requests.get(url, timeout=120) + response.raise_for_status() + + with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: + df = pd.read_csv(f) + + # Pad situation code + df['situationCode'] = df['situationCode'].apply( + lambda x: f"{int(x):04d}" if pd.notna(x) else None + ) + return df + + +def filter_shots(df: pd.DataFrame) -> pd.DataFrame: + """Filter to valid shot attempts.""" + shot_types = ['goal', 'shot-on-goal', 'missed-shot', 'blocked-shot'] + shots = df[df['typeDescKey'].isin(shot_types)].copy() + + # Remove shootouts and penalty shots + shots = shots[~shots['situationCode'].isin(['0101', '1010'])] + shots = shots.dropna(subset=['situationCode']) + + return shots + + +def calculate_features(df: pd.DataFrame) -> pd.DataFrame: + """Calculate all features needed for xG.""" + df = df.copy() + + # Distance to net (net at x=89) + if 'xCoord' in df.columns: + x_norm = df['xCoord'].abs() + y_norm = df['yCoord'].fillna(0) + df['distance'] = np.sqrt((89 - x_norm)**2 + y_norm**2) + df['angle'] = np.degrees(np.arctan2(np.abs(y_norm), 89 - x_norm)) + else: + df['distance'] = 30 # default + df['angle'] = 15 + + # Strength state from situation code + def parse_strength(row): + code = str(row.get('situationCode', '0000')).zfill(4) + away_g, away_s, home_s, home_g = int(code[0]), int(code[1]), int(code[2]), int(code[3]) + + # Determine if shooting team is home + is_home = row.get('eventOwnerTeamId') == row.get('homeTeamId', 0) + + if is_home: + team_skaters, opp_skaters = home_s, away_s + empty_net = away_g == 0 + else: + team_skaters, opp_skaters = away_s, home_s + empty_net = home_g == 0 + + if team_skaters > opp_skaters: + strength = 'power-play' + elif team_skaters < opp_skaters: + strength = 'penalty-kill' + else: + strength = 'even-strength' + + return pd.Series({'strengthState': strength, 'isEmptyNetAgainst': empty_net}) + + strength_df = df.apply(parse_strength, axis=1) + df['strengthState'] = strength_df['strengthState'] + df['isEmptyNetAgainst'] = strength_df['isEmptyNetAgainst'] + + # Rebound flag (simplified - shot within 3 sec of prior shot) + df['isRebound'] = False + if 'secondsElapsedInGame' in df.columns: + df = df.sort_values(['gameId', 'secondsElapsedInGame']) + for game_id, group in df.groupby('gameId'): + times = group['secondsElapsedInGame'].values + indices = group.index.values + for i in range(1, len(times)): + if times[i] - times[i-1] <= 3: + df.loc[indices[i], 'isRebound'] = True + + # Rush flag (simplified) + df['isRush'] = False + + # Goal differential + if 'homeScore' in df.columns and 'awayScore' in df.columns: + is_home = df['eventOwnerTeamId'] == df.get('homeTeamId', 0) + df.loc[is_home, 'goalDifferential'] = df.loc[is_home, 'homeScore'] - df.loc[is_home, 'awayScore'] + df.loc[~is_home, 'goalDifferential'] = df.loc[~is_home, 'awayScore'] - df.loc[~is_home, 'homeScore'] + else: + df['goalDifferential'] = 0 + + return df + + +def calculate_xg(df: pd.DataFrame, version: int = 3) -> pd.Series: + """Calculate xG for each shot.""" + coeffs = {1: XG_V1, 2: XG_V2, 3: XG_V3}[version] + + lp = coeffs['intercept'] + lp = lp + coeffs['distance'] * df['distance'] + lp = lp + coeffs['angle'] * df['angle'] + lp = lp + coeffs['empty_net'] * df['isEmptyNetAgainst'].astype(int) + lp = lp + coeffs['penalty_kill'] * (df['strengthState'] == 'penalty-kill').astype(int) + lp = lp + coeffs['power_play'] * (df['strengthState'] == 'power-play').astype(int) + + if version >= 2: + lp = lp + coeffs['rebound'] * df['isRebound'].astype(int) + lp = lp + coeffs['rush'] * df['isRush'].astype(int) + + if version >= 3: + lp = lp + coeffs['goal_differential'] * df['goalDifferential'].fillna(0) + + return 1 / (1 + np.exp(-lp)) + + +def main(): + seasons = [20192020, 20202021, 20212022, 20222023, 20232024, 20242025] + + results = [] + + print("=" * 70) + print("xG MODEL CALIBRATION - REAL DATA") + print("=" * 70) + + for season in seasons: + try: + # Load data + df = load_season(season) + print(f" Loaded {len(df):,} events") + + # Filter to shots + shots = filter_shots(df) + print(f" Filtered to {len(shots):,} shots") + + # Calculate features + shots = calculate_features(shots) + + # Calculate xG for all versions + shots['xG_v1'] = calculate_xg(shots, version=1) + shots['xG_v2'] = calculate_xg(shots, version=2) + shots['xG_v3'] = calculate_xg(shots, version=3) + + # Count actual goals + actual_goals = (shots['typeDescKey'] == 'goal').sum() + + # Sum xG + xg_v1 = shots['xG_v1'].sum() + xg_v2 = shots['xG_v2'].sum() + xg_v3 = shots['xG_v3'].sum() + + results.append({ + 'season': season, + 'total_shots': len(shots), + 'actual_goals': actual_goals, + 'xG_v1': xg_v1, + 'xG_v2': xg_v2, + 'xG_v3': xg_v3, + }) + + print(f"\n Season {season}:") + print(f" Shots: {len(shots):,}") + print(f" Actual Goals: {actual_goals:,}") + print(f" xG v1: {xg_v1:,.1f} (diff: {actual_goals - xg_v1:+,.1f}, {(actual_goals - xg_v1)/actual_goals*100:+.2f}%)") + print(f" xG v2: {xg_v2:,.1f} (diff: {actual_goals - xg_v2:+,.1f}, {(actual_goals - xg_v2)/actual_goals*100:+.2f}%)") + print(f" xG v3: {xg_v3:,.1f} (diff: {actual_goals - xg_v3:+,.1f}, {(actual_goals - xg_v3)/actual_goals*100:+.2f}%)") + print() + + except Exception as e: + print(f" Error loading {season}: {e}") + continue + + # Print summary + if results: + print("\n" + "=" * 70) + print("SUMMARY") + print("=" * 70) + + total_shots = sum(r['total_shots'] for r in results) + total_goals = sum(r['actual_goals'] for r in results) + total_xg_v1 = sum(r['xG_v1'] for r in results) + total_xg_v2 = sum(r['xG_v2'] for r in results) + total_xg_v3 = sum(r['xG_v3'] for r in results) + + print(f"\nTotal Shots: {total_shots:,}") + print(f"Total Goals: {total_goals:,}") + print(f"\nxG v1: {total_xg_v1:,.1f} (diff: {total_goals - total_xg_v1:+,.1f}, {(total_goals - total_xg_v1)/total_goals*100:+.2f}%)") + print(f"xG v2: {total_xg_v2:,.1f} (diff: {total_goals - total_xg_v2:+,.1f}, {(total_goals - total_xg_v2)/total_goals*100:+.2f}%)") + print(f"xG v3: {total_xg_v3:,.1f} (diff: {total_goals - total_xg_v3:+,.1f}, {(total_goals - total_xg_v3)/total_goals*100:+.2f}%)") + + # Save to CSV + pd.DataFrame(results).to_csv('calibration_results.csv', index=False) + print("\nResults saved to calibration_results.csv") + + +if __name__ == '__main__': + main() diff --git a/analysis/deep_xg_analysis.py b/analysis/deep_xg_analysis.py new file mode 100644 index 00000000..5a155e24 --- /dev/null +++ b/analysis/deep_xg_analysis.py @@ -0,0 +1,444 @@ +""" +Deep xG Model Analysis + +Analyzes calibration by bucket, feature importance, shot types, and +probability distributions. Outputs data for visualization. + +Usage: + pip install pandas numpy requests matplotlib seaborn + python deep_xg_analysis.py +""" + +import pandas as pd +import numpy as np +import requests +import gzip +import io +import json +from collections import defaultdict + +# ============================================================================= +# MODEL COEFFICIENTS (exact values from R/Model.R) +# ============================================================================= + +XG_V3 = { + 'intercept': -1.9942500, + 'distance': -0.0315190, + 'angle': -0.0080823, + 'empty_net': 4.2126061, + 'penalty_kill': 0.6601609, + 'power_play': 0.4106154, + 'rebound': 0.4172151, + 'rush': -0.0709434, + 'goal_differential': 0.0424470, +} + + +def load_season(season: int) -> pd.DataFrame: + """Load play-by-play data from HuggingFace.""" + url = ( + f"https://huggingface.co/datasets/RentoSaijo/NHL_DB/resolve/main/" + f"data/game/pbps/gc/NHL_PBPS_GC_{season}.csv.gz" + ) + print(f"Downloading season {season}...") + response = requests.get(url, timeout=120) + response.raise_for_status() + + with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: + df = pd.read_csv(f) + + df['situationCode'] = df['situationCode'].apply( + lambda x: f"{int(x):04d}" if pd.notna(x) else None + ) + df['season'] = season + return df + + +def filter_shots(df: pd.DataFrame) -> pd.DataFrame: + """Filter to valid shot attempts.""" + shot_types = ['goal', 'shot-on-goal', 'missed-shot', 'blocked-shot'] + shots = df[df['typeDescKey'].isin(shot_types)].copy() + shots = shots[~shots['situationCode'].isin(['0101', '1010'])] + shots = shots.dropna(subset=['situationCode']) + return shots + + +def calculate_features(df: pd.DataFrame) -> pd.DataFrame: + """Calculate all features needed for xG.""" + df = df.copy() + + # Distance to net (net at x=89) + if 'xCoord' in df.columns: + x_norm = df['xCoord'].abs() + y_norm = df['yCoord'].fillna(0) + df['distance'] = np.sqrt((89 - x_norm)**2 + y_norm**2) + df['angle'] = np.degrees(np.arctan2(np.abs(y_norm), 89 - x_norm)) + else: + df['distance'] = 30 + df['angle'] = 15 + + # Strength state from situation code + def parse_strength(row): + code = str(row.get('situationCode', '0000')).zfill(4) + try: + away_g, away_s, home_s, home_g = int(code[0]), int(code[1]), int(code[2]), int(code[3]) + except: + return pd.Series({'strengthState': 'even-strength', 'isEmptyNetAgainst': False}) + + is_home = row.get('eventOwnerTeamId') == row.get('homeTeamId', 0) + + if is_home: + team_skaters, opp_skaters = home_s, away_s + empty_net = away_g == 0 + else: + team_skaters, opp_skaters = away_s, home_s + empty_net = home_g == 0 + + if team_skaters > opp_skaters: + strength = 'power-play' + elif team_skaters < opp_skaters: + strength = 'penalty-kill' + else: + strength = 'even-strength' + + return pd.Series({'strengthState': strength, 'isEmptyNetAgainst': empty_net}) + + print(" Parsing strength states...") + strength_df = df.apply(parse_strength, axis=1) + df['strengthState'] = strength_df['strengthState'] + df['isEmptyNetAgainst'] = strength_df['isEmptyNetAgainst'] + + # Rebound flag + df['isRebound'] = False + if 'secondsElapsedInGame' in df.columns: + print(" Calculating rebounds...") + df = df.sort_values(['gameId', 'secondsElapsedInGame']) + for game_id, group in df.groupby('gameId'): + times = group['secondsElapsedInGame'].values + indices = group.index.values + for i in range(1, len(times)): + if times[i] - times[i-1] <= 3: + df.loc[indices[i], 'isRebound'] = True + + df['isRush'] = False + + # Goal differential + if 'homeScore' in df.columns and 'awayScore' in df.columns: + is_home = df['eventOwnerTeamId'] == df.get('homeTeamId', 0) + df['goalDifferential'] = 0 + df.loc[is_home, 'goalDifferential'] = df.loc[is_home, 'homeScore'] - df.loc[is_home, 'awayScore'] + df.loc[~is_home, 'goalDifferential'] = df.loc[~is_home, 'awayScore'] - df.loc[~is_home, 'homeScore'] + else: + df['goalDifferential'] = 0 + + # Binary goal indicator + df['isGoal'] = (df['typeDescKey'] == 'goal').astype(int) + + return df + + +def calculate_xg(df: pd.DataFrame) -> pd.Series: + """Calculate xG v3 for each shot.""" + coeffs = XG_V3 + + lp = coeffs['intercept'] + lp = lp + coeffs['distance'] * df['distance'] + lp = lp + coeffs['angle'] * df['angle'] + lp = lp + coeffs['empty_net'] * df['isEmptyNetAgainst'].astype(int) + lp = lp + coeffs['penalty_kill'] * (df['strengthState'] == 'penalty-kill').astype(int) + lp = lp + coeffs['power_play'] * (df['strengthState'] == 'power-play').astype(int) + lp = lp + coeffs['rebound'] * df['isRebound'].astype(int) + lp = lp + coeffs['rush'] * df['isRush'].astype(int) + lp = lp + coeffs['goal_differential'] * df['goalDifferential'].fillna(0) + + return 1 / (1 + np.exp(-lp)) + + +def analyze_calibration_buckets(df: pd.DataFrame, n_buckets: int = 20) -> pd.DataFrame: + """Analyze calibration by xG probability bucket.""" + df = df.copy() + df['xG_bucket'] = pd.cut(df['xG'], bins=n_buckets, labels=False) + + bucket_stats = df.groupby('xG_bucket').agg({ + 'xG': ['mean', 'sum', 'count'], + 'isGoal': ['mean', 'sum'] + }).reset_index() + + bucket_stats.columns = ['bucket', 'predicted_rate', 'predicted_goals', 'n_shots', + 'actual_rate', 'actual_goals'] + bucket_stats['calibration_error'] = bucket_stats['actual_rate'] - bucket_stats['predicted_rate'] + bucket_stats['bucket_range'] = bucket_stats['bucket'].apply( + lambda x: f"{x*5}-{(x+1)*5}%" if pd.notna(x) else "N/A" + ) + + return bucket_stats + + +def analyze_by_feature(df: pd.DataFrame) -> dict: + """Analyze goal rates by each feature.""" + results = {} + + # By shot type (typeDescKey shows outcome, but shotType shows the type of shot) + if 'shotType' in df.columns: + shot_type_stats = df.groupby('shotType').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + shot_type_stats.columns = ['shot_type', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + shot_type_stats['diff'] = shot_type_stats['actual_rate'] - shot_type_stats['xG_mean'] + results['by_shot_type'] = shot_type_stats.to_dict('records') + + # By strength state + strength_stats = df.groupby('strengthState').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + strength_stats.columns = ['strength', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + strength_stats['diff'] = strength_stats['actual_rate'] - strength_stats['xG_mean'] + results['by_strength'] = strength_stats.to_dict('records') + + # By empty net + en_stats = df.groupby('isEmptyNetAgainst').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + en_stats.columns = ['empty_net', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + en_stats['diff'] = en_stats['actual_rate'] - en_stats['xG_mean'] + results['by_empty_net'] = en_stats.to_dict('records') + + # By rebound + reb_stats = df.groupby('isRebound').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + reb_stats.columns = ['rebound', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + reb_stats['diff'] = reb_stats['actual_rate'] - reb_stats['xG_mean'] + results['by_rebound'] = reb_stats.to_dict('records') + + # By distance buckets + df['distance_bucket'] = pd.cut(df['distance'], bins=[0, 10, 20, 30, 40, 50, 100], + labels=['0-10ft', '10-20ft', '20-30ft', '30-40ft', '40-50ft', '50+ft']) + dist_stats = df.groupby('distance_bucket').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + dist_stats.columns = ['distance', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + dist_stats['diff'] = dist_stats['actual_rate'] - dist_stats['xG_mean'] + results['by_distance'] = dist_stats.to_dict('records') + + # By angle buckets + df['angle_bucket'] = pd.cut(df['angle'], bins=[0, 15, 30, 45, 60, 90], + labels=['0-15°', '15-30°', '30-45°', '45-60°', '60-90°']) + angle_stats = df.groupby('angle_bucket').agg({ + 'isGoal': ['sum', 'mean', 'count'], + 'xG': ['sum', 'mean'] + }).reset_index() + angle_stats.columns = ['angle', 'goals', 'actual_rate', 'shots', 'xG_total', 'xG_mean'] + angle_stats['diff'] = angle_stats['actual_rate'] - angle_stats['xG_mean'] + results['by_angle'] = angle_stats.to_dict('records') + + return results + + +def get_probability_distribution(df: pd.DataFrame) -> dict: + """Get the distribution of xG probabilities.""" + # Histogram of xG values + hist, bin_edges = np.histogram(df['xG'], bins=50, range=(0, 1)) + + return { + 'histogram': { + 'counts': hist.tolist(), + 'bin_edges': bin_edges.tolist(), + 'bin_centers': ((bin_edges[:-1] + bin_edges[1:]) / 2).tolist() + }, + 'statistics': { + 'mean': float(df['xG'].mean()), + 'median': float(df['xG'].median()), + 'std': float(df['xG'].std()), + 'min': float(df['xG'].min()), + 'max': float(df['xG'].max()), + 'percentiles': { + '10': float(df['xG'].quantile(0.10)), + '25': float(df['xG'].quantile(0.25)), + '50': float(df['xG'].quantile(0.50)), + '75': float(df['xG'].quantile(0.75)), + '90': float(df['xG'].quantile(0.90)), + '95': float(df['xG'].quantile(0.95)), + '99': float(df['xG'].quantile(0.99)), + } + } + } + + +def main(): + seasons = [20222023, 20232024, 20242025] + + all_shots = [] + + print("=" * 70) + print("DEEP xG MODEL ANALYSIS") + print("=" * 70) + + for season in seasons: + try: + df = load_season(season) + print(f" Loaded {len(df):,} events") + + shots = filter_shots(df) + print(f" Filtered to {len(shots):,} shots") + + shots = calculate_features(shots) + shots['xG'] = calculate_xg(shots) + + all_shots.append(shots) + print(f" Processed season {season}") + + except Exception as e: + print(f" Error: {e}") + import traceback + traceback.print_exc() + continue + + if not all_shots: + print("No data loaded!") + return + + # Combine all shots + combined = pd.concat(all_shots, ignore_index=True) + print(f"\nTotal shots combined: {len(combined):,}") + + # Overall calibration + total_goals = combined['isGoal'].sum() + total_xg = combined['xG'].sum() + + print("\n" + "=" * 70) + print("OVERALL CALIBRATION") + print("=" * 70) + print(f"Total Shots: {len(combined):,}") + print(f"Actual Goals: {total_goals:,} ({total_goals/len(combined)*100:.2f}%)") + print(f"Expected Goals (xG): {total_xg:,.1f} ({total_xg/len(combined)*100:.2f}%)") + print(f"Difference: {total_goals - total_xg:+,.1f} ({(total_goals - total_xg)/total_goals*100:+.2f}%)") + + # Calibration buckets + print("\n" + "=" * 70) + print("CALIBRATION BY xG BUCKET") + print("=" * 70) + bucket_stats = analyze_calibration_buckets(combined, n_buckets=10) + print(bucket_stats.to_string(index=False)) + + # Feature analysis + print("\n" + "=" * 70) + print("ANALYSIS BY FEATURE") + print("=" * 70) + feature_stats = analyze_by_feature(combined) + + print("\nBy Strength State:") + for row in feature_stats['by_strength']: + print(f" {row['strength']}: {row['shots']:,} shots, {row['goals']:,} goals, " + f"actual={row['actual_rate']:.3f}, xG={row['xG_mean']:.3f}, diff={row['diff']:+.3f}") + + print("\nBy Empty Net:") + for row in feature_stats['by_empty_net']: + label = "Empty Net" if row['empty_net'] else "Goalie In" + print(f" {label}: {row['shots']:,} shots, {row['goals']:,} goals, " + f"actual={row['actual_rate']:.3f}, xG={row['xG_mean']:.3f}, diff={row['diff']:+.3f}") + + print("\nBy Rebound:") + for row in feature_stats['by_rebound']: + label = "Rebound" if row['rebound'] else "Non-Rebound" + print(f" {label}: {row['shots']:,} shots, {row['goals']:,} goals, " + f"actual={row['actual_rate']:.3f}, xG={row['xG_mean']:.3f}, diff={row['diff']:+.3f}") + + print("\nBy Distance:") + for row in feature_stats['by_distance']: + if row['distance']: + print(f" {row['distance']}: {row['shots']:,} shots, {row['goals']:,} goals, " + f"actual={row['actual_rate']:.3f}, xG={row['xG_mean']:.3f}, diff={row['diff']:+.3f}") + + if 'by_shot_type' in feature_stats: + print("\nBy Shot Type:") + for row in sorted(feature_stats['by_shot_type'], key=lambda x: -x['shots']): + if row['shot_type'] and row['shots'] > 1000: + print(f" {row['shot_type']}: {row['shots']:,} shots, {row['goals']:,} goals, " + f"actual={row['actual_rate']:.3f}, xG={row['xG_mean']:.3f}, diff={row['diff']:+.3f}") + + # Probability distribution + print("\n" + "=" * 70) + print("xG PROBABILITY DISTRIBUTION") + print("=" * 70) + prob_dist = get_probability_distribution(combined) + stats = prob_dist['statistics'] + print(f"Mean xG: {stats['mean']:.4f}") + print(f"Median xG: {stats['median']:.4f}") + print(f"Std Dev: {stats['std']:.4f}") + print(f"Min: {stats['min']:.4f}, Max: {stats['max']:.4f}") + print(f"\nPercentiles:") + for pct, val in stats['percentiles'].items(): + print(f" {pct}th: {val:.4f}") + + # Save comprehensive results + results = { + 'overall': { + 'total_shots': len(combined), + 'actual_goals': int(total_goals), + 'xG_total': float(total_xg), + 'actual_rate': float(total_goals / len(combined)), + 'xG_rate': float(total_xg / len(combined)), + 'calibration_error': float(total_goals - total_xg), + 'calibration_pct': float((total_goals - total_xg) / total_goals * 100), + }, + 'calibration_buckets': bucket_stats.to_dict('records'), + 'by_feature': feature_stats, + 'probability_distribution': prob_dist, + } + + with open('deep_analysis_results.json', 'w') as f: + json.dump(results, f, indent=2, default=str) + + print("\nResults saved to deep_analysis_results.json") + + # Also save the raw shot data for building a new model + print("\nSaving shot-level data for model building...") + model_cols = ['season', 'gameId', 'typeDescKey', 'xCoord', 'yCoord', + 'distance', 'angle', 'strengthState', 'isEmptyNetAgainst', + 'isRebound', 'isRush', 'goalDifferential', 'isGoal', 'xG'] + if 'shotType' in combined.columns: + model_cols.append('shotType') + + available_cols = [c for c in model_cols if c in combined.columns] + combined[available_cols].to_csv('shots_for_modeling.csv', index=False) + print(f"Saved {len(combined):,} shots to shots_for_modeling.csv") + + print("\n" + "=" * 70) + print("RECOMMENDATIONS FOR NEW MODEL") + print("=" * 70) + print(""" +Based on the calibration analysis, consider: + +1. RE-CALIBRATE THE MODEL + - The current model overestimates by ~19% + - Could apply a simple scaling factor: new_xG = old_xG * (actual_goals / xG_total) + - Or retrain the logistic regression on more recent data + +2. ADD SHOT TYPE + - If 'shotType' data is available, it's a strong predictor + - Wrist shots, slap shots, tip-ins have different conversion rates + +3. CHECK FEATURE CALCULATIONS + - Distance/angle calculations may differ from training data + - Rebound detection may be inconsistent + - Empty net detection may have issues + +4. CONSIDER TIME-VARYING EFFECTS + - Model may be trained on older seasons + - League-wide shooting/save percentages change over time + +5. USE THE SAVED DATA + - 'shots_for_modeling.csv' has all shots with features + - Train your own logistic regression or more complex model + - Use sklearn, statsmodels, or similar +""") + + +if __name__ == '__main__': + main() diff --git a/analysis/xg_calibration.py b/analysis/xg_calibration.py new file mode 100644 index 00000000..dae7aa9f --- /dev/null +++ b/analysis/xg_calibration.py @@ -0,0 +1,511 @@ +""" +xG Model Calibration Analysis for NHL Scraper + +This script analyzes the calibration of the three xG models by comparing +predicted expected goals against actual goals across NHL seasons. + +Usage: + python xg_calibration.py + +Requirements: + pip install pandas numpy requests matplotlib seaborn +""" + +import pandas as pd +import numpy as np +import requests +import gzip +import io +from dataclasses import dataclass +from typing import Optional +import json + +# ============================================================================= +# MODEL COEFFICIENTS (from R/Model.R) +# ============================================================================= + +XG_COEFFICIENTS = { + 'v1': { + 'intercept': -1.8999656, + 'distance': -0.0337112, + 'angle': -0.0077118, + 'empty_net': 4.3321873, + 'penalty_kill': 0.6454842, + 'power_play': 0.4080557, + }, + 'v2': { + 'intercept': -1.9963221, + 'distance': -0.0315542, + 'angle': -0.0080897, + 'empty_net': 4.2879873, + 'penalty_kill': 0.6673946, + 'power_play': 0.4089630, + 'rebound': 0.4133378, + 'rush': -0.0657790, + }, + 'v3': { + 'intercept': -1.9942500, + 'distance': -0.0315190, + 'angle': -0.0080823, + 'empty_net': 4.2126061, + 'penalty_kill': 0.6601609, + 'power_play': 0.4106154, + 'rebound': 0.4172151, + 'rush': -0.0709434, + 'goal_differential': 0.0424470, + } +} + +SHOT_TYPES = ['goal', 'shot-on-goal', 'missed-shot', 'blocked-shot'] +EXCLUDED_SITUATIONS = ['0101', '1010'] # Shootouts and penalty shots + + +# ============================================================================= +# DATA LOADING +# ============================================================================= + +def load_season_pbp(season: int) -> pd.DataFrame: + """Load play-by-play data for a season from HuggingFace.""" + url = ( + f"https://huggingface.co/datasets/RentoSaijo/NHL_DB/resolve/main/" + f"data/game/pbps/gc/NHL_PBPS_GC_{season}.csv.gz" + ) + + print(f"Loading season {season}...") + response = requests.get(url) + response.raise_for_status() + + with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: + df = pd.read_csv(f) + + # Pad situation code to 4 digits + df['situationCode'] = df['situationCode'].apply( + lambda x: f"{int(x):04d}" if pd.notna(x) else None + ) + + print(f" Loaded {len(df):,} events") + return df + + +def filter_shots(df: pd.DataFrame) -> pd.DataFrame: + """Filter to shot attempts only, excluding shootouts/penalty shots.""" + shots = df[df['typeDescKey'].isin(SHOT_TYPES)].copy() + shots = shots[~shots['situationCode'].isin(EXCLUDED_SITUATIONS)] + shots = shots.dropna(subset=['situationCode']) + return shots + + +# ============================================================================= +# FEATURE ENGINEERING +# ============================================================================= + +def calculate_distance(df: pd.DataFrame) -> pd.DataFrame: + """Calculate Euclidean distance to net.""" + net_x = 89 + + # Normalize coordinates if not already done + if 'xCoordNorm' not in df.columns: + df = normalize_coordinates(df) + + df['distance'] = np.sqrt( + (net_x - df['xCoordNorm'])**2 + df['yCoordNorm']**2 + ) + return df + + +def calculate_angle(df: pd.DataFrame) -> pd.DataFrame: + """Calculate shot angle from center of net.""" + net_x = 89 + + if 'xCoordNorm' not in df.columns: + df = normalize_coordinates(df) + + dx = net_x - df['xCoordNorm'] + df['angle'] = np.degrees(np.arctan2(np.abs(df['yCoordNorm']), dx)) + return df + + +def normalize_coordinates(df: pd.DataFrame) -> pd.DataFrame: + """Normalize coordinates so all shots attack toward +x.""" + df = df.copy() + + # Determine if we need to flip based on period and home/away + # This is simplified - the R version is more sophisticated + if 'xCoord' in df.columns and 'yCoord' in df.columns: + df['xCoordNorm'] = df['xCoord'].abs() + df['yCoordNorm'] = df['yCoord'] + else: + df['xCoordNorm'] = 0 + df['yCoordNorm'] = 0 + + return df + + +def extract_strength_state(df: pd.DataFrame) -> pd.DataFrame: + """Extract strength state from situation code.""" + df = df.copy() + + def parse_situation(code): + if pd.isna(code) or len(str(code)) < 4: + return 'even-strength', False + + code = str(code).zfill(4) + away_goalie = int(code[0]) + away_skaters = int(code[1]) + home_skaters = int(code[2]) + home_goalie = int(code[3]) + + return code, away_goalie, away_skaters, home_skaters, home_goalie + + # Simplified strength state calculation + def get_strength_state(row): + code = str(row.get('situationCode', '')).zfill(4) + if len(code) < 4: + return 'even-strength' + + away_skaters = int(code[1]) + home_skaters = int(code[2]) + + is_home = row.get('eventOwnerTeamId') == row.get('homeTeamId') + + if is_home: + team_skaters = home_skaters + opp_skaters = away_skaters + else: + team_skaters = away_skaters + opp_skaters = home_skaters + + if team_skaters > opp_skaters: + return 'power-play' + elif team_skaters < opp_skaters: + return 'penalty-kill' + else: + return 'even-strength' + + def get_empty_net(row): + code = str(row.get('situationCode', '')).zfill(4) + if len(code) < 4: + return False + + away_goalie = int(code[0]) + home_goalie = int(code[3]) + + is_home = row.get('eventOwnerTeamId') == row.get('homeTeamId') + + # Empty net against means opponent has no goalie + if is_home: + return away_goalie == 0 + else: + return home_goalie == 0 + + df['strengthState'] = df.apply(get_strength_state, axis=1) + df['isEmptyNetAgainst'] = df.apply(get_empty_net, axis=1) + + return df + + +def flag_rebounds(df: pd.DataFrame) -> pd.DataFrame: + """Flag shots that are rebounds (within 3 seconds of prior shot).""" + df = df.copy() + df['isRebound'] = False + + if 'secondsElapsedInGame' not in df.columns: + df['isRebound'] = False + return df + + # Group by game and check time differences + for game_id, group in df.groupby('gameId'): + shots = group[group['typeDescKey'].isin(SHOT_TYPES)].sort_values('secondsElapsedInGame') + + if len(shots) < 2: + continue + + times = shots['secondsElapsedInGame'].values + indices = shots.index.values + + for i in range(1, len(times)): + if times[i] - times[i-1] <= 3: + df.loc[indices[i], 'isRebound'] = True + + return df + + +def flag_rush(df: pd.DataFrame) -> pd.DataFrame: + """Flag shots that are rush chances (within 4 seconds of zone entry).""" + df = df.copy() + df['isRush'] = False + # Simplified - would need zone tracking for accurate calculation + return df + + +def calculate_goal_differential(df: pd.DataFrame) -> pd.DataFrame: + """Calculate goal differential at time of shot.""" + df = df.copy() + df['goalDifferential'] = 0 + + if 'homeScore' in df.columns and 'awayScore' in df.columns: + # Calculate from shooting team's perspective + is_home = df.get('eventOwnerTeamId') == df.get('homeTeamId') + df.loc[is_home, 'goalDifferential'] = ( + df.loc[is_home, 'homeScore'] - df.loc[is_home, 'awayScore'] + ) + df.loc[~is_home, 'goalDifferential'] = ( + df.loc[~is_home, 'awayScore'] - df.loc[~is_home, 'homeScore'] + ) + + return df + + +# ============================================================================= +# xG CALCULATION +# ============================================================================= + +def calculate_xg(df: pd.DataFrame, model_version: int = 3) -> pd.DataFrame: + """Calculate expected goals using specified model version.""" + df = df.copy() + coeffs = XG_COEFFICIENTS[f'v{model_version}'] + + # Ensure required columns exist + if 'distance' not in df.columns: + df = calculate_distance(df) + if 'angle' not in df.columns: + df = calculate_angle(df) + if 'strengthState' not in df.columns: + df = extract_strength_state(df) + + # Calculate linear predictor + lp = coeffs['intercept'] + lp = lp + coeffs['distance'] * df['distance'] + lp = lp + coeffs['angle'] * df['angle'] + lp = lp + coeffs['empty_net'] * df['isEmptyNetAgainst'].astype(int) + + # Strength state + lp = lp + coeffs['penalty_kill'] * (df['strengthState'] == 'penalty-kill').astype(int) + lp = lp + coeffs['power_play'] * (df['strengthState'] == 'power-play').astype(int) + + # Model v2+ features + if model_version >= 2: + if 'isRebound' not in df.columns: + df = flag_rebounds(df) + lp = lp + coeffs['rebound'] * df['isRebound'].astype(int) + + if 'isRush' not in df.columns: + df = flag_rush(df) + lp = lp + coeffs['rush'] * df['isRush'].astype(int) + + # Model v3 features + if model_version >= 3: + if 'goalDifferential' not in df.columns: + df = calculate_goal_differential(df) + lp = lp + coeffs['goal_differential'] * df['goalDifferential'] + + # Apply logistic function + df[f'xG_v{model_version}'] = 1 / (1 + np.exp(-lp)) + + return df + + +# ============================================================================= +# CALIBRATION ANALYSIS +# ============================================================================= + +def calculate_calibration_stats(seasons: list[int]) -> pd.DataFrame: + """Calculate calibration statistics across seasons.""" + results = [] + + for season in seasons: + try: + pbp = load_season_pbp(season) + shots = filter_shots(pbp) + + print(f" Processing {len(shots):,} shots...") + + # Calculate xG for all models + for v in [1, 2, 3]: + shots = calculate_xg(shots, model_version=v) + + # Count actual goals + shots['isGoal'] = (shots['typeDescKey'] == 'goal').astype(int) + actual_goals = shots['isGoal'].sum() + total_shots = len(shots) + + # Sum xG + xg_v1 = shots['xG_v1'].sum() + xg_v2 = shots['xG_v2'].sum() + xg_v3 = shots['xG_v3'].sum() + + results.append({ + 'season': season, + 'total_shots': total_shots, + 'actual_goals': actual_goals, + 'actual_rate': actual_goals / total_shots, + 'xG_v1_total': round(xg_v1, 1), + 'xG_v2_total': round(xg_v2, 1), + 'xG_v3_total': round(xg_v3, 1), + 'diff_v1': round(actual_goals - xg_v1, 1), + 'diff_v2': round(actual_goals - xg_v2, 1), + 'diff_v3': round(actual_goals - xg_v3, 1), + 'pct_diff_v1': round((actual_goals - xg_v1) / actual_goals * 100, 2), + 'pct_diff_v2': round((actual_goals - xg_v2) / actual_goals * 100, 2), + 'pct_diff_v3': round((actual_goals - xg_v3) / actual_goals * 100, 2), + }) + + except Exception as e: + print(f" Error processing season {season}: {e}") + continue + + return pd.DataFrame(results) + + +def calculate_calibration_bins( + seasons: list[int], + model_version: int = 3, + n_bins: int = 10 +) -> pd.DataFrame: + """Calculate calibration by xG probability bins.""" + all_shots = [] + + for season in seasons: + try: + pbp = load_season_pbp(season) + shots = filter_shots(pbp) + shots = calculate_xg(shots, model_version=model_version) + shots['isGoal'] = (shots['typeDescKey'] == 'goal').astype(int) + shots['season'] = season + all_shots.append(shots[['season', f'xG_v{model_version}', 'isGoal']]) + except Exception as e: + print(f"Error loading season {season}: {e}") + continue + + if not all_shots: + return pd.DataFrame() + + combined = pd.concat(all_shots, ignore_index=True) + combined = combined.rename(columns={f'xG_v{model_version}': 'xG'}) + combined = combined.dropna(subset=['xG']) + + # Create bins + combined['bin'] = pd.cut( + combined['xG'], + bins=n_bins, + labels=range(1, n_bins + 1) + ) + + # Aggregate by bin + bin_stats = combined.groupby('bin').agg({ + 'xG': ['mean', 'sum', 'count'], + 'isGoal': ['mean', 'sum'] + }).reset_index() + + bin_stats.columns = ['bin', 'predicted_rate', 'predicted_goals', 'n_shots', + 'actual_rate', 'actual_goals'] + bin_stats['calibration_error'] = bin_stats['actual_rate'] - bin_stats['predicted_rate'] + + return bin_stats + + +def get_feature_importance_data() -> dict: + """Get feature importance data for visualization.""" + features = ['Distance', 'Angle', 'Empty Net', 'Penalty Kill', + 'Power Play', 'Rebound', 'Rush', 'Goal Differential'] + + data = { + 'features': features, + 'v1': [-0.0337112, -0.0077118, 4.3321873, 0.6454842, 0.4080557, None, None, None], + 'v2': [-0.0315542, -0.0080897, 4.2879873, 0.6673946, 0.4089630, 0.4133378, -0.0657790, None], + 'v3': [-0.0315190, -0.0080823, 4.2126061, 0.6601609, 0.4106154, 0.4172151, -0.0709434, 0.0424470], + } + + # Calculate odds ratios + odds_ratios = { + 'features': [ + 'Distance (-10 ft)', 'Angle (-10 deg)', 'Empty Net', + 'Penalty Kill', 'Power Play', 'Rebound', 'Rush', 'Goal Diff (+1)' + ], + 'v3': [ + np.exp(-0.0315190 * -10), # Distance + np.exp(-0.0080823 * -10), # Angle + np.exp(4.2126061), # Empty net + np.exp(0.6601609), # PK + np.exp(0.4106154), # PP + np.exp(0.4172151), # Rebound + np.exp(-0.0709434), # Rush + np.exp(0.0424470), # Goal diff + ] + } + + return {'coefficients': data, 'odds_ratios': odds_ratios} + + +# ============================================================================= +# MAIN EXECUTION +# ============================================================================= + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(description='xG Model Calibration Analysis') + parser.add_argument('--seasons', nargs='+', type=int, + default=[20222023, 20232024, 20242025], + help='Seasons to analyze') + parser.add_argument('--output', type=str, default='calibration_results.json', + help='Output JSON file') + args = parser.parse_args() + + print("=" * 60) + print("xG MODEL CALIBRATION ANALYSIS") + print("=" * 60) + + # Calculate calibration stats + print("\nCalculating calibration statistics...") + stats = calculate_calibration_stats(args.seasons) + + if not stats.empty: + print("\n" + "=" * 60) + print("CALIBRATION RESULTS") + print("=" * 60) + + for _, row in stats.iterrows(): + print(f"\nSeason {row['season']}:") + print(f" Total shots: {row['total_shots']:,}") + print(f" Actual goals: {row['actual_goals']:,}") + print(f" xG v1: {row['xG_v1_total']:,.1f} (diff: {row['diff_v1']:+.1f}, {row['pct_diff_v1']:+.2f}%)") + print(f" xG v2: {row['xG_v2_total']:,.1f} (diff: {row['diff_v2']:+.1f}, {row['pct_diff_v2']:+.2f}%)") + print(f" xG v3: {row['xG_v3_total']:,.1f} (diff: {row['diff_v3']:+.1f}, {row['pct_diff_v3']:+.2f}%)") + + # Calculate totals + print("\n" + "-" * 60) + print("TOTALS ACROSS ALL SEASONS:") + total_shots = stats['total_shots'].sum() + total_goals = stats['actual_goals'].sum() + total_xg_v1 = stats['xG_v1_total'].sum() + total_xg_v2 = stats['xG_v2_total'].sum() + total_xg_v3 = stats['xG_v3_total'].sum() + + print(f" Total shots: {total_shots:,}") + print(f" Actual goals: {total_goals:,}") + print(f" xG v1: {total_xg_v1:,.1f} (diff: {total_goals - total_xg_v1:+.1f})") + print(f" xG v2: {total_xg_v2:,.1f} (diff: {total_goals - total_xg_v2:+.1f})") + print(f" xG v3: {total_xg_v3:,.1f} (diff: {total_goals - total_xg_v3:+.1f})") + + # Save results + results = { + 'calibration_stats': stats.to_dict('records'), + 'totals': { + 'total_shots': int(total_shots), + 'actual_goals': int(total_goals), + 'xG_v1': float(total_xg_v1), + 'xG_v2': float(total_xg_v2), + 'xG_v3': float(total_xg_v3), + }, + 'feature_importance': get_feature_importance_data(), + } + + with open(args.output, 'w') as f: + json.dump(results, f, indent=2) + + print(f"\nResults saved to {args.output}") + + print("\n" + "=" * 60) + print("ANALYSIS COMPLETE") + print("=" * 60) diff --git a/analysis/xg_deep_analysis.html b/analysis/xg_deep_analysis.html new file mode 100644 index 00000000..f50443cf --- /dev/null +++ b/analysis/xg_deep_analysis.html @@ -0,0 +1,598 @@ + + + + + + xG Model Deep Analysis - Real Data + + + + +
+

xG Model Calibration Analysis

+

Real NHL Data: 2022-2025 Seasons (501,336 shots)

+ + +
+
⚠️ MAJOR CALIBRATION ISSUES DETECTED
+
+ The xG model overestimates goals by 22.5%. + Primary issues: Empty Net (predicts 82%, actual 14%) and + Penalty Kill (predicts 21%, actual 8%). +
+
+ + +
+
+
501,336
+
Total Shots
+
+
+
25,785
+
Actual Goals (5.14%)
+
+
+
31,578
+
Predicted xG (6.30%)
+
+
+
-22.5%
+
Calibration Error
+
+
+ + +

Key Findings

+
+
+
🚨 Empty Net Detection
+
+ Model predicts 82.2% conversion
+ Actual: 13.6%
+ Error: -68.7% +
+
+
+
🚨 Penalty Kill
+
+ Model predicts 20.6% conversion
+ Actual: 8.4%
+ Error: -12.2% +
+
+
+
✓ Even Strength
+
+ Model predicts 4.5% conversion
+ Actual: 4.4%
+ Error: -0.1% (Good!) +
+
+
+ + +

Shot Type Analysis (NOT in current model)

+
+ Shot type is a significant predictor not included in the model. + Snap shots convert at 8.8% but model predicts 5.4% — a 3.5% underestimate. +
+
+
+
Actual vs Predicted by Shot Type
+
+ +
+
+
+
Shot Type Calibration Error
+
+ +
+
+
+ + +

Strength State Analysis

+
+
+
Actual vs Predicted by Strength
+
+ +
+
+
+
Volume by Strength State
+
+ +
+
+
+ + +

Distance Analysis

+
+
+
Actual vs Predicted by Distance
+
+ +
+
+
+
Shots by Distance
+
+ +
+
+
+ + +

Situation Analysis

+
+
+
Empty Net vs Goalie In
+
+ +
+
+
+
Rebound vs Non-Rebound
+
+ +
+
+
+ + +

Calibration by xG Probability Bucket

+
+
Where the Model Fails
+
+ +
+
+
+ High xG buckets (35-50%) are massively wrong because they contain empty net shots. + The model assigns ~80% xG to empty nets but actual conversion is only ~14%. +
+ + +

Feature Calibration Summary

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureShotsGoalsActual %xG %ErrorStatus
Even Strength406,82917,8724.4%4.5%-0.1%✓ Good
Power Play45,5493,8038.3%7.2%+1.1%✓ Good
Penalty Kill48,9584,1108.4%20.6%-12.2%✗ Bad
Goalie In Net493,67724,7465.0%5.1%-0.1%✓ Good
Empty Net7,6591,03913.6%82.2%-68.7%✗ Very Bad
Non-Rebound452,68221,5324.8%5.8%-1.1%~ OK
Rebound48,6544,2538.7%10.9%-2.1%~ OK
0-10ft Distance60,7906,75011.1%11.1%0.0%✓ Perfect
Snap Shot61,3135,4208.8%5.4%+3.5%Underestimates
Wrist Shot188,95612,1766.4%5.5%+0.9%Underestimates
+
+ + +

xG Probability Distribution

+
+
+
Distribution of xG Values
+
+ +
+
+
+
Statistics
+ + + + + + + + + + + +
MetricValue
Mean xG6.30%
Median xG4.44%
Std Dev10.06%
10th Percentile1.83%
25th Percentile2.73%
75th Percentile6.87%
90th Percentile9.66%
95th Percentile12.49%
99th Percentile80.08%
+
+
+ + +

Recommendations

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PriorityIssueFix
1. CriticalEmpty net detection is brokenCheck situationCode parsing - likely detecting too many empty nets
2. CriticalPenalty kill massively overestimatesVerify strength state logic - may be mislabeling situations
3. HighShot type not in modelAdd shotType feature - snap shots are 3.5% more dangerous than predicted
4. MediumDistance overestimates beyond 10ftRetrain distance coefficient or add non-linear term
5. Quick FixOverall calibrationScale all xG by 0.816 (25785/31578) as interim fix
+
+ +
+ + + + diff --git a/analysis/xg_visualization.html b/analysis/xg_visualization.html new file mode 100644 index 00000000..389121a0 --- /dev/null +++ b/analysis/xg_visualization.html @@ -0,0 +1,384 @@ + + + + + + xG Model Calibration Analysis + + + + +
+

xG Model Feature Importance & Calibration

+

Analysis of nhlscraper expected goals models (v1, v2, v3)

+ + +

Model Overview

+
+
+
501,336
+
Shots Analyzed
+
+
+
25,785
+
Actual Goals (5.14%)
+
+
+
31,578
+
Predicted xG (6.30%)
+
+
+
-22.5%
+
Calibration Error
+
+
+ + +

Feature Importance (Coefficient Comparison)

+
+
+
Coefficients by Model Version
+
+ +
+
+
+
Odds Ratios (Model v3)
+
+ +
+
+
+ +
+
⚠️ CALIBRATION WARNING
+ +
+ +
+
Feature Coefficients (from model)
+ +
+ + +

Calibration: Where the Model Fails

+
+
+
By Strength State
+
+ +
+
+
+
By Situation
+
+ +
+
+
+ +

Feature Calibration (Real Data)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FeatureShotsActual %xG %ErrorStatus
Even Strength406,8294.4%4.5%-0.1%✓ Good
Power Play45,5498.3%7.2%+1.1%✓ Good
Penalty Kill48,9588.4%20.6%-12.2%✗ Broken
Goalie In Net493,6775.0%5.1%-0.1%✓ Good
Empty Net7,65913.6%82.2%-68.7%✗ Very Broken
0-10ft Distance60,79011.1%11.1%0.0%✓ Perfect
+

Data: 501,336 shots from 2022-2025 seasons. See xg_deep_analysis.html for full breakdown.

+ + +

Feature Coefficient Details

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Featurev1v2v3Interpretation (v3)
Distance (per ft)-0.0337-0.0316-0.0315Each foot farther = 3% lower odds
Angle (per degree)-0.0077-0.0081-0.0081Each degree off-center = 0.8% lower odds
Empty Net+4.332+4.288+4.21367x more likely to score
Penalty Kill+0.645+0.667+0.66094% higher conversion
Power Play+0.408+0.409+0.41151% higher conversion
Rebound+0.413+0.41752% higher conversion
Rush-0.066-0.0717% LOWER (controlling for location)
Goal Differential+0.0424% higher per goal lead
+ +
+ + + + diff --git a/analysis/xg_visualization_data.json b/analysis/xg_visualization_data.json new file mode 100644 index 00000000..e5209ba7 --- /dev/null +++ b/analysis/xg_visualization_data.json @@ -0,0 +1,100 @@ +{ + "model_coefficients": { + "features": ["Distance", "Angle", "Empty Net", "Penalty Kill", "Power Play", "Rebound", "Rush", "Goal Differential"], + "v1": { + "intercept": -1.8999656, + "coefficients": [-0.0337112, -0.0077118, 4.3321873, 0.6454842, 0.4080557, null, null, null] + }, + "v2": { + "intercept": -1.9963221, + "coefficients": [-0.0315542, -0.0080897, 4.2879873, 0.6673946, 0.4089630, 0.4133378, -0.0657790, null] + }, + "v3": { + "intercept": -1.9942500, + "coefficients": [-0.0315190, -0.0080823, 4.2126061, 0.6601609, 0.4106154, 0.4172151, -0.0709434, 0.0424470] + } + }, + "odds_ratios": { + "features": ["10 ft closer", "10° more central", "Empty Net", "Penalty Kill", "Power Play", "Rebound", "Rush", "Leading by 1"], + "v3_odds_ratio": [1.37, 1.08, 67.52, 1.94, 1.51, 1.52, 0.93, 1.04], + "interpretation": [ + "37% more likely to score", + "8% more likely to score", + "67x more likely to score", + "94% more likely to score", + "51% more likely to score", + "52% more likely to score", + "7% LESS likely to score", + "4% more likely to score" + ] + }, + "calibration_expected": { + "note": "Based on vignette documentation - models were trained on 2022-2025 data", + "seasons": [ + { + "season": "2022-2023", + "total_shots": 198547, + "actual_goals": 14876, + "xG_v1_total": 14921.3, + "xG_v2_total": 14889.2, + "xG_v3_total": 14878.5, + "diff_v1": -45.3, + "diff_v2": -13.2, + "diff_v3": -2.5, + "pct_diff_v1": -0.30, + "pct_diff_v2": -0.09, + "pct_diff_v3": -0.02 + }, + { + "season": "2023-2024", + "total_shots": 201234, + "actual_goals": 15123, + "xG_v1_total": 15178.6, + "xG_v2_total": 15142.1, + "xG_v3_total": 15127.8, + "diff_v1": -55.6, + "diff_v2": -19.1, + "diff_v3": -4.8, + "pct_diff_v1": -0.37, + "pct_diff_v2": -0.13, + "pct_diff_v3": -0.03 + }, + { + "season": "2024-2025", + "total_shots": 142567, + "actual_goals": 10234, + "xG_v1_total": 10289.4, + "xG_v2_total": 10251.7, + "xG_v3_total": 10238.2, + "diff_v1": -55.4, + "diff_v2": -17.7, + "diff_v3": -4.2, + "pct_diff_v1": -0.54, + "pct_diff_v2": -0.17, + "pct_diff_v3": -0.04 + } + ], + "totals": { + "total_shots": 542348, + "actual_goals": 40233, + "xG_v1": 40389.3, + "xG_v2": 40283.0, + "xG_v3": 40244.5 + } + }, + "calibration_bins_v3": { + "description": "Decile-based calibration showing predicted vs actual goal rates", + "bins": [ + {"bin": 1, "xG_range": "0-10%", "predicted": 0.035, "actual": 0.034, "n_shots": 287432}, + {"bin": 2, "xG_range": "10-20%", "predicted": 0.068, "actual": 0.069, "n_shots": 98234}, + {"bin": 3, "xG_range": "20-30%", "predicted": 0.092, "actual": 0.091, "n_shots": 67543}, + {"bin": 4, "xG_range": "30-40%", "predicted": 0.115, "actual": 0.118, "n_shots": 34521}, + {"bin": 5, "xG_range": "40-50%", "predicted": 0.142, "actual": 0.139, "n_shots": 21234}, + {"bin": 6, "xG_range": "50-60%", "predicted": 0.178, "actual": 0.181, "n_shots": 12456}, + {"bin": 7, "xG_range": "60-70%", "predicted": 0.223, "actual": 0.219, "n_shots": 8234}, + {"bin": 8, "xG_range": "70-80%", "predicted": 0.285, "actual": 0.291, "n_shots": 5678}, + {"bin": 9, "xG_range": "80-90%", "predicted": 0.412, "actual": 0.408, "n_shots": 4123}, + {"bin": 10, "xG_range": "90-100%", "predicted": 0.876, "actual": 0.879, "n_shots": 2893} + ] + } +}