diff --git a/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1 b/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 similarity index 99% rename from PAX_Purview_Audit_Log_Processor_v1.11.11.ps1 rename to PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 index 1d905bb..2a4e6c3 100644 --- a/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1 +++ b/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 @@ -1,5 +1,5 @@ # Portable Audit eXporter (PAX) - Purview Audit Log Processor -# Version: v1.11.11 +# Version: v1.11.12 # Requirements: PowerShell 7+ for default Graph API mode; PowerShell 5.1 supported ONLY with -UseEOM (serial Exchange Online Management mode, no parallel query/explosion). # Default Activity Type: CopilotInteraction (captures ALL M365 Copilot usage including all M365 apps and Teams meetings) # DSPM for AI activity types (specified via -ActivityTypes): AIInteraction, ConnectedAIAppInteraction, AIAppInteraction @@ -979,13 +979,17 @@ 2. Single activity type: -ActivityTypes CopilotInteraction (only one activity type selected) **CSV Mode Behavior:** - • Union-merges current-run rows with the target file keyed on RecordId (non-rollup) or - Message_Id_Raw (rollup); rows in the target but missing from the current run are kept - with In_Latest_Append = FALSE. + • Union-merges current-run rows with the target file keyed on RecordId (non-rollup) or, + for the rollup Fact CSV, on the full grain + Message_Id_Raw composite key (rollup Fact + rows FAN OUT — many rows can share one Message_Id_Raw, one per distinct grain); rows in + the target but missing from the current run are kept with In_Latest_Append = FALSE. • Date_Added / Latest_Append_Date / In_Latest_Append provenance columns are maintained on every row. NOTE: these three columns apply ONLY to the row-identity merges — the non-rollup raw audit CSV (keyed on RecordId) and the CopilotInteraction rollup Fact - CSV (keyed on Message_Id_Raw), where one row maps to exactly one record. They are + CSV (keyed on the grain + Message_Id_Raw composite). In the rollup Fact CSV a single + Message_Id_Raw can map to MULTIPLE rows (one per distinct grain, e.g. per accessed + resource), so append dedup keys on the whole grain plus Message_Id_Raw — never on + Message_Id_Raw alone (doing so would collapse fan-out rows and lose data). They are intentionally NOT added to the M365 Usage bundle rollup (-IncludeM365Usage + -Rollup), whose rows are additive aggregates summed across runs — see the per-row provenance caveat in the M365 Rollup Anchoring section below. @@ -2231,7 +2235,7 @@ $m365UsageActivityBundle = @( ) | Select-Object -Unique # Script version constant (must appear after param/help to keep param() valid as first executable block) -$ScriptVersion = '1.11.11' +$ScriptVersion = '1.11.12' function Invoke-PaxVersionCheck { # Informational, non-blocking, failure-isolated version check against the public PAX repo. @@ -3793,7 +3797,7 @@ if (($IncludeAgent365Info -or $OnlyAgent365Info)) { } # Agent 365 + app-only auth (AppRegistration certificate, AppRegistration client secret, -# ManagedIdentity): SUPPORTED as of v1.11.11 (defect D3). The Microsoft Graph Agent Package +# ManagedIdentity): The Microsoft Graph Agent Package # Management API exposes an APPLICATION permission (CopilotPackages.Read.All app-role; plus # Application.Read.All for developer-name resolution) per Microsoft Learn "List Copilot packages" # (https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api). The app-only token @@ -4479,8 +4483,17 @@ _NONGRAIN_ATTRS_AIO_BASE: tuple[str, ...] = ( "Value_Outcome", "ActivityDate", ) -# AIO carried attrs = the v3.1.0 base set + the trailing raw reconciliation keys. -_NONGRAIN_ATTRS_AIO: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + _RAW_ID_ATTRS +# AIO carried attrs = the v3.1.0 base set + a stable user-identity column + the +# trailing raw reconciliation keys. +_NONGRAIN_ATTRS_AIO: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( + # Stable, deid-consistent user identity for AIO. Mirrors the + # AIBV [Audit_UserId_Normalized] value (deid_upn -> normalize_user_id), so it + # is deterministically de-identified under -Deidentify and never exposes a raw + # UPN. Gives the cross-run append merge key a stable user component in place of + # the per-run UserKey INT surrogate. Placed BEFORE the raw keys so + # Message_Id_Raw / ThreadId_Raw stay the trailing reconciliation columns. + "User_Id_Normalized", +) + _RAW_ID_ATTRS # AIBV non-grain carried attrs = AIO base set + AIBV-only offloaded columns, with # the raw reconciliation keys appended LAST so they remain the trailing two columns @@ -6273,6 +6286,11 @@ def explode_record( "Behavior_Source": "", "Value_Outcome": "", "ActivityDate": interaction_date_str, + # Stable, deid-consistent user identity (AIO parity with the + # AIBV [Audit_UserId_Normalized] value). Emitted only for the AIO profile + # (AIBV's header carries Audit_UserId_Normalized instead), so for AIBV this + # key is a harmless extra that its fact-header selection ignores. + "User_Id_Normalized": audit_user_id_norm, # Cross-run append reconciliation key (trailing). Constant per record; # Message_Id_Raw (per message) is injected in the emit loop below. "ThreadId_Raw": thread_id_raw, @@ -10682,6 +10700,61 @@ function Merge-UsersCsv { } } +function Get-FactCompositeKeyColumns { + <# + .SYNOPSIS + Return the grain-composite append dedup key column list for a rolled-up + Interactions Fact CSV, derived from its header columns. + .DESCRIPTION + FIX 1 (v1.11.12). Rolled-up fact rows FAN OUT: many rows share one + Message_Id_Raw, one per distinct grain (e.g. per AccessedResource). Cross-run + append dedup must therefore key on the FULL grain PLUS Message_Id_Raw; keying + on Message_Id_Raw alone collapses every fan-out row for a message to one and + silently discards the rest on merge. + + Grain columns use their stable, cross-run-comparable forms: the per-run INT + surrogates 'UserKey' and 'ThreadId' are replaced by the deid-consistent + normalized user identity (AIO: 'User_Id_Normalized' (FIX 5); AIBV: the + pre-existing 'Audit_UserId_Normalized') and by 'ThreadId_Raw' respectively. + The remaining names mirror the embedded processor grain (GRAIN_KEYS_AIO / + GRAIN_KEYS_AIBV) verbatim. Profile is detected from the header (AIBV carries + 'Is_Agent_Activity'). Returns @() when the header is not a recognizable fact + header (no Message_Id_Raw) so the caller falls back to its own guard. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string[]] $HeaderColumns) + + if (-not ($HeaderColumns -contains 'Message_Id_Raw')) { return @() } + $isAibv = ($HeaderColumns -contains 'Is_Agent_Activity') + $userIdCol = if ($isAibv) { 'Audit_UserId_Normalized' } else { 'User_Id_Normalized' } + $cols = [System.Collections.Generic.List[string]]::new() + $cols.Add($userIdCol) # replaces the per-run 'UserKey' INT surrogate + $cols.Add('InteractionDate') + $cols.Add('AgentId') + $cols.Add('AgentName') + $cols.Add('AppHost') + $cols.Add('Environment') + $cols.Add('License Status') + $cols.Add('Context_Type') + $cols.Add('Behavior_Category') + $cols.Add('Behavior_Enriched') + $cols.Add('AI_Model') + $cols.Add('Is_Sensitive') + $cols.Add('Autonomy_Pattern') + $cols.Add('AppIdentity_AppId') + $cols.Add('AISystemPlugin_Name') + $cols.Add('ThreadId_Raw') # replaces the per-run 'ThreadId' INT surrogate + if ($isAibv) { + $cols.Add('Is_Agent_Activity') + $cols.Add('Web_Grounded_Signal') + $cols.Add('Workflow_Action') + } + $cols.Add('Message_Id_Raw') + # Emit as a flat string[]; callers collect with @(...) (an empty return above + # unrolls to nothing, which @(...) normalizes to an empty array). + return $cols.ToArray() +} + function Merge-FactCsv { <# .SYNOPSIS @@ -10711,12 +10784,33 @@ function Merge-FactCsv { [Parameter(Mandatory)] [string] $CurrentFactCsv, [Parameter()] [string] $OutputPath, [Parameter()] [string] $KeyColumn = 'Message_Id_Raw', + [Parameter()] [string[]] $CompositeKeyColumn = @(), [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd') ) if (-not (Test-Path -LiteralPath $CurrentFactCsv -PathType Leaf)) { throw "Merge-FactCsv: current Fact CSV not found: '$CurrentFactCsv'" } if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentFactCsv } + # Grain-composite dedup key. When -CompositeKeyColumn is + # supplied, a row's key is the U+001F-joined values of those columns (ASCII Unit + # Separator 0x1F is a control char that never appears in audit field data), so + # fan-out fact rows (many per Message_Id_Raw) are each keyed distinctly instead of + # collapsing to one. Otherwise the single -KeyColumn is used (RecordId raw-audit + # path, unchanged). $getRowKey is invoked for every target/current row below. + $useCompositeKey = ($null -ne $CompositeKeyColumn -and @($CompositeKeyColumn).Count -gt 0) + $compositeSep = [char]0x1F + $getRowKey = { + param($row) + if ($useCompositeKey) { + $parts = foreach ($c in $CompositeKeyColumn) { + $pp = $row.PSObject.Properties[$c] + if ($pp) { [string]$pp.Value } else { '' } + } + return ($parts -join $compositeSep) + } + $kp = $row.PSObject.Properties[$KeyColumn] + if ($kp) { [string]$kp.Value } else { '' } + } # Use script:Import-CsvDeduped for header-dupe safety (parallel to Merge-UsersCsv). $targetRows = @() if (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) { @@ -10724,8 +10818,8 @@ function Merge-FactCsv { } # APPEND SAFETY (data-loss guard): if the target file exists with content but parsed to # zero rows, the read failed (parse/encoding/memory) or the key cannot be matched. - # Overwriting would replace the customer's file with current-run rows only (the TDOT - # shrink). Abort instead so the existing target is left untouched; the caller keeps the + # Overwriting would replace the existing target file with current-run rows only (a + # row-count shrink). Abort instead so the existing target is left untouched; the caller keeps the # fresh current-run CSV on disk for a manual merge. if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetFactCsv).Length -gt 0)) { throw "Merge-FactCsv: target '$TargetFactCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data). Verify the file's schema/key column ('$KeyColumn')." @@ -10741,11 +10835,13 @@ function Merge-FactCsv { # get blanks for target-only fields). if ($targetRows.Count -gt 0) { $targetHeaders = @($targetRows[0].PSObject.Properties.Name) - if ($KeyColumn -notin $targetHeaders) { + $requiredKeyCols = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + $missingKeyCols = @($requiredKeyCols | Where-Object { $_ -notin $targetHeaders }) + if ($missingKeyCols.Count -gt 0) { Microsoft.PowerShell.Utility\Write-Host ( - ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column '{0}'. " + + ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column(s) '{0}'. " + "Cannot classify Retained / New / Departed rows reliably; treating ALL current-run rows as New. " + - "Target: {1}") -f $KeyColumn, $TargetFactCsv + "Target: {1}") -f ($missingKeyCols -join ', '), $TargetFactCsv ) -ForegroundColor Yellow } } @@ -10753,18 +10849,16 @@ function Merge-FactCsv { $targetByKey = @{} foreach ($r in $targetRows) { - $kProp = $r.PSObject.Properties[$KeyColumn] - $k = if ($kProp) { $kProp.Value } else { $null } - if (-not [string]::IsNullOrWhiteSpace([string]$k) -and -not $targetByKey.ContainsKey([string]$k)) { - $targetByKey[[string]$k] = $r + $k = [string](& $getRowKey $r) + if (-not [string]::IsNullOrWhiteSpace($k) -and -not $targetByKey.ContainsKey($k)) { + $targetByKey[$k] = $r } } $currentByKey = @{} foreach ($r in $currentRows) { - $kProp = $r.PSObject.Properties[$KeyColumn] - $k = if ($kProp) { $kProp.Value } else { $null } - if (-not [string]::IsNullOrWhiteSpace([string]$k) -and -not $currentByKey.ContainsKey([string]$k)) { - $currentByKey[[string]$k] = $r + $k = [string](& $getRowKey $r) + if (-not [string]::IsNullOrWhiteSpace($k) -and -not $currentByKey.ContainsKey($k)) { + $currentByKey[$k] = $r } } @@ -10789,20 +10883,20 @@ function Merge-FactCsv { # 1. Current-run rows (retained + new). foreach ($r in $currentRows) { - $kProp = $r.PSObject.Properties[$KeyColumn] - $k = if ($kProp) { [string]$kProp.Value } else { '' } + $k = [string](& $getRowKey $r) $obj = [ordered]@{} foreach ($c in $hdrOrder) { $obj[$c] = '' } foreach ($p in $r.PSObject.Properties) { $obj[$p.Name] = $p.Value } if ($k -and $targetByKey.ContainsKey($k)) { $tr = $targetByKey[$k] - # When dedup-keyed on Message_Id_Raw, target's Message_Id INT wins - # (continuity across runs; embedded Python seed-mid-map normally aligns - # this, but enforce here so a seed-prep failure still produces a - # continuous union). For other key columns (e.g. RecordId on the raw - # audit CSV) there is no surrogate-INT continuity contract. - if ($KeyColumn -eq 'Message_Id_Raw') { + # When dedup-keyed on Message_Id_Raw (single or as part of the grain- + # composite key), target's Message_Id INT wins (continuity across runs; + # embedded Python seed-mid-map normally aligns this, but enforce here so a + # seed-prep failure still produces a continuous union). For other single key + # columns (e.g. RecordId on the raw audit CSV) there is no surrogate-INT + # continuity contract. + if ($KeyColumn -eq 'Message_Id_Raw' -or ($useCompositeKey -and (@($CompositeKeyColumn) -contains 'Message_Id_Raw'))) { $trMid = $tr.PSObject.Properties['Message_Id'] if ($trMid -and -not [string]::IsNullOrWhiteSpace([string]$trMid.Value)) { $obj['Message_Id'] = $trMid.Value @@ -14087,7 +14181,6 @@ function Connect-PurviewAudit { # sign-in completes, so the log shows BOTH phases honestly side-by-side. Capture the # Phase 1 context now into script-scope vars so the combined emitter can use them. # - # v1.11.11 (D3): Agent 365 now runs on the SAME app-only context as the audit phase # (app-only modes) - or the same delegated context (delegated modes) - so there is no # separate Phase 2 context to combine and nothing to defer. Always emit the auth-context # display inline below for every auth mode. @@ -17407,12 +17500,12 @@ function Connect-Agent365InteractiveContext { function Invoke-Agent365EarlyInteractiveSignIn { <# .SYNOPSIS - No-op retained for call-site stability (defect D3, v1.11.11). + No-op retained for call-site stability. .DESCRIPTION Historically this performed an eager up-front interactive DELEGATED sign-in for the Agent 365 phase under -Auth AppRegistration, because the catalog endpoint was assumed to - have no app-only Graph scope. As of v1.11.11 the Agent Package Management API is read with + have no app-only Graph scope. The Agent Package Management API is read with the APPLICATION app-role CopilotPackages.Read.All (+ Application.Read.All) on the app / managed-identity service principal, so NO interactive sign-in is required in ANY auth mode: - app-only modes (AppRegistration cert/secret, ManagedIdentity) reuse the existing @@ -17513,6 +17606,35 @@ function Get-Agent365Packages { return $results.ToArray() } +function Invoke-Agent365GraphWithRetry { + <# + .SYNOPSIS + Throttle-aware Graph GET for the Agent 365 catalog read path. Retries on HTTP 429 + and 5xx with exponential backoff (up to 5 attempts, min(60, 2^attempt) seconds), + honoring a Retry-After header when present. Non-throttle / non-5xx errors rethrow + immediately so callers keep their existing skip-with-warning behavior. Mirrors the + backoff convention used by the chunked remote-upload path. + #> + param([Parameter(Mandatory = $true)][string]$Uri) + $attempt = 0 + while ($true) { + try { + return Invoke-MgGraphRequest -Method GET -Uri $Uri -ErrorAction Stop + } catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 0) { if ($_.Exception.Message -match '429|Too Many Requests|TooManyRequests') { $status = 429 } } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } +} + function Get-Agent365PackageDetail { <# .SYNOPSIS @@ -17524,7 +17646,7 @@ function Get-Agent365PackageDetail { Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue } catch {} try { - return Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + return Invoke-Agent365GraphWithRetry -Uri $uri } catch { Write-LogHost (" WARNING: Agent 365 detail fetch failed for '{0}': {1}" -f $PackageId, $_.Exception.Message) -ForegroundColor Yellow return $null @@ -17549,7 +17671,7 @@ function Resolve-Agent365DeveloperName { $resolved = '' try { $appUri = "https://graph.microsoft.com/v1.0/applications?`$filter=appId eq '$AppId'&`$select=id,displayName,publisherDomain" - $appResp = Invoke-MgGraphRequest -Method GET -Uri $appUri -ErrorAction Stop + $appResp = Invoke-Agent365GraphWithRetry -Uri $appUri if ($appResp -and $appResp.value -and $appResp.value.Count -gt 0) { $app = $appResp.value[0] if ($app.publisherDomain) { $resolved = $app.publisherDomain } @@ -17558,7 +17680,7 @@ function Resolve-Agent365DeveloperName { if (-not $resolved -and $app.id) { try { $ownerUri = "https://graph.microsoft.com/v1.0/applications/$($app.id)/owners?`$select=userPrincipalName,displayName" - $ownerResp = Invoke-MgGraphRequest -Method GET -Uri $ownerUri -ErrorAction Stop + $ownerResp = Invoke-Agent365GraphWithRetry -Uri $ownerUri if ($ownerResp -and $ownerResp.value -and $ownerResp.value.Count -gt 0) { $resolved = $ownerResp.value[0].displayName if (-not $resolved) { $resolved = $ownerResp.value[0].userPrincipalName } @@ -17930,17 +18052,17 @@ function Invoke-Agent365Phase { $idx = 0 foreach ($p in $listed) { $idx++ - $pid = $null - try { $pid = $p.id } catch {} - if (-not $pid) { try { $pid = $p.titleId } catch {} } - if (-not $pid) { continue } - $detail = Get-Agent365PackageDetail -PackageId $pid + $pkgId = $null + try { $pkgId = $p.id } catch {} + if (-not $pkgId) { try { $pkgId = $p.titleId } catch {} } + if (-not $pkgId) { continue } + $detail = Get-Agent365PackageDetail -PackageId $pkgId if (-not $detail) { continue } try { $row = ConvertTo-Agent365Row -Package $detail -AuditEnrichment $script:Agent365AuditEnrichment [void]$rows.Add($row) } catch { - Write-LogHost (" WARNING: Row build failed for package '{0}': {1}" -f $pid, $_.Exception.Message) -ForegroundColor Yellow + Write-LogHost (" WARNING: Row build failed for package '{0}': {1}" -f $pkgId, $_.Exception.Message) -ForegroundColor Yellow } if (($idx % 25) -eq 0) { Write-LogHost (" ... {0}/{1} packages processed" -f $idx, $listed.Count) -ForegroundColor DarkGray @@ -20235,9 +20357,6 @@ else { $auditContextLbl = if ($isManagedId) { 'APP-ONLY (managed identity / application permissions)' } elseif ($isAppOnlyAuth) { 'APP-ONLY (application permissions)' } else { 'DELEGATED (interactive user sign-in)' } - # v1.11.11 (D3): Agent 365 no longer opens a separate delegated Phase 2 context - app-only modes - # reuse the app-only audit context and delegated modes reuse their delegated context. There is no - # longer a dual-context run, so the auth-context summary shows a single context for every mode. Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green Write-LogHost " QUERY MODE: Microsoft Graph Security API (Default)" -ForegroundColor Green @@ -20261,14 +20380,6 @@ else { Write-LogHost " Permissions Required for THIS run:" -ForegroundColor White Write-LogHost " (Yellow = required this run; DarkGray = not needed this run)" -ForegroundColor Gray - # Build conditional Tag legend so only relevant tags are shown: - # [App-only] appears under -Auth AppRegistration OR -Auth ManagedIdentity (both are app-only) - # [Delegated] appears only under a non-app-only (delegated) auth mode. As of v1.11.11 Agent 365 - # uses APPLICATION app-roles under app-only auth, so app-only runs surface no - # [Delegated] tag even when Agent 365 is included. - # [Role] appears only when Agent 365 is in a DELEGATED run (an app-only run needs the - # CopilotPackages.Read.All app-role, not an Entra directory role on a user) - # [Azure RBAC] appears only when an -OutputPath* value is a OneLake URL (Fabric uses storage RBAC, not Graph) $tagLegendLines = New-Object System.Collections.Generic.List[string] if ($isAppOnlyAuth) { if ($isManagedId) { @@ -23664,10 +23775,6 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit Import-Module ExchangeOnlineManagement -Force } - # Banner 7a: app-only Agent 365 informational notice (AppRegistration certificate/secret or - # ManagedIdentity + Agent 365). As of v1.11.11 (D3) the Agent 365 catalog is read with the - # APPLICATION app-role CopilotPackages.Read.All on the service principal - no interactive - # sign-in is performed and the whole run stays on a single app-only Graph context. if (($IncludeAgent365Info -or $OnlyAgent365Info) -and ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity')) { Write-LogHost "" Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Cyan @@ -23908,10 +24015,6 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } # PAX4A-INGEST-END - # Agent 365 no longer requires an eager interactive sign-in (v1.11.11, D3): app-only modes - # (AppRegistration cert/secret, ManagedIdentity) reuse the existing application context and - # delegated modes already hold their Agent 365 scopes from the initial sign-in. The call - # below is a retained no-op kept for wiring stability; it never prompts in any auth mode. if (-not $UseEOM) { $null = Invoke-Agent365EarlyInteractiveSignIn } @@ -32262,7 +32365,7 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # the UserInfo destination. When -OutputPathUserInfo supplies a file-form leaf, # the rolled-up leaf ('_Users.csv') does NOT carry the current # run timestamp, so the remote upload sweep's timestamp wildcard misses it - # (D2 / NatWest: '-AppendFile + -OutputPathUserInfo + -Rollup' shipped only + # (previously, '-AppendFile + -OutputPathUserInfo + -Rollup' produced only # 2 artifacts, dropping the Entra Users dim). Register the leaf explicitly — # exactly like the M365 sidecar leaves — so the upload sweep includes it; # per-data-type routing (Get-DataTypeForOutputFile -> 'UserInfo') then lands @@ -32303,19 +32406,35 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread try { $rollupPurviewStem = [System.IO.Path]::GetFileNameWithoutExtension($rollupPurviewCsv) $rollupFactCsv = Join-Path $rollupOutputDir ("{0}_Interactions.csv" -f $rollupPurviewStem) - # OD2 (append data-safety): a pre-v1.11.11 seed has no 'Message_Id_Raw' - # dedup key and cannot reconcile on real message identity. Merging into - # such a target would classify every existing row as departed and (when the - # seed is smaller than this run) silently overwrite it with current-only - # rows. Probe the target header; if the key is absent, SKIP the in-place - # merge, leave the seed untouched, and write this run's rollup to a new - # timestamped file so the customer can re-baseline. The run still succeeds. $rollupAfSkipMerge = $false + $rollupAfSkipReason = '' + $rollupAfCompositeKey = @() if ((Test-Path -LiteralPath $rollupFactCsv) -and (Test-Path -LiteralPath $AppendFile -PathType Leaf)) { try { $rollupAfHdrLine = @(Get-Content -LiteralPath $AppendFile -TotalCount 1) $rollupAfCols = if ($rollupAfHdrLine.Count -gt 0) { ($rollupAfHdrLine[0] -split ',') | ForEach-Object { $_.Trim().Trim('"') } } else { @() } - if (-not ($rollupAfCols -contains 'Message_Id_Raw')) { $rollupAfSkipMerge = $true } + # Derive the grain-composite dedup key from the FRESH + # rollup fact header (always current-schema), then require the TARGET to + # carry every composite column. Fact rows fan out (many per + # Message_Id_Raw, one per grain), so append dedup MUST key on the full + # grain (UserKey/ThreadId in stable raw form) + Message_Id_Raw or fan-out + # rows collapse and data is lost. Missing composite columns on the target + # (e.g. an older AIO seed with no User_Id_Normalized) route to the + # same data-loss-safe re-baseline skip as an older seed with no reconciliation key. + $rollupCurHdrLine = @(Get-Content -LiteralPath $rollupFactCsv -TotalCount 1) + $rollupCurCols = if ($rollupCurHdrLine.Count -gt 0) { ($rollupCurHdrLine[0] -split ',') | ForEach-Object { $_.Trim().Trim('"') } } else { @() } + $rollupAfCompositeKey = @(Get-FactCompositeKeyColumns -HeaderColumns $rollupCurCols) + if (-not ($rollupAfCols -contains 'Message_Id_Raw')) { + $rollupAfSkipMerge = $true + $rollupAfSkipReason = "target lacks the 'Message_Id_Raw' dedup key (pre-v1.11.11 seed)" + } + elseif ($rollupAfCompositeKey.Count -gt 0) { + $rollupAfMissingKeyCols = @($rollupAfCompositeKey | Where-Object { $_ -notin $rollupAfCols }) + if ($rollupAfMissingKeyCols.Count -gt 0) { + $rollupAfSkipMerge = $true + $rollupAfSkipReason = ("target is missing grain-composite dedup column(s) [{0}] (pre-v1.11.12 seed)" -f ($rollupAfMissingKeyCols -join ', ')) + } + } } catch { # Header probe failed; fall through to Merge-FactCsv whose own @@ -32328,13 +32447,13 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $rollupAfRebaselinePath = Join-Path $rollupOutputDir $rollupAfRebaselineName Move-Item -LiteralPath $rollupFactCsv -Destination $rollupAfRebaselinePath -Force $_rollupAfDisplay = if ($script:AppendRaw.ContainsKey('Purview') -and $script:AppendRaw['Purview']) { $script:AppendRaw['Purview'] } else { $AppendFile } - Write-LogHost ("Rollup: -AppendFile: target lacks the 'Message_Id_Raw' dedup key (pre-v1.11.11 seed); append SKIPPED to prevent data loss.") -ForegroundColor Yellow + Write-LogHost ("Rollup: -AppendFile: {0}; append SKIPPED to prevent data loss." -f $rollupAfSkipReason) -ForegroundColor Yellow Write-LogHost ("Rollup: -> Target left unchanged: {0}" -f $_rollupAfDisplay) -ForegroundColor Yellow Write-LogHost ("Rollup: -> This run's rollup written to: {0}" -f (Get-DisplayPath -LocalPath $rollupAfRebaselinePath)) -ForegroundColor Yellow - Write-LogHost "Rollup: -> To re-baseline: use this new file as your -AppendFile target going forward (it carries Message_Id_Raw); subsequent appends will reconcile on real message identity." -ForegroundColor Yellow + Write-LogHost "Rollup: -> To re-baseline: use this new file as your -AppendFile target going forward (it carries Message_Id_Raw and the full grain-composite key); subsequent appends reconcile on real grain + message identity." -ForegroundColor Yellow } elseif (Test-Path -LiteralPath $rollupFactCsv) { - $rollupFactMergeStats = Merge-FactCsv -TargetFactCsv $AppendFile -CurrentFactCsv $rollupFactCsv -KeyColumn 'Message_Id_Raw' -OutputPath $AppendFile + $rollupFactMergeStats = Merge-FactCsv -TargetFactCsv $AppendFile -CurrentFactCsv $rollupFactCsv -KeyColumn 'Message_Id_Raw' -CompositeKeyColumn $rollupAfCompositeKey -OutputPath $AppendFile Write-LogHost ("Rollup: -AppendFile merge: Retained={0:N0} New={1:N0} Departed={2:N0} Union={3:N0}" -f $rollupFactMergeStats.Retained, $rollupFactMergeStats.New, $rollupFactMergeStats.Departed, $rollupFactMergeStats.Union) -ForegroundColor Green $_rollupAfDisplay = if ($script:AppendRaw.ContainsKey('Purview') -and $script:AppendRaw['Purview']) { $script:AppendRaw['Purview'] } else { $AppendFile } Write-LogHost ("Appended to: {0}" -f $_rollupAfDisplay) -ForegroundColor White diff --git a/README.md b/README.md index d5c65ad..5b06ee8 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@

PAX PowerShell Script  |  PAX Cookbook  |  PAX Cookbook Mini-Kitchen

-**⬇️ Download the script:** [`PAX_Purview_Audit_Log_Processor_v1.11.11.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.11/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1)  |  Release Date: July 2, 2026 +**⬇️ Download the script:** [`PAX_Purview_Audit_Log_Processor_v1.11.12.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.12/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1)  |  Release Date: July 3, 2026 **📖 Script Resources:** [Latest Documentation](https://github.com/microsoft/PAX/blob/release/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md) | [Latest Release Notes](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) diff --git a/release_documentation/.gitkeep b/release_documentation/.gitkeep index d0b498b..f26214d 100644 --- a/release_documentation/.gitkeep +++ b/release_documentation/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-02 (Purview v1.11.11, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.33) \ No newline at end of file +# Last updated: 2026-07-03 (Purview v1.11.12, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.34) \ No newline at end of file diff --git a/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md b/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md index 04dfca9..523c6c0 100644 --- a/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md +++ b/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md @@ -1,8 +1,8 @@ # Portable Audit eXporter (PAX) -
Purview Audit Log Processor -> **📥 Quick Start:** Download the script → [`PAX_Purview_Audit_Log_Processor_v1.11.11.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.11/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1) +> **📥 Quick Start:** Download the script → [`PAX_Purview_Audit_Log_Processor_v1.11.12.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.12/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1) > -> **📅 Script v1.11.11 Release Date:** July 2, 2026 +> **📅 Script v1.11.12 Release Date:** July 3, 2026 > > **📋 Release Notes:** See what's new → [v1.11.x Release Notes](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) | [All Release Notes](https://github.com/microsoft/PAX/tree/release/release_notes/Purview_Audit_Log_Processor) > @@ -10,7 +10,7 @@ > > **📚 Documentation Archive:** [All Documentation](https://github.com/microsoft/PAX/tree/release/release_documentation/Purview_Audit_Log_Processor) -**Documentation Version:** v1.11.x (Current Script Version: v1.11.11) +**Documentation Version:** v1.11.x (Current Script Version: v1.11.12) **Audience:** IT admins, security/compliance analysts, BI/data teams **Runtime:** PowerShell 7+ (required for default Graph API mode); PowerShell 5.1 supported only with `-UseEOM` **License:** MIT @@ -406,7 +406,7 @@ The **Purview Audit Reader** role is only required for EOM mode (`-UseEOM`) and ### Download the Script -- **Script:** [PAX_Purview_Audit_Log_Processor_v1.11.11.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.11/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1) +- **Script:** [PAX_Purview_Audit_Log_Processor_v1.11.12.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.12/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1) - **Release Notes:** [v1.11.x](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) Save the downloaded script to a working directory (e.g., `C:\Scripts\PAX\`). @@ -3561,9 +3561,9 @@ Adding `-Deidentify` to a rollup run anonymizes the rolled-up output as well — When you point a rollup run at an existing rollup file with `-AppendFile`, PAX merges this run's rolled-up interactions into that file instead of writing a brand-new one, so you can grow a single dataset across many runs (for example, a monthly refresh onto a running history). -**Reconciliation on stable message identity.** The append merge reconciles on each interaction's **stable message identity**, so overlapping records between runs are matched and de-duplicated, brand-new records are added, and records that are only in the existing file are preserved. The result is an exact union — no interaction is dropped, and overlaps are never double-counted. +**Reconciliation on the full analytical grain + stable message identity.** A single interaction (message) can produce **several** rolled-up rows — one per distinct combination of the analytical grain (for example, per accessed resource). The append merge reconciles on the **full grain together with each interaction's stable message identity**, so every one of those rows is matched independently: overlapping rows between runs are matched and de-duplicated, brand-new rows are added, and rows that are only in the existing file are preserved. The result is an exact union — no row is dropped (even when several rows share the same message), and overlaps are never double-counted. -**One-time re-baseline for older seed files.** Append files created by **earlier versions of PAX** were written without the stable identity key the reconciliation relies on. To protect your data, PAX **does not silently merge** onto one of these older files: +**One-time re-baseline for older seed files.** Append files created by **earlier versions of PAX** were written without every column the current reconciliation relies on — older files may lack the stable message identity key entirely, and files created **before v1.11.12** may lack a grain-composite column the fan-out-safe merge needs (for example, the AIO rolled-up interactions file gained a stable user-identity column in v1.11.12). To protect your data, PAX **does not silently merge** onto one of these older files: - The existing file is **left completely untouched**. - This run's rolled-up output is written to a **new, timestamped file** alongside it. @@ -3571,6 +3571,8 @@ When you point a rollup run at an existing rollup file with `-AppendFile`, PAX m **What this means for you:** an append file you started with an **earlier version** needs a **one-time re-baseline** — generate a fresh rollup file with the current version once and use that as your new append target. From then on, every `-AppendFile` run reconciles and grows correctly. **No data is lost** in the process: your original file is preserved as-is, and this run's data is safely written to the new file. +> **Note (v1.11.12):** if you have been appending onto an **AIO** rolled-up interactions file created before v1.11.12, expect this one-time re-baseline on your next run — the fan-out-safe merge needs a stable user-identity column that older AIO files do not carry. Your existing file is left untouched and the run writes a fresh, timestamped file to re-baseline from. AIBV files created in v1.11.11 already carry the needed identity column. + **The rolled-up Users dimension always uploads.** When a rollup run also produces the Users dimension (the org / licensing companion to the interactions file), that Users file is uploaded correctly in **all four** combinations of interactions-append and Users destination — whether the interactions stream is appending or not, and whether the Users destination is `-OutputPathUserInfo` or `-AppendUserInfo`. It uploads exactly once in every case. diff --git a/release_notes/.gitkeep b/release_notes/.gitkeep index d0b498b..f26214d 100644 --- a/release_notes/.gitkeep +++ b/release_notes/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-02 (Purview v1.11.11, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.33) \ No newline at end of file +# Last updated: 2026-07-03 (Purview v1.11.12, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.34) \ No newline at end of file diff --git a/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md b/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md index e6e8c31..c0999e3 100644 --- a/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md +++ b/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md @@ -2,8 +2,8 @@ ## Release Information -- **Latest Version:** 1.11.11 -- **Latest Release Date:** July 2, 2026 +- **Latest Version:** 1.11.12 +- **Latest Release Date:** July 3, 2026 - **Released By:** Microsoft Copilot Growth ROI Advisory Team (copilot-roi-advisory-team-gh@microsoft.com) --- @@ -12,13 +12,29 @@ Download the script below. For questions or issues, refer to the documentation. -- **PAX Purview Audit Log Processor Script v1.11.11:** [PAX_Purview_Audit_Log_Processor_v1.11.11.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.11/PAX_Purview_Audit_Log_Processor_v1.11.11.ps1) +- **PAX Purview Audit Log Processor Script v1.11.12:** [PAX_Purview_Audit_Log_Processor_v1.11.12.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.12/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1) - **Documentation v1.11.x (Markdown):** [PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md](https://github.com/microsoft/PAX/blob/release/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md) --- ## Overview +### v1.11.12 + +Version 1.11.12 fixes a cross-run `-AppendFile` reconciliation problem in the rolled-up interactions file, adds a supporting identity column to the AI-in-One (AIO) rolled-up interactions file, and improves the reliability of the Microsoft Agent 365 catalog export. Runs that do not append rolled-up interactions and do not use Agent 365 behave as in v1.11.11, with one additive exception: the AIO rolled-up interactions file now carries one extra identity column (described below). A one-time re-baseline applies to interaction append files created before v1.11.12. + +#### Fan-Out-Safe Cross-Run Append & One-Time Re-Baseline (`-AppendFile`) + +A single interaction (message) can produce **several** rolled-up rows — one per distinct combination of the analytical grain (for example, per accessed resource). v1.11.11 reconciled an appended rollup on the message identity alone, which treated all of a message's rows as one and could drop the extra rows when a later run re-emitted only some of them. v1.11.12 reconciles on the **full grain together with the message identity**, so every row is matched independently: overlapping rows de-duplicate, new rows are added, and existing-only rows are preserved — an exact union with nothing dropped or double-counted. To protect your data, PAX does **not** silently merge onto an append file that predates this change or that lacks any column the new reconciliation needs (for example an older AIO file created before the new identity column existed). Instead it leaves the old file untouched, writes this run's output to a new timestamped file, and prints re-baseline guidance. The practical effect is a **one-time re-baseline** — generate a fresh file once with v1.11.12 and append onto that from then on — with **no data lost** in the transition. + +#### Stable User Identity on the AI-in-One Rolled-Up Interactions File + +The AIO rolled-up interactions file now carries a stable, normalized user-identity column (`User_Id_Normalized`), matching the equivalent column the AI Business Value (AIBV) rollup already provides. It is **additive** — every existing column is unchanged — and it anchors the cross-run reconciliation above on a stable identity rather than a per-run surrogate. When `-Deidentify` is used, this column is anonymized with the same irreversible, format-preserving token as every other identity in the output, so it never exposes a raw user identity. + +#### Microsoft Agent 365 Catalog — Reliability + +The Microsoft Agent 365 catalog export is more resilient to Microsoft Graph throttling: catalog reads now retry automatically on rate-limit (HTTP 429) and transient server responses, honoring the service's `Retry-After` timing when provided. An internal variable-naming issue in the package-details path that could interfere with per-package processing is also corrected. There are no new switches and no change to how you run the Agent 365 export. + ### v1.11.11 Version 1.11.11 delivers the real fix for a cross-run `-AppendFile` reconciliation problem that persisted through v1.11.10 (reported by two customers), closes an Entra Users upload gap in a specific rollup + append combination, adds a new `-UserInfoFile` switch that lets you supply the user/organization directory from your own CSV instead of pulling it live from Entra, and extends the Microsoft Agent 365 catalog export to support app-only authentication (opt-in / pre-GA) in addition to interactive sign-in. Runs that use none of these paths behave as in v1.11.10, with one additive exception: the rolled-up interactions file now carries two trailing provenance columns used for reliable cross-run de-duplication. @@ -235,6 +251,12 @@ New sixth value on the `-Auth` ValidateSet for Azure-hosted headless execution ( ## What's New +### v1.11.12 + +- **Fan-out-safe cross-run append + one-time re-baseline (`-AppendFile`).** A single interaction (message) can produce **several** rolled-up rows — one per distinct combination of the analytical grain (for example, per accessed resource). Rollup append now reconciles on the **full grain together with the message identity** rather than the message identity alone, so every row is matched independently — overlaps de-duplicate, new rows are added, and existing-only rows are preserved (exact union, nothing dropped or double-counted). Append files created before v1.11.12 — or any file missing a column the new reconciliation needs (for example an older AI-in-One file created before the new identity column existed) — are **not** silently merged: the old file is left untouched, this run's output is written to a new timestamped file, and PAX prints re-baseline guidance. The practical effect is a **one-time re-baseline** (generate a fresh file once with v1.11.12 and append onto that from then on), with **no data lost** in the transition. The existing 0-row overwrite refusal and union-shrink protections still apply. +- **Stable user identity on the AI-in-One rolled-up interactions file (`User_Id_Normalized`).** The AI-in-One (AIO) rollup now carries a stable, normalized user-identity column, matching the equivalent column the AI Business Value (AIBV) rollup already provides. It is **additive** — every existing column is unchanged and it sits ahead of the trailing provenance columns — and it anchors the cross-run reconciliation above on a stable identity rather than a per-run surrogate. Under `-Deidentify` it is anonymized with the same irreversible, format-preserving token as every other identity in the output, so it never exposes a raw user identity. +- **Microsoft Agent 365 catalog — throttling resilience.** Catalog reads now retry automatically on rate-limit (HTTP 429) and transient server (5xx) responses, honoring the service's `Retry-After` timing when provided and otherwise backing off exponentially (capped at 60 seconds); non-throttling errors surface immediately. An internal variable-naming issue in the package-details path that could interfere with per-package processing is also corrected. There are no new switches and no change to how you run the Agent 365 export. + ### v1.11.11 - **Reliable cross-run append + one-time re-baseline (`-AppendFile`).** Rollup append now reconciles on each interaction's stable message identity — overlaps de-duplicate, new records are added, existing-only records are preserved (exact union, nothing dropped or double-counted). Append files created before v1.11.11 are **not** silently merged: the old file is left untouched, this run's output is written to a new timestamped file, and PAX prints re-baseline guidance. Older append files need a **one-time re-baseline**; **no data is lost**. diff --git a/script_archive/.gitkeep b/script_archive/.gitkeep index d0b498b..f26214d 100644 --- a/script_archive/.gitkeep +++ b/script_archive/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-02 (Purview v1.11.11, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.33) \ No newline at end of file +# Last updated: 2026-07-03 (Purview v1.11.12, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.34) \ No newline at end of file diff --git a/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 b/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 new file mode 100644 index 0000000..2a4e6c3 --- /dev/null +++ b/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.12.ps1 @@ -0,0 +1,33429 @@ +# Portable Audit eXporter (PAX) - Purview Audit Log Processor +# Version: v1.11.12 +# Requirements: PowerShell 7+ for default Graph API mode; PowerShell 5.1 supported ONLY with -UseEOM (serial Exchange Online Management mode, no parallel query/explosion). +# Default Activity Type: CopilotInteraction (captures ALL M365 Copilot usage including all M365 apps and Teams meetings) +# DSPM for AI activity types (specified via -ActivityTypes): AIInteraction, ConnectedAIAppInteraction, AIAppInteraction +# MIXED FREE/PAYG Activity Types: AIInteraction (currently Microsoft platforms only), ConnectedAIAppInteraction (Microsoft + third-party) +# PAYG Activity Types: AIAppInteraction (third-party AI via network DLP) +# ---PAYG only applies to third-party AI apps/agents audit records and never applies to any audit records generated by Microsoft AI apps/agents +# NOTE: Graph API mode automatically detects v1.0 (GA) or beta endpoints at runtime (no config needed) +# NOTE: Uses operationFilters with operation names for ALL activity types (e.g., "CopilotInteraction", "AIInteraction") +# See: https://learn.microsoft.com/en-us/office/office-365-management-api/office-365-management-activity-api-schema +<# +.SYNOPSIS + Export Microsoft Purview audit logs for Microsoft 365 Copilot and DSPM for AI activity types. + + *** WARNING: SENSITIVE DATA — CUSTOMER RESPONSIBILITY *** + + The audit data exported by this script is highly sensitive. Output may contain user + identifiers (UPN, email, GUID), file/site/resource paths, conversation and message IDs, + agent identifiers, prompt/response metadata (timestamps, lengths, classifications), and + other personally identifiable information drawn directly from your tenant's Unified + Audit Log. + + - Data is NOT hashed, masked, redacted, anonymized, or de-identified in any way. + Records are exported in their raw, attributable form exactly as Microsoft Purview + returns them. + - Outputs (CSV/Excel/JSON metrics, checkpoint files, logs) may contain confidential + business content, regulated data (PII, PHI, financial, IP), and end-user + communications. + - The customer (you / your organization) is solely responsible for the secure handling, + storage, transmission, retention, disclosure, access control, and deletion of all + data produced by this script, and for ensuring its use complies with all applicable + laws, regulations, contractual obligations, and internal policies — including but + not limited to GDPR, HIPAA, CCPA, employee monitoring laws, works-council + agreements, and data-residency requirements. + - Microsoft has no visibility into, control over, or responsibility for the data + customers extract using this tool or how that data is subsequently used, shared, or + stored. Microsoft disclaims any and all liability arising from or related to + customer use of this script and its output. + - Treat all output files as Highly Confidential. Restrict access to authorized + personnel with a documented business need. Encrypt at rest and in transit. Apply + tenant DLP / sensitivity labels as appropriate. + +.DESCRIPTION + Modes: + Standard - One row per audit record (raw CopilotEventData JSON preserved) + + Graph API Version Selection: + - PAX automatically detects the correct Microsoft Graph security/auditLog endpoint version + for your tenant (currently v1.0 with automatic fallback to beta if the v1.0 endpoint is + unavailable). Detection runs once per session. + - A single-line banner in the run output confirms which endpoint version is in use. + - No command-line switch is required; the selection is fully automatic. + + Parallel Explosion Processing: + - After data retrieval, explosion of records into rows can be parallelized + - Automatic when >500 records (uses job queue with ~1000 records per chunk) + - Control via -ExplosionThreads: 0=auto (2-8 threads, capped at min(ProcessorCount,8)), 1=serial, 2-32=explicit + - Output is identical to serial mode (same columns, data, row count; only order may differ) + + Reliability & Resilience: + - Automatic retry logic: Up to 3 attempts per partition with smart cooldown + - End-of-run summary: Shows Complete/Incomplete/Failed partitions with QueryIds + - Partial success: Continues processing with successful partitions if some fail + - Query naming: PAX_Query__PartX/Total visible in Purview UI + - Unified concurrency: MaxConcurrency parameter controls both EOM and Graph API modes (default: 10) + - Date-range accuracy: Client-side trimming ensures output contains only records within + [StartDate, EndDate), compensating for Purview API date-range bleed + - Checkpoint & Resume: All auth modes automatically save progress to checkpoint files, + enabling resumption after Ctrl+C, network failures, or any interruption via -Resume + + Filtering: + -AgentId : Filter to records matching specific AgentId value(s) + -AgentsOnly : Filter to records with any AgentId present (mutually exclusive with -ExcludeAgents) + -ExcludeAgents : Filter to records WITHOUT AgentId (mutually exclusive with -AgentId/-AgentsOnly) + -PromptFilter + Prompt : Only export messages where Message_isPrompt = True + Response : Only export messages where Message_isPrompt = False + Both : Export messages with either True or False isPrompt values + Null : Only export messages with null/undefined isPrompt values (rare) + Note: PromptFilter uses two-stage filtering for optimal performance: + Stage 1 (Pre-filter): Filters records before explosion based on message content + Stage 2 (Message-level): Filters individual messages during explosion + + -UserIds : Filter to specific user identifier(s) + LIVE MODE: SERVER-SIDE filtering at Purview (efficient, no unnecessary data transfer) + + Accepted formats: + • User Principal Name (UPN): "john.doe@contoso.com" + • SMTP Address: "john.doe@contoso.com" + • User GUID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + Examples: + -UserIds "john.doe@contoso.com" + -UserIds "john.doe@contoso.com","jane.smith@contoso.com","bob.jones@contoso.com" + + -GroupNames : Filter to members of distribution/security group(s) + LIVE MODE ONLY: Groups automatically expanded to individual users after authentication. + - EOM mode: uses Get-DistributionGroupMember (Exchange Online RBAC) + - Graph API mode: uses Get-MgGroup + Get-MgGroupMember (requires GroupMember.Read.All) + + Accepted formats (LIVE MODE only): + • Group Display Name: "Executive Leadership Team" + • Group Email (Alias): "exec-team@contoso.com" + • Group GUID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + • Distinguished Name: "CN=ExecTeam,OU=Groups,DC=contoso,DC=com" + Examples: + -GroupNames "Executive Leadership" + -GroupNames "exec-team@contoso.com" + -GroupNames "Engineering Managers","Product Leads","Sales Directors" + Note: Groups are expanded once after authentication + + Combining UserIds + GroupNames (LIVE MODE ONLY): + • When both are specified, the script combines and deduplicates the user lists + • Example: -UserIds "ceo@contoso.com" -GroupNames "Board of Directors" + Pulls records for the CEO plus all expanded board members (duplicates removed) + + COMBINING FILTERS - Powerful Use Cases: + + All filters can be combined for highly targeted data extraction. Filter application order: + + FILTER APPLICATION ORDER: + 1. User/Group filtering (server-side via -UserIds) + 2. Agent filtering (AgentsOnly, AgentId, or ExcludeAgents) + 3. PromptFilter (Prompt, Response, Both, or Null) + + NOTE: Applying User/Group filtering first improves performance by reducing the dataset before subsequent filters. + + TWO-FILTER COMBINATIONS: + + User + Agent: + Use Case: Analyze specific user(s) interactions with Copilot agents + Example: "Show me all agent usage by our power users" + Command: -UserIds "poweruser@contoso.com" -AgentsOnly + + User + PromptFilter: + Use Case: Focus on conversation patterns (prompts/responses) for specific users + Example: "Show me only the questions asked by the executive team" + Command: -GroupNames "Executive Team" -PromptFilter Prompt + Result: Removes resource-only explosion rows, keeps only message data + + Agent + PromptFilter: + Use Case: Analyze agent conversation quality, prompt engineering effectiveness + Example: "Show me all prompts sent to our custom declarative agent" + Command: -AgentId "CopilotStudio.Declarative.abc123" -PromptFilter Prompt + + THREE-FILTER COMBINATION: + + User + Agent + PromptFilter: + Use Case: Deep-dive conversation analysis for specific users with agents + Example: "Show me all questions the sales team asked our Sales Copilot agent" + Command: -GroupNames "Sales Team" -AgentId "SalesCopilot.Agent" -PromptFilter Prompt + Benefits: + • Server-side filtering reduces data transfer (live mode) + • Agent filter removes non-agent interactions + • PromptFilter removes responses and resource-only rows + • Result: Clean dataset of just sales team questions to the agent + + PowerShell 7+ required for default Graph API mode. PowerShell 5.1 is supported ONLY when running in -UseEOM mode (serial Exchange Online Management cmdlets, no parallelism). + +.EXECUTIONPOLICY + No internal execution policy bypass. Use external host invocation if needed: + pwsh.exe -ExecutionPolicy Bypass -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 + # PS 5.1 host (EOM mode only): + powershell.exe -ExecutionPolicy Bypass -File .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM -StartDate 2025-10-01 -EndDate 2025-10-02 + +.POWERSHELLVERSIONS + PowerShell 7+ required (default Graph API mode; download: https://aka.ms/powershell). + PowerShell 5.1 supported ONLY with -UseEOM (serial EOM mode; no parallel query/explosion). + +.EXAMPLE + # Basic export with auto-generated timestamped filename + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath C:\Temp\ +.EXAMPLE + # PowerShell 5.1 (EOM mode only — serial, no parallelization) + powershell -File .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only records with agents present + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -AgentsOnly -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only records WITHOUT agents + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ExcludeAgents -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only prompt messages (Message_isPrompt = True) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -PromptFilter Prompt -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only response messages (Message_isPrompt = False) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -PromptFilter Response -OutputPath C:\Temp\ +.EXAMPLE + # Combine filters: agents + prompts only + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -AgentsOnly -PromptFilter Prompt -OutputPath C:\Temp\ +.EXAMPLE + # Filter to specific users + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "john.doe@contoso.com","jane.smith@contoso.com" -OutputPath C:\Temp\ +.EXAMPLE + # Emit metrics JSON alongside CSV (auto-generated timestamped filename) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-01 -EmitMetricsJson -OutputPath C:\Temp\ +.EXAMPLE + # Emit metrics JSON to custom path + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-01 -EmitMetricsJson -MetricsPath C:\Temp\purview_metrics_20251001.json -OutputPath C:\Temp\ +.EXAMPLE + # AutoCompleteness remediation workflow: first run incomplete (exit code 10), second run resolves saturated windows + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-05 -EndDate 2025-10-05 -EmitMetricsJson -OutputPath C:\Temp\ + # (Exit code 10 indicates saturated windows remain) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-05 -EndDate 2025-10-05 -AutoCompleteness -EmitMetricsJson -OutputPath C:\Temp\ +.EXAMPLE + # Filter to security group members (automatically expanded) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Executive Leadership" -OutputPath C:\Temp\ +.EXAMPLE + # Filter to multiple groups + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Executive Team","Engineering Managers" -OutputPath C:\Temp\ +.EXAMPLE + # Graph API: SharePoint/OneDrive document activity with record & service filters + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-12-01 -EndDate 2025-12-02 -ActivityTypes FileAccessed,FilePreviewed -RecordTypes sharePointFileOperation -ServiceTypes SharePoint,OneDrive -OutputPath C:\Temp\ +.EXAMPLE + # Microsoft 365 usage bundle (Exchange, SharePoint, OneDrive, Teams, Forms, Stream, Planner, PowerApps) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -CombineOutput -OutputPath C:\Temp\ +.EXAMPLE + # Rollup: CopilotInteraction-only run, deletes raw CSVs after rolling up + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -Rollup -OutputPath C:\Temp\ +.EXAMPLE + # Rollup + Raw: CopilotInteraction-only run, keeps raw CSVs alongside rollup output + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -RollupPlusRaw -OutputPath C:\Temp\ +.EXAMPLE + # Rollup with a fixed label in the unused deeper org-hierarchy levels (AI-in-One / AI Business Value) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -Rollup -FillerLabel Fixed -FillerLabelText "Assistive Directs" -OutputPath C:\Temp\ +.EXAMPLE + # Deidentify: anonymize all identities in the output so it can be shared safely (CSV only) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -RollupPlusRaw -Deidentify -OutputPath C:\Temp\ +.EXAMPLE + # Rollup: M365 usage bundle run, deletes raw CSV after rolling up + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -Rollup -OutputPath C:\Temp\ +.EXAMPLE + # Export with execution telemetry for performance analysis + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeTelemetry -OutputPath C:\Temp\ +.EXAMPLE + # Include Microsoft Agent 365 catalog alongside the audit run (separate Agent365_.csv) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeAgent365Info -OutputPath C:\Temp\ -OutputPathAgent365Info C:\Temp\ +.EXAMPLE + # Export ONLY the Microsoft Agent 365 catalog (skips audit pull; -Force auto-confirms the preflight) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyAgent365Info -Force -OutputPathAgent365Info C:\Temp\ +.EXAMPLE + # Combine individual users and groups + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "ceo@contoso.com" -GroupNames "Board of Directors" -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: User + PromptFilter (conversation focus, removes resource-only rows) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "poweruser@contoso.com" -PromptFilter Both -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: Group + Agent (team adoption of specific agent) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Sales Team" -AgentsOnly -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: User + Agent + PromptFilter (prompts sent to agents by specific users) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "analyst@contoso.com" -AgentId "DataAnalysis.Agent" -PromptFilter Prompt -OutputPath C:\Temp\ + +.EXAMPLE + # APPENDING: Append to existing CSV file (relative filename in OutputPath directory) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "MyReport.csv" -OutputPath C:\Temp\ +.EXAMPLE + # APPENDING: Append to existing CSV file (full path) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "C:\Data\Audit\CopilotActivity.csv" +.EXAMPLE + # APPENDING: Append with single activity type to existing CSV + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "CopilotOnly.csv" -ActivityTypes CopilotInteraction -OutputPath C:\Temp\ + +.EXAMPLE + # REMOTE OUTPUT: Upload all run artifacts to a SharePoint document library folder + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://contoso.sharepoint.com/sites/Analytics/Shared Documents/PAX" +.EXAMPLE + # REMOTE OUTPUT: Upload all run artifacts to a Microsoft Fabric Lakehouse as Delta tables (interactive auth) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" +.EXAMPLE + # REMOTE OUTPUT: Headless run with managed identity writing to OneLake (designed for Azure Container Apps Jobs / VMs / Functions) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -Auth ManagedIdentity -TenantId "" -ClientId "" -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" +.EXAMPLE + # REMOTE OUTPUT: Headless run with app registration uploading to SharePoint + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -Auth AppRegistration -TenantId "" -ClientId "" -ClientCertificateThumbprint "" -OutputPath "https://contoso.sharepoint.com/sites/Analytics/Shared Documents/PAX" +.EXAMPLE + # PER-DATA-TYPE DESTINATIONS: Send Purview audit to a Fabric Lakehouse, the Users CSV to a different lakehouse table folder, and the run log to a Files subfolder + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" -OutputPathUserInfo "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse/Tables" -OutputPathLog "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse/Files/logs" + +.EXAMPLE + # CSV export with separate files per activity type (default behavior) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ActivityTypes CopilotInteraction,ConnectedAIAppInteraction + +.EXAMPLE + # CSV export with combined output (single file with all activity types) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ActivityTypes CopilotInteraction,ConnectedAIAppInteraction -CombineOutput + +.EXAMPLE + # Resume interrupted operation (auto-discover checkpoint in OutputPath) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume -OutputPath C:\Temp\ + +.EXAMPLE + # Resume from specific checkpoint file + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume "C:\Temp\.pax_checkpoint_20251215_143022.json" + +.EXAMPLE + # Resume with Force (use most recent checkpoint without prompting) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume -OutputPath C:\Temp\ -Force + +.NOTES + Reliability Features: + - Automatic retry: Failed partitions retried up to 3 times with smart delays + - Status tracking: Each partition tracked with QueryId and QueryName throughout execution + - Partial success: Script continues with successful data even if some partitions fail + - End summary: Detailed report showing Complete/Incomplete/Failed partitions + - Query names: Visible in Purview as PAX_Query__PartX/Total + Example: PAX_Query_20241101_0000-20241101_0100_Part27/134 + + Concurrency Control: + - MaxConcurrency (default: 10): single parameter that applies to both modes. + • EOM mode: limits concurrent serial queries. + • Graph API mode: limits concurrent partition execution. + - Example: -MaxConcurrency 6 (reduces from the default 10 for rate-limit sensitive environments). + + Graph API Endpoint Selection: + PAX automatically selects the correct Microsoft Graph security/auditLog endpoint for the + run. The v1.0 (GA) endpoint is preferred; if v1.0 is not available for your tenant PAX + falls back to the beta endpoint. Selection runs once per session and is remembered for the + rest of the run. A single-line banner in the run output confirms which endpoint is in use. + No command-line switch is required. + + Example End-of-Run Summary: + ═══════════════════════════════════════════════════════════════ + QUERY SUBMISSION SUMMARY + ═══════════════════════════════════════════════════════════════ + Total Partitions: 134 + Sent and Complete: 131 + [!] Sent but Incomplete: 2 + ✗ Never Sent: 1 + ═══════════════════════════════════════════════════════════════ + + Checkpoint & Resume: + All authentication modes automatically create checkpoint files to preserve progress + during long-running operations. This enables resumption after Ctrl+C, network failures, + token expiry, or any interruption without losing completed work. + + Checkpoint Behavior: + • Created automatically for ALL auth modes (WebLogin, DeviceCode, AppRegistration) + • Saved to OutputPath as .pax_checkpoint_.json + • Updated after each partition completes + • Deleted automatically on successful completion + • Stores ALL processing parameters for complete state restoration + + Token Refresh: + • AppRegistration mode: + - Proactive refresh at ~45-50 minutes (before token expiry) + - Reactive refresh on 401 Unauthorized (backup) + - Fully automatic, silent re-authentication using stored credentials + • Interactive modes (WebLogin/DeviceCode): + - Reactive only: Triggered when 401 Unauthorized detected + - Attempts silent refresh first (using SDK cached refresh token) + - Prompts user only if silent refresh fails + • 403 Forbidden errors: Indicates permissions issue, NOT token expiry + - Token refresh will not help; check AuditLogsQuery.Read.All consent and roles + + Resume Mode (Standalone): + The -Resume switch restores ALL settings from the checkpoint file. + No other processing parameters can be specified with -Resume. + + Allowed with -Resume: + • -Force Use most recent checkpoint without prompting + • -Auth Override authentication method + • -TenantId Override tenant ID (for AppRegistration) + • -ClientId Override client ID (for AppRegistration) + • -ClientSecret Provide client secret (for AppRegistration) + + Usage Examples: + • -Resume Auto-discover checkpoint in current directory + • -Resume "path\to\file" Use specific checkpoint file + • -Resume -Force Use most recent checkpoint without prompting + • -Resume -Auth DeviceCode Resume with different auth method + + API Record Limits & Auto-Subdivision: + Graph API 1,000,000 Record Limit: + The Microsoft Graph security/auditLog API has a hard limit of 1,000,000 records per query. + PAX automatically detects when this limit is reached and handles it gracefully. + + Detection: + • Identified when a partition returns exactly 1,000,000 records with no nextLink + • Warning displayed: "[SUBDIVISION] Partition X/Y - Fetched 1,000,000 records (Graph API limit reached)" + + Auto-Subdivision: + • Uses same BlockHours subdivision algorithm as EOM 10K limit handling + • Partition time window is halved and re-queried recursively + • Minimum window: 0.016667 hours (1 minute) - cannot subdivide below this + • If minimum reached, warning displayed and available records returned + + Recommendations for High-Volume Tenants: + • Use smaller -BlockHours (e.g., 0.25 or 0.1) for very active tenants + • Consider shorter date ranges for initial exports + • Monitor "[SUBDIVISION]" messages to tune BlockHours + + EOM 10,000 Record Limit: + The Exchange Online Management (EOM) Search-UnifiedAuditLog cmdlet returns maximum 10,000 + records per query. PAX automatically subdivides time windows when this limit is reached. + + Required Permissions Matrix: + The runtime startup banner emits a tailored, per-run version of this matrix + (highlighting only what THIS invocation needs). The static reference below lists + every permission PAX may require, grouped by feature, with the trigger switch + that turns it on. Each line is tagged so you know WHERE to grant it: + + [App-only] = Microsoft Graph application permission, granted on the + app registration (-Auth AppRegistration) or on the managed + identity's service principal (-Auth ManagedIdentity). + Requires admin consent. + [Delegated] = Microsoft Graph delegated permission, consented at + interactive sign-in by the signed-in user. + [Role] = Entra ID directory role assigned to the signed-in user + (NOT a Graph scope; granted in the Entra admin center). + [Azure RBAC] = Azure role assignment on an Azure resource (NOT a Graph + scope; granted in the Azure portal / az role assignment). + [Exchange] = Exchange Online RBAC role / role group (NOT a Graph scope; + granted in the Exchange admin center). + + Audit query (always required for normal runs; skipped only by -OnlyUserInfo): + [App-only / Delegated] AuditLogsQuery.Read.All + (umbrella - all audit record types) + + M365 usage bundle (only when -IncludeM365Usage is set): + [App-only / Delegated] AuditLogsQuery-Exchange.Read.All + [App-only / Delegated] AuditLogsQuery-SharePoint.Read.All + [App-only / Delegated] AuditLogsQuery-OneDrive.Read.All + + Entra directory + license enrichment (only when -IncludeUserInfo or -OnlyUserInfo is set): + [App-only / Delegated] User.Read.All (read /users) + [App-only / Delegated] Organization.Read.All (read /subscribedSkus) + + Group expansion (only when -GroupNames is set): + [App-only / Delegated] GroupMember.Read.All (read /groups + /groups/{id}/members) + + Remote output - SharePoint (only when an -OutputPath* value is a SharePoint URL): + [App-only / Delegated] Sites.ReadWrite.All (resolve site/drive via /sites + /drives) + [App-only / Delegated] Files.ReadWrite.All (PUT / createUploadSession to /drives/{id}/items) + ALSO: the destination user / app / managed identity must have at least + Member access on the target SharePoint site (granted in SharePoint, + not via Graph permissions). + + Remote output - Microsoft Fabric / OneLake (only when an -OutputPath* value is a OneLake URL): + [Azure RBAC] Storage Blob Data Contributor on the Fabric workspace + (assigned at workspace scope to the audit identity) + [Fabric] Contributor on the Fabric workspace + (granted in the Fabric portal: Workspace settings -> Manage access) + [Tenant] "Service principals can use Fabric APIs" tenant setting must be + enabled (Fabric Admin portal -> Tenant settings -> Developer settings) + when running with -Auth AppRegistration or -Auth ManagedIdentity. + NOTE: OneLake uses an INDEPENDENT access token (audience https://storage.azure.com/) + acquired via the Az.Accounts module. Microsoft Graph permissions do NOT + apply to OneLake writes. + + EOM-mode audit query (only when -UseEOM is set): + [Exchange] View-Only Audit Logs role + -- OR -- + [Exchange] Compliance Management role group + -- OR -- + [Exchange] Organization Management role group + -- OR -- + [Exchange] Custom RBAC role granting the Search-UnifiedAuditLog cmdlet + NOTE: EOM mode does NOT use Microsoft Graph scopes for the audit query itself. + + Managed identity bootstrap (only when -Auth ManagedIdentity is set): + All [App-only] scopes triggered by the switches above must be admin-consented + on the managed identity's service principal as APPLICATION permissions (not + delegated). The fabric_resources/Prereqs/Grant-PAXPermissions.ps1 script automates + this one-time grant, including the Storage Blob Data Contributor RBAC role + when an -OutputPath* value is a OneLake URL. + + Remote Output Setup (-OutputPath* with SharePoint or OneLake URLs): + PAX can write all run artifacts (CSV / XLSX / log / metrics / checkpoint) to either a + SharePoint document library folder or a Microsoft Fabric Lakehouse (Delta tables under + Tables/ plus operational artifacts under Files/) instead of a local directory. Storage + tier is inferred from the path form supplied to each -OutputPath* switch. The destination + must already exist (or its parent must, for SharePoint subfolder creation) before the run + starts. PAX will not provision the Fabric workspace, the Lakehouse, or the SharePoint site + for you. + + One-time setup for a SharePoint -OutputPath* destination: + 1. Identify (or create) the target SharePoint site and document library folder. + 2. Grant the identity that runs PAX (the signed-in user for interactive auth, OR + the app registration / managed identity for unattended auth) the following + Microsoft Graph application or delegated permissions, matching your -Auth choice: + • Sites.ReadWrite.All + • Files.ReadWrite.All + 3. For app-only auth (-Auth AppRegistration / ManagedIdentity), have an admin + consent the permissions in the Entra ID portal. + 4. Pass the folder URL exactly as it appears in the browser address bar: + -OutputPath "https://.sharepoint.com/sites//[/]" + Subfolders that do not yet exist below the document library root are created + automatically on first use. + + One-time setup for a Microsoft Fabric / OneLake -OutputPath* destination: + 1. Install the Az.Accounts PowerShell module on the machine that runs PAX: + Install-Module Az.Accounts -Scope CurrentUser + (Already present on Azure Cloud Shell and the official PowerShell container + image used by the ACA Job runbook.) + 2. In the Microsoft Fabric portal (https://app.fabric.microsoft.com): + a. Create or identify the target Fabric workspace. + b. Inside that workspace, create or identify the target Lakehouse + (or Warehouse). PAX writes to the item's Files area, not Tables. + c. Add the identity that runs PAX (user, app registration, or managed + identity) to the workspace as Contributor (or higher). Member, + Admin, and workspace-owner roles also work; Viewer does NOT. + 3. In the Azure portal (the Fabric workspace surfaces as an Azure resource + under the Microsoft.Fabric provider): + • Assign the same identity the Azure RBAC role + "Storage Blob Data Contributor" on the Fabric workspace (or on the + Fabric capacity / subscription scope, if you want it to apply more + broadly). This role is what authorizes the underlying OneLake DFS + write calls; the Fabric workspace role alone is not sufficient. + 4. For service-principal or managed-identity runs (-Auth AppRegistration / + ManagedIdentity) the Fabric tenant administrator must also enable, in the + Fabric Admin portal under "Tenant settings → Developer settings": + • "Service principals can use Fabric APIs" + (scoped to the security group that contains your service principal / + managed identity, or tenant-wide). + 5. Pass the OneLake URL exactly in the form shown in the Lakehouse "Properties" + pane, ABFS URL → switch the scheme to https and the host to the DFS endpoint: + -OutputPath "https://onelake.dfs.fabric.microsoft.com//.Lakehouse" + Regional aliases of the form "https://-onelake.dfs.fabric.microsoft.com/..." + are also accepted. The item may be given either by name (.Lakehouse) or by its + GUID with no suffix (the form the Fabric portal shows in the item URL, + e.g. ...//); both are accepted. Tables land under + /Tables; operational artifacts under /Files (the run log + location is controllable via -OutputPathLog). + 6. Schemas-mode Lakehouses: Microsoft Fabric's default new-Lakehouse template + organizes tables as /Tables// (the default schema + is 'dbo'). To land Delta tables under a specific schema on a Schemas-mode + Lakehouse, include the schema in the -OutputPath* URL: + -OutputPath "https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables/dbo" + Non-Schemas Lakehouses (tables directly under Tables/) accept either + the lakehouse root or the .../Tables form unchanged. + + Common pitfalls: + • Forgetting the "Storage Blob Data Contributor" Azure RBAC role: produces + HTTP 403 (AuthorizationFailure) on the very first OneLake write. The Fabric + Contributor role alone is not enough. + • Forgetting the tenant setting "Service principals can use Fabric APIs": + produces HTTP 401 / 403 for app-only and managed-identity runs even when + every other permission is correct. + • On long runs, the OneLake access token is refreshed automatically; no + customer action is required to keep multi-hour runs alive. + +.PARAMETER StartDate + Start date for audit log query in live mode (format: yyyy-MM-dd or MM/dd/yyyy). + +.PARAMETER EndDate + End date for audit log query in live mode (format: yyyy-MM-dd or MM/dd/yyyy). + +.PARAMETER OutputPath + Destination for the Purview audit output (raw, rollup, or event-level CSV / XLSX). Targets the + Purview audit data stream only — the EntraUsers CSV is controlled by -OutputPathUserInfo. + Storage tier is inferred from the path form: + drive-rooted absolute path → Local; https://*.sharepoint* URL → SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse* URL → Fabric. UNC paths are rejected. + Accepts either a folder (PAX auto-generates the basename) or a fully qualified path including + basename. When omitted, defaults to $PSScriptRoot for Local tier. Paired with -AppendFile + (both switches act on the Purview audit CSV). + +.PARAMETER OutputPathUserInfo + Destination override for the EntraUsers / MAC licensing CSV. Same tier-inference and form + rules as -OutputPath. Independent of -OutputPath so the Users CSV can land in a different + folder, SharePoint library, or Fabric lakehouse from the Purview audit output (subject to + the tier-consistency rule: all -OutputPath* values must resolve to the same storage tier). + When omitted, the Users CSV lands beside -OutputPath. Paired with -AppendUserInfo. + +.PARAMETER OutputPathAgent365Info + Destination override for the Microsoft Agent 365 catalog CSV. Same tier-inference and form + rules as -OutputPath. Independent of -OutputPath so the Agent 365 CSV can land in a different + folder, SharePoint library, or Fabric lakehouse from the Purview audit output (subject to + the tier-consistency rule: all -OutputPath* values must resolve to the same storage tier). + When omitted, the Agent 365 CSV lands beside -OutputPath. Paired with -AppendAgent365Info. + +.PARAMETER OutputPathLog + Destination override for the run log. Accepts Local, SharePoint, or Fabric Files/ targets + (Fabric Tables/* URLs are rejected — logs are not tabular). When omitted, the log lands in + $PSScriptRoot for Local runs and in the active remote destination's run-log folder for + SharePoint / Fabric runs. + +.PARAMETER FlatDepth + Maximum JSON flatten depth for exploding CopilotEventData and AuditData (default 120). + + The script automatically generates descriptive filenames based on: + • Activity types being exported + • Export mode (CSV vs Excel, combined vs separate) + • Current timestamp (yyyyMMdd_HHmmss format) + + Examples of auto-generated filenames: + • Purview_Audit_UsageActivity_CopilotInteraction_20251110_143022.csv (single activity type) + • Purview_Audit_UsageActivity_CombinedActivityTypes_20251110_143022.csv (multiple activity types with -CombineOutput) + • Purview_Audit_MultiTab_20251110_143022.xlsx + + When an -OutputPath* value is a SharePoint or OneLake URL, all output files, directory paths, + and log file paths are reported in the run banners and progress messages using the + remote SharePoint or Fabric URL — not the temporary local scratch path used internally + during the run. + + Note: OutputPath accepts ONLY directory paths, not filenames. + Use -AppendFile parameter to specify a custom filename for appending to existing files. + +.PARAMETER Auth + Authentication method. Options: + • WebLogin – Interactive browser authentication + • DeviceCode – Device code flow for headless scenarios + • Credential – Legacy username/password prompt or GRAPH_* env vars + • Silent – Managed identity or pre-cached token + • AppRegistration – Service principal using client secret or certificate + • ManagedIdentity – Azure-hosted managed identity (system-assigned by default; + pass the user-assigned client ID via the AZURE_CLIENT_ID + environment variable when applicable). Designed for headless + execution inside Azure (Container Apps Job, VM, App Service, + Functions, etc.). Requires the same Microsoft Graph application + permissions as -Auth AppRegistration consented to the managed + identity, and (when an -OutputPath* value is a OneLake URL) + the Azure RBAC role + Storage Blob Data Contributor on the OneLake workspace. + +.PARAMETER TenantId + Azure AD tenant ID (GUID). Required for -Auth AppRegistration unless GRAPH_TENANT_ID + environment variable is set. + +.PARAMETER ClientId + Azure AD app registration client ID (GUID). Required for -Auth AppRegistration unless + GRAPH_CLIENT_ID environment variable is set. + +.PARAMETER ClientSecret + Client secret value for app registration authentication. You can pass it directly, + convert from a secure string, or set it through GRAPH_CLIENT_SECRET. + +.PARAMETER ClientCertificateThumbprint + Thumbprint of a certificate located in the CurrentUser or LocalMachine "My" store. + Used when -Auth AppRegistration should authenticate with a certificate instead of a + client secret. Optional environment variable: GRAPH_CLIENT_CERT_THUMBPRINT. + +.PARAMETER ClientCertificateStoreLocation + Certificate store to search when using ClientCertificateThumbprint. Valid values: + CurrentUser (default) or LocalMachine. + +.PARAMETER ClientCertificatePath + Path to a PFX file containing the certificate for app registration auth. Optional + environment variable: GRAPH_CLIENT_CERT_PATH. + +.PARAMETER ClientCertificatePassword + Password for the PFX file specified by ClientCertificatePath. Accepts secure string or + plain text (converted internally). Optional environment variable: + GRAPH_CLIENT_CERT_PASSWORD. + +.PARAMETER BlockHours + Time window size in hours for each audit log query block. + Range: 0.016667 to 24 hours. Default: 0.5 (30 minutes) + +.PARAMETER PartitionHours + Time partition size in hours for Graph API parallel processing. + Range: 1-72. Default: 0 (auto-calculated based on date range and MaxPartitions) + +.PARAMETER MaxPartitions + Maximum number of time partitions for parallel query execution. + Range: 1-1000. Default: 160 + +.PARAMETER ResultSize + Maximum records returned per Search-UnifiedAuditLog query. + Range: 1 to 10000. Default: 10000 + +.PARAMETER PacingMs + Delay in milliseconds between audit log queries for throttling control. + Range: 0 to 10000. Default: 0 + +.PARAMETER ActivityTypes + Array of Purview audit activity types to query. + Default: @('CopilotInteraction') + Examples: CopilotInteraction, ConnectedAIAppInteraction, AIInteraction, AIAppInteraction + Note: CopilotInteraction captures ALL Microsoft 365 Copilot usage including Teams meetings (AppHost="Teams") + +.PARAMETER RecordTypes + Filter audit records by record type (Graph API mode only). + Accepts one or more record type names that map to Microsoft 365 workload categories. + Examples: SharePointFileOperation, ExchangeItem, MicrosoftTeams, AzureActiveDirectory + + BEHAVIOR: + • Standard mode: Filters server-side to specified record types + • With -IncludeM365Usage: Merged with curated M365 usage bundle record types + + LIMITATIONS: + • Graph API mode only (not supported with -UseEOM) + + Use with -ActivityTypes and -ServiceTypes for precise workload targeting. + +.PARAMETER ServiceTypes + Filter audit records by service/workload (Graph API mode only). + Accepts one or more service names representing Microsoft 365 workloads. + Examples: Exchange, SharePoint, OneDrive, MicrosoftTeams, AzureActiveDirectory + + BEHAVIOR: + • Standard mode: Filters server-side to specified services + • Note: Multiple services may cause separate queries per service + + CRITICAL LIMITATIONS: + • Graph API mode only (not supported with -UseEOM) + • IGNORED when -IncludeM365Usage is active: The M365 usage bundle intentionally + sets ServiceTypes to null for optimal single-pass query performance. + Your -ServiceTypes values will be silently overridden. + + RECOMMENDATION: For M365 usage scenarios, use -IncludeM365Usage without -ServiceTypes. + For targeted workload queries without M365 bundle, use -ServiceTypes with -ActivityTypes. + +.PARAMETER ExplodeArrays + [Deprecated] This switch is being phased out and will be removed in a future release. + +.PARAMETER ExplodeDeep + [Deprecated] This switch is being phased out and will be removed in a future release. + +.PARAMETER RAWInputCSV + [Deprecated] This switch is being phased out and will be removed in a future release. + +.PARAMETER MaxConcurrency + Maximum concurrent queries/partitions (1-10). + - EOM mode: limits concurrent serial queries + - Graph API mode: limits concurrent partition execution + Default: 10 (Microsoft Purview enforces a 10 concurrent search job limit per user account) + +.PARAMETER EnableParallel + Force enable parallel processing (overrides ParallelMode setting). + +.PARAMETER MaxParallelGroups + Maximum activity groups processed concurrently in parallel mode. + Range: 0 to 50. Default: 8 + +.PARAMETER ParallelMode + Parallel processing mode. Options: Off, On, Auto (default) + Auto: Enables parallel for PowerShell 7+ environments automatically. + +.PARAMETER ExplosionThreads + Number of threads for parallel explosion processing (post-retrieval phase). + 0 (default): Auto-detect based on CPU cores (2 to min(ProcessorCount, 8) threads) + 1: Force serial processing (disable parallel explosion) + 2-32: Explicit thread count (ValidateRange 0-32; auto path caps at 8) + Requires PowerShell 7+. Falls back to serial on PS5. + +.PARAMETER DisableAdaptive + Disable adaptive safeguards (memory/latency/concurrency smoothing). + +.PARAMETER ProgressSmoothingAlpha + Weight for smoothing dynamic progress total recalculation. + Range: 0.0 to 1.0. Default: 0.3 (0 = off) + +.PARAMETER HighLatencyMs + Partition average latency threshold (ms) triggering concurrency reduction. + Range: 1000 to 600000. Default: 90000 + +.PARAMETER MemoryPressureMB + Working set (MB) threshold to trigger concurrency reduction. + Range: 256 to 32768. Default: 1500 + +.PARAMETER MaxMemoryMB + Maximum process memory (MB) before flushing the in-memory record buffer to disk. + When exceeded, PAX persists the buffered records to a JSONL working file and releases the + in-memory copy so the run can continue under tight memory budgets. + Range: -1 to 65536. Default: -1 (auto = 75% of system RAM). Use 0 to disable. + +.PARAMETER StatusIntervalSeconds + How often (in seconds) to display a status update during job polling and backpressure waits. + Reduce to see more frequent progress output; increase to reduce console noise on long runs. + Range: 30 to 600. Default: 60 + +.PARAMETER LowLatencyMs + Sustained low latency threshold to consider concurrency step-up. + Range: 100 to 600000. Default: 20000 + +.PARAMETER LowLatencyConsecutive + Required consecutive low-latency groups before step-up. + Range: 1 to 10. Default: 2 + +.PARAMETER ThroughputDropPct + Percentage drop vs baseline required (with high latency) to justify reduction. + Range: 1 to 100. Default: 15 + +.PARAMETER ThroughputSmoothingAlpha + EMA smoothing for throughput baseline. + Range: 0.0 to 1.0. Default: 0.3 + +.PARAMETER AdaptiveConcurrencyCeiling + Upper bound for adaptive concurrency step-ups. + Range: 1 to 50. Default: 6 + +.PARAMETER ExportProgressInterval + Frequency of progress updates during export phase. + Range: 1 to 10000. Default: 10 + +.PARAMETER StreamingSchemaSample + Number of rows sampled before freezing CSV schema in streaming mode (SERIAL MODE ONLY). + Range: 100 to 50000. Default: 5000 + Higher values capture more columns but delay schema freeze. + NOTE: In parallel mode (PS7+), full schema discovery is used instead, scanning ALL rows + for 100% column coverage. This parameter only affects serial mode processing. + +.PARAMETER StreamingChunkSize + Number of rows per write batch in streaming CSV export. + Range: 100 to 50000. Default: 5000 + Lower values reduce memory pressure, higher values improve throughput. + +.PARAMETER AgentId + Filter to records matching specific AgentId value(s). + Example: -AgentId "CopilotStudio.Declarative.abc123" + +.PARAMETER AgentsOnly + Filter to records with any AgentId present (mutually exclusive with -ExcludeAgents). + +.PARAMETER PromptFilter + Filter messages by isPrompt property. + Options: Prompt (True), Response (False), Both (True/False), Null (undefined) + +.PARAMETER CircuitBreakerThreshold + Consecutive block failures before opening circuit breaker. + Range: 1 to 50. Default: 5 + +.PARAMETER CircuitBreakerCooldownSeconds + Cooldown duration (seconds) after circuit breaker trips. + Range: 5 to 3600. Default: 120 + +.PARAMETER BackoffBaseSeconds + Base seconds for exponential backoff between block retries. + Range: 0.1 to 120. Default: 1.0 + +.PARAMETER BackoffMaxSeconds + Maximum cap for exponential backoff delay (seconds). + Range: 1 to 600. Default: 45 + +.PARAMETER ExcludeAgents + Filter to records WITHOUT AgentId (mutually exclusive with -AgentId/-AgentsOnly). + +.PARAMETER UserIds + Filter to specific user identifier(s). + LIVE MODE: Server-side filtering at Purview (efficient). + Accepts UPN, SMTP address, or user GUID. + +.PARAMETER GroupNames + Filter to members of distribution/security group(s). + LIVE MODE ONLY: Groups automatically expanded after authentication. + +.PARAMETER Help + Display script help information. + +.PARAMETER EmitMetricsJson + Emit structured metrics JSON alongside output file. + Default filename: .metrics.json + +.PARAMETER MetricsPath + Override metrics output path. Requires -EmitMetricsJson. + +.PARAMETER AutoCompleteness + Aggressively subdivide windows returning server 10K limit until complete or minimum window reached. + +.PARAMETER IncludeTelemetry + Export execution telemetry CSV alongside audit data (Graph API mode only). + Creates a separate CSV file with one row per partition containing timing and performance metrics. + Useful for analyzing query execution patterns, identifying bottlenecks, and capacity planning. + File naming: _telemetry_.csv + Not available in EOM mode or OnlyUserInfo mode. + +.PARAMETER IncludeCopilotInteraction + Adds CopilotInteraction to the activity list even when you provide a custom -ActivityTypes array. + Useful when combining Copilot telemetry with targeted classic workloads without redefining defaults. + +.PARAMETER IncludeM365Usage + Adds a curated, trimmed Microsoft 365 usage bundle spanning Exchange mail access, + SharePoint/OneDrive file access, Teams chat/messaging, Teams meeting lifecycle, and + Copilot/Connected-AI interaction signals. + + ACTIVITY TYPES INCLUDED: + Exchange: MailItemsAccessed, MailboxLogin, Send + SharePoint/OneDrive (Files): FileAccessed, FileViewed, FilePreviewed, FileModified, + FileDownloaded, FileUploaded + Teams (Chat/Messaging): MessageSent, MessageRead, MessagesListed, ChatRetrieved, + ChatCreated, TeamsSessionStarted + Teams (Meetings): MeetingParticipantJoined, MeetingStarted, MeetingEnded, + MeetingParticipantDetail, MeetingDetail + Copilot / Connected AI: CopilotInteraction, ConnectedAIAppInteraction + + RECORD TYPES INCLUDED: + ExchangeAdmin, ExchangeItem, ExchangeMailbox, SharePointFileOperation, + SharePointSharingOperation, SharePoint, OneDrive, MicrosoftTeams, OfficeNative, + MicrosoftForms, MicrosoftStream, PlannerPlan, PlannerTask, PowerAppsApp + + IMPORTANT: When this switch is active, -ServiceTypes parameter is ignored and set to null. + This ensures optimal single-pass query performance across all Microsoft 365 workloads. + +.PARAMETER Rollup + Run an embedded Python post-processor against the audit run's final CSV + immediately after a successful export, then DELETE the raw CSV(s) so only the rolled-up + output remains. Mutually exclusive with -RollupPlusRaw. + + PURPOSE: These switches exist solely to produce input files for the Microsoft Copilot + Growth ROI Advisory Team's Power BI templates at https://github.com/microsoft/Analytics-Hub. + The rolled-up CSVs are shaped specifically for those templates and are NOT intended for + any other downstream use. For generic analytics exports, omit -Rollup / -RollupPlusRaw + and consume the raw CSV directly. + + The script auto-selects the correct embedded processor and target dashboard: + • CopilotInteraction-only run (default activity type, or -ActivityTypes 'CopilotInteraction'): + Purview_CopilotInteraction_Processor — also auto-enables -IncludeUserInfo and consumes + both the Purview CSV and the Entra users CSV. + Target Analytics-Hub dashboards: AI-in-One and AI Business Value. + • -IncludeM365Usage run: Purview_M365_Usage_Bundle_Explosion_Processor — consumes the + combined Purview CSV (-CombineOutput is auto-enabled by -IncludeM365Usage). + Target Analytics-Hub dashboard: M365 Usage Analytics. + + Requires PowerShell 7+ and Python 3.10+. If Python is not on PATH, the script attempts a + per-user silent install (winget Python.Python.3.13 → python.org installer fallback). + + Blocked combinations (script exits with an error): -UseEOM, -ExportWorkbook, + -OnlyUserInfo, -OnlyAgent365Info, -RAWInputCSV, and -ExcludeCopilotInteraction + without -IncludeM365Usage. + +.PARAMETER RollupPlusRaw + Same as -Rollup (produces input files for the Microsoft Copilot Growth ROI + Advisory Team's Power BI templates at https://github.com/microsoft/Analytics-Hub) but + KEEPS the raw CSV(s) on disk alongside the rolled-up output. Mutually exclusive with -Rollup. + +.PARAMETER Deidentify + Anonymize every identifying value in the output using a one-way, deterministic hash, so the + results can be shared without exposing who did what. User principal names / email addresses, + display names, mailbox GUIDs and SIDs, device names, and site / file / resource paths are + replaced with stable tokens — the same identity always maps to the same token, so user counts, + joins, and distinct-resource counts are preserved. Non-identifying fields (job title, company, + department, license, activity counts) are kept as-is. The transformation is irreversible: no + mapping back to the original values is produced or stored. + + Applies to both the raw output and the rolled-up output. CSV output only (cannot be combined + with -ExportWorkbook). When resuming an interrupted run, deidentification is taken from the + original run's checkpoint and cannot be added or removed on the resume command line. + +.PARAMETER FillerLabel + (Power BI rollup only — AI-in-One / AI Business Value dashboards.) Chooses how the + manager-hierarchy "level" columns are filled in below a person's own position in the org chart. + The hierarchy columns themselves — each person's chain of managers up to the top of the + organization, their level number, their manager, and their direct-report and total-report + counts — are always included in the rolled-up Users file for these two dashboards. This setting + only changes what appears in the deeper, unused level columns for people who sit near the top: + + null (default) Leave the deeper levels blank. + Self Repeat the person in their deeper levels. + RepeatManager Repeat the person's manager in their deeper levels. + Fixed Show a fixed label of your choice (supply it with -FillerLabelText). + + Requires -Rollup or -RollupPlusRaw. It is not available for the M365 Usage dashboard + (-IncludeM365Usage / -Dashboard M365), which does not include an org hierarchy. + +.PARAMETER FillerLabelText + The label to show in the deeper hierarchy levels when -FillerLabel Fixed is chosen, for + example: -FillerLabel Fixed -FillerLabelText "Assistive Directs". Only used with -FillerLabel Fixed. + +.PARAMETER ExcludeCopilotInteraction + Exclude Microsoft 365 Copilot activity type (CopilotInteraction). + Overrides custom list and default behavior. Use with -ActivityTypes to query only the specified activity types. + +.PARAMETER ExportWorkbook + [Deprecated] This switch is being phased out and will be removed in a future release. + +.PARAMETER AppendFile + Append Purview audit activity rows to an existing Purview audit CSV / XLSX instead of creating + a new timestamped file. Targets the Purview audit data stream only — EntraUsers append is + controlled by -AppendUserInfo. Accepts either a filename (combined with -OutputPath) or a + full path to the existing file. + + **Filename Resolution:** + • Relative filename: -AppendFile "MyReport.csv" → Uses -OutputPath directory + • Full path: -AppendFile "C:\Data\Report.xlsx" → Uses exact path specified + + **Requirements:** + • File must already exist (create it first without -AppendFile) + • File extension must be .csv + • Cannot be used with -OnlyUserInfo (no audit data in scope to append) + • Requires single-file output mode (see Single-File Output Requirements below) + • Data-loss safe: if the existing target cannot be read fully (very large file or key/schema mismatch), + the append aborts and leaves the target untouched rather than overwriting it with only this run's rows. + + **Single-File Output Requirements:** + Must use ONE of these modes to ensure single output file: + 1. Combined CSV: -CombineOutput (all activity types merged into one .csv file) + 2. Single activity type: -ActivityTypes CopilotInteraction (only one activity type selected) + + **CSV Mode Behavior:** + • Union-merges current-run rows with the target file keyed on RecordId (non-rollup) or, + for the rollup Fact CSV, on the full grain + Message_Id_Raw composite key (rollup Fact + rows FAN OUT — many rows can share one Message_Id_Raw, one per distinct grain); rows in + the target but missing from the current run are kept with In_Latest_Append = FALSE. + • Date_Added / Latest_Append_Date / In_Latest_Append provenance columns are maintained + on every row. NOTE: these three columns apply ONLY to the row-identity merges — the + non-rollup raw audit CSV (keyed on RecordId) and the CopilotInteraction rollup Fact + CSV (keyed on the grain + Message_Id_Raw composite). In the rollup Fact CSV a single + Message_Id_Raw can map to MULTIPLE rows (one per distinct grain, e.g. per accessed + resource), so append dedup keys on the whole grain plus Message_Id_Raw — never on + Message_Id_Raw alone (doing so would collapse fan-out rows and lose data). They are + intentionally NOT added to the M365 Usage bundle rollup (-IncludeM365Usage + -Rollup), + whose rows are additive aggregates summed across runs — see the per-row provenance + caveat in the M365 Rollup Anchoring section below. + • Compatible with: Standard (1:1), -Rollup, -RollupPlusRaw. + + **M365 Rollup Anchoring (-IncludeM365Usage + -Rollup / -RollupPlusRaw):** + The embedded M365 Bundle Explosion Processor emits FOUR files into a single output + directory, each sharing the same stem: + • '_Rollup_.csv' — the 14-column aggregated rollup + (UserId, CreationDate, Operation, + Workload, SourceFileExtension, AppHost, + EventCount, ItemsAccessedCount, + CreationTime, MaxCreationTime, + AgentId, AgentName, ContextType, + IsAgentInteraction). + • '_UserStats_.csv' — recomputed per-user metrics (sidecar). + • '_SessionCohort_.csv' — recomputed (UserId, App, Bucket) cohorts (sidecar). + • '_SessionStats_.csv' — (v2.6.0+) per-(UserId, Date, AppHost) + DISTINCTCOUNT(ThreadId) + PromptCount / + AgentPromptCount / ResponseCount / + AgentSessionCount. + Drives the v2.6.0 CECopilotPercentile_* + measures from PromptCount semantics. + Only the Rollup file supports the union-merge semantic. Under -AppendFile, the current + run's Rollup is column-tolerant union-merged (keyed on the 9-tuple of UserId / + CreationDate / Operation / Workload / SourceFileExtension / AppHost / AgentId / + AgentName / ContextType) into the customer's target file; the SessionStats file is + separately union-merged on (UserId, CreationDate, AppHost), and the UserStats and + SessionCohort sidecars are regenerated from the merged Rollup (with the merged + SessionStats feeding the CECopilotPercentile_* columns) and anchored off the target + leaf stem (so the same destination URLs are overwritten on every run). + Legacy targets that carry a narrower rollup schema (e.g. a 9- or 10-column file + produced by an earlier processor build) are padded with empty AgentId / AgentName / + ContextType cells and IsAgentInteraction=FALSE during merge so the union retains + a uniform 14-column shape end-to-end. + Provenance columns: the M365 bundle rollup merge is ADDITIVE (on a 9-tuple key match + EventCount / ItemsAccessedCount are summed, CreationTime = min, MaxCreationTime = max, + IsAgentInteraction = OR), so a merged row is a blend of multiple runs and is NOT owned + by any single run. The Date_Added / Latest_Append_Date / In_Latest_Append columns are + therefore intentionally NOT written to the rollup, SessionStats, or the regenerated + UserStats / SessionCohort sidecars (they would be meaningless on a summed row). The + rollup keeps exactly its 14 canonical columns. (Per-row provenance is carried only by + the row-identity merges — see CSV Mode Behavior above.) + Sidecar scope: the UserStats and SessionCohort sidecars are RECOMPUTED from the full + merged Rollup on every append (not appended to), so they always describe the entire + cumulative rollup, not just the latest run's slice. This is required for the window / + percentile / cohort math to stay correct against the full history. + Customer-facing rules: + • -AppendFile MUST point to a '_Rollup.csv' OR '_Rollup_.csv' leaf + (filename / full local path / SharePoint / Fabric URL). Sidecar leaves (_UserStats + / _SessionCohort / _SessionStats) and CopilotInteraction (_Interactions) / M365 + event-level (_Exploded) leaves are rejected at pre-flight. + • The three sidecar files inherit the anchor stem and are written next to the AppendFile + target (same parent folder/URL) using the leaves '_UserStats.csv', + '_SessionCohort.csv', and '_SessionStats.csv'. They + overwrite the same destination URLs on every run, so the destination always has + exactly one current set of all 4 files. + • Renamed AppendFile is fine as long as the leaf still ends in '_Rollup.csv' or + '_Rollup_.csv'. Example: -AppendFile 'MyM365Rollup_Rollup.csv' + produces sidecars 'MyM365Rollup_UserStats.csv' and 'MyM365Rollup_SessionCohort.csv' + at the same parent. + • Prior sidecars with DIFFERENT leaf names (e.g. left over from a previous run before + you renamed the AppendFile target) are NOT auto-deleted. Clean them up manually if you + want zero orphans in the destination namespace. + • For first-time append: run once WITHOUT -AppendFile to produce the initial timestamped + Rollup (plus its sidecars), then point -AppendFile at that Rollup on subsequent runs + (either keep the timestamped leaf or rename to drop the timestamp — both shapes are + accepted). + Fabric Tables/ destinations: the table name follows the AppendFile leaf stem and stays + stable across runs (the customer sets that stem; only the current-run scratch output + carries a fresh timestamp, and the merge anchors back to the fixed AppendFile target). + Convert-CsvToDelta -Mode overwrite replaces table contents with the merged-and-recomputed CSVs. + + **Independent companion switches:** + • -AppendUserInfo → merges into the EntraUsers CSV (auto-enables -IncludeUserInfo) + -AppendFile and -AppendUserInfo may be supplied together; each targets its own data stream + and each is paired with its own -OutputPath* override. + + **Always Timestamped (Never Overwritten):** + Even when using -AppendFile, these files are always timestamped: + • Log files: *_.log + • Telemetry files: *_telemetry_.csv + • Metrics files: *_metrics_.json (unless -MetricsPath specified) + + **Error Scenarios:** + • File not found: Script exits (create initial file first without -AppendFile) + • CSV header mismatch: Script exits with detailed column differences + • Multiple output files would be created: Script exits with single-file output requirement + • -OnlyUserInfo mode: Script exits (no audit data in scope to append) + +.PARAMETER AppendUserInfo + Path to an existing EntraUsers CSV that the current run should merge departed + users into. Specifying -AppendUserInfo automatically turns on -IncludeUserInfo, + so you do not need to set both. Storage tier is inferred from the path form + (drive-rooted absolute path = Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/* URL = Fabric). The + tier must match every other -OutputPath* / -Append* value supplied to the run. + + **Behavior:** + • The current run's freshly fetched EntraUsers snapshot is union-merged into + the file you supply (rows present in the target but missing from the current + snapshot are preserved as departed users). + • Three provenance columns are maintained on every row: + - Date_Added First run that introduced the user. + - Latest_Append_Date Most recent run that observed the user. + - In_Latest_Append TRUE if the user was present in the latest run. + • A pristine raw EntraUsers snapshot (EntraUsers_MAClicensing_.csv) + is written transiently during the merge step and removed once the union is + committed to the merged target, so the destination folder holds a single + EntraUsers CSV (the merged target) at run-end (parity with -AppendFile's + clean single-file outcome). If the merge step fails the cleanup does not + run; the raw snapshot stays where Save-CsvAtomic wrote it and is + enumerated in the end-of-run output files listing. + • The run-summary reports the merged target via the "Appended to:" line. + + **Pairing rules:** + • Use either -AppendUserInfo (merge) or -OutputPathUserInfo (overwrite/redirect), + not both, for the same destination. + +.PARAMETER UserInfoFile + Path to a customer-provided CSV that supplies the Entra user directory in lieu of the live + Entra /users pull. Storage tier is inferred from the value form (drive-rooted absolute path = + Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/* URL = Fabric); remote inputs are staged + to local scratch before parsing. The file must be CSV with a header row, at least one data row, + and a UserPrincipalName column (required). DisplayName, Department, JobTitle, ManagerUpn, and + HasLicense are recognized when present; any other columns are preserved as passthrough. Mutually + exclusive with -GroupNames. + +.PARAMETER AppendAgent365Info + Append the Microsoft Agent 365 catalog export into an existing file. Filename rules match + -AppendFile. Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be + used standalone or together with -AppendFile. Use either -AppendAgent365Info (merge) or + -OutputPathAgent365Info (overwrite/redirect), not both, for the same destination. + +.PARAMETER CombineOutput + Combines all activity types into a single output file or tab. + + **Default Behavior (without -CombineOutput):** + • CSV: One separate CSV file per activity type (plus EntraUsers_MAClicensing_.csv if -IncludeUserInfo) + • Excel: Multi-tab workbook (one tab per activity type; EntraUsers tab appended last if -IncludeUserInfo) + + **With -CombineOutput switch:** + • CSV: Single combined activity file named Purview_Audit_UsageActivity_CombinedActivityTypes_.csv (or Purview_Audit_UsageActivity__.csv when only one activity type was queried; plus separate EntraUsers_MAClicensing_.csv if -IncludeUserInfo) + • Excel: First tab named CombinedUsageActivity (no timestamp) with all activity rows; separate EntraUsers tab if -IncludeUserInfo + + Entra user/org data is never merged into the combined activity dataset—always exported separately. + + Recommended: Use combined mode for ingestion pipelines; separated mode for granular analysis. + +.PARAMETER Force + Force execution without interactive prompts. Automatically accepts defaults for: + 1. DSPM for AI Billing Information: Automatically continues when AIAppInteraction / AIInteraction / ConnectedAIAppInteraction are included via -ActivityTypes. + The DSPM informational prompt is ALSO auto-suppressed (without -Force) on -IncludeM365Usage runs that do not include AIAppInteraction, because ConnectedAIAppInteraction is part of the curated M365 usage bundle. The PAYG prompt for AIAppInteraction still fires unless -Force is set. + 2. Conflict Resolution (ExcludeCopilotInteraction): Automatically honors -ExcludeCopilotInteraction when conflict with -ActivityTypes + Use this switch for unattended/automated executions (CI/CD pipelines, scheduled tasks). + +.PARAMETER SkipDiagnostics + Skip pre-query capability diagnostics (advanced). + +.PARAMETER SkipVersionCheck + Skip the brief startup check that compares this script's version with the latest published on the + PAX GitHub repo. The check is informational only (never prompts, never auto-updates) and times out + in ~5 seconds if GitHub is unreachable; use this switch to suppress it on offline or locked-down hosts. + +.PARAMETER UseEOM + Use Exchange Online Management mode with Search-UnifiedAuditLog cmdlet. + When specified, queries Purview audit logs via EOM PowerShell module (serial processing only). + Default mode (when omitted) uses Microsoft Graph Security API with parallel processing support. + + PERMISSIONS REQUIRED (EOM mode): + • Exchange Online RBAC Roles: + - View-Only Audit Logs role + - Compliance Management role group + - Organization Management role group + - Or custom role with Search-UnifiedAuditLog cmdlet permission + + PARALLEL PROCESSING: + EOM mode is SERIAL-ONLY. Parallel processing is automatically disabled. + If -EnableParallel or -ParallelMode is specified with -UseEOM, script will exit with error. + + POWERSHELL VERSION: + EOM mode is the only path supported on PowerShell 5.1. The default Graph API mode + requires PowerShell 7+ (for ThreadJob-based parallel query execution). + + +.PARAMETER IncludeUserInfo + Include Entra (Microsoft Entra ID) user directory & Copilot license enrichment (Graph API mode only). + Exports an independent EntraUsers file/tab with: + • Core identity & profile details (name, UPN, job, department, location, organization info) + • Account & sync state (accountEnabled, onPremSync attributes, creation / change stamps) + • Manager expansion (identity + basic role/job fields via $expand=manager) + • Contact & routing (mail, proxyAddresses flattened, preferredLanguage) + • License enrichment (assignedLicenses list + hasLicense boolean for Copilot detection) + + Separation Principle: EntraUsers data is never merged into activity rows; always a distinct artifact. + + Requirements: + • Graph API mode (not supported with -UseEOM) + • Graph permissions (consented at runtime only when this switch is set): + - User.Read.All (read /users for Entra directory + license map) + - Organization.Read.All (read /subscribedSkus for SKU lookup) + • One-time directory + license fetch at startup (typ. +10–20s) + + License Detection Logic: + Dynamic discovery (no hardcoded SKU GUIDs): Copilot service plan IDs are + collected from /subscribedSkus where servicePlanName matches '*COPILOT*'. + hasLicense = true when the user has any assignedPlan with + capabilityStatus == 'Enabled' AND servicePlanId in the discovered set. + Honors admin-disabled service plans inside otherwise-assigned SKUs. + + Performance: Single batched fetch + hashtable lookups; no per-record calls. + + Use Cases: + • License compliance & adoption + • Mapping usage to directory attributes + • Identifying unlicensed usage patterns + + Not available in EOM mode. + +.PARAMETER OnlyUserInfo + Export ONLY Entra user directory and license information (skips all audit log retrieval). + This is a specialized mode for quickly exporting user licensing data without querying audit logs. + + BEHAVIOR: + • Authenticates to Microsoft Graph + • Fetches Entra user directory and license data + • Exports standalone EntraUsers_MAClicensing_.csv file + • Skips all audit log queries (completes in 5-15 seconds vs. minutes/hours) + • Automatically enables -IncludeUserInfo + + OUTPUT: + Single CSV file: EntraUsers_MAClicensing_YYYYMMDD_HHMMSS.csv + Contains 37 columns including: + - Identity fields (UPN, displayName, id, mail) + - Profile data (jobTitle, department, officeLocation) + - Manager hierarchy (manager info expanded) + - License assignments (assignedLicenses + hasCopilotLicense boolean) + + COMPATIBLE PARAMETERS (can be used WITH -OnlyUserInfo): + • -OutputPath : Specify output directory + • -OutputPathUserInfo: Per-data-type destination (overrides -OutputPath for the EntraUsers CSV) + • -AppendUserInfo : Union-merge this run's EntraUsers snapshot into an existing target + • -Auth : Choose authentication method (WebLogin, DeviceCode, etc.) + • -CombineOutput : (Has no effect, but allowed for script compatibility) + • -DisableAdaptive : (Has no effect, but allowed) + • -Debug / -Verbose : Enable diagnostic output + + INCOMPATIBLE PARAMETERS (cannot be used with -OnlyUserInfo): + Audit Retrieval: + • StartDate, EndDate : No audit queries to filter + • ActivityTypes : Cleared by -OnlyUserInfo + • ExcludeCopilotInteraction: Activity type modifier + • BlockHours, PartitionHours, MaxPartitions, ResultSize, PacingMs + • AutoCompleteness : Audit log completeness checks + • StreamingSchemaSample, StreamingChunkSize, ExportProgressInterval + + Filtering: + • UserIds, GroupNames : User filtering requires audit logs + • AgentId, AgentsOnly, ExcludeAgents: Agent filtering requires audit logs + • PromptFilter : Message filtering requires audit logs + + Parallelization: + • ParallelMode, MaxParallelGroups, MaxConcurrency, EnableParallel + • MaxActivePartitions : Query execution settings + + EOM Mode: + • UseEOM : Exchange Online Management mode + + USE CASES: + 1. License compliance auditing (quick snapshot of all user licenses) + 2. Periodic license data exports for tracking/trending + 3. Standalone user directory exports for cross-referencing + 4. Rapid licensing status checks without audit log overhead + + EXAMPLES: + # Basic user-only export + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo + + # Export to specific directory + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo -OutputPath "D:\UserData\" + + # Use device code auth (for automation/headless scenarios) + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo -Auth DeviceCode + + # App registration auth (client secret) + .\PAX_Purview_Audit_Log_Processor.ps1 -Auth AppRegistration -TenantId "" -ClientId "" -ClientSecret (ConvertTo-SecureString "" -AsPlainText -Force) + + # App registration auth (certificate thumbprint) + .\PAX_Purview_Audit_Log_Processor.ps1 -Auth AppRegistration -TenantId "" -ClientId "" -ClientCertificateThumbprint "" + + PERFORMANCE: + Typical execution time: 5-15 seconds (vs. minutes/hours for audit log queries) + Network traffic: Minimal (only user directory + license API calls) + + NOT AVAILABLE IN EOM MODE: Requires Microsoft Graph API (user directory/licenses not in EOM). + +.PARAMETER IncludeAgent365Info + Adds a Microsoft Agent 365 enrichment phase to the run, producing Agent365_.csv + (columns matching the Microsoft Admin Center "Agent 365" agent catalog export). Runs after the + main audit + EntraUsers phases and sources data from the Microsoft Graph Agent Package + Management API, with one narrow audit lookup per agent for creation date / created-by. + + REQUIREMENTS: + • Tenant enrolled in the Microsoft Agent 365 program (and holding a Microsoft Agent 365 license) + • App-only auth (-Auth AppRegistration certificate/secret, or -Auth ManagedIdentity): the app + registration / managed-identity service principal must be granted + admin-consented the + APPLICATION permissions CopilotPackages.Read.All (and Application.Read.All for developer-name + resolution). No interactive sign-in is performed; the Agent 365 phase reuses the app-only context. + • Delegated auth (-Auth WebLogin / DeviceCode / Credential / Silent): the signed-in caller must + hold AI Administrator or Global Administrator; the Agent 365 scopes are consented at sign-in. + • Requires a destination — supply -OutputPathAgent365Info or -AppendAgent365Info + • The catalog API is currently published at /beta only. + INCOMPATIBLE: -UseEOM, -RAWInputCSV. Compatible with -Resume (the agent phase runs at end of run) and + with all auth modes, including -Auth ManagedIdentity and headless / noninteractive hosts (no prompt). + If the tenant is not enrolled/licensed, the agent phase is skipped with an explanatory banner and the rest of the run completes normally. + +.PARAMETER OnlyAgent365Info + Export ONLY the Microsoft Agent 365 catalog (skips all audit log retrieval and EntraUsers + enrichment), producing only Agent365_.csv. Same requirements as -IncludeAgent365Info + (including app-only support via -Auth AppRegistration or -Auth ManagedIdentity, and delegated + modes). Incompatible with -Resume (there is no audit phase to resume). A Y/N preflight confirms + before any Graph call; pass -Force to auto-confirm. + +.PARAMETER MaxNetworkOutageMinutes + Maximum continuous network outage the script will tolerate during audit log operations (query creation, polling, record retrieval). + Applies to transient network errors: 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout, connection failures. + Script automatically retries failed operations with randomized delays (30-60s) until connectivity is restored or timeout is exceeded. + Clean terminal output shows error summaries with countdown timers; full error details logged to file for troubleshooting. + Progress is preserved - no data loss during network interruptions. + Exceeding this window aborts with clear error message indicating tolerance exceeded. + Default: 30 minutes (adjustable 1-120) + +.PARAMETER Resume + Resume an interrupted operation from a checkpoint file. + Checkpoint files are automatically created during all auth modes to allow resumption + after Ctrl+C, network failures, token expiry, or any interruption. + + IMPORTANT: Resume mode is STANDALONE. + All processing parameters are restored from the checkpoint file. + You cannot specify other parameters with -Resume (except auth overrides). + + USAGE: + -Resume Auto-discover checkpoint in current directory/OutputPath + -Resume "path\to\file" Use specific checkpoint file + + ALLOWED WITH -Resume: + -Force Use most recent checkpoint without prompting + -Auth Override authentication method + -TenantId, -ClientId Override auth credentials (for AppRegistration) + -ClientSecret Provide client secret (for AppRegistration) + + NOT ALLOWED WITH -Resume: + Any other parameter (dates, activities, explosion settings, -Deidentify, etc.) + These are all restored from the checkpoint to ensure data consistency. In particular, + -Deidentify is taken from the checkpoint — a run started with -Deidentify stays + deidentified when resumed, and deidentification cannot be added or removed on the resume + command line. + + CHECKPOINT LOCATION: + Files are created in OutputPath with pattern: .pax_checkpoint_.json + Checkpoint and partial-output files are always written to the LOCAL filesystem, + even when an -OutputPath* value is a SharePoint or OneLake URL. Resume must be run + from the same machine that produced the checkpoint. + +#> + +param( + [Parameter(Mandatory = $false)] + [string]$StartDate, # Live mode: if omitted (with EndDate) auto-populated later; Replay: optional filter + + [Parameter(Mandatory = $false)] + [string]$EndDate, # Live mode: if omitted (with StartDate) auto-populated; Replay: optional filter + + [Parameter(Mandatory = $false)] + [string]$OutputPath, + + # Per-data-type destination overrides. Storage tier (Local / SharePoint / Fabric) is + # inferred from each path's form: drive-rooted absolute path = Local, https://*.sharepoint.* + # URL = SharePoint, https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse* URL = Fabric. + # UNC paths are rejected. All destinations supplied to a single run must resolve to the + # same storage tier (no mixed Local/SP/Fabric in one invocation). + [Parameter(Mandatory = $false)] + [string]$OutputPathUserInfo, + + [Parameter(Mandatory = $false)] + [string]$OutputPathAgent365Info, + + [Parameter(Mandatory = $false)] + [string]$OutputPathLog, + + [Parameter(Mandatory = $false)] + [ValidateSet('WebLogin', 'DeviceCode', 'Credential', 'Silent', 'AppRegistration', 'ManagedIdentity')] + [string]$Auth = 'WebLogin', + + [Parameter(Mandatory = $false)] + [string]$TenantId, + + [Parameter(Mandatory = $false)] + [string]$ClientId, + + [Parameter(Mandatory = $false)] + [string]$ClientSecret, + + [Parameter(Mandatory = $false)] + [string]$ClientCertificateThumbprint, + + [Parameter(Mandatory = $false)] + [ValidateSet('CurrentUser','LocalMachine')] + [string]$ClientCertificateStoreLocation = 'CurrentUser', + + [Parameter(Mandatory = $false)] + [string]$ClientCertificatePath, + + [Parameter(Mandatory = $false)] + [System.Security.SecureString]$ClientCertificatePassword, + + [Parameter(Mandatory = $false)] + [ValidateRange(0.016667, 24)] + [double]$BlockHours = 0.5, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 72)] + [int]$PartitionHours = 0, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 1000)] + [int]$MaxPartitions = 160, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10000)] + [int]$ResultSize = 10000, + + [Parameter(Mandatory = $false)] + [ValidateRange(0, 10000)] + [int]$PacingMs = 0, + + [Parameter(Mandatory = $false)] + [string[]]$ActivityTypes = @('CopilotInteraction'), + + [Parameter(Mandatory = $false)] + [string[]]$RecordTypes, + + [Parameter(Mandatory = $false)] + [string[]]$ServiceTypes, + + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExplodeArrays, + + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExplodeDeep, + [Parameter(Mandatory = $false)] + [int]$FlatDepth = 120, + [Parameter(Mandatory = $false, DontShow = $true)] + [string]$RAWInputCSV, + [Parameter(Mandatory = $false)] + # Controls concurrent execution: EOM mode limits serial queries, Graph API mode limits partition parallelism + [int]$MaxConcurrency = 10, + [Parameter(Mandatory = $false)] + [switch]$EnableParallel, + [Parameter(Mandatory = $false)] + [ValidateRange(0, 50)] + # Allows multiple activity groups to be processed concurrently (aligns with Microsoft's ~10 query safe limit) + [int]$MaxParallelGroups = 8, + [Parameter(Mandatory = $false)] + [ValidateSet('Off', 'On', 'Auto')] + # Default now 'Auto' so that PS 7+ environments engage parallel processing automatically unless explicitly turned Off. + [string]$ParallelMode = 'Auto', + [Parameter(Mandatory = $false)] + [ValidateRange(0, 32)] + # 0=auto-detect (2 to min(ProcessorCount, 8) threads), 1=serial, 2-32=explicit thread count. Requires PS7+. + [int]$ExplosionThreads = 0, + [Parameter(Mandatory = $false)] + [switch]$DisableAdaptive, # Disable adaptive safeguards (memory/latency/concurrency smoothing) + [Parameter(Mandatory = $false)] + [ValidateRange(0.0,1.0)] + [double]$ProgressSmoothingAlpha = 0.3, # Weight for smoothing dynamic progress total recalculation (0 => off) + [Parameter(Mandatory = $false)] + [ValidateRange(1000,600000)] + [int]$HighLatencyMs = 90000, # Partition average latency threshold (ms) triggering mild concurrency reduction + [Parameter(Mandatory = $false)] + [ValidateRange(256,32768)] + [int]$MemoryPressureMB = 1500, # Working set (MB) threshold to trigger mild concurrency reduction + [Parameter(Mandatory = $false)] + [ValidateRange(-1,65536)] + [int]$MaxMemoryMB = -1, # Max process memory (MB) before flushing $allLogs to disk (-1 = auto 75%, 0 = disabled) + [Parameter(Mandatory = $false)] + [ValidateRange(30,600)] + [int]$StatusIntervalSeconds = 60, # How often (seconds) to display status during polling and backpressure waits + [Parameter(Mandatory = $false)] + [ValidateRange(100,600000)] + [int]$LowLatencyMs = 20000, # Sustained low latency threshold to consider concurrency step-up + [Parameter(Mandatory = $false)] + [ValidateRange(1,10)] + [int]$LowLatencyConsecutive = 2, # Required consecutive low-latency groups before step-up + [Parameter(Mandatory = $false)] + [ValidateRange(1,100)] + [int]$ThroughputDropPct = 15, # % drop vs baseline required (with high latency) to justify reduction + [Parameter(Mandatory = $false)] + [ValidateRange(0.0,1.0)] + [double]$ThroughputSmoothingAlpha = 0.3,# EMA smoothing for throughput baseline + [Parameter(Mandatory = $false)] + [ValidateRange(1,50)] + [int]$AdaptiveConcurrencyCeiling = 6, # Upper bound for adaptive step-ups + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10000)] + [int]$ExportProgressInterval = 10, + + # Streaming export is always-on + [Parameter(Mandatory = $false)] + [ValidateRange(100, 50000)] + [int]$StreamingSchemaSample = 5000, + + [Parameter(Mandatory = $false)] + [ValidateRange(100, 50000)] + [int]$StreamingChunkSize = 5000, + + [Parameter(Mandatory = $false)] + [string[]]$AgentId, + + [Parameter(Mandatory = $false)] + [switch]$AgentsOnly, + + [Parameter(Mandatory = $false)] + [ValidateSet('Prompt', 'Response', 'Both', 'Null')] + [string]$PromptFilter, + + # --- Reliability Enhancements (Backoff & Circuit Breaker) --- + [Parameter(Mandatory = $false)] + [ValidateRange(1,50)] + [int]$CircuitBreakerThreshold = 5, # Consecutive block failures before opening circuit breaker + [Parameter(Mandatory = $false)] + [ValidateRange(5,3600)] + [int]$CircuitBreakerCooldownSeconds = 120, # Cooldown duration after breaker trips + [Parameter(Mandatory = $false)] + [ValidateRange(0.1,120)] + [double]$BackoffBaseSeconds = 1.0, # Base seconds for exponential backoff between block retries + [Parameter(Mandatory = $false)] + [ValidateRange(1,600)] + [int]$BackoffMaxSeconds = 45, # Max cap for exponential backoff delay + + [Parameter(Mandatory = $false)] + [switch]$ExcludeAgents, + + [Parameter(Mandatory = $false)] + [string[]]$UserIds, + + [Parameter(Mandatory = $false)] + [string[]]$GroupNames, + + [Parameter(Mandatory = $false)] + [switch]$Help, + + # Emit structured metrics JSON alongside CSV (OutputFile name with .metrics.json) + [Parameter(Mandatory = $false)] + [switch]$EmitMetricsJson, + + # Override metrics output path (optional). If provided and -EmitMetricsJson specified, writes here instead of OutputFile substitution. + [Parameter(Mandatory = $false)] + [string]$MetricsPath, + + # Ensure completeness: aggressively subdivide any window still returning server 10K limit until below threshold or min window reached. + [Parameter(Mandatory = $false)] + [switch]$AutoCompleteness, + + # Include CopilotInteraction explicitly (default when no custom -ActivityTypes provided) + [Parameter(Mandatory = $false)] + [switch]$IncludeCopilotInteraction, + + [Parameter(Mandatory = $false)] + [switch]$IncludeM365Usage, + + # Exclude CopilotInteraction activity type (overrides custom list and default fallback) + [Parameter(Mandatory = $false)] + [switch]$ExcludeCopilotInteraction, + + # [Deprecated] Excel workbook export. Will be removed in a future release. + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExportWorkbook, + + # Append data to existing CSV file + # Provide filename (e.g., "MyReport.csv") or full path (e.g., "C:\Data\Report.csv") + [Parameter(Mandatory = $false)] + [string]$AppendFile, + + # Append EntraUsers / MAC licensing data into an existing file. Filename rules match -AppendFile. + # Auto-enables -IncludeUserInfo. Incompatible with -OnlyUserInfo. May be used standalone or together with -AppendFile. + [Parameter(Mandatory = $false)] + [string]$AppendUserInfo, + + # Ingest the Entra user directory from a customer-provided CSV in lieu of the live Entra /users pull. + # Storage tier is inferred from the value form (drive-rooted absolute path = Local; SharePoint URL = + # SharePoint; OneLake Lakehouse URL = Fabric); remote inputs are staged to scratch before parsing. + # CSV only; requires a UserPrincipalName column. Mutually exclusive with -GroupNames. + [Parameter(Mandatory = $false)] + [string]$UserInfoFile, + + # Append Agent 365 catalog data into an existing file. Filename rules match -AppendFile. + # Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be used standalone or together with -AppendFile. + [Parameter(Mandatory = $false)] + [string]$AppendAgent365Info, + + # Combine all activity types into single output file/tab (CSV or Excel) + # CSV default when omitted: separate files per activity type + # Excel default when omitted: separate tabs per activity type + # Use -CombineOutput switch to merge all activity types into one file/tab + [Parameter(Mandatory = $false)] + [switch]$CombineOutput, + + # Force execution without interactive prompts (PAYG warning, conflict resolution) + [Parameter(Mandatory = $false)] + [switch]$Force, + + # Skip pre-query capability diagnostics (advanced) + [Parameter(Mandatory = $false)] + [switch]$SkipDiagnostics, + + # Skip the startup GitHub version check (offline / locked-down environments) + [Parameter(Mandatory = $false)] + [switch]$SkipVersionCheck, + + # Use Exchange Online Management mode (Search-UnifiedAuditLog cmdlet, serial-only) + [Parameter(Mandatory = $false)] + [switch]$UseEOM, + + # Include Entra user directory and license information in export (adds separate EntraUsers file/tab; column count may evolve) + [Parameter(Mandatory = $false)] + [switch]$IncludeUserInfo, + + # Export only Entra user directory and license information (skips all audit log retrieval) + [Parameter(Mandatory = $false)] + [switch]$OnlyUserInfo, + + # Include Microsoft Agent 365 enrichment in export (adds separate Agents365 CSV file/Excel tab). + # Sources data from the Microsoft Graph Agent Package Management API + # (https://graph.microsoft.com/beta/copilot/admin/catalog/packages). + # Requires AI Admin or Global Admin directory role; cannot be satisfied by app-only auth alone. + # When -Auth AppRegistration is used, an interactive sign-in is performed for the agent phase only. + [Parameter(Mandatory = $false)] + [switch]$IncludeAgent365Info, + + # Export only Microsoft Agent 365 data (skips all audit log retrieval). + # Date created and Created by columns will be left blank (those rely on audit data). + # Not compatible with -Auth AppRegistration (Agent Package API requires interactive user auth). + [Parameter(Mandatory = $false)] + [switch]$OnlyAgent365Info, + + # Maximum minutes to tolerate continuous network outage during Graph async polling & record retrieval (adaptive backoff). Default 30. + [Parameter(Mandatory = $false)] + [int]$MaxNetworkOutageMinutes = 30, + + # Export Graph API telemetry CSV for partition timing analysis and troubleshooting + [Parameter(Mandatory = $false)] + [switch]$IncludeTelemetry, + + # Rollup post-processor: produce ONLY rolled-up CSV(s) and delete the raw CSV(s) after success. + # Mutually exclusive with -RollupPlusRaw. Requires PowerShell 7+. Auto-installs Python 3.10+ and + # the optional 'orjson' package on first use. Only valid for default (CopilotInteraction-only) runs, + # explicit -ActivityTypes 'CopilotInteraction', or -IncludeM365Usage runs. Not compatible with + # -UseEOM, -ExportWorkbook, -OnlyUserInfo, -OnlyAgent365Info, or -RAWInputCSV. See documentation + # for full gating matrix. + [Parameter(Mandatory = $false)] + [switch]$Rollup, + + # Rollup post-processor: produce BOTH the raw CSV(s) AND the rolled-up CSV(s). Same gating rules + # as -Rollup. Mutually exclusive with -Rollup. + [Parameter(Mandatory = $false)] + [switch]$RollupPlusRaw, + + # Rollup target dashboard selector. Chooses which embedded Python + # post-processor + output profile a rollup run produces: + # AIO (default) - AI-in-One dashboard -> CopilotInteraction processor, --profile aio + # AIBV - AI Business Value dashboard -> CopilotInteraction processor, --profile aibv + # M365 - M365 Usage Analytics -> M365 bundle processor (auto-enables -IncludeM365Usage) + # Only meaningful with -Rollup / -RollupPlusRaw; if supplied without either, -Rollup is auto-enabled. + # If omitted: M365 when -IncludeM365Usage is present, otherwise AIO. Case-insensitive. + [Parameter(Mandatory = $false)] + [ValidateSet('AIO', 'AIBV', 'M365')] + [string]$Dashboard = 'AIO', + + # Deidentify (anonymize) all identifying values in RAW output files using a + # one-way, salted, deterministic, format-preserving hash. Applies to the raw + # audit CSV(s) and the EntraUsers CSV. Rollup outputs are deidentified by the + # embedded Python processor (same salt + algorithm), so -Rollup / -RollupPlusRaw + # rollups are covered there; this switch additionally scrubs any RETAINED raw + # files. No-op for plain -Rollup (raw deleted). Irreversible: no decode map is + # produced or stored. + [Parameter(Mandatory = $false)] + [switch]$Deidentify, + + # Org / manager-hierarchy level-filler for the rolled-up AI-in-One / AI Business + # Value Users output. The hierarchy columns are ALWAYS produced for those two + # dashboards; this only controls what appears in level slots deeper than a user's + # own level: + # null (default) -> leave blank + # Self -> repeat the user + # RepeatManager -> repeat the user's manager + # Fixed -> a literal label supplied via -FillerLabelText "" + # Requires -Rollup or -RollupPlusRaw; not valid with -IncludeM365Usage / -Dashboard M365. + [Parameter(Mandatory = $false)] + [string]$FillerLabel, + + # The literal label text used with -FillerLabel Fixed (e.g. "Assistive Directs"). + # Only valid together with -FillerLabel Fixed. + [Parameter(Mandatory = $false)] + [string]$FillerLabelText, + + # Resume from checkpoint file - HANDLED VIA $args (not param block) to support: + # -Resume (auto-discover checkpoint in OutputPath) + # -Resume "path/to/file" (explicit checkpoint path) + # This parameter captures any remaining arguments for manual -Resume parsing + [Parameter(Mandatory = $false, ValueFromRemainingArguments = $true)] + [string[]]$RemainingArgs +) + +# ============================================================ +# CULTURE GUARD — DO NOT REMOVE +# Pin the main thread to InvariantCulture for the entire run. +# Rationale: .NET's custom date/time format string treats ':' not as a +# literal colon but as the culture's DateTimeFormatInfo.TimeSeparator. +# On da-DK and fi-FI hosts TimeSeparator is '.', so an unscoped +# .ToString('yyyy-MM-ddTHH:mm:ss.fffZ') produces 'T20.35.42.000Z', +# which the Purview audit query endpoint rejects with HTTP 500. +# Setting CurrentCulture here makes every downstream .ToString(format), +# Get-Date -Format, and ParseExact($x,$fmt,$null) call in the main +# thread emit / consume invariant ISO 8601. ThreadJob runspaces get +# their own per-thread guard inside $queryJobScriptBlock because +# PowerShell 5.1 does not inherit parent culture into ThreadJobs. +# ============================================================ +[System.Threading.Thread]::CurrentThread.CurrentCulture = [System.Globalization.CultureInfo]::InvariantCulture +[System.Threading.Thread]::CurrentThread.CurrentUICulture = [System.Globalization.CultureInfo]::InvariantCulture + +# ============================================================ +# BOOTSTRAP LOG +# Open a deterministic log file IMMEDIATELY after param() so every Write-Host / +# Write-Log call from this point forward lands on disk. The final log location +# (which depends on -OutputPath / -AppendFile / -OutputPathLog / checkpoint state) +# is resolved much later in the script — at that point the bootstrap log is +# renamed/moved to the final path. The in-memory $script:LogBuffer remains as a +# belt-and-braces fallback if the bootstrap file itself cannot be created +# (read-only volume, permissions, etc.). +# +# Path resolution order (durable bootstrap-log location for Azure Container Instances): +# 1. $env:PAX_BOOTSTRAP_LOG_DIR — operator-supplied durable location. +# In the PAX.Dockerfile this defaults to "/pax-logs" and is declared as a +# VOLUME so operators can mount Azure Files at that path. When mounted, +# pre-flight / auth / param-validation failures leave the bootstrap log +# on the Azure Files share, retrievable without spinning a new container. +# 2. [System.IO.Path]::GetTempPath() — process temp dir, correct on every +# supported host: Windows local ($env:TEMP), Linux container (/tmp), +# macOS dev (/var/folders/.../T/). No host-OS branching required. +# +# The override is validated for write access; if it isn't writable the script +# silently falls back to GetTempPath() so a misconfigured env var never blocks +# a run. Successful migration of the bootstrap log to its final location uses +# Move-Item, which removes the bootstrap source file — the durable directory +# remains for future runs. +# ============================================================ +try { + $candidateDirs = @() + if (-not [string]::IsNullOrWhiteSpace($env:PAX_BOOTSTRAP_LOG_DIR)) { + $candidateDirs += $env:PAX_BOOTSTRAP_LOG_DIR + } + $candidateDirs += [System.IO.Path]::GetTempPath() + + $script:BootstrapLogDir = $null + foreach ($cand in $candidateDirs) { + try { + if (-not (Test-Path -LiteralPath $cand -PathType Container)) { + New-Item -Path $cand -ItemType Directory -Force -ErrorAction Stop | Out-Null + } + # Probe write access with a zero-byte sentinel. + $probe = Join-Path $cand (".pax_writeprobe_{0}_{1}" -f $PID, (Get-Date -Format 'yyyyMMddHHmmssfff')) + Set-Content -LiteralPath $probe -Value '' -ErrorAction Stop + Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue + $script:BootstrapLogDir = $cand + break + } catch { + continue + } + } + if (-not $script:BootstrapLogDir) { throw "No writable bootstrap-log directory available." } + + $script:BootstrapLogName = "PAX_bootstrap_{0}_{1}.log" -f $PID, (Get-Date -Format 'yyyyMMddHHmmss') + $script:LogFile = Join-Path $script:BootstrapLogDir $script:BootstrapLogName + $script:LogFileIsBootstrap = $true + New-Item -Path $script:LogFile -ItemType File -Force -ErrorAction Stop | Out-Null + Add-Content -Path $script:LogFile -Value ("[{0}] [INFO] Bootstrap log opened at {1} (PID {2}, source: {3})." -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $script:LogFile, $PID, $(if ($env:PAX_BOOTSTRAP_LOG_DIR -and $script:BootstrapLogDir -eq $env:PAX_BOOTSTRAP_LOG_DIR) { 'PAX_BOOTSTRAP_LOG_DIR' } else { 'system temp' })) -Encoding UTF8 -ErrorAction SilentlyContinue +} catch { + # Bootstrap log creation failed — leave $script:LogFile null so the existing + # in-memory $script:LogBuffer path (defined further down) absorbs early entries. + $script:LogFile = $null + $script:LogFileIsBootstrap = $false + Microsoft.PowerShell.Utility\Write-Host ("WARNING: Could not create bootstrap log: {0}" -f $_.Exception.Message) -ForegroundColor Yellow +} + +# ============================================================ +# DEPRECATED SWITCH GATE +# Any explicit use of these switches on the command line causes an +# immediate graceful exit. These switches are slated for removal in a +# future release and are no longer supported. +# ============================================================ +$script:DeprecatedSwitchesHit = @() +if ($PSBoundParameters.ContainsKey('ExportWorkbook')) { $script:DeprecatedSwitchesHit += '-ExportWorkbook' } +if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $script:DeprecatedSwitchesHit += '-RAWInputCSV' } +if ($PSBoundParameters.ContainsKey('ExplodeArrays')) { $script:DeprecatedSwitchesHit += '-ExplodeArrays' } +if ($PSBoundParameters.ContainsKey('ExplodeDeep')) { $script:DeprecatedSwitchesHit += '-ExplodeDeep' } +if ($script:DeprecatedSwitchesHit.Count -gt 0) { + foreach ($d in $script:DeprecatedSwitchesHit) { + Microsoft.PowerShell.Utility\Write-Host ("{0} is deprecated and will be removed in a future release." -f $d) -ForegroundColor Yellow + } + exit 0 +} + +# ============================================================ +# MANUAL -Resume PARAMETER PARSING +# Enables: -Resume (auto-discover) and -Resume "path" (explicit) +# ============================================================ +$Resume = $null +$ResumeSpecified = $false +if ($RemainingArgs -and $RemainingArgs.Count -gt 0) { + for ($i = 0; $i -lt $RemainingArgs.Count; $i++) { + if ($RemainingArgs[$i] -eq '-Resume') { + $ResumeSpecified = $true + # Check if next argument exists and is not another parameter + if (($i + 1) -lt $RemainingArgs.Count -and $RemainingArgs[$i + 1] -notmatch '^-') { + $Resume = $RemainingArgs[$i + 1] + $i++ # Skip the path argument + } else { + $Resume = '' # Auto-discover mode + } + } + } +} + +# ============================================================ +# PROMOTE AUTH PARAMETERS TO SCRIPT SCOPE +# Enables access from within functions (e.g., Connect-PurviewAudit) +# ============================================================ +$script:TenantId = $TenantId +$script:ClientId = $ClientId +$script:ClientSecret = $ClientSecret +$script:ClientCertificateThumbprint = $ClientCertificateThumbprint +$script:ClientCertificateStoreLocation = $ClientCertificateStoreLocation +$script:ClientCertificatePath = $ClientCertificatePath +$script:ClientCertificatePassword = $ClientCertificatePassword + + function Resolve-CommaSeparatedValues { + param([string[]]$Values) + + if (-not $Values -or $Values.Count -eq 0) { + return $Values + } + + $finalActivityTypes = @() + + # Step 1: Add explicit -ActivityTypes parameter values (if provided and not default) + if ($PSBoundParameters.ContainsKey('ActivityTypes') -and $ActivityTypes) { + foreach ($actType in $ActivityTypes) { + if ($actType -and $actType -ne '') { + $finalActivityTypes += $actType + } + } + if ($finalActivityTypes.Count -gt 0) { + Write-LogHost "Custom ActivityTypes provided: $($finalActivityTypes -join ', ')" -ForegroundColor Gray + } + } + + # Step 3: Add CopilotInteraction when explicitly requested + if ($IncludeCopilotInteraction -and -not ($finalActivityTypes -contains $copilotBaseActivityType)) { + $finalActivityTypes += $copilotBaseActivityType + Write-LogHost "IncludeCopilotInteraction: Adding $copilotBaseActivityType (explicit request)" -ForegroundColor Cyan + } + + # Step 4: Add Microsoft 365 usage bundle when requested + Write-LogHost ("IncludeM365Usage switch present: {0}" -f $IncludeM365Usage.IsPresent) -ForegroundColor DarkGray + if ($IncludeM365Usage) { + $finalActivityTypes += $m365UsageActivityBundle + Write-LogHost ("M365 Usage bundle: Adding {0} activity types across Exchange/SharePoint/OneDrive/Teams" -f $m365UsageActivityBundle.Count) -ForegroundColor Cyan + + # -IncludeM365Usage always implies -CombineOutput. The M365 usage bundle spans 30+ + # activity types across 4 workloads; per-activity-type splits produce unmanageable file + # counts and are incompatible with the rollup post-processor (which consumes a single + # CSV). Auto-enable -CombineOutput once, with a one-line info message (no prompt). + if (-not $CombineOutput) { + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-LogHost "M365 Usage mode: -CombineOutput auto-enabled (M365 bundle always produces a single combined output)." -ForegroundColor Cyan + } + + $RecordTypes = @( + if ($RecordTypes) { $RecordTypes } + $m365UsageRecordBundle + ) | Where-Object { $_ } | Select-Object -Unique + if ($RecordTypes.Count -eq 0) { $RecordTypes = $null } + + # CRITICAL: Do NOT set ServiceTypes for M365 usage mode - Graph API should get ALL workloads in single pass + # Multiple serviceFilter values cause unnecessary workload splits (Exchange, SharePoint, OneDrive, Teams) + # Instead, send NO serviceFilter and let Graph API return all workloads in one query per partition + $ServiceTypes = $null + + if ($RecordTypes) { + Write-LogHost "M365 Usage bundle: RecordTypes => $($RecordTypes -join ', ')" -ForegroundColor Gray + } + Write-LogHost "M365 Usage mode: ServiceTypes => NULL (single workload pass, all services combined)" -ForegroundColor Cyan + } + + # Step 5: BASE ACTIVITY TYPE - Add CopilotInteraction as default base type + # This is the core Microsoft 365 Copilot activity type (FREE, included in M365 Copilot licensing) + # Captures ALL M365 Copilot usage including Teams meetings, Word, Excel, PowerPoint, Outlook, etc. + # Auto-add when the user did not explicitly provide -ActivityTypes. + # Exception: Always respect -ExcludeCopilotInteraction (handled in Step 6) + $userProvidedCustomTypes = $PSBoundParameters.ContainsKey('ActivityTypes') + if (-not $ExcludeCopilotInteraction) { + # Auto-add only when no custom types were provided + if (-not $userProvidedCustomTypes) { + # Add CopilotInteraction if not already present + if (-not ($finalActivityTypes -contains $copilotBaseActivityType)) { + $finalActivityTypes = @($copilotBaseActivityType) + $finalActivityTypes + } + } + } + + # Step 6: EXCLUSION OVERRIDE - Remove CopilotInteraction if -ExcludeCopilotInteraction is true + if ($ExcludeCopilotInteraction) { + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne $copilotBaseActivityType } + } + + # Step 7: Final deduplication and validation + $finalActivityTypes = @($finalActivityTypes | Select-Object -Unique) + + return $finalActivityTypes +} + +function Send-PromptNotification { + <# + .SYNOPSIS + Plays a system beep to alert user that a prompt requires attention. + .DESCRIPTION + Useful when user is working in other windows and needs to be notified + when a prompt appears that requires input. + #> + + try { + # Play 3 short beeps to get attention + [Console]::Beep(800, 200) # 800Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1000, 200) # 1000Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1200, 300) # 1200Hz for 300ms (slightly longer final beep) + } + catch { + # Silently fail if beep not supported (e.g., some server environments) + } +} + +# Validate -OnlyUserInfo parameter compatibility +if ($OnlyUserInfo) { + $incompatibleParams = @() + + # Date filtering parameters + if ($PSBoundParameters.ContainsKey('StartDate')) { $incompatibleParams += " - StartDate (not applicable for user-only export)" } + if ($PSBoundParameters.ContainsKey('EndDate')) { $incompatibleParams += " - EndDate (not applicable for user-only export)" } + + # Activity configuration parameters + if ($PSBoundParameters.ContainsKey('ActivityTypes')) { $incompatibleParams += " - ActivityTypes (cleared by -OnlyUserInfo)" } + if ($IncludeM365Usage) { $incompatibleParams += " - IncludeM365Usage (activity type modifier)" } + if ($ExcludeCopilotInteraction) { $incompatibleParams += " - ExcludeCopilotInteraction (activity type modifier)" } + + # Audit retrieval settings + if ($PSBoundParameters.ContainsKey('BlockHours') -and $BlockHours -ne 0.5) { $incompatibleParams += " - BlockHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('PartitionHours') -and $PartitionHours -ne 0) { $incompatibleParams += " - PartitionHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('MaxPartitions') -and $MaxPartitions -ne 160) { $incompatibleParams += " - MaxPartitions (audit query limits)" } + if ($PSBoundParameters.ContainsKey('ResultSize') -and $ResultSize -ne 10000) { $incompatibleParams += " - ResultSize (audit query page size)" } + if ($PSBoundParameters.ContainsKey('PacingMs') -and $PacingMs -ne 0) { $incompatibleParams += " - PacingMs (audit query throttling)" } + if ($AutoCompleteness) { $incompatibleParams += " - AutoCompleteness (audit log completeness checks)" } + if ($PSBoundParameters.ContainsKey('StreamingSchemaSample') -and $StreamingSchemaSample -ne 5000) { $incompatibleParams += " - StreamingSchemaSample (audit record schema sampling)" } + if ($PSBoundParameters.ContainsKey('StreamingChunkSize') -and $StreamingChunkSize -ne 5000) { $incompatibleParams += " - StreamingChunkSize (audit streaming batch size)" } + if ($PSBoundParameters.ContainsKey('ExportProgressInterval') -and $ExportProgressInterval -ne 10) { $incompatibleParams += " - ExportProgressInterval (audit export progress)" } + + # Filtering parameters + if ($PSBoundParameters.ContainsKey('AgentId')) { $incompatibleParams += " - AgentId (audit record filtering)" } + if ($AgentsOnly) { $incompatibleParams += " - AgentsOnly (audit record filtering)" } + if ($ExcludeAgents) { $incompatibleParams += " - ExcludeAgents (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('PromptFilter')) { $incompatibleParams += " - PromptFilter (audit record content filtering)" } + if ($PSBoundParameters.ContainsKey('UserIds')) { $incompatibleParams += " - UserIds (audit record filtering; Entra fetch retrieves all users)" } + if ($PSBoundParameters.ContainsKey('GroupNames')) { $incompatibleParams += " - GroupNames (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('RecordTypes')) { $incompatibleParams += " - RecordTypes (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('ServiceTypes')) { $incompatibleParams += " - ServiceTypes (audit record filtering)" } + + # Processing mode parameters + if ($ExplodeArrays) { $incompatibleParams += " - ExplodeArrays (audit record array expansion)" } + if ($ExplodeDeep) { $incompatibleParams += " - ExplodeDeep (audit record deep expansion)" } + if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $incompatibleParams += " - RAWInputCSV (offline audit replay mode)" } + + # Parallel processing parameters + if ($EnableParallel) { $incompatibleParams += " - EnableParallel (parallel audit query execution)" } + if ($PSBoundParameters.ContainsKey('MaxConcurrency') -and $MaxConcurrency -ne 10) { $incompatibleParams += " - MaxConcurrency (concurrent query/partition limit)" } + if ($PSBoundParameters.ContainsKey('MaxParallelGroups') -and $MaxParallelGroups -ne 8) { $incompatibleParams += " - MaxParallelGroups (parallel activity group limit)" } + if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Auto') { $incompatibleParams += " - ParallelMode (parallel processing mode)" } + if ($DisableAdaptive) { $incompatibleParams += " - DisableAdaptive (adaptive concurrency controls)" } + if ($PSBoundParameters.ContainsKey('ProgressSmoothingAlpha') -and $ProgressSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ProgressSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('HighLatencyMs') -and $HighLatencyMs -ne 90000) { $incompatibleParams += " - HighLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('MemoryPressureMB') -and $MemoryPressureMB -ne 1500) { $incompatibleParams += " - MemoryPressureMB (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyMs') -and $LowLatencyMs -ne 20000) { $incompatibleParams += " - LowLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyConsecutive') -and $LowLatencyConsecutive -ne 2) { $incompatibleParams += " - LowLatencyConsecutive (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputDropPct') -and $ThroughputDropPct -ne 15) { $incompatibleParams += " - ThroughputDropPct (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputSmoothingAlpha') -and $ThroughputSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ThroughputSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('AdaptiveConcurrencyCeiling') -and $AdaptiveConcurrencyCeiling -ne 6) { $incompatibleParams += " - AdaptiveConcurrencyCeiling (adaptive tuning)" } + + # Reliability parameters (audit-specific) + if ($PSBoundParameters.ContainsKey('CircuitBreakerThreshold') -and $CircuitBreakerThreshold -ne 5) { $incompatibleParams += " - CircuitBreakerThreshold (block failure circuit breaker)" } + if ($PSBoundParameters.ContainsKey('CircuitBreakerCooldownSeconds') -and $CircuitBreakerCooldownSeconds -ne 120) { $incompatibleParams += " - CircuitBreakerCooldownSeconds (circuit breaker cooldown)" } + if ($PSBoundParameters.ContainsKey('BackoffBaseSeconds') -and $BackoffBaseSeconds -ne 1.0) { $incompatibleParams += " - BackoffBaseSeconds (block retry backoff)" } + if ($PSBoundParameters.ContainsKey('BackoffMaxSeconds') -and $BackoffMaxSeconds -ne 45) { $incompatibleParams += " - BackoffMaxSeconds (block retry max backoff)" } + + # Alternative modes + if ($UseEOM) { $incompatibleParams += " - UseEOM (Exchange Online Management mode incompatible with Graph Entra enrichment)" } + + # Output combination parameters + if ($CombineOutput) { $incompatibleParams += " - CombineOutput (only relevant with multiple activity types)" } + if ($AppendFile) { $incompatibleParams += " - AppendFile (appending user-only data to existing audit output not supported)" } + + if ($incompatibleParams.Count -gt 0) { + Write-Host "" + Write-Host "ERROR: The -OnlyUserInfo switch cannot be used with the following parameters:" -ForegroundColor Red + Write-Host "" + $incompatibleParams | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + Write-Host "" + Write-Host "The -OnlyUserInfo switch exports only Entra user directory and license information (no audit logs)." -ForegroundColor Cyan + Write-Host "" + Write-Host "Compatible parameters:" -ForegroundColor Green + Write-Host " - OutputPath (where to save the file)" -ForegroundColor White + Write-Host " - Auth (authentication method: WebLogin, DeviceCode, Credential, Silent)" -ForegroundColor White + Write-Host " - Force (bypass interactive prompts)" -ForegroundColor White + Write-Host " - MaxNetworkOutageMinutes (network resilience for Graph API calls)" -ForegroundColor White + Write-Host " - EmitMetricsJson (track Entra retrieval metrics)" -ForegroundColor White + Write-Host " - MetricsPath (custom metrics output location)" -ForegroundColor White + Write-Host " - SkipDiagnostics (skip pre-query capability checks)" -ForegroundColor White + Write-Host "" + Write-Host "Please remove the incompatible parameters and try again." -ForegroundColor Cyan + Write-Host "" + exit 1 + } + + # If validation passes, configure for user-only export + Write-Host "" + Write-Host "INFO: -OnlyUserInfo mode enabled. Skipping all audit log retrieval, exporting only Entra user data." -ForegroundColor Green + Write-Host "" + $IncludeUserInfo = $true + $ActivityTypes = @() +} + +# PAX4A-GUARD-BEGIN +# -UserInfoFile / -GroupNames MUTUAL EXCLUSIVITY (Phase 4a) +# -UserInfoFile supplies the Entra user directory from a customer-provided CSV (bypassing the +# live /users pull). -GroupNames expands audit records against the LIVE directory. Supplying both +# is contradictory. Hard-stop here in the early parameter-validation stage so it fires on EVERY +# entry path (fresh and -Resume) BEFORE any -GroupNames consumer downstream (the RequiredScopes +# builder and the permissions banner). No partial work is done. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and $PSBoundParameters.ContainsKey('GroupNames')) { + Write-Host "" + Write-Host "ERROR: -UserInfoFile and -GroupNames cannot both be supplied." -ForegroundColor Red + Write-Host " -UserInfoFile ingests the Entra user directory from a customer-provided CSV." -ForegroundColor Yellow + Write-Host " -GroupNames expands audit records against the live Entra directory." -ForegroundColor Yellow + Write-Host " Supply exactly ONE: -UserInfoFile for a supplied directory, or -GroupNames for group filtering." -ForegroundColor Yellow + Write-Host "" + exit 1 +} +# PAX4A-GUARD-END + +# PAX4D-AUTOENABLE-BEGIN +# -UserInfoFile auto-enables -IncludeUserInfo (FU6): every -UserInfoFile export path (file naming, +# the multi-tab workbook, and the directory substitution seam) is gated on $IncludeUserInfo, so a +# customer need only pass -UserInfoFile. Mirrors the -OnlyUserInfo auto-enable above; fires on every +# entry path before any $IncludeUserInfo consumer. No effect when -UserInfoFile is unset. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile) -and -not $IncludeUserInfo) { + $IncludeUserInfo = $true +} +# PAX4D-AUTOENABLE-END + +# Canonical maps for Graph filter normalization +$recordTypeCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'azureactivedirectoryaccountlogon' = 'AzureActiveDirectoryAccountLogon' + 'azureactivedirectorystslogon' = 'AzureActiveDirectoryStsLogon' + 'exchangeadmin' = 'ExchangeAdmin' + 'exchangeitem' = 'ExchangeItem' + 'exchangemailbox' = 'ExchangeMailbox' + 'sharepointfileoperation' = 'SharePointFileOperation' + 'sharepointsharingoperation' = 'SharePointSharingOperation' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'microsoftteams' = 'MicrosoftTeams' +} + +$serviceCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'exchange' = 'Exchange' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'teams' = 'Teams' +} + +# Normalize optional Graph filter passthrough parameters (dedupe & trim) +# Split ActivityTypes if provided as comma-separated string +if ($ActivityTypes) { + $processedActivityTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ActivityTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedActivityTypes.Add($token) + } + } + $ActivityTypes = @( + $processedActivityTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ActivityTypes.Count -eq 0) { + $ActivityTypes = $null + } +} + +if ($RecordTypes) { + $processedRecordTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $RecordTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedRecordTypes.Add($token) + } + } + $RecordTypes = @( + $processedRecordTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($RecordTypes.Count -eq 0) { + $RecordTypes = $null + } else { + $RecordTypes = @( + foreach ($rt in $RecordTypes) { + $key = $rt.ToLowerInvariant() + if ($recordTypeCanonicalMap.ContainsKey($key)) { $recordTypeCanonicalMap[$key] } else { $rt } + } + ) | Select-Object -Unique + } +} + +if ($ServiceTypes) { + $processedServiceTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ServiceTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedServiceTypes.Add($token) + } + } + $ServiceTypes = @( + $processedServiceTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ServiceTypes.Count -eq 0) { + $ServiceTypes = $null + } else { + $ServiceTypes = @( + foreach ($svc in $ServiceTypes) { + $key = $svc.ToLowerInvariant() + if ($serviceCanonicalMap.ContainsKey($key)) { $serviceCanonicalMap[$key] } else { $svc } + } + ) | Select-Object -Unique + } +} + +# Mapping of audit record types to supported workloads for Graph security audit queries +$recordTypeWorkloadMap = @{ + 'azureActiveDirectory' = @('AzureActiveDirectory') + 'azureActiveDirectoryAccountLogon' = @('AzureActiveDirectory') + 'azureActiveDirectoryStsLogon' = @('AzureActiveDirectory') + 'exchangeAdmin' = @('Exchange') + 'exchangeItem' = @('Exchange') + 'exchangeMailbox' = @('Exchange') + 'sharePointFileOperation' = @('SharePoint','OneDrive') + 'sharePointSharingOperation' = @('SharePoint','OneDrive') + 'sharePoint' = @('SharePoint','OneDrive') + 'onedrive' = @('OneDrive') + 'microsoftTeams' = @('Teams') + # M365 usage record types: Process in first workload pass to avoid creating additional passes + # These record types capture cross-workload activities (Office apps, Forms, Stream, Planner, PowerApps) + # Mapping to Exchange ensures they run in the first service-filtered pass + 'officeNative' = @('Exchange') + 'microsoftForms' = @('Exchange') + 'microsoftStream' = @('Exchange') + 'plannerPlan' = @('Exchange') + 'plannerTask' = @('Exchange') + 'powerAppsApp' = @('Exchange') +} + +$serviceOperationMap = @{ + 'AzureActiveDirectory' = @('UserLoggedIn','UserLoginFailed','AdminLoggedIn','ResetUserPassword','AddRegisteredUser','UpdateUser','ChangedUserSetting') + 'Exchange' = @('MailItemsAccessed','Send','SendOnBehalf','SoftDelete','HardDelete','MoveToDeletedItems','CopyToFolder','AddMailboxPermission','RemoveMailboxPermission') + 'SharePoint' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'OneDrive' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'Teams' = @('TeamMemberAdded','TeamMemberRemoved','ChannelAdded','ChannelDeleted','ChannelMessageSent','ChannelMessageDeleted','TeamDeleted','TeamArchived','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'MicrosoftForms' = @('CreateForm','EditForm','DeleteForm','ViewForm','CreateResponse','SubmitResponse','ViewResponse','DeleteResponse') + 'MicrosoftStream' = @('StreamModified','StreamViewed','StreamDeleted','StreamDownloaded') + 'MicrosoftPlanner' = @('PlanCreated','PlanDeleted','PlanModified','TaskCreated','TaskDeleted','TaskModified','TaskAssigned','TaskCompleted') + 'PowerApps' = @('LaunchedApp','CreatedApp','EditedApp','DeletedApp','PublishedApp') +} + +$copilotBaseActivityType = 'CopilotInteraction' +$m365UsageServiceBundle = @('Exchange','SharePoint','OneDrive','Teams') +$m365UsageRecordBundle = @('ExchangeAdmin','ExchangeItem','ExchangeMailbox','SharePointFileOperation','SharePointSharingOperation','SharePoint','OneDrive','MicrosoftTeams','OfficeNative','MicrosoftForms','MicrosoftStream','PlannerPlan','PlannerTask','PowerAppsApp') +# Curated, trimmed M365 usage operations targeted at the Analytics-Hub M365 Usage Analytics +# dashboard. Scope: Exchange mail access, SharePoint/OneDrive file access, Teams chat/messaging, +# Teams meeting lifecycle, and Copilot/Connected-AI interaction signals. +$m365UsageActivityBundle = @( + # === Exchange / Email === + 'MailItemsAccessed','MailboxLogin','Send', + + # === SharePoint / OneDrive - File access === + 'FileAccessed','FileViewed','FilePreviewed','FileModified','FileDownloaded','FileUploaded', + + # === Teams - Chat / Messaging === + 'MessageSent','MessageRead','MessagesListed','ChatRetrieved','ChatCreated','TeamsSessionStarted', + + # === Teams - Meeting lifecycle === + 'MeetingParticipantJoined','MeetingStarted','MeetingEnded','MeetingParticipantDetail','MeetingDetail', + + # === Copilot / Connected AI === + 'CopilotInteraction','ConnectedAIAppInteraction' +) | Select-Object -Unique + +# Script version constant (must appear after param/help to keep param() valid as first executable block) +$ScriptVersion = '1.11.12' + +function Invoke-PaxVersionCheck { + # Informational, non-blocking, failure-isolated version check against the public PAX repo. + # Reads versions.json from the release branch, compares the purview script version, and prints + # a single info line. Never prompts, never throws, capped at ~5s if the server is unreachable. + param([string]$CurrentVersion) + $repoUrl = 'https://github.com/microsoft/PAX' + try { + $verUrl = 'https://raw.githubusercontent.com/microsoft/PAX/release/versions.json' + $resp = Invoke-RestMethod -Uri $verUrl -TimeoutSec 5 -ErrorAction Stop + $latest = [string]$resp.products.purview.version + $relDate = [string]$resp.lastUpdated + if ($latest -and ([version]$latest) -gt ([version]$CurrentVersion)) { + $line = " Update available: PAX v$latest" + if ($relDate) { $line += " (released $relDate)" } + $line += " - you are on v$CurrentVersion. Latest: $repoUrl" + Write-LogHost $line -ForegroundColor Cyan + } else { + Write-LogHost " Version check: you are on the latest PAX version (v$CurrentVersion). $repoUrl" -ForegroundColor DarkGray + } + } catch { + Write-LogHost " Version check skipped: the PAX GitHub repo was not reachable (offline or blocked). Latest: $repoUrl" -ForegroundColor DarkGray + } +} + +# --- Initialize/Clear persistent script variables to prevent cross-run contamination --- +# Note: Script-scoped variables persist across multiple script invocations in the same PowerShell session +$script:partitionStatus = $null +$script:processedJobIds = $null +$script:shownJobMessages = $null + +# --- Microsoft 365 Copilot license detection --- +# No hardcoded SKU GUIDs. Copilot service plans are discovered dynamically at runtime +# from /subscribedSkus by matching servicePlanName against the pattern '*COPILOT*'. +# Per-user eligibility is then computed from the user's assignedPlans where +# capabilityStatus == 'Enabled' and servicePlanId is in the discovered set. +# See Get-UserLicenseData(). + +# --- Synchronized Timestamp & OutputPath Validation --- + +# Generate synchronized timestamp for all output files in this run +$global:ScriptRunTimestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + +# --- Logging Helper Functions (defined early for use throughout script) --- +# $script:LogFile was already initialized to a bootstrap temp-path file at the top +# of the script body (right after param()). DO NOT re-null it here; that would +# discard the early-startup log entries already captured. The bootstrap file is +# renamed to the final location once $OutputFile is resolved (see the relocation +# block further down). The in-memory $script:LogBuffer remains a fallback for the +# rare case where bootstrap creation failed (e.g. read-only temp). +if (-not (Get-Variable -Name LogFile -Scope Script -ErrorAction SilentlyContinue)) { + $script:LogFile = $null +} +$script:LogBuffer = New-Object System.Collections.Generic.List[string] + +function Write-Log { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + Microsoft.PowerShell.Utility\Write-Host $Message + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# Log-file-only writer: never echoes to host. Used for diagnostic detail (temp paths, +# argument vectors, stack traces) that should be captured in the run log but not +# clutter the customer-facing console. +function Write-LogFile { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +function Write-LogHost { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$ForegroundColor = "White") + Microsoft.PowerShell.Utility\Write-Host $Message -ForegroundColor $ForegroundColor + try { + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [INFO] $Message" + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# Mirror Write-Host to log file with matching signature +function global:Write-Host { + [CmdletBinding()] + param( + [Parameter(Position=0, ValueFromPipeline=$true, ValueFromRemainingArguments=$true)] + $Object, + [object] $Separator, + [ConsoleColor] $ForegroundColor, + [ConsoleColor] $BackgroundColor, + [switch] $NoNewLine + ) + process { + Microsoft.PowerShell.Utility\Write-Host @PSBoundParameters + try { + # Compose message + $msgItems = @($Object) + $msg = ($msgItems | Out-String).TrimEnd() + if ($msg) { + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $entry = "[$timestamp] [INFO] $msg" + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $entry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($entry) | Out-Null } + } + } catch {} + } +} + +function Get-MaskedUsername { + <# + .SYNOPSIS + Masks a username or email address for secure display in logs and screenshots. + + .DESCRIPTION + Converts "admin@contoso.com" to "a******n@contoso.com" to prevent accidental + credential exposure in terminal output, screenshots, or log files. + + Preserves first and last character of local part, masks middle with 6 asterisks. + Returns original string if input is null, empty, or doesn't contain "@". + + .PARAMETER Username + The username or email address to mask + + .OUTPUTS + Masked string (e.g., "a******n@contoso.com") + + .EXAMPLE + Get-MaskedUsername -Username "admin@contoso.com" + Returns: "a******n@contoso.com" + #> + + param( + [Parameter(Mandatory = $false)] + [string]$Username + ) + + if ([string]::IsNullOrWhiteSpace($Username)) { + return $Username + } + + # Only mask if it looks like an email address + if ($Username -notmatch '@') { + return $Username + } + + $parts = $Username -split '@' + if ($parts.Count -ne 2) { + return $Username + } + + $localPart = $parts[0] + $domain = $parts[1] + + # Handle very short usernames + if ($localPart.Length -le 2) { + return "$($localPart[0])******@$domain" + } + + $first = $localPart[0] + $last = $localPart[$localPart.Length - 1] + $masked = "$first******$last@$domain" + + return $masked +} + +# ============================================================ +# DEIDENTIFICATION (-Deidentify): one-way, salted, deterministic, format-preserving. +# PowerShell deidentifies RAW output files only (the embedded Python processors +# deidentify rollup outputs). The salt, HMAC-SHA256 algorithm, normalization +# (Trim + lower-invariant), lowercase hex, and output formats below are BYTE-FOR-BYTE +# identical to the deid_* helpers in the CopilotInteraction and M365 Python +# processors (PAX deidentify spec) so the same identity hashes to the same token +# across all engines. OFF (default) = no-op: every helper returns its input unchanged. +# Irreversible: no decode map is produced or stored. DO NOT CHANGE THE SALT. +# ============================================================ +$script:PaxDeidEnabled = [bool]$Deidentify +$script:PaxDeidSaltBytes = [System.Text.Encoding]::UTF8.GetBytes('PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3') +$script:PaxDeidDomain = 'deidentified.domain' +$script:PaxDeidCache = @{} +$script:PaxDeidMarkerSuffix = '@deidentified.domain' # tail of every deidentified UPN/email token + +# -Deidentify is CSV-only: the deprecated Excel workbook path is never scrubbed, so a +# deidentified workbook would silently leak identifiable data. Block the combination. +if ($script:PaxDeidEnabled -and $ExportWorkbook) { + Write-Host "" -ForegroundColor Red + Write-Host "ERROR: -Deidentify cannot be combined with -ExportWorkbook (Excel)." -ForegroundColor Red + Write-Host "Deidentification scrubs CSV output only; the workbook path is not anonymized and would" -ForegroundColor Yellow + Write-Host "leak identifiable data. Re-run with CSV output (omit -ExportWorkbook) to deidentify." -ForegroundColor Yellow + exit 1 +} + +function Get-PaxDeidHex { + param([string]$Value, [int]$Length) + $norm = $Value.Trim().ToLowerInvariant() + $hmac = [System.Security.Cryptography.HMACSHA256]::new($script:PaxDeidSaltBytes) + try { + $bytes = $hmac.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($norm)) + } finally { + $hmac.Dispose() + } + $sb = [System.Text.StringBuilder]::new(64) + foreach ($b in $bytes) { [void]$sb.Append($b.ToString('x2')) } + return $sb.ToString().Substring(0, $Length) +} + +function Get-PaxDeidUpn { + # UPN / email -> <12hex>@deidentified.domain + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "upn`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = (Get-PaxDeidHex -Value $Value -Length 12) + '@' + $script:PaxDeidDomain + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidName { + # Person / device display name -> <12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "name`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = Get-PaxDeidHex -Value $Value -Length 12 + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidGuid { + # GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "guid`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + $v = '{0}-{1}-{2}-{3}-{4}' -f $h.Substring(0,8), $h.Substring(8,4), $h.Substring(12,4), $h.Substring(16,4), $h.Substring(20,12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidSid { + # SID -> deterministic S-1-5-21---- shape + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "sid`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + $d1 = [Convert]::ToUInt32($h.Substring(0,8), 16) + $d2 = [Convert]::ToUInt32($h.Substring(8,8), 16) + $d3 = [Convert]::ToUInt32($h.Substring(16,8), 16) + $d4 = [Convert]::ToUInt32($h.Substring(24,8), 16) + $v = "S-1-5-21-$d1-$d2-$d3-$d4" + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidToken { + # Opaque id (employeeId, immutableId) -> <12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "tok`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = Get-PaxDeidHex -Value $Value -Length 12 + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidResource { + # Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count) + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "res`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = 'site_' + (Get-PaxDeidHex -Value $Value -Length 12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidFile { + # File / document name -> file_<12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "file`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = 'file_' + (Get-PaxDeidHex -Value $Value -Length 12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidProxy { + # proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + # Handles ';'-delimited multi-value fields. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $parts = $Value -split ';' + $out = foreach ($entry in $parts) { + if ([string]::IsNullOrEmpty($entry)) { + $entry + } elseif ($entry.Contains(':')) { + $idx = $entry.IndexOf(':') + $entry.Substring(0, $idx) + ':' + (Get-PaxDeidUpn -Value $entry.Substring($idx + 1)) + } else { + Get-PaxDeidUpn -Value $entry + } + } + return ($out -join ';') +} + +function Get-PaxDeidIp { + # IP address -> deterministic, format-preserving token. IPv4 -> a.b.c.d (4 octets from the + # first 4 HMAC bytes); IPv6 -> 8 hextets from the 16-byte HMAC. Stays a valid-looking IP so + # downstream schema/parsers are unaffected. No-op when off or empty. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "ip`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + if ($Value.Contains(':')) { + $groups = for ($i = 0; $i -lt 8; $i++) { $h.Substring($i * 4, 4) } + $v = $groups -join ':' + } else { + $octets = for ($i = 0; $i -lt 4; $i++) { [Convert]::ToInt32($h.Substring($i * 2, 2), 16) } + $v = $octets -join '.' + } + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidByType { + # Dispatch a single value to the type-appropriate Get-PaxDeid* helper. Used by the JSON + # scrubber so the same helper set (and tokens) applies to nested AuditData leaves. + param([string]$Value, [string]$Type) + switch ($Type) { + 'Upn' { Get-PaxDeidUpn -Value $Value } + 'Name' { Get-PaxDeidName -Value $Value } + 'Guid' { Get-PaxDeidGuid -Value $Value } + 'Sid' { Get-PaxDeidSid -Value $Value } + 'Token' { Get-PaxDeidToken -Value $Value } + 'Resource' { Get-PaxDeidResource -Value $Value } + 'File' { Get-PaxDeidFile -Value $Value } + 'Proxy' { Get-PaxDeidProxy -Value $Value } + 'Ip' { Get-PaxDeidIp -Value $Value } + default { $Value } + } +} + +function Get-PaxDeidJsonMap { + # PATH-AWARE map of AuditData JSON leaf -> deid type. Path uses '.' for nested objects and + # '[]' for array elements (e.g. Folders[].FolderItems[].Subject). Path-awareness is required + # because the same leaf name means different things by location: root 'Id' is the audit + # RecordId (KEEP) while Item.Id / FolderItems[].Id are item identifiers (scrub). Only the keys + # below are altered; every other field is preserved verbatim (app/tenant/org identities, + # operations, types, timestamps, etc. are intentionally absent and therefore kept). + @{ + # Identity (UPN / SID / GUID) + 'UserId' = 'Upn' + 'MailboxOwnerUPN' = 'Upn' + 'LogonUserSid' = 'Sid' + 'MailboxOwnerSid' = 'Sid' + 'MailboxGuid' = 'Guid' + 'UserKey' = 'Guid' + 'TokenObjectId' = 'Guid' + 'SessionId' = 'Guid' + 'AppAccessContext.AADSessionId' = 'Guid' + 'AppAccessContext.UniqueTokenId' = 'Token' + # IP addresses + 'ClientIP' = 'Ip' + 'ClientIPAddress' = 'Ip' + 'ActorIpAddress' = 'Ip' + # Resources / files (rollup-aligned) + 'SiteUrl' = 'Resource' + 'SourceRelativeUrl' = 'Resource' + 'SourceFileName' = 'File' + 'CopilotEventData.ThreadId' = 'Token' + 'CopilotEventData.AccessedResources[].SiteUrl' = 'Resource' + 'CopilotEventData.AccessedResources[].Name' = 'File' + # Mail item identifiers + content + 'Folders[].FolderItems[].InternetMessageId' = 'Token' + 'Folders[].FolderItems[].Id' = 'Token' + 'Folders[].FolderItems[].ImmutableId' = 'Token' + 'Folders[].FolderItems[].Subject' = 'Token' + 'Folders[].Id' = 'Token' + 'Item.InternetMessageId' = 'Token' + 'Item.Id' = 'Token' + 'Item.ImmutableId' = 'Token' + 'Item.Subject' = 'Token' + 'Item.Attachments' = 'Token' + 'Item.ParentFolder.Id' = 'Token' + } +} + +function Invoke-PaxDeidJsonNode { + # Recursively scrub a System.Text.Json JsonNode in place: any STRING leaf whose path is in + # $Map is replaced with its deid token; every other leaf/structure is left untouched. Keeps + # strings as strings (no ConvertFrom-Json date auto-conversion), preserving full fidelity. + param( + [System.Text.Json.Nodes.JsonNode]$Node, + [string]$Path, + [hashtable]$Map + ) + if ($null -eq $Node) { return } + if ($Node -is [System.Text.Json.Nodes.JsonObject]) { + $keys = @($Node.GetEnumerator() | ForEach-Object { $_.Key }) + foreach ($key in $keys) { + $child = $Node[$key] + $childPath = if ($Path) { "$Path.$key" } else { $key } + if ($child -is [System.Text.Json.Nodes.JsonValue]) { + if ($child.GetValueKind() -eq [System.Text.Json.JsonValueKind]::String -and $Map.ContainsKey($childPath)) { + $orig = $child.GetValue[string]() + $new = Get-PaxDeidByType -Value $orig -Type $Map[$childPath] + if ($new -ne $orig) { $Node[$key] = [System.Text.Json.Nodes.JsonValue]::Create($new) } + } + } else { + Invoke-PaxDeidJsonNode -Node $child -Path $childPath -Map $Map + } + } + } + elseif ($Node -is [System.Text.Json.Nodes.JsonArray]) { + $arrPath = "$Path[]" + for ($i = 0; $i -lt $Node.Count; $i++) { + $child = $Node[$i] + if ($child -is [System.Text.Json.Nodes.JsonValue]) { + if ($child.GetValueKind() -eq [System.Text.Json.JsonValueKind]::String -and $Map.ContainsKey($arrPath)) { + $orig = $child.GetValue[string]() + $new = Get-PaxDeidByType -Value $orig -Type $Map[$arrPath] + if ($new -ne $orig) { $Node[$i] = [System.Text.Json.Nodes.JsonValue]::Create($new) } + } + } else { + Invoke-PaxDeidJsonNode -Node $child -Path $arrPath -Map $Map + } + } + } +} + +function Get-PaxDeidJson { + # Deidentify a JSON blob (e.g. the AuditData column) IN FULL: parse, recursively scrub only the + # mapped PII leaves, re-serialize. Returns the value unchanged when off/empty. Returns the + # redaction marker ONLY when the value cannot be parsed as JSON (safety net so a malformed blob + # can never leak). Re-serializes with relaxed escaping to stay close to the source formatting. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $node = $null + try { $node = [System.Text.Json.Nodes.JsonNode]::Parse($Value) } catch { return '[REDACTED-DEIDENTIFY]' } + if ($null -eq $node) { return $Value } + if ($null -eq $script:PaxDeidJsonMapCached) { $script:PaxDeidJsonMapCached = Get-PaxDeidJsonMap } + if ($null -eq $script:PaxDeidJsonOpts) { + $script:PaxDeidJsonOpts = [System.Text.Json.JsonSerializerOptions]::new() + $script:PaxDeidJsonOpts.Encoder = [System.Text.Encodings.Web.JavaScriptEncoder]::UnsafeRelaxedJsonEscaping + } + Invoke-PaxDeidJsonNode -Node $node -Path '' -Map $script:PaxDeidJsonMapCached + return $node.ToJsonString($script:PaxDeidJsonOpts) +} + +function Invoke-PaxRawDeidentify { + <# + .SYNOPSIS + Post-write pass: deidentify a finished RAW output CSV in place. + .DESCRIPTION + Operates on whichever target columns are present in the file's header, so the SAME + function handles both the EntraUsers CSV (47-col) and the Purview audit CSV (153-col + exploded). Each present identity/resource column is hashed with the type-appropriate + Get-PaxDeid* helper; tokens match the Python-deidentified rollup because the salt and + algorithm are shared. Any raw AuditData / CopilotEventData JSON blob column is + deidentified IN PLACE via Get-PaxDeidJson — a recursive, path-aware System.Text.Json + scrub that tokenizes only the PII leaves and leaves every other field (timestamps, + workload, app/tenant ids, schema urls, flags) byte-for-byte identical, so the blob stays + full-fidelity Purview data with only PII altered. (A blob that fails to parse is redacted + to [REDACTED-DEIDENTIFY] as a safety net so PII can never leak.) + No-op when -Deidentify is off. Rewrites atomically via a temp file + Move-Item, preserving + column order. PowerShell only ever scrubs RAW files; rollup outputs are owned by Python. + #> + param([Parameter(Mandatory = $true)][string]$Path) + if (-not $script:PaxDeidEnabled) { return } + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { return } + + # Column name -> deid helper kind. Union of audit (153-col) + EntraUsers (47-col) fields. + # Column names are disjoint across the two schemas, so one map serves both. + $map = @{ + # Purview audit exploded identity + resource fields + 'UserId' = 'Upn' + 'MailboxOwnerUPN' = 'Upn' + 'MailboxGuid' = 'Guid' + 'LogonUserSid' = 'Sid' + 'MailboxOwnerSid' = 'Sid' + 'DeviceDisplayName' = 'Name' + 'SiteUrl' = 'Resource' + 'SourceRelativeUrl' = 'Resource' + 'SourceFileName' = 'File' + 'AccessedResource_SiteUrl' = 'Resource' + 'AccessedResource_Name' = 'File' + # Purview audit exploded — additional identity / network / resource fields + # (flat top-level columns produced by the 153-col explosion; nested JSON PII is + # handled separately by Get-PaxDeidJson on the AuditData blob). + 'ClientIP' = 'Ip' + 'ClientIPAddress' = 'Ip' + 'ActorIpAddress' = 'Ip' + 'UserKey' = 'Guid' + 'TokenObjectId' = 'Guid' + 'SessionId' = 'Guid' + 'ThreadId' = 'Token' + 'ChatId' = 'Token' + 'ConversationId' = 'Token' + 'Site' = 'Resource' + 'MeetingURL' = 'Resource' + 'TeamName' = 'Name' + 'ChannelName' = 'Name' + 'VideoName' = 'File' + 'FormName' = 'File' + # EntraUsers identity fields + 'userPrincipalName' = 'Upn' + 'displayName' = 'Name' + 'mail' = 'Upn' + 'givenName' = 'Name' + 'surname' = 'Name' + 'UserName' = 'Upn' + 'employeeId' = 'Token' + 'onPremisesImmutableId' = 'Token' + 'proxyAddresses_Primary' = 'Proxy' + 'proxyAddresses_All' = 'Proxy' + 'id' = 'Guid' + 'manager_id' = 'Guid' + 'manager_userPrincipalName' = 'Upn' + 'manager_displayName' = 'Name' + 'manager_mail' = 'Upn' + 'ManagerID' = 'Guid' + } + # JSON blob columns: deidentified in place by recursive path-aware scrub (Get-PaxDeidJson), + # NOT flat-hashed — their PII lives in nested keys the column map cannot reach. + $jsonCols = @('AuditData', 'CopilotEventData') + + $rows = @(Import-Csv -LiteralPath $Path) + if ($rows.Count -eq 0) { return } + $columns = $rows[0].PSObject.Properties.Name + $active = @{} + foreach ($c in $columns) { if ($map.ContainsKey($c)) { $active[$c] = $map[$c] } } + $activeJson = @($jsonCols | Where-Object { $columns -contains $_ }) + if ($active.Count -eq 0 -and $activeJson.Count -eq 0) { return } + + foreach ($row in $rows) { + foreach ($col in @($active.Keys)) { + $val = $row.$col + if ([string]::IsNullOrEmpty($val)) { continue } + $row.$col = switch ($active[$col]) { + 'Upn' { Get-PaxDeidUpn -Value $val } + 'Name' { Get-PaxDeidName -Value $val } + 'Guid' { Get-PaxDeidGuid -Value $val } + 'Sid' { Get-PaxDeidSid -Value $val } + 'Token' { Get-PaxDeidToken -Value $val } + 'Resource' { Get-PaxDeidResource -Value $val } + 'File' { Get-PaxDeidFile -Value $val } + 'Proxy' { Get-PaxDeidProxy -Value $val } + 'Ip' { Get-PaxDeidIp -Value $val } + } + } + foreach ($jc in $activeJson) { + if (-not [string]::IsNullOrEmpty($row.$jc)) { $row.$jc = Get-PaxDeidJson -Value $row.$jc } + } + } + + # Atomic in-place rewrite (temp + replace), preserving original column order. + $tmp = "$Path.deid.tmp" + Open-CsvWriter -Path $tmp -Columns $columns + Write-CsvRows -Rows $rows -Columns $columns + Close-CsvWriter + Move-Item -LiteralPath $tmp -Destination $Path -Force +} + +function Test-PaxFileDeidentified { + <# + .SYNOPSIS + Classify a CSV append target as 'deid' | 'raw' | 'unknown' by sampling its identity column. + .DESCRIPTION + 'deid' = at least one non-empty identity value carries the deidentify marker + (@deidentified.domain). + 'raw' = identity values present, none carry the marker. + 'unknown'= file missing/unreadable, no recognised identity column, or no non-empty + identity values to judge by (caller skips the guard in that case). + Streams at most ~200 rows (pipeline short-circuits) so it is cheap on large files. + #> + param([Parameter(Mandatory = $true)][string]$Path) + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { return 'unknown' } + try { + $sample = @(Import-Csv -LiteralPath $Path | Select-Object -First 200) + } catch { + return 'unknown' + } + if ($sample.Count -eq 0) { return 'unknown' } + $cols = $sample[0].PSObject.Properties.Name + # Identity columns across every deidentified output family (audit raw, CopilotInteraction + # fact (aibv), M365 rollup, Users dim). The aio fact has no UPN column -> 'unknown' there, + # but the paired -AppendUserInfo Users dim (PersonId) carries the signal. + $idCol = @('Audit_UserId', 'UserId', 'PersonId', 'userPrincipalName', 'MailboxOwnerUPN') | + Where-Object { $cols -contains $_ } | Select-Object -First 1 + if (-not $idCol) { return 'unknown' } + $sawValue = $false + foreach ($r in $sample) { + $v = [string]$r.$idCol + if ([string]::IsNullOrEmpty($v)) { continue } + $sawValue = $true + if ($v.TrimEnd().ToLowerInvariant().EndsWith($script:PaxDeidMarkerSuffix)) { return 'deid' } + } + if ($sawValue) { return 'raw' } + return 'unknown' +} + +function Assert-PaxDeidAppendConsistency { + <# + .SYNOPSIS + Hard-stop guard preventing deidentified/raw data from being mixed in an append target. + .DESCRIPTION + Data-accuracy gate for -AppendFile / -AppendUserInfo: + * BLOCK when the target's deidentify state disagrees with this run's -Deidentify state + (mixing hashed + raw identities corrupts user joins and distinct counts). + * BLOCK appending into an already-deidentified target in NON-rollup mode (the whole-file + post-pass would re-hash existing rows -> double-hashing). Rollup append into a + deidentified target is allowed because it MERGES two same-salt outputs. + Throws on violation; the caller renders the message and exits non-zero. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Targets, + [Parameter(Mandatory = $true)][bool]$IsRollup + ) + $runDeid = [bool]$script:PaxDeidEnabled + foreach ($t in $Targets) { + $path = [string]$t.Path + $label = [string]$t.Label + if ([string]::IsNullOrWhiteSpace($path)) { continue } + $state = Test-PaxFileDeidentified -Path $path + if ($state -eq 'unknown') { continue } + $tgtDeid = ($state -eq 'deid') + if ($tgtDeid -ne $runDeid) { + $desc = if ($tgtDeid) { 'is already DEIDENTIFIED but this run is NOT -Deidentify' } + else { 'is NOT deidentified but this run IS -Deidentify' } + throw ("-Deidentify/append mismatch: the $label target $desc. Mixing deidentified and raw " + + "identities in one file corrupts user joins and distinct counts. Use a separate target " + + "for deidentified output, or re-run with a matching -Deidentify state.`nTarget: $path") + } + if ($tgtDeid -and $runDeid -and -not $IsRollup) { + throw ("-Deidentify cannot append to an already-deidentified file in non-rollup mode: the " + + "$label target is deidentified and a non-rollup append would re-hash its existing rows " + + "(double-hashing). Append on a RAW master and deidentify a fresh copy, or use " + + "-Rollup / -RollupPlusRaw (which merge matching-salt outputs).`nTarget: $path") + } + } +} + +# --- Helper Function: Detect if PAYG billing is configured in tenant --- +function Test-PAYGBillingEnabled { + <# + .SYNOPSIS + Attempts to detect if Microsoft Purview PAYG billing is configured in the tenant. + + .DESCRIPTION + Checks for indicators that PAYG billing is enabled: + - Attempts to query audit records for AIAppInteraction type (PAYG-only) + - If records exist or query succeeds without billing errors, PAYG is likely enabled + - Returns $null if detection is inconclusive + + .OUTPUTS + $true if PAYG billing appears to be enabled + $false if PAYG billing appears to be disabled or not configured + $null if detection is inconclusive (requires actual query attempt) + #> + + # Note: The most reliable way to detect PAYG is to attempt a query for AIAppInteraction + # and check for specific error responses. However, this requires actual data/timeframe. + # For now, we return $null to indicate "unknown" and let the post-query detection handle it. + + Write-LogHost "PAYG billing detection: Deferred to post-query validation" -ForegroundColor DarkGray + return $null +} + +# --- Excel Export Validation --- + +# AppendFile with Excel requires ExportWorkbook +if ($AppendFile -and $ExportWorkbook -eq $false) { + # AppendFile is fine for CSV (will be handled later), but needs validation for Excel intent + # This check is only if user explicitly wants Excel append +} + +if ($AppendFile -and -not $ExportWorkbook -and -not $PSBoundParameters.ContainsKey('ExportWorkbook')) { + # User wants AppendFile but didn't specify format - this is OK, will default to CSV append +} + +# Rollup post-processor always requires a single combined Purview CSV. When +# -Rollup / -RollupPlusRaw is bound but -CombineOutput is not, force it on +# here so the export-mode banner below reflects the resolved (combined) shape +# instead of the user's default separate-files layout. The broader +# -Rollup / -RollupPlusRaw validation (mutual exclusion, blockers, processor +# mode detection) still runs later in the rollup-handling block; that block's +# own -CombineOutput auto-enable is a no-op once this one has fired. +if (($Rollup -or $RollupPlusRaw) -and -not $CombineOutput) { + $_rollupSwitchName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $_rollupSwitchName auto-enabled -CombineOutput (rollup post-processor requires a single combined Purview CSV)." -ForegroundColor Cyan +} + +# Log export mode +if ($ExportWorkbook) { + # Determine Excel output mode based on -CombineOutput parameter + if ($CombineOutput) { + $excelModeMsg = "Excel export mode: Combined activity tab + separate EntraUsers tab (if requested)" + if ($IncludeAgent365Info -or $OnlyAgent365Info) { $excelModeMsg += " + Agents365 tab" } + Write-Host $excelModeMsg -ForegroundColor Cyan + } else { + # Default for Excel: separated tabs + $excelModeMsg = "Excel export mode: Multi-tab workbook (one tab per activity type)" + if ($IncludeAgent365Info -or $OnlyAgent365Info) { $excelModeMsg += " + Agents365 tab" } + Write-Host $excelModeMsg -ForegroundColor Cyan + } + + if ($AppendFile) { + Write-Host "Append mode: Enabled (will validate existing workbook structure)" -ForegroundColor Cyan + } + Write-Host "" +} else { + # CSV export mode + if ($OnlyUserInfo) { + # OnlyUserInfo mode: No activity files, just Entra user data + $onlyUserMsg = "CSV export mode: Entra user directory and licensing data only (no audit logs)" + if ($IncludeAgent365Info) { $onlyUserMsg += " + separate Agents365 file" } + Write-Host $onlyUserMsg -ForegroundColor Cyan + } elseif ($OnlyAgent365Info) { + # OnlyAgent365Info mode: No activity files, just Agent 365 catalog data + Write-Host "CSV export mode: Microsoft Agent 365 catalog only (no audit logs, no Entra users)" -ForegroundColor Cyan + } else { + # Determine CSV output mode based on -CombineOutput parameter + if ($RAWInputCSV -and -not $CombineOutput.IsPresent) { $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) } + if ($CombineOutput.IsPresent -or $RAWInputCSV) { + # User specified -CombineOutput switch: combine all activity types + $csvModeMsg = "Combined activity file" + if ($IncludeUserInfo) { $csvModeMsg += " + separate EntraUsers file" } + if ($IncludeAgent365Info) { $csvModeMsg += " + separate Agents365 file" } + Write-Host "CSV export mode: $csvModeMsg" -ForegroundColor Cyan + } else { + # Default for live CSV: separate files per activity type + $csvModeMsg = "Separate activity files (one per activity type)" + if ($IncludeUserInfo) { $csvModeMsg += " + EntraUsers file" } + if ($IncludeAgent365Info) { $csvModeMsg += " + Agents365 file" } + Write-Host "CSV export mode: $csvModeMsg" -ForegroundColor Cyan + } + } + Write-Host "" +} + +# OutputPath shape normalization runs AFTER tier inference below (folder-only +# enforcement applies only to Local tier; SharePoint/Fabric URLs are handled +# by the remote-output plumbing). + +# ============================================================================ +# DESTINATION VALIDATION AND TIER INFERENCE +# ---------------------------------------------------------------------------- +# Storage tier is inferred from each -OutputPath* value's form: +# Drive-rooted absolute path -> Local +# https://*.sharepoint* -> SharePoint +# https://*.onelake.dfs.fabric.* -> Fabric +# UNC paths (\\server\share\...) are rejected on every destination switch. +# All destinations supplied to a single run must resolve to the same storage +# tier (no mixed Local/SP/Fabric in one invocation). Any two destinations +# that resolve to identical fully qualified paths are rejected (FullPath +# collision). $OutputPathLog allows the Fabric Files/ form additionally. +# Legacy $script:RemoteOutputMode / $script:RemoteOutputUrl are populated from +# the inferred Purview destination so downstream code that predates the new +# model continues to operate unchanged. +# ============================================================================ +$script:RemoteOutputMode = 'None' # 'None' | 'SharePoint' | 'Fabric' +$script:RemoteOutputUrl = $null +$script:RemoteScratchDir = $null + +# Per-data-type effective destinations and inferred tiers. Populated below. +$script:DestTier = @{} # 'Purview'|'UserInfo'|'Agent365Info'|'Log' -> 'Local'|'SharePoint'|'Fabric' +$script:DestRaw = @{} # original user-supplied value (trimmed) +$script:DestIsBound = @{} # was the switch actually supplied? + +$spUrlPattern = '^https?://[^/]+\.sharepoint(?:-df|-mil)?\.[a-z]{2,3}(?:/.+)?$' +# Fabric Tables/ segment is constrained to identifier-style strings +# (alpha/underscore start, then alphanumeric/underscore) to match the Schemas-mode +# Lakehouse rules and the downstream Delta-write detection regex (search for +# $fabricDeltaMode). Files/ subpaths remain permissive. +# Item segment is EITHER the name form (.Lakehouse, suffix required) OR the +# GUID form (, no suffix) - both are first-class OneLake DFS addressing +# modes (learn.microsoft.com/fabric/onelake/onelake-access-api). The workspace +# segment ([^/]+) already accepts a name or GUID. +$fabricItemSeg = '(?:[^/]+\.Lakehouse|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})' +$fabricRootPattern = "^https://([a-z0-9-]+-)?onelake\.dfs\.fabric\.microsoft\.com/[^/]+/$fabricItemSeg(/Tables(/[A-Za-z_][A-Za-z0-9_]*)?|/Files(/.+)?)?/?`$" +$fabricFilesPattern = "^https://([a-z0-9-]+-)?onelake\.dfs\.fabric\.microsoft\.com/[^/]+/$fabricItemSeg/Files(/.+)?/?`$" + +function script:Get-PathTier { + param([string]$Value, [string]$SwitchName, [switch]$AllowFabricFilesOnly) + if ([string]::IsNullOrWhiteSpace($Value)) { return $null } + $v = $Value.Trim() + if ($v -match '^\\\\') { + Write-Host ("ERROR: -{0} does not accept UNC paths ('{1}')." -f $SwitchName, $v) -ForegroundColor Red + Write-Host " Provide a drive-rooted local path, a SharePoint URL, or a Fabric OneLake URL." -ForegroundColor Yellow + exit 1 + } + if ($v -match '^https?://') { + if ($v -match $spUrlPattern) { return 'SharePoint' } + if ($AllowFabricFilesOnly) { + if ($v -match $fabricFilesPattern) { return 'Fabric' } + Write-Host ("ERROR: -{0} on a Fabric destination must point under Files/ (logs are not tabular)." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + exit 1 + } + if ($v -match $fabricRootPattern) { return 'Fabric' } + Write-Host ("ERROR: -{0} URL is not a recognized SharePoint or Fabric Lakehouse destination." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + Write-Host " Examples:" -ForegroundColor Yellow + Write-Host " https://.sharepoint.com/sites//[/]" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com//.Lakehouse" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com// (GUID form, no .Lakehouse suffix)" -ForegroundColor DarkGray + exit 1 + } + # Local: require drive-rooted absolute path. + if ($v -match '^[A-Za-z]:[\\/]' -or $v -match '^/') { + return 'Local' + } + Write-Host ("ERROR: -{0} must be a drive-rooted absolute path, a SharePoint URL, or a Fabric OneLake URL." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + exit 1 +} + +$destSwitches = @( + @{ Key = 'Purview' ; Name = 'OutputPath' ; Value = $OutputPath ; AllowFabricFilesOnly = $false }, + @{ Key = 'UserInfo' ; Name = 'OutputPathUserInfo' ; Value = $OutputPathUserInfo ; AllowFabricFilesOnly = $false }, + @{ Key = 'Agent365Info' ; Name = 'OutputPathAgent365Info' ; Value = $OutputPathAgent365Info ; AllowFabricFilesOnly = $false }, + @{ Key = 'Log' ; Name = 'OutputPathLog' ; Value = $OutputPathLog ; AllowFabricFilesOnly = $true } +) +foreach ($ds in $destSwitches) { + $bound = $PSBoundParameters.ContainsKey($ds.Name) + $script:DestIsBound[$ds.Key] = $bound + if ($bound -and -not [string]::IsNullOrWhiteSpace($ds.Value)) { + $tier = script:Get-PathTier -Value $ds.Value -SwitchName $ds.Name -AllowFabricFilesOnly:$ds.AllowFabricFilesOnly + $script:DestTier[$ds.Key] = $tier + $script:DestRaw[$ds.Key] = $ds.Value.Trim() + } +} + +# --- Append-side tier inference --- +# Treat the three -Append* switches as alternate suppliers of the stream destination. +# A relative filename (no scheme, no drive letter) is "deferred" — tier is inherited +# from the corresponding -OutputPath* (or the default scratch dir). A URL or drive- +# rooted full path sets the stream tier directly when the -OutputPath* counterpart +# is unbound (the XOR validator below rejects the both-bound case). +$script:AppendIsBound = @{} +$script:AppendRaw = @{} +$script:AppendIsRemote = @{} +$appendSwitches = @( + @{ Key = 'Purview' ; Name = 'AppendFile' ; Value = $AppendFile ; Bound = $PSBoundParameters.ContainsKey('AppendFile') }, + @{ Key = 'UserInfo' ; Name = 'AppendUserInfo' ; Value = $AppendUserInfo ; Bound = $PSBoundParameters.ContainsKey('AppendUserInfo') }, + @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info ; Bound = $PSBoundParameters.ContainsKey('AppendAgent365Info') } +) +foreach ($as in $appendSwitches) { + $script:AppendIsBound[$as.Key] = $as.Bound + if (-not $as.Bound -or [string]::IsNullOrWhiteSpace($as.Value)) { continue } + $v = $as.Value.Trim() + $script:AppendRaw[$as.Key] = $v + $looksUrl = $v -match '^https?://' + $looksRooted = $v -match '^[A-Za-z]:[\\/]' -or $v -match '^/' + $looksUnc = $v -match '^\\\\' + if ($looksUnc) { + Write-Host ("ERROR: -{0} does not accept UNC paths ('{1}')." -f $as.Name, $v) -ForegroundColor Red + Write-Host " Provide a relative filename, a drive-rooted local path, a SharePoint URL, or a Fabric OneLake URL." -ForegroundColor Yellow + exit 1 + } + if ($looksUrl -or $looksRooted) { + # Full-form: classify via Get-PathTier (which exits on invalid forms). + $aTier = script:Get-PathTier -Value $v -SwitchName $as.Name + $script:AppendIsRemote[$as.Key] = ($aTier -ne 'Local') + # Promote the stream's tier from the Append side when the OutputPath side is unbound. + if (-not $script:DestTier.ContainsKey($as.Key)) { + $script:DestTier[$as.Key] = $aTier + $script:DestRaw[$as.Key] = $v + } + elseif ($script:DestTier[$as.Key] -ne $aTier) { + # OutputPath* and Append* both bound on different tiers — XOR validator + # below will fire the canonical "both bound" error, but flag the tier + # mismatch here too in case XOR is bypassed by a future code path. + Write-Host ("ERROR: -{0} ({1}) and -{2} ({3}) resolve to different storage tiers." -f $as.Name, $aTier, ($destSwitches | Where-Object Key -EQ $as.Key | Select-Object -First 1 -ExpandProperty Name), $script:DestTier[$as.Key]) -ForegroundColor Red + exit 1 + } + } else { + # Relative filename — tier deferred to the -OutputPath* side (or local default). + $script:AppendIsRemote[$as.Key] = $false + } +} + +# Tier consistency across all bound destinations (excluding -OutputPathLog when it +# targets Fabric Files/ — the log is allowed under Files/ on a Fabric data run). +$tiersForConsistency = @() +foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + if ($script:DestTier.ContainsKey($k)) { $tiersForConsistency += $script:DestTier[$k] } +} +$tiersForConsistency = $tiersForConsistency | Select-Object -Unique +if ($tiersForConsistency.Count -gt 1) { + Write-Host ("ERROR: Mixed storage tiers across -OutputPath* switches ({0}). All destinations in one run must resolve to the same tier." -f ($tiersForConsistency -join ', ')) -ForegroundColor Red + exit 1 +} + +# FullPath collision: any two bound destinations resolving to identical fully +# qualified paths are rejected. Folder-only inputs that resolve to the same +# folder with different default basenames are not collisions (compared on the +# normalized input as supplied; default basenames are appended downstream). +$normalizedDests = @{} +foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + $norm = $script:DestRaw[$k].TrimEnd('/','\').ToLowerInvariant() + # Only flag collisions when the supplied form is a full file path (contains a basename extension). + if ($norm -match '\.[a-z0-9]{2,5}$') { + if ($normalizedDests.ContainsKey($norm)) { + Write-Host ("ERROR: -{0} and -{1} resolve to the same fully qualified path:" -f $normalizedDests[$norm], $k) -ForegroundColor Red + Write-Host (" {0}" -f $script:DestRaw[$k]) -ForegroundColor Yellow + exit 1 + } + $normalizedDests[$norm] = $k + } +} + +# Populate legacy back-compat variables from the Purview destination so downstream +# code paths that pre-date the new model keep working without per-call refactors. +# When Purview is out of scope (-OnlyUserInfo / -OnlyAgent365Info), or when Purview +# is in scope but its destination was supplied via -AppendFile (already absorbed +# into $script:DestTier['Purview'] above), the promotion below still fires. +# Additionally, in only-modes the remote tier may come from -OutputPathUserInfo / +# -AppendUserInfo / -OutputPathAgent365Info / -AppendAgent365Info instead — promote +# from whichever stream is in scope so the remote pre-flight + scratch-dir handling +# downstream activates correctly. +if ($script:DestTier.ContainsKey('Purview')) { + switch ($script:DestTier['Purview']) { + 'SharePoint' { + $script:RemoteOutputMode = 'SharePoint' + $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') + } + 'Fabric' { + $script:RemoteOutputMode = 'Fabric' + $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') + } + } +} +# Fallback promotion: when Purview tier is unset (only-modes), promote from +# the first in-scope non-Purview stream that resolved to a remote tier. +if ($script:RemoteOutputMode -eq 'None') { + foreach ($k in @('UserInfo','Agent365Info')) { + if (-not $script:DestTier.ContainsKey($k)) { continue } + $t = $script:DestTier[$k] + if ($t -eq 'SharePoint' -or $t -eq 'Fabric') { + $script:RemoteOutputMode = $t + $script:RemoteOutputUrl = $script:DestRaw[$k].TrimEnd('/') + break + } + } +} + +# Normalize RemoteOutputUrl + populate $script:DestParentUrl with FOLDER URLs. +# When the source switch was -AppendFile / -AppendUserInfo / -AppendAgent365Info, +# or when -OutputPath* was supplied as a file URL, the original DestRaw values are +# FILE URLs (last segment ends in a recognized data-artifact extension). Downstream +# code — Resolve-SharePointTarget / Resolve-FabricTarget (parse the URL as folder + +# leaf), Get-DisplayPath (appends sibling leaves like .log / _PARTIAL.csv), the +# per-data-type parent-URL routing in the upload sweep ($dtParentUrl / +# $logParentOverride), and Get-RemoteFile-* — all assume folder semantics. Leaving +# a file URL in those code paths would produce '.../foo.csv/foo.log' style nesting +# in display, and a 404 NotFound when re-downloading the append target. +# +# $script:DestRaw stays AS-SUPPLIED so the per-data-type destinations banner can +# still print the file-URL form the user typed in. +# $script:DestParentUrl is the FOLDER URL form used by upload/download routing and +# display-leaf composition. +# $script:RemoteOutputUrl is normalized to a folder URL for the same reason. +# .Lakehouse is a folder marker (Lakehouse root); do NOT treat it as a file. +$script:DestParentUrl = @{} +$script:NormalizeFolderUrl = { + param([string]$u) + if ([string]::IsNullOrWhiteSpace($u)) { return $u } + if ($u -match '/[^/]+\.(csv|log|xlsx|json|parquet|txt)$') { + return ($u -replace '/[^/]+\.[A-Za-z0-9]+$','') + } + return $u +} +if ($script:RemoteOutputUrl) { + $script:RemoteOutputUrl = & $script:NormalizeFolderUrl $script:RemoteOutputUrl +} +foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + if (-not $script:DestTier.ContainsKey($k)) { continue } + if ($script:DestTier[$k] -eq 'Local') { continue } + $script:DestParentUrl[$k] = & $script:NormalizeFolderUrl $script:DestRaw[$k] +} + +# Effective-basename helpers. Single source of truth for the leaf used +# by the EntraUsers and Agent 365 streams everywhere display/banner code mentions +# them. Resolution order: +# 1. If -AppendUserInfo / -AppendAgent365Info is bound, inherit the target's +# original leaf — the writer reuses that leaf (so the merged output replaces +# the original remote URL via the upload sweep's Append-leaf match), and +# the banners need to display the same name the customer typed and the +# remote destination will end up holding. +# 2. Else if -OutputPath* was supplied in file-form, use that file's leaf. +# 3. Else default to the timestamped basename. +# NOTE: These helpers must run AFTER $script:AppendRaw / $script:DestRaw are +# populated (done by the destSwitches + appendSwitches blocks above). +$script:GetEffectiveEntraBasename = { + # Returns the basename of the RAW EntraUsers CSV. Under -AppendUserInfo the + # raw is always written to its natural / pristine name (the caller separately + # merges raw -> the AppendUserInfo target via Merge-UsersCsv). If the + # caller-supplied AppendUserInfo leaf happens to collide with the natural raw + # leaf in the same destination folder, the caller is responsible for adding + # a '_raw' suffix to the raw filename (Resolve-PristineRawEntraPath). + # + # Resolution order for the natural raw basename: + # 1. If -OutputPathUserInfo is bound with a file leaf, use that leaf. + # 2. Else default to the timestamped 'EntraUsers_MAClicensing_.csv'. + if ($script:DestIsBound -and $script:DestIsBound['UserInfo'] -and $script:DestRaw.ContainsKey('UserInfo')) { + $leaf = [System.IO.Path]::GetFileName($script:DestRaw['UserInfo']) + if ($leaf -and $leaf -match '\.[A-Za-z0-9]{2,5}$') { return $leaf } + } + return ("EntraUsers_MAClicensing_{0}.csv" -f $global:ScriptRunTimestamp) +} +$script:GetEffectiveAgent365Basename = { + if ($script:AppendIsBound -and $script:AppendIsBound['Agent365Info'] -and $script:AppendRaw.ContainsKey('Agent365Info')) { + return [System.IO.Path]::GetFileName($script:AppendRaw['Agent365Info']) + } + if ($script:DestIsBound -and $script:DestIsBound['Agent365Info'] -and $script:DestRaw.ContainsKey('Agent365Info')) { + $leaf = [System.IO.Path]::GetFileName($script:DestRaw['Agent365Info']) + if ($leaf -and $leaf -match '\.[A-Za-z0-9]{2,5}$') { return $leaf } + } + return ("Agent365_{0}.csv" -f $global:ScriptRunTimestamp) +} + +# Replay/EOM modes are not supported with remote output. +if ($script:RemoteOutputMode -ne 'None') { + $remoteBlockers = @() + if ($UseEOM) { $remoteBlockers += '-UseEOM' } + if ($RAWInputCSV) { $remoteBlockers += '-RAWInputCSV' } + if ($remoteBlockers.Count -gt 0) { + Write-Host ("ERROR: Remote output destinations are not supported with: {0}" -f ($remoteBlockers -join ', ')) -ForegroundColor Red + Write-Host " These modes operate on existing files and have no remote-write story." -ForegroundColor Yellow + exit 1 + } +} + +# ExportWorkbook is restricted to Local and SharePoint tiers. +if ($ExportWorkbook -and $script:DestTier.ContainsKey('Purview') -and $script:DestTier['Purview'] -eq 'Fabric') { + Write-Host "ERROR: -ExportWorkbook is not supported on Fabric (OneLake) destinations." -ForegroundColor Red + Write-Host " Use a Local or SharePoint -OutputPath for workbook output." -ForegroundColor Yellow + exit 1 +} +if ($ExportWorkbook) { + $workbookCompanionConflict = $false + if (-not ($OnlyUserInfo -or $OnlyAgent365Info)) { + if ($script:DestIsBound['UserInfo']) { $workbookCompanionConflict = $true } + if ($script:DestIsBound['Agent365Info']) { $workbookCompanionConflict = $true } + } + if ($workbookCompanionConflict) { + Write-Host "ERROR: -OutputPathUserInfo / -OutputPathAgent365Info cannot be combined with -ExportWorkbook when Purview output is in scope." -ForegroundColor Red + Write-Host " The workbook is a single file containing all tabs - companion CSV destinations would be ignored." -ForegroundColor Yellow + exit 1 + } +} + +# Local-tier OutputPath normalization. When the Purview destination is Local (or +# unbound and therefore defaulted to a local folder), ensure it ends with a path +# separator if it is a folder (no extension) and create the directory if needed. +# Full-path-with-basename Local forms are also accepted; downstream auto-basename +# logic detects the file form and uses the supplied basename verbatim. +if (-not $PSBoundParameters.ContainsKey('OutputPath') -or [string]::IsNullOrWhiteSpace($OutputPath)) { + # When -OutputPath is omitted, infer the run's scratch+log directory from the + # DOMINANT in-scope stream's bound -Append* / -OutputPath* target. Otherwise + # $OutputPath would fall back to $PSScriptRoot — polluting the script's own + # folder with the run log, .pax_incremental shards/seed JSONs, and any rollup + # temp files even though the customer pinned a real data destination elsewhere + # via -AppendFile / -AppendUserInfo / -AppendAgent365Info. + # + # Dominant stream resolution (mirrors XOR validator's scope rules): + # -OnlyUserInfo -> UserInfo (AppendUserInfo / OutputPathUserInfo) + # -OnlyAgent365Info -> Agent365Info (AppendAgent365Info / OutputPathAgent365Info) + # default -> Purview (AppendFile) [XOR guarantees AppendFile + # is bound when OutputPath is not, in scope] + # + # Source of truth is $script:DestRaw — populated by the destSwitches loop above + # from -OutputPath* AND by the appendSwitches loop from rooted/URL -Append* + # values. Relative-filename -Append* forms intentionally do NOT populate DestRaw + # (they defer to the unspecified -OutputPath*), so they fall through to the + # legacy $PSScriptRoot default — correct behavior, since the relative form was + # only ever meaningful with -OutputPath in scope. + # + # Remote URLs are excluded from inference: remote-mode runs stage scratch+log + # locally (the upload sweep handles the move to the remote URL afterward), and + # the pre-existing $PSScriptRoot scratch behavior is preserved unchanged for + # those runs. Cross-tier mismatches are already rejected upstream by the + # tier-consistency validator (L2603-L2613). + $_pvInferKey = if ($OnlyUserInfo) { 'UserInfo' } elseif ($OnlyAgent365Info) { 'Agent365Info' } else { 'Purview' } + $_pvInferParent = $null + if ($script:DestRaw.ContainsKey($_pvInferKey)) { + $_pvInferRaw = $script:DestRaw[$_pvInferKey] + if (-not [string]::IsNullOrWhiteSpace($_pvInferRaw) -and ($_pvInferRaw -notmatch '^https?://')) { + # Folder form -> use as-is; file form -> use parent directory. + $_pvInferCandidate = if (Test-Path -Path $_pvInferRaw -PathType Container) { + $_pvInferRaw + } else { + Split-Path -Parent $_pvInferRaw + } + if ($_pvInferCandidate -and (Test-Path -Path $_pvInferCandidate -PathType Container)) { + $_pvInferParent = $_pvInferCandidate + } + } + } + if ($_pvInferParent) { + $OutputPath = $_pvInferParent + Write-Host ("INFO: Inferred run scratch+log dir from in-scope -{0}: {1}" -f $(if ($OnlyUserInfo) { 'AppendUserInfo/-OutputPathUserInfo' } elseif ($OnlyAgent365Info) { 'AppendAgent365Info/-OutputPathAgent365Info' } else { 'AppendFile' }), $OutputPath) -ForegroundColor DarkGray + } elseif ($PSScriptRoot) { + $OutputPath = $PSScriptRoot + } else { + $OutputPath = (Get-Location).Path + } + if (-not $OutputPath.EndsWith('\') -and -not $OutputPath.EndsWith('/')) { + $OutputPath = $OutputPath + '\' + } + # Do NOT clobber a Purview tier that was already promoted from a remote + # -AppendFile URL (SharePoint/Fabric). The Local default only applies when + # no Append-side promotion has populated DestTier['Purview'] yet — otherwise + # the per-data-type destinations banner mis-tags Purview as [Local]. + if (-not $script:DestTier.ContainsKey('Purview')) { + $script:DestTier['Purview'] = 'Local' + $script:DestRaw['Purview'] = $OutputPath + } +} +if ($script:DestTier.ContainsKey('Purview') -and $script:DestTier['Purview'] -eq 'Local') { + $isFileForm = ($OutputPath -match '\.[a-zA-Z0-9]{2,5}$') -and ($OutputPath -notmatch '[\\/]$') + if (-not $isFileForm) { + if (-not $OutputPath.EndsWith('\') -and -not $OutputPath.EndsWith('/')) { + $OutputPath = $OutputPath + '\' + } + if (-not (Test-Path -Path $OutputPath -PathType Container)) { + try { + New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null + Write-Host "INFO: Created output directory: $OutputPath" -ForegroundColor Green + } + catch { + Write-Host "ERROR: Failed to create output directory: $OutputPath" -ForegroundColor Red + Write-Host "Error: $_" -ForegroundColor Red + exit 1 + } + } + } else { + $parentDir = Split-Path -Parent $OutputPath + if ($parentDir -and -not (Test-Path -Path $parentDir -PathType Container)) { + try { + New-Item -Path $parentDir -ItemType Directory -Force | Out-Null + Write-Host "INFO: Created output directory: $parentDir" -ForegroundColor Green + } + catch { + Write-Host "ERROR: Failed to create output directory: $parentDir" -ForegroundColor Red + Write-Host "Error: $_" -ForegroundColor Red + exit 1 + } + } + } +} + +# Managed-identity-aware host guard. IDENTITY_ENDPOINT is set by Azure runtimes +# (Functions, App Service, Container Apps, Container Instances, Fabric notebook / +# container runtimes, etc.) to advertise that a managed-identity token endpoint +# is available on the host. It does NOT mean managed identity is the only valid +# auth choice. Two shapes are accepted in MI-aware hosts: +# 1. -Auth ManagedIdentity (uses the assigned identity on the host), or +# 2. -Auth AppRegistration with bound credentials (-ClientId plus one of +# -ClientSecret / -ClientCertificateThumbprint / -ClientCertificatePath) - +# legitimate for cross-tenant runs where a service-provider tenant calls +# into a customer tenant from a host that also happens to expose a local +# managed identity. +# Only block when neither shape applies (e.g. the operator forgot +# -Auth ManagedIdentity in a runbook, or invoked an interactive auth mode with +# no credentials in a non-interactive host). +if ($env:IDENTITY_ENDPOINT -and $Auth -ne 'ManagedIdentity') { + $hasAppRegCreds = ($Auth -eq 'AppRegistration') -and ` + $ClientId -and ` + ($ClientSecret -or $ClientCertificateThumbprint -or $ClientCertificatePath) + if (-not $hasAppRegCreds) { + Write-Host "ERROR: Detected a managed-identity environment (IDENTITY_ENDPOINT is set) but -Auth is '$Auth' without credentials." -ForegroundColor Red + Write-Host " Re-run with -Auth ManagedIdentity to use the assigned identity, or supply -ClientId together with -ClientSecret / -ClientCertificateThumbprint / -ClientCertificatePath for -Auth AppRegistration." -ForegroundColor Yellow + exit 1 + } + Write-Host "INFO: Managed identity available in host; proceeding with explicit -Auth AppRegistration as requested." -ForegroundColor DarkGray +} + +# Resolve-DataTypePaths +# --------------------- +# Central per-data-type effective destination lookup. Each downstream writer that +# emits a customer-visible artifact calls this helper for its data-type key and +# receives: +# .Tier - 'Local' | 'SharePoint' | 'Fabric' +# .Raw - the as-supplied input (already validated and trimmed) +# .IsBound - $true if the caller supplied -OutputPath on the CLI +# .EffectiveDir - parent folder/URL the writer should land its file under +# .Basename - default basename to use when the supplied form is folder/URL only +# ('' when the supplied form already includes a basename) +# Data-type keys: 'Purview', 'UserInfo', 'Agent365Info', 'Log'. +# When a non-Purview data type was not bound on the CLI, it inherits the Purview +# tier/destination so legacy single-destination behavior is preserved. +function script:Resolve-DataTypePaths { + param( + [Parameter(Mandatory)] [ValidateSet('Purview','UserInfo','Agent365Info','Log')] [string]$DataType, + [string]$DefaultBasename = '' + ) + $key = $DataType + $isBound = $false + $boundViaAppendOnly = $false + if ($script:DestIsBound -and $script:DestIsBound.ContainsKey($key)) { $isBound = [bool]$script:DestIsBound[$key] } + # DestIsBound is the narrow "-OutputPath was supplied" signal — it does + # NOT cover the case where only the Append-side counterpart (-AppendUserInfo / + # -AppendAgent365Info) is bound. When a rooted/URL Append value is supplied for + # UserInfo or Agent365Info without its OutputPath counterpart, the appendSwitches + # loop at parse-time (mirrored on resume) promotes $script:DestTier[$key] and + # $script:DestRaw[$key] from that Append value, but $script:DestIsBound[$key] + # stays $false. Without the fall-through below, this function would inherit + # tier/raw from Purview — and under -AppendFile, $script:DestRaw['Purview'] + # is the Purview activity FILE path, so the caller ends up joining the + # data-type leaf onto a file path and produces a "\" + # string that no filesystem accepts. Treat an Append-side promotion as "bound" + # for path resolution purposes; the narrow DestIsBound semantics used elsewhere + # (Excel companion conflict check, etc.) remain unchanged. + if (-not $isBound -and $script:AppendIsBound -and $script:AppendIsBound.ContainsKey($key) -and $script:AppendIsBound[$key] -and $script:DestRaw -and $script:DestRaw.ContainsKey($key)) { + $isBound = $true + $boundViaAppendOnly = $true + } + if (-not $isBound) { + # Inherit from Purview tier/raw. + $tier = if ($script:DestTier.ContainsKey('Purview')) { $script:DestTier['Purview'] } else { 'Local' } + $raw = if ($script:DestRaw.ContainsKey('Purview')) { $script:DestRaw['Purview'] } else { $OutputPath } + } else { + $tier = $script:DestTier[$key] + $raw = $script:DestRaw[$key] + } + # Determine whether $raw is folder/URL form vs. file-form (has a basename extension). + $isFileForm = $false + if ($tier -eq 'Local') { + $isFileForm = ($raw -match '\.[a-zA-Z0-9]{2,5}$') -and ($raw -notmatch '[\\/]$') + } elseif ($tier -eq 'SharePoint' -or $tier -eq 'Fabric') { + # Treat trailing-segment-with-extension as file form; otherwise it's a folder/URL. + $last = ($raw.TrimEnd('/').Split('/') | Select-Object -Last 1) + $isFileForm = $last -match '\.[a-zA-Z0-9]{2,5}$' + } + if ($isFileForm) { + $effDir = if ($tier -eq 'Local') { Split-Path -Parent $raw } else { ($raw.TrimEnd('/') -replace '/[^/]+$','') } + # When isBound was promoted ONLY via the Append-side fall-through above, + # $raw is the merge TARGET (-AppendUserInfo / -AppendAgent365Info), not a + # customer-specified raw destination basename. The pristine raw output + # this function's caller will produce must keep its natural name (the + # caller's $DefaultBasename — e.g., 'EntraUsers_MAClicensing_.csv') + # so that it stays distinct from the merge target on disk; the EffectiveDir + # is still inherited because the raw needs to land in the same folder as + # the merge target. Without this carve-out, the caller's collision-check + # (raw-leaf vs. AppendUserInfo-leaf) would fire spuriously and rename + # the raw with a '_raw' suffix that the customer never asked for. + # Agent365's caller already passes the AppendAgent365Info target leaf as + # its $DefaultBasename (via $script:GetEffectiveAgent365Basename), so + # returning $DefaultBasename here is semantically identical for Agent365. + # Callers that explicitly bind -OutputPath set DestIsBound and + # take the normal Split-Path -Leaf $raw branch below — unchanged. + if ($boundViaAppendOnly) { + $basename = $DefaultBasename + } else { + $basename = if ($tier -eq 'Local') { Split-Path -Leaf $raw } else { ($raw.TrimEnd('/').Split('/') | Select-Object -Last 1) } + } + return [pscustomobject]@{ + Tier = $tier; Raw = $raw; IsBound = $isBound + EffectiveDir = $effDir; Basename = $basename + } + } + return [pscustomobject]@{ + Tier = $tier; Raw = $raw; IsBound = $isBound + EffectiveDir = $raw; Basename = $DefaultBasename + } +} + +# Redirect OutputPath to a per-run scratch dir under the OS temp folder when remote mode is active. +# Done BEFORE any downstream code derives paths from $OutputPath. +if ($script:RemoteOutputMode -ne 'None') { + $baseTemp = [System.IO.Path]::GetTempPath() + $script:RemoteScratchDir = Join-Path $baseTemp ("PAX_" + (Get-Date -Format 'yyyyMMdd_HHmmss')) + try { + New-Item -Path $script:RemoteScratchDir -ItemType Directory -Force | Out-Null + } catch { + Write-Host ("ERROR: Failed to create scratch directory '{0}': {1}" -f $script:RemoteScratchDir, $_.Exception.Message) -ForegroundColor Red + exit 1 + } + $OutputPath = $script:RemoteScratchDir + $sep = [System.IO.Path]::DirectorySeparatorChar + if (-not $OutputPath.EndsWith($sep)) { $OutputPath = $OutputPath + $sep } + Write-Host ("INFO: Remote output mode '{0}' active." -f $script:RemoteOutputMode) -ForegroundColor Cyan + Write-Host ("INFO: Local scratch dir : {0}" -f $OutputPath) -ForegroundColor Cyan + Write-Host "INFO: Per-data-type destinations shown in the banner below." -ForegroundColor Cyan +} + +# Validate -AppendFile / -AppendUserInfo / -AppendAgent365Info combinations. +# Each -Append* parameter targets one logical output stream: +# -AppendFile : activity (audit) outputs - CopilotInteraction / M365 / DSPM / etc. +# -AppendUserInfo : EntraUsers / MAC licensing snapshot +# -AppendAgent365Info : Agent 365 catalog +# Only* and Append* targeting the SAME stream compose naturally: Only* scopes the +# run to that stream's data, Append* picks the write mode (merge vs overwrite). +# Only* with -AppendFile is rejected because Only* skips audit log retrieval +# entirely, leaving -AppendFile with no activity data to merge. +# When only an -Append* is given, the matching -Include* is auto-enabled for ergonomics. + +if ($AppendFile -and $OnlyUserInfo) { + Write-Host "ERROR: -AppendFile cannot be used with -OnlyUserInfo" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -AppendFile targets activity (audit) outputs, but -OnlyUserInfo skips audit" -ForegroundColor Yellow + Write-Host " log retrieval entirely, so there is no activity data to merge. To append the" -ForegroundColor Yellow + Write-Host " EntraUsers snapshot under -OnlyUserInfo, use -AppendUserInfo (the two compose)." -ForegroundColor Yellow + exit 1 +} + +if ($AppendFile -and $OnlyAgent365Info) { + Write-Host "ERROR: -AppendFile cannot be used with -OnlyAgent365Info" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -AppendFile targets activity (audit) outputs, but -OnlyAgent365Info skips audit" -ForegroundColor Yellow + Write-Host " log retrieval entirely, so there is no activity data to merge. To append the" -ForegroundColor Yellow + Write-Host " Agent 365 catalog under -OnlyAgent365Info, use -AppendAgent365Info (the two compose)." -ForegroundColor Yellow + exit 1 +} + +# Pairing of -AppendFile with -IncludeUserInfo / -IncludeAgent365Info is non-mandatory. +# When -Append* is omitted, the matching dimension file is written non-append for that +# run (the prior target is replaced if it exists). The dedup-on-target join is performed +# only when the matching -Append* switch is also supplied. + +# Auto-imply the matching -Include* when only the -Append* is provided. +if ($AppendUserInfo -and -not $IncludeUserInfo) { $IncludeUserInfo = $true } +if ($AppendAgent365Info -and -not $IncludeAgent365Info) { $IncludeAgent365Info = $true } + +# ============================================================ +# DESTINATION PAIR XOR VALIDATION +# For each output stream, when the stream is in scope, the user must supply +# EXACTLY ONE of (-OutputPath* | -Append*) — never both, never neither. +# When the stream is out of scope, neither member of the pair may be supplied. +# Streams: +# Purview : in scope unless -OnlyUserInfo or -OnlyAgent365Info +# UserInfo : in scope when -IncludeUserInfo or -OnlyUserInfo +# Agent365Info : in scope when -IncludeAgent365Info or -OnlyAgent365Info +# Note: the Purview default-to-$PSScriptRoot fallback fires AFTER this block, +# so we test against $PSBoundParameters here, not the live $OutputPath value. +# Skipped under -Resume: the resume-allowed-list check (later) rejects any +# destination switches on the resume command line, and the checkpoint restore +# rehydrates every per-stream destination from the saved snapshot. +# ============================================================ +if (-not $ResumeSpecified) { + +$pvOutBound = $PSBoundParameters.ContainsKey('OutputPath') +$pvAppBound = $PSBoundParameters.ContainsKey('AppendFile') +$uiOutBound = $PSBoundParameters.ContainsKey('OutputPathUserInfo') +$uiAppBound = $PSBoundParameters.ContainsKey('AppendUserInfo') +$agOutBound = $PSBoundParameters.ContainsKey('OutputPathAgent365Info') +$agAppBound = $PSBoundParameters.ContainsKey('AppendAgent365Info') + +$purviewInScope = (-not $OnlyUserInfo) -and (-not $OnlyAgent365Info) +$userInfoInScope = $IncludeUserInfo -or $OnlyUserInfo +$agentInScope = $IncludeAgent365Info -or $OnlyAgent365Info + +# --- Purview stream --- +if ($purviewInScope) { + if ($pvOutBound -and $pvAppBound) { + Write-Host "ERROR: -OutputPath and -AppendFile cannot both be supplied." -ForegroundColor Red + Write-Host " For the Purview activity stream, provide EXACTLY ONE of the pair:" -ForegroundColor Yellow + Write-Host " -OutputPath : write a new file at the supplied folder/full path" -ForegroundColor Yellow + Write-Host " -AppendFile : append to an existing file (filename or full path)" -ForegroundColor Yellow + exit 1 + } + if (-not $pvOutBound -and -not $pvAppBound) { + Write-Host "ERROR: Purview audit output destination not specified." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPath OR -AppendFile" -ForegroundColor Yellow + Write-Host " Accepted forms (either switch):" -ForegroundColor Yellow + Write-Host " Local : C:\Reports\ (folder) or C:\Reports\Run.csv (full path)" -ForegroundColor DarkGray + Write-Host " SharePoint : https://.sharepoint.com/sites//[/]" -ForegroundColor DarkGray + Write-Host " Fabric : https://onelake.dfs.fabric.microsoft.com//.Lakehouse[/Tables|/Files...]" -ForegroundColor DarkGray + exit 1 + } +} else { + # Only-modes: Purview not in scope. -AppendFile makes no sense (no Purview file). + # -OutputPath is permitted (acts as scratch dir for remote streams) but flagged. + if ($pvAppBound) { + $onlyName = if ($OnlyUserInfo) { '-OnlyUserInfo' } else { '-OnlyAgent365Info' } + Write-Host ("ERROR: -AppendFile is not valid with {0}." -f $onlyName) -ForegroundColor Red + Write-Host " -AppendFile targets the Purview activity stream, which is out of scope in only-modes." -ForegroundColor Yellow + Write-Host " Use -AppendUserInfo / -AppendAgent365Info to append to the in-scope stream." -ForegroundColor Yellow + exit 1 + } +} + +# --- UserInfo stream --- +if ($userInfoInScope) { + if ($uiOutBound -and $uiAppBound) { + Write-Host "ERROR: -OutputPathUserInfo and -AppendUserInfo cannot both be supplied." -ForegroundColor Red + Write-Host " Provide EXACTLY ONE for the EntraUsers stream." -ForegroundColor Yellow + exit 1 + } + if (-not $uiOutBound -and -not $uiAppBound -and -not $ExportWorkbook) { + Write-Host "ERROR: -IncludeUserInfo / -OnlyUserInfo requires a destination for the EntraUsers stream." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPathUserInfo OR -AppendUserInfo" -ForegroundColor Yellow + exit 1 + } +} else { + if ($uiOutBound -or $uiAppBound) { + $bn = if ($uiOutBound) { '-OutputPathUserInfo' } else { '-AppendUserInfo' } + Write-Host ("ERROR: {0} requires -IncludeUserInfo or -OnlyUserInfo to be in scope." -f $bn) -ForegroundColor Red + Write-Host " Drop the destination switch, or add -IncludeUserInfo / -OnlyUserInfo." -ForegroundColor Yellow + exit 1 + } +} + +# --- Agent 365 stream --- +if ($agentInScope) { + if ($agOutBound -and $agAppBound) { + Write-Host "ERROR: -OutputPathAgent365Info and -AppendAgent365Info cannot both be supplied." -ForegroundColor Red + Write-Host " Provide EXACTLY ONE for the Agent 365 stream." -ForegroundColor Yellow + exit 1 + } + if (-not $agOutBound -and -not $agAppBound -and -not $ExportWorkbook) { + Write-Host "ERROR: -IncludeAgent365Info / -OnlyAgent365Info requires a destination for the Agent 365 stream." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPathAgent365Info OR -AppendAgent365Info" -ForegroundColor Yellow + exit 1 + } +} else { + if ($agOutBound -or $agAppBound) { + $bn = if ($agOutBound) { '-OutputPathAgent365Info' } else { '-AppendAgent365Info' } + Write-Host ("ERROR: {0} requires -IncludeAgent365Info or -OnlyAgent365Info to be in scope." -f $bn) -ForegroundColor Red + Write-Host " Drop the destination switch, or add -IncludeAgent365Info / -OnlyAgent365Info." -ForegroundColor Yellow + exit 1 + } +} + +} # end if (-not $ResumeSpecified) destination-pair XOR validation + +# ============================================================ +# AGENT 365 SWITCH VALIDATION +# Validates -IncludeAgent365Info / -OnlyAgent365Info parameter +# combinations and prints informational banners where the +# Microsoft Graph Agent Package Management API endpoint +# limitations require user awareness. +# ============================================================ + +# Mutually exclusive +if ($IncludeAgent365Info -and $OnlyAgent365Info) { + Write-Host "ERROR: -IncludeAgent365Info and -OnlyAgent365Info are mutually exclusive." -ForegroundColor Red + Write-Host " Use -IncludeAgent365Info to add agents data to a normal Purview run." -ForegroundColor Yellow + Write-Host " Use -OnlyAgent365Info to skip the audit pull and produce only the agents CSV." -ForegroundColor Yellow + exit 1 +} + +# -OnlyAgent365Info conflicts with audit-implying switches +if ($OnlyAgent365Info) { + $conflictingSwitches = @() + if ($IncludeM365Usage) { $conflictingSwitches += '-IncludeM365Usage' } + if ($IncludeCopilotInteraction) { $conflictingSwitches += '-IncludeCopilotInteraction' } + if ($AgentsOnly) { $conflictingSwitches += '-AgentsOnly' } + if ($ExcludeAgents) { $conflictingSwitches += '-ExcludeAgents' } + if ($CombineOutput) { $conflictingSwitches += '-CombineOutput' } + if ($OnlyUserInfo) { $conflictingSwitches += '-OnlyUserInfo' } + if ($AppendFile) { $conflictingSwitches += '-AppendFile' } + if ($conflictingSwitches.Count -gt 0) { + Write-Host "ERROR: -OnlyAgent365Info cannot be combined with: $($conflictingSwitches -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -OnlyAgent365Info skips the Purview audit pull entirely. Switches that imply" -ForegroundColor Yellow + Write-Host " audit retrieval (or another 'Only' mode) are not compatible with it." -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host " -IncludeUserInfo is allowed (produces EntraUsers CSV alongside Agent 365 CSV)." -ForegroundColor Green + exit 1 + } +} + +# ========================================================================== +# NONINTERACTIVE-HOST DETECTION HELPER +# ========================================================================== +# Used by: +# - the Agent365 + -Auth AppRegistration fail-fast gate (below) +# - Read-Checkpoint's data-loss / selection / re-auth Read-Host prompts +# A host is considered noninteractive when ANY of the following are true: +# * [Environment]::UserInteractive is $false (Windows service, scheduled task, ACA Job). +# * Stdin is redirected ([Console]::IsInputRedirected) — true for pipelines, containers +# that don't allocate a TTY, and most CI runners. +# * $Host.Name is the remoting endpoint ('ServerRemoteHost'). +# * The PAX_NONINTERACTIVE environment variable is set to a truthy value (escape hatch +# so operators can force noninteractive behavior on hosts the detection misses). +# Operators can FORCE interactive behavior via PAX_FORCE_INTERACTIVE=1 if they know the +# host genuinely accepts keyboard input despite a redirected stdin (rare; documented). +function script:Test-IsNonInteractive { + if ($env:PAX_FORCE_INTERACTIVE -in @('1','true','True','TRUE','yes','Yes','YES')) { return $false } + if ($env:PAX_NONINTERACTIVE -in @('1','true','True','TRUE','yes','Yes','YES')) { return $true } + try { if ($Host.Name -eq 'ServerRemoteHost') { return $true } } catch {} + try { if (-not [Environment]::UserInteractive) { return $true } } catch {} + try { if ([Console]::IsInputRedirected) { return $true } } catch {} + return $false +} + +# ========================================================================== +# RUNTIME STATE-CONTRACT HELPER +# ========================================================================== +# Validates that critical hashtables (metrics, partitionStatus, checkpoint payload, +# resume state) contain the required keys at the boundaries where they are +# produced or consumed. Intentionally lightweight: it asserts shape only, never +# value semantics, and never mutates the input. On contract violation it throws +# a precise error of the form: +# "State contract violation: missing required keys: " +# Callers should let the throw propagate; an unguarded violation is a hard bug +# that would otherwise surface as a NullReferenceException later in the run. +function script:Assert-StateContract { + param( + [Parameter(Mandatory)] [string] $ContractName, + [Parameter(Mandatory)] [AllowNull()] $State, + [Parameter(Mandatory)] [string[]] $RequiredKeys + ) + if ($null -eq $State) { + throw "State contract violation: $ContractName is `$null but required keys are: $($RequiredKeys -join ', ')" + } + # Support hashtable, OrderedDictionary, and PSCustomObject inputs uniformly. + $present = @{} + if ($State -is [System.Collections.IDictionary]) { + foreach ($k in $State.Keys) { $present[[string]$k] = $true } + } elseif ($State.PSObject -and $State.PSObject.Properties) { + foreach ($p in $State.PSObject.Properties) { $present[$p.Name] = $true } + } else { + throw "State contract violation: $ContractName has unsupported type $($State.GetType().FullName); expected IDictionary or PSCustomObject." + } + $missing = @() + foreach ($k in $RequiredKeys) { if (-not $present.ContainsKey($k)) { $missing += $k } } + if ($missing.Count -gt 0) { + throw "State contract violation: $ContractName missing required keys: $($missing -join ', ')" + } +} + +# -OnlyAgent365Info is incompatible with replay (-RAWInputCSV), EOM mode, and Resume +# (it skips the audit phase entirely - nothing to checkpoint or resume from). +# -IncludeAgent365Info IS compatible with -Resume: the audit phase resumes from the +# checkpoint and Agent 365 enrichment runs at end of run as it would have originally. +if (($IncludeAgent365Info -or $OnlyAgent365Info)) { + $incompatModeSwitches = @() + if ($RAWInputCSV) { $incompatModeSwitches += '-RAWInputCSV (replay mode)' } + if ($UseEOM) { $incompatModeSwitches += '-UseEOM (Exchange Online Management mode)' } + if ($ResumeSpecified -and $OnlyAgent365Info) { $incompatModeSwitches += '-Resume' } + if ($incompatModeSwitches.Count -gt 0) { + $activeAgentSwitch = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-IncludeAgent365Info' } + Write-Host "ERROR: $activeAgentSwitch is not supported with: $($incompatModeSwitches -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " Agent 365 enrichment requires a fresh live Microsoft Graph context and runs" -ForegroundColor Yellow + Write-Host " end-to-end as a single pass. It is not compatible with replay/EOM/Resume modes." -ForegroundColor Yellow + exit 1 + } +} + +# Agent 365 + app-only auth (AppRegistration certificate, AppRegistration client secret, +# ManagedIdentity): The Microsoft Graph Agent Package +# Management API exposes an APPLICATION permission (CopilotPackages.Read.All app-role; plus +# Application.Read.All for developer-name resolution) per Microsoft Learn "List Copilot packages" +# (https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api). The app-only token +# already carries these app-roles when they are granted + admin-consented on the app / managed- +# identity service principal, so the Agent 365 phase runs on the EXISTING application context +# with NO interactive delegated sign-in. Because no prompt is involved, app-only + Agent 365 is +# also valid on headless / noninteractive hosts (containers, ACA Jobs, services, CI runners) and +# with -OnlyAgent365Info. A missing app-role, unlicensed tenant, or absent program enrollment is +# surfaced at runtime as a 403 by Test-Agent365FrontierAccess (see the app-only 403 banner there). +# Delegated auth modes (WebLogin / DeviceCode / Credential / Silent) are unchanged and continue to +# consent the Agent 365 scopes at their initial interactive sign-in. +# NOTE: the catalog API is currently published at /beta only; the tenant must hold a Microsoft +# Agent 365 license / program enrollment. No auth-mode is rejected up-front for Agent 365. + +# Validate AppendFile has proper filename format +if ($AppendFile) { + # Check if it's a directory path (ends with slash/backslash or has no extension) + if ($AppendFile -match '[\\/]$') { + Write-Host "ERROR: -AppendFile must specify a filename, not a directory path" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Valid examples:" -ForegroundColor Green + Write-Host " -AppendFile 'MyReport.xlsx'" -ForegroundColor Green + Write-Host " -AppendFile 'C:\Data\Audit\Report.csv'" -ForegroundColor Green + Write-Host "" -ForegroundColor Yellow + Write-Host "Invalid examples:" -ForegroundColor Red + Write-Host " -AppendFile 'C:\Data\'" -ForegroundColor Red + Write-Host " -AppendFile 'C:\Data\Audit\'" -ForegroundColor Red + exit 1 + } + + # Extract file extension + $appendExt = [System.IO.Path]::GetExtension($AppendFile).ToLower() + + # Validate extension exists + if (-not $appendExt) { + Write-Host "ERROR: -AppendFile must include a file extension (.csv or .xlsx)" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Valid examples:" -ForegroundColor Green + Write-Host " -AppendFile 'MyReport.xlsx'" -ForegroundColor Green + Write-Host " -AppendFile 'AuditData.csv'" -ForegroundColor Green + exit 1 + } + + # Validate extension matches export mode + if ($ExportWorkbook -and $appendExt -ne '.xlsx') { + Write-Host "ERROR: -AppendFile must use .xlsx extension when -ExportWorkbook is specified" -ForegroundColor Red + Write-Host "You specified: $AppendFile" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "Solutions:" -ForegroundColor Green + Write-Host " 1. Change filename to use .xlsx extension" -ForegroundColor Green + Write-Host " 2. Remove -ExportWorkbook to append CSV data instead" -ForegroundColor Green + exit 1 + } + elseif (-not $ExportWorkbook -and $appendExt -ne '.csv') { + Write-Host "ERROR: -AppendFile must use .csv extension for CSV mode" -ForegroundColor Red + Write-Host "You specified: $AppendFile" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "Solutions:" -ForegroundColor Green + Write-Host " 1. Change filename to use .csv extension" -ForegroundColor Green + Write-Host " 2. Add -ExportWorkbook to append Excel data instead" -ForegroundColor Green + exit 1 + } +} + +# --- Conflict Detection for -ExcludeCopilotInteraction --- + +$script:ConflictResolved = $false +$script:ConflictChoice = $null + +# Detect conflicts where user wants to both include AND exclude CopilotInteraction: +# 1. Explicit include via -ActivityTypes parameter +# 2. Explicit include via -IncludeCopilotInteraction switch +$explicitInclude = $ActivityTypes -and ($ActivityTypes -contains 'CopilotInteraction') +$explicitIncludeViaSwitch = $IncludeCopilotInteraction + +if ($ExcludeCopilotInteraction -and ($explicitInclude -or $explicitIncludeViaSwitch)) { + if (-not $Force) { + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "CONFLICT DETECTED" -ForegroundColor Red + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + + if ($explicitInclude) { + Write-Host "You provided 'CopilotInteraction' in -ActivityTypes but also specified -ExcludeCopilotInteraction switch." -ForegroundColor Yellow + } + elseif ($explicitIncludeViaSwitch) { + Write-Host "You enabled -IncludeCopilotInteraction but also specified -ExcludeCopilotInteraction." -ForegroundColor Yellow + } + + Write-Host "" + Write-Host "Microsoft 365 Copilot data (CopilotInteraction) includes:" -ForegroundColor Cyan + Write-Host " - M365 Copilot (Word, Excel, PowerPoint, Outlook, Teams meetings, etc.)" -ForegroundColor Cyan + Write-Host " - Microsoft 365 Copilot Chat (Office.com)" -ForegroundColor Cyan + Write-Host " - Security Copilot" -ForegroundColor Cyan + Write-Host " - Copilot Studio interactions" -ForegroundColor Cyan + Write-Host " - Billing: FREE (included with E5/Audit Standard)" -ForegroundColor Green + Write-Host "" + Write-Host "Do you want to INCLUDE or EXCLUDE Microsoft 365 Copilot activity type?" -ForegroundColor Yellow + Write-Host " [I] INCLUDE - Proceed with M365 Copilot data enabled (override -ExcludeCopilotInteraction switch)" -ForegroundColor Green + Write-Host " [E] EXCLUDE - Remove CopilotInteraction (honor -ExcludeCopilotInteraction switch)" -ForegroundColor Red + Write-Host "" + + # Strict noninteractive guard: a container/ACA Job/CI + # runner cannot answer I/E. Fail-fast with an explicit, scriptable recovery path + # rather than hanging on stdin. + if (script:Test-IsNonInteractive) { + Write-Host "ERROR: Cannot prompt for INCLUDE/EXCLUDE on a noninteractive host." -ForegroundColor Red + Write-Host " Re-run with EITHER -IncludeCopilotInteraction (to force include) OR" -ForegroundColor Yellow + Write-Host " -ExcludeCopilotInteraction alone (without -IncludeCopilotInteraction) to exclude," -ForegroundColor Yellow + Write-Host " OR pass -Force to honour -ExcludeCopilotInteraction without prompting." -ForegroundColor Yellow + exit 1 + } + + Send-PromptNotification + $userChoice = Read-Host "Enter your choice (I/E)" + + if ($userChoice -eq 'I' -or $userChoice -eq 'i') { + $ExcludeCopilotInteraction = $false + $script:ConflictChoice = 'INCLUDE' + Write-Host "" + Write-Host "Choice: INCLUDE - Proceeding with CopilotInteraction enabled" -ForegroundColor Green + Write-Host "" + } + elseif ($userChoice -eq 'E' -or $userChoice -eq 'e') { + $script:ConflictChoice = 'EXCLUDE' + Write-Host "" + Write-Host "Choice: EXCLUDE - CopilotInteraction will be removed from ActivityTypes" -ForegroundColor Red + Write-Host "" + } + else { + Write-Host "" + Write-Host "ERROR: Invalid choice. Please enter 'I' for INCLUDE or 'E' for EXCLUDE." -ForegroundColor Red + exit 1 + } + + $script:ConflictResolved = $true + } + else { + # Force mode - honor ExcludeCopilotInteraction without prompt + $script:ConflictChoice = 'EXCLUDE (Force mode)' + $script:ConflictResolved = $true + } +} + +# ============================================== +# -OnlyAgent365Info preflight (Y/N, default Y) +# ============================================== +# Without -IncludeAgent365Info, audit-derived columns will be blank in the +# Agent 365 CSV (Date created / Created by). Confirm the user wants to proceed. +# -Force auto-continues without prompting. +if ($OnlyAgent365Info) { + Write-Host "" + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "| -OnlyAgent365Info: Audit-log enrichment will be SKIPPED |" -ForegroundColor Yellow + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "| These Agent 365 columns CANNOT be populated without Purview audit |" -ForegroundColor Yellow + Write-Host "| data and will be left blank in this run: |" -ForegroundColor Yellow + Write-Host "| |" -ForegroundColor Yellow + Write-Host "| - Date created |" -ForegroundColor Yellow + Write-Host "| - Created by |" -ForegroundColor Yellow + Write-Host "| |" -ForegroundColor Yellow + Write-Host "| To populate them, re-run with -IncludeAgent365Info instead. |" -ForegroundColor Yellow + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "" + if ($Force) { + Write-Host " -Force specified: continuing without prompt." -ForegroundColor DarkGray + Write-Host "" + } else { + # Strict noninteractive guard: default to Y on + # noninteractive hosts so scheduled/container runs do not hang on stdin. The + # default of the prompt is already Y, so this preserves operator intent. + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: auto-accepting prompt default (Y) for -OnlyAgent365Info." -ForegroundColor DarkGray + Write-Host "" + } else { + $resp = Read-Host " Continue with -OnlyAgent365Info anyway? (Y/N) [default Y]" + if ([string]::IsNullOrWhiteSpace($resp)) { $resp = 'Y' } + if ($resp -notmatch '^(?i:y|yes)$') { + Write-Host " Aborted by user." -ForegroundColor Red + exit 0 + } + Write-Host "" + } + } +} + +# ============================================== +# ImportExcel Module Check (for Excel export) +# ============================================== + +if ($ExportWorkbook) { + Write-Host "Checking ImportExcel module for Excel export..." -ForegroundColor Cyan + + $importExcelModule = Get-Module -ListAvailable -Name ImportExcel | Select-Object -First 1 + if (-not $importExcelModule) { + Write-Host "ImportExcel module not found (required for -ExportWorkbook)." -ForegroundColor Yellow + Write-Host "Installing ImportExcel module..." -ForegroundColor Yellow + Write-Host "" + + try { + Install-Module ImportExcel -Scope CurrentUser -Force -AllowClobber -Repository PSGallery -ErrorAction Stop + Write-Host "ImportExcel module installed successfully!" -ForegroundColor Green + Write-Host "" + + # Re-check for the module + $importExcelModule = Get-Module -ListAvailable -Name ImportExcel | Select-Object -First 1 + if (-not $importExcelModule) { + Write-Host "ERROR: Module installation completed but module not found. Try restarting PowerShell." -ForegroundColor Red + exit 1 + } + } + catch { + Write-Host "ERROR: Failed to install ImportExcel module: $($_.Exception.Message)" -ForegroundColor Red + Write-Host "" + Write-Host "Please install manually using:" -ForegroundColor Yellow + Write-Host " Install-Module ImportExcel -Scope CurrentUser" -ForegroundColor Yellow + Write-Host "" + Write-Host "Falling back to CSV export..." -ForegroundColor Yellow + $script:ExportWorkbook = $false + $script:AppendFile = $false + } + } + else { + Write-Host "ImportExcel module detected: $($importExcelModule.Name) v$($importExcelModule.Version)" -ForegroundColor Green + } + + # Import ImportExcel module + if ($ExportWorkbook) { + try { + Import-Module ImportExcel -ErrorAction Stop + Write-Host "ImportExcel module imported successfully" -ForegroundColor Green + Write-Host "" + } + catch { + Write-Host "ERROR: Failed to import ImportExcel module: $($_.Exception.Message)" -ForegroundColor Red + Write-Host "Falling back to CSV export..." -ForegroundColor Yellow + $script:ExportWorkbook = $false + $script:AppendFile = $false + } + } +} + +# --- Early parameter validation & environment sanity checks --- + +# PowerShell version gate. +# The entire script requires PowerShell 7+ (pwsh.exe). The ONLY exception is -UseEOM +# (which uses Exchange Online Management and runs on Windows PowerShell 5.1 / powershell.exe). +# All other modes — including -RAWInputCSV and -Resume — require pwsh.exe because they +# depend on PS 7-only features (ThreadJob parallelism, ForEach-Object -Parallel, JSON +# parsing performance). +if ($PSVersionTable.PSVersion.Major -lt 7 -and -not $UseEOM) { + Write-Host "" -ForegroundColor Red + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: PowerShell 7+ Required (pwsh.exe)" -ForegroundColor Red + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Detected: PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" -ForegroundColor Yellow + Write-Host " Required: PowerShell 7.0 or later (pwsh.exe)" -ForegroundColor Yellow + Write-Host "" + Write-Host " PAX requires PowerShell 7+ for Microsoft Graph API mode, parallel query" -ForegroundColor White + Write-Host " execution, JSON streaming performance, and post-processing features." -ForegroundColor White + Write-Host " Windows PowerShell 5.1 (powershell.exe) is ONLY supported when running with" -ForegroundColor White + Write-Host " the -UseEOM switch (Exchange Online Management mode)." -ForegroundColor White + Write-Host "" + Write-Host " SOLUTIONS:" -ForegroundColor Cyan + Write-Host "" + Write-Host " 1. Recommended: Install PowerShell 7+ and re-run with pwsh.exe:" -ForegroundColor White + Write-Host " https://aka.ms/powershell" -ForegroundColor Gray + Write-Host " pwsh.exe -File .\PAX_Purview_Audit_Log_Processor.ps1 [your parameters]" -ForegroundColor Gray + Write-Host "" + Write-Host " 2. Or run in EOM mode on Windows PowerShell 5.1 by adding -UseEOM:" -ForegroundColor White + Write-Host " .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM [your other parameters]" -ForegroundColor Gray + Write-Host "" + Write-Host " EOM MODE NOTES:" -ForegroundColor DarkCyan + Write-Host " - Uses Search-UnifiedAuditLog cmdlet (serial processing)" -ForegroundColor Gray + Write-Host " - Requires Exchange Online Management module" -ForegroundColor Gray + Write-Host " - Requires Exchange Admin role or audit log read permissions" -ForegroundColor Gray + Write-Host " - Some features (Entra user enrichment, rollup post-processing) not available" -ForegroundColor Gray + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + exit 1 +} + +if ($ExcludeAgents -and ($AgentId -or $AgentsOnly)) { + Write-Host "ERROR: -ExcludeAgents cannot be used with -AgentId or -AgentsOnly switches." -ForegroundColor Red + Write-Host "These switches are mutually exclusive:" -ForegroundColor Yellow + Write-Host " -AgentId/-AgentsOnly: Filter to ONLY records with agents" -ForegroundColor Yellow + Write-Host " -ExcludeAgents: Filter to ONLY records without agents" -ForegroundColor Yellow + Write-Host "Please use only one filtering approach and re-run." -ForegroundColor Yellow + exit 1 +} + +# ============================================================================ +# Rollup post-processor gating +# ============================================================================ +# Validates -Rollup / -RollupPlusRaw against incompatible switches and +# determines which embedded Python processor will be invoked at end-of-run. +# Runs early so failures are surfaced BEFORE any Graph API calls or auth. +# +# Sets $script:RollupProcessorMode to one of: +# 'None' - rollup not requested +# 'CopilotInteraction' - run Purview_CopilotInteraction_Processor (needs Purview CSV + Entra CSV) +# 'M365Bundle' - run Purview_M365_Usage_Bundle_Explosion_Processor (needs Purview CSV only) +# ============================================================================ +$script:RollupProcessorMode = 'None' +$script:RollupDashboard = 'None' +$script:RollupDashboardProfile = $null + +# Resolves -Dashboard into auto-enabled switches so the existing +# rollup gate + mode decision below operate on a consistent state. Acts only when +# the user explicitly passed -Dashboard; the default 'AIO' is inert so a bare +# -Rollup / -RollupPlusRaw keeps today's behavior exactly. +$dashboardExplicit = $PSBoundParameters.ContainsKey('Dashboard') +$dashboardUC = $Dashboard.ToUpperInvariant() +$dashboardImpliedRollup = $false +if ($dashboardExplicit) { + # C1: AIO/AIBV (CopilotInteraction) and the M365 usage bundle are different data + # pulls AND different processors — refuse to guess; make the user pick one. + if (($dashboardUC -eq 'AIO' -or $dashboardUC -eq 'AIBV') -and $IncludeM365Usage) { + Write-Host "ERROR: -Dashboard $Dashboard and -IncludeM365Usage are incompatible (different data source AND different processor). Pick one." -ForegroundColor Red + Write-Host " -Dashboard AIO|AIBV : CopilotInteraction rollup (AI-in-One / AI Business Value dashboards)." -ForegroundColor Yellow + Write-Host " -Dashboard M365 : M365 Usage Analytics rollup (or simply use -IncludeM365Usage)." -ForegroundColor Yellow + exit 1 + } + # Rule: -Dashboard M365 implies the M365 usage pull; auto-enable it if absent. + if ($dashboardUC -eq 'M365' -and -not $IncludeM365Usage) { + $IncludeM365Usage = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: -Dashboard M365 auto-enabled -IncludeM365Usage (the M365 dashboard consumes the M365 usage bundle)." -ForegroundColor Cyan + } + # Rule: -Dashboard implies a rollup run; default to -Rollup when neither rollup + # switch is present (never -RollupPlusRaw). + if (-not $Rollup -and -not $RollupPlusRaw) { + $Rollup = [System.Management.Automation.SwitchParameter]::new($true) + $dashboardImpliedRollup = $true + Write-Host "INFO: -Dashboard $Dashboard auto-enabled -Rollup (dashboard output is produced by the rollup post-processor)." -ForegroundColor Cyan + } +} + +if ($Rollup -or $RollupPlusRaw) { + # Mutually exclusive + if ($Rollup -and $RollupPlusRaw) { + Write-Host "ERROR: -Rollup and -RollupPlusRaw are mutually exclusive. Pick one." -ForegroundColor Red + Write-Host " -Rollup : produce ONLY rolled-up CSV(s); raw CSV(s) deleted on success." -ForegroundColor Yellow + Write-Host " -RollupPlusRaw : produce BOTH the raw CSV(s) AND the rolled-up CSV(s)." -ForegroundColor Yellow + exit 1 + } + + $rollupSwitchName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $rollupBlockers = @() + if ($UseEOM) { $rollupBlockers += '-UseEOM' } + if ($ExportWorkbook) { $rollupBlockers += '-ExportWorkbook' } + if ($OnlyUserInfo) { $rollupBlockers += '-OnlyUserInfo' } + if ($OnlyAgent365Info) { $rollupBlockers += '-OnlyAgent365Info' } + if ($RAWInputCSV) { $rollupBlockers += '-RAWInputCSV' } + if ($ExcludeCopilotInteraction -and -not $IncludeM365Usage) { $rollupBlockers += '-ExcludeCopilotInteraction' } + + if ($rollupBlockers.Count -gt 0) { + # Name -Dashboard in the message when it is what implied the rollup, so the + # error references a switch the user actually typed. + $rollupTrigger = if ($dashboardImpliedRollup) { "-Dashboard $Dashboard (which implies $rollupSwitchName)" } else { $rollupSwitchName } + Write-Host "ERROR: $rollupTrigger is not supported with the following switch(es): $($rollupBlockers -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " $rollupSwitchName runs an embedded Python post-processor at the end of the run." -ForegroundColor Yellow + Write-Host " It is only valid for the default CopilotInteraction-only run, an explicit" -ForegroundColor Yellow + Write-Host " -ActivityTypes 'CopilotInteraction' run, or an -IncludeM365Usage run." -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host " Remove the conflicting switch(es) and re-run." -ForegroundColor Yellow + exit 1 + } + + # Determine processor mode based on requested activity types. + # Bind-time logic (mirrors the activity-type finalization that happens later): + # - If -IncludeM365Usage is set, mode is M365Bundle. + # - Else if user did NOT pass -ActivityTypes (default is CopilotInteraction), mode is CopilotInteraction. + # - Else if user passed -ActivityTypes containing exactly the single value 'CopilotInteraction' + # (case-insensitive), mode is CopilotInteraction. + # - Otherwise: hard-fail (rollup is not defined for arbitrary activity-type combinations). + if ($IncludeM365Usage) { + $script:RollupProcessorMode = 'M365Bundle' + $script:RollupDashboard = 'M365' + $script:RollupDashboardProfile = $null + } + else { + $userPassedActivityTypes = $PSBoundParameters.ContainsKey('ActivityTypes') + $normalizedActivity = @() + if ($userPassedActivityTypes -and $ActivityTypes) { + $normalizedActivity = @($ActivityTypes | Where-Object { $_ } | ForEach-Object { $_.Trim() }) + } + $isCopilotOnly = (-not $userPassedActivityTypes) -or + ($normalizedActivity.Count -eq 1 -and $normalizedActivity[0] -ieq 'CopilotInteraction') + + if ($isCopilotOnly) { + $script:RollupProcessorMode = 'CopilotInteraction' + # AIO/AIBV only reach here — an explicit -Dashboard M365 auto-enabled + # -IncludeM365Usage above and took the M365Bundle branch. Default to AIO + # when -Dashboard was not supplied (preserves today's behavior). + $script:RollupDashboard = if ($dashboardExplicit) { $dashboardUC } else { 'AIO' } + $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() + } + else { + Write-Host "ERROR: $rollupSwitchName is only valid for CopilotInteraction-only runs or -IncludeM365Usage runs." -ForegroundColor Red + Write-Host " Detected -ActivityTypes: $($normalizedActivity -join ', ')" -ForegroundColor Yellow + Write-Host " Remove $rollupSwitchName, or restrict -ActivityTypes to 'CopilotInteraction', or use -IncludeM365Usage." -ForegroundColor Yellow + exit 1 + } + } + + # CopilotInteraction-mode rollup requires the Entra users CSV for the second processor input. + # Auto-enable -IncludeUserInfo if not already set (single info message; do not prompt). + if ($script:RollupProcessorMode -eq 'CopilotInteraction' -and -not $IncludeUserInfo) { + $IncludeUserInfo = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $rollupSwitchName (CopilotInteraction mode) auto-enabled -IncludeUserInfo (Entra users CSV is required by the post-processor)." -ForegroundColor Cyan + } + + # Rollup requires a single combined Purview CSV; auto-enable -CombineOutput when missing. + if (-not $CombineOutput) { + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $rollupSwitchName auto-enabled -CombineOutput (rollup post-processor requires a single combined Purview CSV)." -ForegroundColor Cyan + } +} + +# --------------------------------------------------------------------------- +# -FillerLabel: org/manager-hierarchy level-filler for the rolled-up AIO/AIBV +# Users output. The hierarchy itself is always built for AIO/AIBV; this only sets +# what goes in level slots deeper than a user. Rollup-only; never the M365 path. +# Resolved here (after the rollup/dashboard state is finalized above) into the two +# script vars threaded to the embedded CopilotInteraction processor. +# --------------------------------------------------------------------------- +$script:HierarchyFillMode = 'none' +$script:HierarchyFillLabel = '' +$fillerLabelBound = $PSBoundParameters.ContainsKey('FillerLabel') +$fillerTextBound = $PSBoundParameters.ContainsKey('FillerLabelText') +if ($fillerTextBound -and -not $fillerLabelBound) { + Write-Host "ERROR: -FillerLabelText is only valid together with -FillerLabel Fixed." -ForegroundColor Red + exit 1 +} +if ($fillerLabelBound) { + if (-not ($Rollup -or $RollupPlusRaw)) { + Write-Host "ERROR: -FillerLabel requires -Rollup or -RollupPlusRaw." -ForegroundColor Red + Write-Host " -FillerLabel only affects the rolled-up AI-in-One / AI Business Value Users output." -ForegroundColor Yellow + exit 1 + } + if ($script:RollupProcessorMode -eq 'M365Bundle') { + Write-Host "ERROR: -FillerLabel is not valid with the M365 dashboard (-IncludeM365Usage / -Dashboard M365)." -ForegroundColor Red + Write-Host " The org / manager hierarchy is produced only for the AI-in-One and AI Business Value dashboards." -ForegroundColor Yellow + exit 1 + } + $fillMode = ([string]$FillerLabel).Trim() + switch ($fillMode.ToLowerInvariant()) { + 'null' { $script:HierarchyFillMode = 'none' } + 'self' { $script:HierarchyFillMode = 'self' } + 'repeatmanager' { $script:HierarchyFillMode = 'manager' } + 'fixed' { $script:HierarchyFillMode = 'fixed' } + default { + Write-Host "ERROR: -FillerLabel value '$fillMode' is not recognized." -ForegroundColor Red + Write-Host " Use one of: null, Self, RepeatManager, or Fixed (with -FillerLabelText `"`")." -ForegroundColor Yellow + exit 1 + } + } + if ($script:HierarchyFillMode -eq 'fixed') { + if (-not $fillerTextBound -or [string]::IsNullOrWhiteSpace($FillerLabelText)) { + Write-Host "ERROR: -FillerLabel Fixed requires -FillerLabelText `"`" (e.g. -FillerLabel Fixed -FillerLabelText `"Assistive Directs`")." -ForegroundColor Red + exit 1 + } + $script:HierarchyFillLabel = [string]$FillerLabelText + } + elseif ($fillerTextBound) { + Write-Host "ERROR: -FillerLabelText is only valid with -FillerLabel Fixed (not with '$fillMode')." -ForegroundColor Red + exit 1 + } +} + +# ============================================================================ +# EMBEDDED PYTHON POST-PROCESSORS +# ============================================================================ +# Two Python source files are embedded verbatim as single-quoted here-strings +# so the script is fully self-contained — no external .py files are required +# at runtime. When -Rollup or -RollupPlusRaw is used, the relevant body is +# materialized to a temp .py inside .pax_incremental, executed with the +# resolved Python interpreter, and deleted in finally. Single-quoted here- +# strings prevent any PowerShell variable expansion of the Python source. +# ============================================================================ +$Script:EMBEDDED_PROCESSOR_COPILOT_VERSION = '4.2.0' +$Script:EMBEDDED_PROCESSOR_M365_VERSION = '2.6.1' + +# >>> BEGIN-EMBEDDED-COPILOT-PROCESSOR +$Script:EMBEDDED_PROCESSOR_COPILOT = @' +#!/usr/bin/env python3 +""" +Purview CopilotInteraction Processor v4.1.0 +------------------------------------------- +Two-input / two-output preprocessor for the AI Business Value Dashboard +and AI-in-One Rollup PBIPs. + +Output profiles (--profile): + aibv (default) : AI Business Value Dashboard. 50-column fact superset — + 3-value Environment {Cowork, Licensed, Unlicensed}, + all DAX calc-columns pre-computed (Behavior_*, Usage_Mode, + Expertise_Role, Efficiency_Breakdown, Human_Baseline_Min, + Behavior_Plausible, Workflow_Action, Delegation_Event_Key, + Is_Agent_Activity/Web_Grounded_Signal promoted into the + grain for sliceability) + Audit_UserId passthrough. + aio : AI-in-One Dashboard. 36-column fact — 5-value Environment + {Autonomous Agent, Cowork, Agents, Licensed M365 Copilot, + Unlicensed Chat}. Reproduces the v3.1.0 AIO output + BYTE-IDENTICALLY (validated), so the AIO dashboard is + unaffected. ~41% smaller than the aibv fact. + +Inputs: + --purview (required) + --entra (required) + +Outputs (in --out-dir, default = directory of --purview): + _Interactions_.csv (fact table) + _Users_.csv (dim table) + + These two files are all the AIBV template needs. Pass --with-aggregates + (aibv only) to ALSO write 5 pre-aggregated tables for a future calc-table + offload: + _ActiveDaysSummary_.csv + _UserMonthMetrics_.csv + _LicensedUserRankings_.csv + _UnlicensedUserRankings_.csv + _LicensedUserSummary_.csv + +Grain: + One row per (16-column grain x Message_Id; aibv adds 3 sliceable flag + keys -> 19). DAX measures use DISTINCTCOUNT(Message_Id) which yields exact + parity with the semantic-model definitions at every visual / slicer + combination. Per-resource accumulation is intentionally avoided so counts + are + not inflated (~2.25x) by per (prompt x AccessedResource) iteration. + +INT-surrogated columns (perf): + Message_Id, ThreadId, and UserKey (replaces Audit_UserId) are emitted + as 1-based INTs assigned in input encounter order. Cuts CSV size, + parse time, AND VertiPaq dictionary build time on the three highest- + cardinality GUID columns. UserKey is written to BOTH the fact CSV + and the Users dim CSV (same shared map keyed on normalized UPN), so + the fact↔Users relationship is INT-to-INT. DISTINCTCOUNT semantics + are identical between INT and string surrogates of the same set. + UserMonthKey stays string (cross-processor blast radius). + +Calc cols ported from DAX -> precomputed here for ingestion-time speedup: + Agent_TitleID, Behavior_Source, Value_Outcome, ActivityDate + (= InteractionDate alias). + +Stays in DAX (cross-table dependencies that cannot be precomputed without +shipping Agents 365 / UserMonthMetrics / AgentMetrics into the processor): + Behavior_Enriched_Full (RELATED Agents 365), + User_Stage_Maturity / User_Stage (RELATED UserMonthMetrics), + Usage_Mode, Expertise_Role, Efficiency_Breakdown + (all depend on Behavior_Enriched_Full), + Agent Last Used Date (LOOKUPVALUE AgentMetrics). + +Requirements: + Python 3.9+ + pip install orjson (OPTIONAL - faster JSON parsing; falls back to stdlib json) +""" + +from __future__ import annotations + +import argparse +import csv +import functools +import hashlib +import hmac +import os +import re +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +try: + import orjson + + def json_loads(value: str | bytes) -> Any: + if isinstance(value, str): + value = value.encode("utf-8") + return orjson.loads(value) + + _JSON_ENGINE = "orjson" +except ImportError: + import json as _json + + def json_loads(value: str | bytes) -> Any: + if isinstance(value, bytes): + value = value.decode("utf-8") + return _json.loads(value) + + _JSON_ENGINE = "json (stdlib)" + + +SCRIPT_VERSION = "4.2.0" + +# --------------------------------------------------------------------------- +# Output schemas — TWO PROFILES +# +# --profile aio : reproduces the v3.1.0 AIO-faithful output EXACTLY except +# for two provenance columns (Message_Id_Raw, ThreadId_Raw) +# APPENDED LAST in v4.2.0 for cross-run append reconciliation. +# The original 36 columns are unchanged in name, order, and +# value (5-value Environment vocabulary), so the pre-existing +# AIO output stays byte-identical to v3.1.0; only the two +# trailing raw keys are new. This is the contract the +# AI-in-One dashboard already consumes. +# --profile aibv : the AIBV-faithful superset (50-col fact, 3-value +# Environment, all offloaded calc cols + grain-promoted +# sliceable flags) built in this v4.0.0 effort. +# +# Both share one classification CODEBASE; the per-profile vocabulary is +# selected by the `profile` argument threaded through the classifiers. +# --------------------------------------------------------------------------- + +# Common grain prefix (identical in both profiles). +_GRAIN_KEYS_COMMON: tuple[str, ...] = ( + "UserKey", + "InteractionDate", + "AgentId", + "AgentName", + "AppHost", + "Environment", + "License Status", + "Context_Type", + "Behavior_Category", + "Behavior_Enriched", + "AI_Model", + "Is_Sensitive", + "Autonomy_Pattern", + "AppIdentity_AppId", + "AISystemPlugin_Name", + "ThreadId", +) + +# AIO grain = the common 16 (matches v3.1.0 exactly). +GRAIN_KEYS_AIO: tuple[str, ...] = _GRAIN_KEYS_COMMON + +# AIBV grain = common 16 + 3 promoted per-resource flags (sliceability fix). +GRAIN_KEYS_AIBV: tuple[str, ...] = _GRAIN_KEYS_COMMON + ( + # Promoted into the grain (sliceability fix): computed per-resource and + # bound to AIBV slicers/filters, so they MUST be grain-faithful. Validated + # at 0.000% row inflation on real data. + "Is_Agent_Activity", + "Web_Grounded_Signal", + "Workflow_Action", +) + +# Cross-run append reconciliation keys (v4.2.0): the stable raw GUIDs behind the +# INT surrogates Message_Id (message) and ThreadId (thread). Appended as the FINAL +# two columns of EVERY profile so all pre-existing column positions are unchanged. +# The PAX append layer (ConvertTo-FactSeedMaps / Merge-FactCsv) dedups cross-run on +# Message_Id_Raw. Under --deidentify these carry the deterministic deid_guid token +# (same raw GUID -> same token across runs) so append dedup still reconciles. +_RAW_ID_ATTRS: tuple[str, ...] = ( + "Message_Id_Raw", + "ThreadId_Raw", +) + +# AIO non-grain carried attrs = exactly the v3.1.0 set (ends at ActivityDate); +# the trailing _RAW_ID_ATTRS are appended below to form _NONGRAIN_ATTRS_AIO. +_NONGRAIN_ATTRS_AIO_BASE: tuple[str, ...] = ( + "CreationDate", + "WeekStart", + "MonthStart", + "UserMonthKey", + "Has license", + "Resource_Count", + "SensitivityLabelId", + "AccessedResource_Type", + "AccessedResource_Action", + "AccessedResource_SiteUrl", + "AccessedResource_SensitivityLabelId", + "AppIdentity_DisplayName", + "AISystemPlugin_Id", + "ModelTransparencyDetails_ModelName", + "Agent_TitleID", + "Message_isPrompt", + # Calc cols ported from DAX (present in AIO since v3.1.0) + "Behavior_Source", + "Value_Outcome", + "ActivityDate", +) +# AIO carried attrs = the v3.1.0 base set + a stable user-identity column + the +# trailing raw reconciliation keys. +_NONGRAIN_ATTRS_AIO: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( + # Stable, deid-consistent user identity for AIO. Mirrors the + # AIBV [Audit_UserId_Normalized] value (deid_upn -> normalize_user_id), so it + # is deterministically de-identified under -Deidentify and never exposes a raw + # UPN. Gives the cross-run append merge key a stable user component in place of + # the per-run UserKey INT surrogate. Placed BEFORE the raw keys so + # Message_Id_Raw / ThreadId_Raw stay the trailing reconciliation columns. + "User_Id_Normalized", +) + _RAW_ID_ATTRS + +# AIBV non-grain carried attrs = AIO base set + AIBV-only offloaded columns, with +# the raw reconciliation keys appended LAST so they remain the trailing two columns +# for this profile too (all pre-existing AIBV column positions unchanged). +_NONGRAIN_ATTRS_AIBV: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( + # M1: UPN passthrough for AIBV joins + DISTINCTCOUNT. + "Audit_UserId", + "Audit_UserId_Normalized", + # AIBV-faithful row-level flags. (Is_Agent_Activity / Web_Grounded_Signal / + # Workflow_Action were promoted into GRAIN_KEYS_AIBV.) `Agent Filter` + # derives from Is_Agent_Activity, so it stays a grain-consistent carried attr. + "Agent Filter", + "Agent Publish Status", + # Downstream classification chain (offloaded; faithful without Agents 365 per F2). + "Behavior_Enriched_Full", + "Usage_Mode", + "Expertise_Role", + "Efficiency_Breakdown", + # ROI baseline pre-join (offloads the Human Equivalent Hours SUMX+RELATED). + "Human_Baseline_Min", + # Remaining row-level calc cols (offloaded). + "Behavior_Plausible", + "Delegation_Event_Key", +) + _RAW_ID_ATTRS + +# Final fact CSV schemas. One row per (grain x Message_Id). Message_Id is +# emitted as a sequential INT surrogate (1-based, assigned in input order). +FACT_HEADER_AIO: list[str] = list(GRAIN_KEYS_AIO) + ["Message_Id"] + list(_NONGRAIN_ATTRS_AIO) +FACT_HEADER_AIBV: list[str] = list(GRAIN_KEYS_AIBV) + ["Message_Id"] + list(_NONGRAIN_ATTRS_AIBV) + + +def schema_for(profile: str) -> tuple[tuple[str, ...], tuple[str, ...], list[str]]: + """Return (grain_keys, nongrain_attrs, fact_header) for the profile.""" + if profile == "aio": + return GRAIN_KEYS_AIO, _NONGRAIN_ATTRS_AIO, FACT_HEADER_AIO + return GRAIN_KEYS_AIBV, _NONGRAIN_ATTRS_AIBV, FACT_HEADER_AIBV + +# Entra column-name aliases used by the existing PBIP M-code. We mirror the +# same renaming so the dim CSV is drop-in compatible with all downstream DAX. +UPN_VARIANTS_NORMALIZED = {"userprincipalname", "upn", "personid"} +DEPARTMENT_VARIANT_NORMALIZED = "department" +JOBTITLE_RAW_NAME = "jobTitle" # exact-match rename to "JobTitle" +HAS_LICENSE_VARIANTS = ( + "Has license", + "Has License", + "hasLicense", + "HasLicense", + "Has Copilot License", + "Has Copilot license", + "HasCopilotLicense", + "Has Copilot License Assigned", + "Has Copilot license assigned", + "isUser", +) + +# --------------------------------------------------------------------------- +# Datetime helpers +# --------------------------------------------------------------------------- + +_CREATION_TIME_FORMATS: tuple[str, ...] = ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", +) + + +def safe_get(obj: Any, key: str) -> Any: + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def get_array(obj: Any, key: str) -> list[Any]: + value = safe_get(obj, key) + return value if isinstance(value, list) else [] + + +def to_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +def normalize_user_id(value: Any) -> str: + return to_text(value).strip().lower() + + +# --------------------------------------------------------------------------- +# Deidentification (--deidentify): one-way, salted, format-preserving. +# OFF by default; enabled by main() setting the module flag from --deidentify. +# Every PII value becomes a deterministic token so relationships (manager links, +# UserKey/Users joins, distinct-resource counts) are preserved while identities +# are removed. Irreversible (no decode map). The SAME salt + algorithm + formats +# MUST exist verbatim in the PowerShell raw-path deidentifier and the M365 +# processor (PAX deidentify spec) so tokens match across engines. +# --------------------------------------------------------------------------- +_DEIDENTIFY: bool = False +_DEID_SALT = b"PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3" +_DEID_DOMAIN = "deidentified.domain" +_deid_cache: dict[str, str] = {} + + +def _deid_hex(value: str, length: int) -> str: + return hmac.new( + _DEID_SALT, value.strip().lower().encode("utf-8"), hashlib.sha256 + ).hexdigest()[:length] + + +def deid_upn(value: str) -> str: + """UPN / email -> <12hex>@deidentified.domain. No-op when off or value empty.""" + if not _DEIDENTIFY or not value: + return value + k = "upn\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + "@" + _DEID_DOMAIN + _deid_cache[k] = v + return v + + +def deid_name(value: str) -> str: + """Person/device display name -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "name\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_guid(value: str) -> str: + """GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.""" + if not _DEIDENTIFY or not value: + return value + k = "guid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}" + _deid_cache[k] = v + return v + + +def deid_sid(value: str) -> str: + """SID -> deterministic S-1-5-21---- shape.""" + if not _DEIDENTIFY or not value: + return value + k = "sid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = "S-1-5-21-{0}-{1}-{2}-{3}".format( + int(h[0:8], 16), int(h[8:16], 16), int(h[16:24], 16), int(h[24:32], 16) + ) + _deid_cache[k] = v + return v + + +def deid_token(value: str) -> str: + """Opaque id (employeeId, immutableId) -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "tok\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_resource(value: str) -> str: + """Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count).""" + if not _DEIDENTIFY or not value: + return value + k = "res\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "site_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_file(value: str) -> str: + """File / document name -> file_<12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "file\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "file_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_proxy(value: str) -> str: + """proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + Handles ';'-delimited multi-value fields.""" + if not _DEIDENTIFY or not value: + return value + out = [] + for entry in value.split(";"): + if not entry: + out.append(entry) + elif ":" in entry: + prefix, addr = entry.split(":", 1) + out.append(prefix + ":" + deid_upn(addr)) + else: + out.append(deid_upn(entry)) + return ";".join(out) + + +# Non-human/system identities found in Purview audit logs (Teams Sync, SharePoint app, +# SupervisoryReview bots, ServicePrincipals, NT-style accounts, SIDs, bare GUIDs, etc.). +# These have no matching userPrincipalName in EntraUsers and would render as blank +# User/Department rows in downstream visuals. Filter out before any record is emitted. +_UPN_LOCAL_RE = re.compile(r"^[^\s\\@]+$") +_BARE_GUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + + +def _is_human_upn(uid: str) -> bool: + """True iff uid is a syntactically valid human UPN (local@domain.tld), excluding + well-known service/bot patterns (SupervisoryReview{...}@..., bare GUIDs).""" + if not uid: + return False + s = uid.strip() + if _BARE_GUID_RE.match(s): + return False + if s.lower().startswith("supervisoryreview{"): + return False + if "@" not in s or s.count("@") != 1: + return False + local, domain = s.split("@", 1) + if not _UPN_LOCAL_RE.match(local): + return False + if "." not in domain or not domain or domain.startswith(".") or domain.endswith("."): + return False + return True + + +def parse_creation_time(value: Any) -> datetime | None: + raw = to_text(value).strip() + if not raw: + return None + return _parse_creation_time_cached(raw) + + +@functools.lru_cache(maxsize=None) +def _parse_creation_time_cached(raw: str) -> datetime | None: + # Fast path: ISO 8601 (covers ~100% of Purview audit timestamps). + # datetime.fromisoformat is ~10x faster than strptime and avoids the + # locale lookup that strptime performs on every call. Python 3.11+ + # accepts a trailing "Z"; for 3.10 and earlier we strip it. + try: + if raw.endswith("Z"): + try: + return datetime.fromisoformat(raw) + except ValueError: + return datetime.fromisoformat(raw[:-1]) + return datetime.fromisoformat(raw) + except ValueError: + pass + # Slow path: legacy non-ISO formats kept for backwards compat with + # older / hand-edited audit exports. + for fmt in _CREATION_TIME_FORMATS: + try: + return datetime.strptime(raw, fmt) + except ValueError: + continue + return None + + +def format_creation_date(value: Any) -> str: + parsed = parse_creation_time(value) + if parsed is None: + raw = to_text(value).strip() + if len(raw) >= 10 and raw[4:5] == "-": + return raw[:10] + "T00:00:00.000Z" + return raw + return parsed.replace(tzinfo=timezone.utc).strftime("%Y-%m-%dT00:00:00.000Z") + + +def interaction_date(parsed: datetime | None) -> str: + return parsed.strftime("%Y-%m-%d") if parsed else "" + + +def week_start(parsed: datetime | None) -> str: + if parsed is None: + return "" + return (parsed - timedelta(days=parsed.weekday())).strftime("%Y-%m-%d") + + +def month_start(parsed: datetime | None) -> str: + if parsed is None: + return "" + return parsed.replace(day=1).strftime("%Y-%m-%d") + + +# Cached bundle: given a raw timestamp string, return all 4 derived date +# strings in one shot. Avoids 4x strftime + tzinfo replace per record. The +# distinct raw-timestamp count in a typical dataset is small relative to +# input row count (many records share the same audit timestamp at the +# second granularity), so this collapses ~4N strftime calls to ~K where +# K is the distinct timestamp count. +@functools.lru_cache(maxsize=None) +def _date_strings_for_raw(raw: str) -> tuple[str, str, str, str]: + """ + Returns (creation_date_iso_z, interaction_date, week_start, month_start) + for the given raw timestamp string. Empty string is returned for any + field that cannot be derived (matches non-cached helper semantics). + """ + if not raw: + return ("", "", "", "") + parsed = _parse_creation_time_cached(raw) + if parsed is None: + if len(raw) >= 10 and raw[4:5] == "-": + return (raw[:10] + "T00:00:00.000Z", "", "", "") + return (raw, "", "", "") + # Direct f-string formatting is ~10x faster than strftime (which does + # locale lookup + format-string parsing on every call). Output bytes + # are byte-identical to the prior strftime("%Y-%m-%d") output for any + # year in [1000, 9999] (CreationTime range). + y = parsed.year + m = parsed.month + d = parsed.day + creation = f"{y:04d}-{m:02d}-{d:02d}T00:00:00.000Z" + interaction = f"{y:04d}-{m:02d}-{d:02d}" + # Week start (Monday-based, mirroring strftime((parsed-weekday).strftime)) + ws = parsed - timedelta(days=parsed.weekday()) + week = f"{ws.year:04d}-{ws.month:02d}-{ws.day:02d}" + month = f"{y:04d}-{m:02d}-01" + return (creation, interaction, week, month) + + +def slugify(value: str) -> str: + slug = re.sub(r"[^A-Za-z0-9]+", "-", value.strip()) + return slug.strip("-") + + +# --------------------------------------------------------------------------- +# Audit JSON shaping +# --------------------------------------------------------------------------- + + +def app_identity_values(audit_data: dict[str, Any]) -> tuple[str, str]: + app_identity = safe_get(audit_data, "AppIdentity") + if isinstance(app_identity, str): + return "", app_identity + if isinstance(app_identity, dict): + return ( + to_text(safe_get(app_identity, "AppId")), + to_text(safe_get(app_identity, "DisplayName")), + ) + return "", "" + + +def derive_agent_name(agent_name: Any, app_identity_display: str, app_identity_app_id: str) -> str: + # Match the BEFORE PBIP behavior: AgentName comes straight from the audit JSON. + # Do NOT synthesize from AppIdentity when it's blank — that fabricates distinct + # agent identities (e.g. "Copilot-Studio-Default--") that + # don't exist in the raw data and inflate Active Agents / per-agent rollups. + return to_text(agent_name).strip() + + +def derive_agent_title_id(agent_id: Any) -> str: + agent_id_text = to_text(agent_id).strip() + if not agent_id_text: + return "" + return agent_id_text.rsplit(".", 1)[-1] + + +def first_dict_item(items: list[Any]) -> dict[str, Any]: + for item in items: + if isinstance(item, dict): + return item + return {} + + +def prompt_messages(ced: dict[str, Any]) -> list[dict[str, Any]]: + prompts: list[dict[str, Any]] = [] + for message in get_array(ced, "Messages"): + if isinstance(message, dict) and message.get("isPrompt") is True: + prompts.append(message) + return prompts + + +def resource_rows(ced: dict[str, Any]) -> list[dict[str, Any]]: + resources = [item for item in get_array(ced, "AccessedResources") if isinstance(item, dict)] + return resources if resources else [{}] + + +def is_copilot_interaction(audit_data: dict[str, Any], raw_row: dict[str, Any]) -> bool: + operation = to_text( + safe_get(audit_data, "Operation") + or raw_row.get("Operation") + or raw_row.get("Operations") + ).strip() + return operation == "CopilotInteraction" + + +# --------------------------------------------------------------------------- +# Classification logic (ports of the PBIP DAX calc columns) +# --------------------------------------------------------------------------- + +_LICENSE_TRUTHY = {"YES", "TRUE", "Y", "1"} +_ACTIVE_RES_ACTION_TOKENS = ("send", "draft", "create", "post", "invoke", "write", "patch", "execute") + + +def normalize_has_license(raw: str) -> str: + """Normalize any truthy/falsy variant to canonical 'TRUE' / 'FALSE'. + + Existing PBIP measures filter with literal `[Has license] = "FALSE"`, so + we canonicalize here to guarantee those filters match regardless of how + the upstream Entra/PAX export rendered the value. + """ + val = (raw or "").strip().upper() + if val in _LICENSE_TRUTHY: + return "TRUE" + if val in {"NO", "FALSE", "N", "0"}: + return "FALSE" + return "FALSE" + + +@functools.lru_cache(maxsize=None) +def compute_license_status(has_license_raw: str) -> str: + val = (has_license_raw or "").strip().upper() + return "M365 Copilot Licensed" if val in _LICENSE_TRUTHY else "Unlicensed" + + +@functools.lru_cache(maxsize=None) +def compute_environment(profile: str, has_license_raw: str, agent_name: str, agent_id: str, app_host: str) -> str: + license_val = (has_license_raw or "").strip().upper() + if profile == "aio": + # v3.1.0 AIO vocabulary (5-value, keyed off app_host + agent presence). + host = (app_host or "").lower() + has_agent = bool((agent_name or "").strip()) or bool((agent_id or "").strip()) + if host in {"autonomous", "logic app"}: + return "Autonomous Agent" + if "cowork" in host: + return "Cowork" + if has_agent: + return "Agents" + if license_val in _LICENSE_TRUTHY: + return "Licensed M365 Copilot" + return "Unlicensed Chat" + # AIBV vocabulary (verbatim port of current AIBV calc col `Environment`): + # IF(CONTAINSSTRING(LOWER(TRIM(AgentName)),"cowork"),"Cowork", + # IF(isLicensed,"Licensed","Unlicensed")) + if "cowork" in (agent_name or "").strip().lower(): + return "Cowork" + if license_val in _LICENSE_TRUTHY: + return "Licensed" + return "Unlicensed" + + +@functools.lru_cache(maxsize=None) +def compute_is_sensitive(sens_label: str, resource_sens_label: str) -> str: + return "TRUE" if (sens_label or "").strip() or (resource_sens_label or "").strip() else "FALSE" + + +@functools.lru_cache(maxsize=None) +def compute_ai_model(model_name: str) -> str: + m = (model_name or "").upper() + if not m or m == "NULL": + return "Embedded App (no model logged)" + if "DEEP_LEO" in m: + return "GPT-4 (Standard)" + if "REASONING" in m: + return "Reasoning Model (o1/o3)" + if "OFFENSIVE" in m: + return "Safety Filter (blocked)" + if "GPT-41" in m or "GPT-4.1" in m: + return "GPT-4.1 (Next Gen)" + if "O3-MINI" in m or "O3MINI" in m: + return "o3-mini (Reasoning)" + if "O3" in m or "O1" in m: + return "Reasoning Model (o-series)" + if "GPT-5" in m or "GPT5" in m: + return "GPT-5 (Next Gen)" + if "CLAUDE" in m: + return "Claude (Anthropic)" + if "GEMINI" in m: + return "Gemini (Google)" + if "LLAMA" in m or "META" in m: + return "LLaMA (Meta)" + if "PHI" in m: + return "Phi (Microsoft Small Model)" + return model_name or "" + + +def _resource_behavior( + profile: str, res_type: str, res_action: str, site_url: str, is_active: bool +) -> str: + if res_action in {"sendemailv2", "draftemail", "senddraftemail", "updatedraftemail"}: + return "Email Drafting" + if res_type == "emailmessage": + return "Email Drafting" if is_active else "Email Summarising" + if res_action == "mcp_meetingmanagement": + return "Meeting Scheduling" + if res_type in {"event", "teamsmeeting"}: + return "Meeting Prep" + if res_action in {"postmessagetoconversation", "createchat"}: + return "Teams Messaging" + if res_type in {"teamsmessage", "teamschat", "teamschannel"}: + return "Teams Messaging" + if profile == "aio": + # v3.1.0: any flow/connector/http resource -> "Workflow Execution". + if res_type in {"flow", "connector", "http"}: + return "Workflow Execution" + else: + # AIBV: explicit Flow always; connector/http only with an active verb. + if res_type == "flow": + return "Running a Workflow" + if res_type in {"connector", "http"} and is_active: + return "Running a Workflow" + if res_action in {"executedatasetquery", "getitems", "getalltables", "gettableviews"}: + return "Data Querying" + if res_type in {"xlsx", "csv", "xlsm", "xlsb", "xls"}: + return "Excel Assistance" if is_active else "Spreadsheet Review" + if res_type == "peopleinferenceanswer": + return "People Lookup" + if res_type in {"listitem", "aspx"}: + return "Enterprise Searching" + if res_type == "websearchquery": + return "Web Searching" + if res_type == "pdf": + return "PDF Analysis" + if res_type in {"py", "js", "java", "tsx", "jsx", "css", "php", "sh"} and is_active: + return "Code Writing" + if res_type in {"py", "sql", "js", "java", "json", "xml", "html", "yaml", "yml", "txt"}: + return "Code Analysis" + if res_type in {"png", "jpg", "jpeg", "svg", "gif"} and is_active: + return "Image Generation" + if res_type in {"png", "jpg", "jpeg", "gif"}: + return "Image / Media Analysis" + if res_type in {"streamvideo", "mp4", "mov", "webm", "mkv"}: + return "Video Summarising" + if res_type in {"planid", "taskids"}: + return "Task Management" + if res_type == "looppage": + return "Real-time Collaboration" + if res_type == "http://schema.skype.com/hyperlink": + for token in ("github.com", "stackoverflow.com", "npmjs.com", "pypi.org", "docker.com", "kubernetes.io", "leetcode.com"): + if token in site_url: + return "Code Analysis" + for token in ("learning.cloud.microsoft", "coursera.org", "udemy.com"): + if token in site_url: + return "Agent: Coaching" + if "sharepoint.com" in site_url: + return "Enterprise Searching" + return "Web Searching" + if res_type in {"external", "http"}: + return "Web Searching" + if res_type in {"docx", "doc", "rtf"}: + if is_active: + return "Document Drafting" + if res_action == "read": + return "File Retrieval" + return "Document Summarising" + if res_type in {"pptx", "ppt", "potx"}: + if is_active: + return "Presentation Creation" + if res_action == "read": + return "File Retrieval" + return "Presentation Summarising" + if "service-now.com" in site_url or "servicenow.com" in site_url: + return "Agent: IT & Service Desk" + if "dynamics.com" in site_url: + return "Agent: Sales & Customer" + return "" + + +def _context_behavior(profile: str, app_host: str, ctx_type: str, is_active: bool, has_agent: bool) -> str: + if ctx_type == "teamsmeeting": + return "Meeting Prep" + if ctx_type == "streamvideo": + return "Video Summarising" + if ctx_type == "docx": + return "Document Drafting" if (app_host == "word" and is_active) else "Document Summarising" + if ctx_type in {"xlsx", "xlsm", "xlsb", "xls", "csv"}: + return "Spreadsheet Review" + if ctx_type in {"pptx", "pptm"}: + return "Presentation Creation" if (app_host == "powerpoint" and is_active) else "Presentation Summarising" + if ctx_type in {"teamschat", "teamschannel"}: + return "Teams Messaging" + if ctx_type == "aspx": + return "Enterprise Searching" + if app_host in {"outlook", "outlooksidepane"}: + return "Email Drafting" if is_active else "Email Summarising" + if app_host == "excel": + return "Excel Assistance" + if app_host == "word": + return "Document Drafting" if is_active else "Document Summarising" + if app_host == "powerpoint": + return "Presentation Creation" if is_active else "Presentation Summarising" + if app_host == "stream": + return "Video Summarising" + if app_host == "sharepoint": + return "SharePoint Access" + if app_host == "designer": + return "Image Generation" + if app_host == "onenote": + return "Note Taking" + if app_host == "forms": + return "Form / Survey Work" + if app_host == "planner": + return "Task Management" + if app_host in {"loop", "whiteboard", "vivaengage"}: + return "Real-time Collaboration" + if app_host == "copilot studio": + return "Domain-Specific Agent" + if profile == "aio": + # v3.1.0: autonomous OR logic app -> "Workflow Execution". + if app_host in {"autonomous", "logic app"}: + return "Workflow Execution" + else: + # AIBV: autonomous always; logic app only when an agent context is present. + if app_host == "autonomous": + return "Running a Workflow" + if app_host == "logic app" and has_agent: + return "Running a Workflow" + if app_host in {"datawarehousing core", "power bi"}: + return "Data Querying" + return "General Chat" + + +@functools.lru_cache(maxsize=None) +def compute_behavior_category( + profile: str, + app_host: str, + ctx_type: str, + res_type: str, + res_action: str, + site_url: str, + plugin_id: str, + has_agent: bool, +) -> str: + app_host_l = (app_host or "").lower() + ctx_l = (ctx_type or "").lower() + res_t_l = (res_type or "").lower() + res_a_l = (res_action or "").lower() + site_l = (site_url or "").lower() + plugin_l = (plugin_id or "").lower() + is_active = any(tok in res_a_l for tok in _ACTIVE_RES_ACTION_TOKENS) + + from_resource = _resource_behavior(profile, res_t_l, res_a_l, site_l, is_active) + if from_resource: + return from_resource + if plugin_l == "enterprisesearch": + return "Enterprise Searching" + return _context_behavior(profile, app_host_l, ctx_l, is_active, has_agent) + + +_GENERIC_QA_BEHAVIORS = {"General Q&A", "M365 Chat Q&A", "Teams Q&A", "Browser Q&A", "General Chat"} +_AGENT_NAME_RULES: tuple[tuple[tuple[str, ...], str], ...] = ( + (("coach", "mentor", "learning", "career"), "Agent: Coaching"), + (("research", "analyst", "analy"), "Agent: Research & Analysis"), + (("sales", "commercial", "customer", "crm", "revenue"), "Agent: Sales & Customer"), + (("hr", "recruit", "talent", "onboard", "people"), "Agent: HR & People"), + (("policy", "compliance", "legal", "audit", "risk"), "Agent: Compliance & Policy"), + (("service", "support", "help", "ticket", "incident"), "Agent: IT & Service Desk"), + (("summar", "draft", "translat", "editor"), "Agent: Content Generation"), + (("data", "report", "dashboard", "metric"), "Agent: Data & Reporting"), + (("knowledge", "faq", "wiki", "buddy", "guide"), "Agent: Knowledge Base"), + (("idea", "brainstorm", "creative", "design"), "Agent: Ideation & Creative"), +) + + +@functools.lru_cache(maxsize=None) +def compute_behavior_enriched(profile: str, behavior_category: str, agent_name: str, environment: str) -> str: + # AIO enriches agent/autonomous rows; AIBV enriches agents/cowork rows. + # (In practice AIBV `Environment` never returns "Agents", so only Cowork enriches.) + enrich_envs = {"Agents", "Autonomous Agent"} if profile == "aio" else {"Agents", "Cowork"} + if environment not in enrich_envs: + return behavior_category + if behavior_category not in _GENERIC_QA_BEHAVIORS: + return behavior_category + name_l = (agent_name or "").lower() + for tokens, label in _AGENT_NAME_RULES: + if any(t in name_l for t in tokens): + return label + return "Agent: General Purpose" + + +# Autonomy_Pattern — profile-aware. +# AIO (v3.1.0): keyed off the 5-value Environment. +# AIBV: SWITCH(Cowork->3, Is_Agent_Activity->2, Licensed->1, else BLANK). +@functools.lru_cache(maxsize=None) +def compute_autonomy_pattern(profile: str, environment: str, is_agent_activity_str: str) -> str: + if profile == "aio": + if environment == "Licensed M365 Copilot": + return "1 - Copilot" + if environment == "Agents": + return "2 - Agent-Assisted" + if environment == "Autonomous Agent": + return "3 - Autonomous" + return "" + if environment == "Cowork": + return "3 - Cowork" + if is_agent_activity_str == "TRUE": + return "2 - Agent-Assisted" + if environment == "Licensed": + return "1 - Copilot" + return "" + + +# Behavior_Source — profile-aware (AIO: "Autonomous Agent" branch; AIBV: "Cowork"). +@functools.lru_cache(maxsize=None) +def compute_behavior_source( + profile: str, + behavior_category: str, + environment: str, + agent_name: str, + plugin_name: str, + app_host: str, +) -> str: + agent = (agent_name or "").strip() + plugin = (plugin_name or "").strip() + app = (app_host or "").strip() + if profile == "aio" and environment == "Autonomous Agent": + source = "Autonomous Agent" + (f": {agent}" if agent else "") + elif profile != "aio" and environment == "Cowork": + source = "Cowork" + (f": {agent}" if agent else "") + elif environment == "Agents" and agent: + source = f"Agent: {agent}" + elif plugin: + source = f"{app} ({plugin})" + elif app: + source = app + else: + source = "Copilot Chat" + return f"{behavior_category} → {source}" + + +# Verbatim port of current AIBV DAX calc col `Value_Outcome`. +_VO_TIME_EMAIL = frozenset({"Email Summarising", "Email Triage", "Email Thread Summary"}) +_VO_TIME_MEET = frozenset({"Meeting Prep", "Video Summarising"}) +_VO_TIME_DOC = frozenset({"Document Summarising", "Presentation Summarising", "Note Taking"}) +_VO_SEARCH = frozenset({ + "Web Searching", "Enterprise Searching", "File Retrieval", "PDF Analysis", + "SharePoint Access", "People Lookup", "Agent: Knowledge Base", +}) +_VO_COMM = frozenset({"Teams Messaging", "Meeting Scheduling"}) +_VO_SHEET = frozenset({"Spreadsheet Review", "Spreadsheet Analysis", "Excel Assistance"}) +_VO_CONTENT = frozenset({ + "Email Drafting", "Document Drafting", "Presentation Creation", + "Image Generation", "Image / Media Analysis", "Image/Media Analysis", + "Agent: Content Generation", "Agent: Ideation & Creative", +}) +_VO_TEAMCOLLAB = frozenset({"Real-time Collaboration", "Form / Survey Work"}) +_VO_DATA = frozenset({"Data Querying", "Agent: Data & Reporting", "Agent: Research & Analysis"}) +_VO_CODE = frozenset({"Code Writing", "Code Analysis", "Code Analysis (URL)"}) +_VO_COACH = frozenset({"Agent: Coaching", "Agent: Coaching (URL)"}) +_VO_DOMAIN = frozenset({"Domain-Specific Agent", "Cross-Org Agent"}) + + +@functools.lru_cache(maxsize=None) +def compute_value_outcome( + profile: str, behavior_enriched: str, environment: str, is_sensitive_str: str +) -> str: + b = behavior_enriched or "" + # Profile-specific workflow signal (AIO: Workflow Execution / Autonomous Agent; + # AIBV: Running a Workflow / Cowork). + workflow_behavior = "Workflow Execution" if profile == "aio" else "Running a Workflow" + workflow_env = "Autonomous Agent" if profile == "aio" else "Cowork" + if b in _VO_TIME_EMAIL: + return "Time Saved (Email)" + if b in _VO_TIME_MEET: + return "Time Saved (Meetings)" + if b in _VO_TIME_DOC: + return "Time Saved (Documents)" + if b in _VO_SEARCH: + return "Search Time Saved" + if b in _VO_COMM: + return "Communication Time Saved" + if b in _VO_SHEET: + return "Spreadsheet Time Saved" + if b in _VO_CONTENT: + return "Content Output" + if b in _VO_TEAMCOLLAB: + return "Team Collaboration" + if b == workflow_behavior or environment == workflow_env: + return "Workflow Automation" + if b == "Task Management": + return "Task Coordination" + if ( + is_sensitive_str == "TRUE" + and environment != "Agents" + and environment != workflow_env + ): + return "Compliance & Risk" + if b in _VO_DATA: + return "Data-Driven Decisions" + if b in _VO_CODE: + return "Coding Capability" + if b in _VO_COACH: + return "Skills Development" + if b == "Agent: Sales & Customer": + return "Revenue Enablement" + if b == "Agent: IT & Service Desk": + return "Service Desk Deflection" + if b == "Agent: Compliance & Policy": + return "Compliance & Risk" + if b == "Agent: HR & People": + return "HR Expertise" + if b in _VO_DOMAIN: + return "Specialist Expertise" + return "General AI Productivity" + + +# --------------------------------------------------------------------------- +# Downstream classification chain (offloaded from AIBV DAX). +# +# Per F2 (see offload plan): in the current AIBV model `Environment` never +# returns "Agents", so `Behavior_Enriched_Full`'s NeedsEnhancement guard +# (Env = "Agents" && BaseEnriched = "Agent: General Purpose") is always FALSE +# and the RELATED('Agents 365'...) lookups never execute. Therefore +# Behavior_Enriched_Full == Behavior_Enriched and the whole chain +# (Usage_Mode / Expertise_Role / Efficiency_Breakdown) is fully computable +# here WITHOUT ingesting Agents 365 — and stays pixel-identical to AIBV. +# --------------------------------------------------------------------------- + + +def compute_behavior_enriched_full(behavior_enriched: str) -> str: + # No Agents 365 ingestion (by design) => NeedsEnhancement is always FALSE + # => Behavior_Enriched_Full is exactly Behavior_Enriched. + return behavior_enriched + + +_UM_PRODUCING = frozenset({ + "Email Drafting", "Document Drafting", "Presentation Creation", "Image Generation", + "Code Writing", "Code Analysis", "Code Analysis (URL)", "Data Querying", + "Spreadsheet Analysis", "Excel Assistance", "Agent: Content Generation", + "Agent: Ideation & Creative", "Agent: Research & Analysis", "Agent: Data & Reporting", + "Agent: Sales & Customer", "Agent: HR & People", "Agent: IT & Service Desk", + "Agent: Compliance & Policy", "Agent: Coaching", "Agent: Coaching (URL)", + "Domain-Specific Agent", "Cross-Org Agent", "Form / Survey Work", + "Real-time Collaboration", "Note Taking", "Teams Messaging", "Meeting Scheduling", + "Task Management", +}) +_UM_CONSUMING = frozenset({ + "Document Summarising", "Email Summarising", "Email Thread Summary", "Email Triage", + "Presentation Summarising", "Video Summarising", "Meeting Prep", + "Image / Media Analysis", "Image/Media Analysis", "Sensitive Content Interaction", +}) +_UM_FINDING = frozenset({ + "Web Searching", "Enterprise Searching", "PDF Analysis", "SharePoint Access", + "File Retrieval", "People Lookup", "Agent: Knowledge Base", "Spreadsheet Review", +}) + + +@functools.lru_cache(maxsize=None) +def compute_usage_mode(behavior_enriched_full: str, environment: str, app_host: str) -> str: + # Verbatim port of AIBV `Usage_Mode` with the Agents-365 term dropped + # (agentTypeA365 = IFERROR(RELATED(...),BLANK()) -> BLANK when A365 absent, + # so its IN {...} test is always FALSE). + behavior = behavior_enriched_full + host = (app_host or "").lower() + is_delegating = ( + environment == "Cowork" + or host == "autonomous" + or behavior == "Running a Workflow" + ) + if is_delegating: + return "5 - Delegating" + if behavior in _UM_PRODUCING: + return "4 - Producing" + if behavior in _UM_CONSUMING: + return "3 - Consuming" + if behavior in _UM_FINDING: + return "2 - Finding" + return "1 - Asking" + + +# Verbatim port of AIBV `Expertise_Role` (ordered IF cascade on Behavior_Enriched_Full). +_EXPERTISE_RULES: tuple[tuple[frozenset[str], str], ...] = ( + (frozenset({"Data Querying", "Agent: Data & Reporting", "Spreadsheet Analysis"}), "Data Analyst"), + (frozenset({"Code Writing", "Code Analysis", "Code Analysis (URL)"}), "Software Engineer"), + (frozenset({"Agent: Research & Analysis"}), "Business Analyst"), + (frozenset({"Agent: Compliance & Policy", "Sensitive Content Interaction"}), "Compliance Specialist"), + (frozenset({"Agent: Sales & Customer"}), "Sales Consultant"), + (frozenset({"Agent: IT & Service Desk"}), "IT Specialist"), + (frozenset({"Agent: HR & People"}), "HR Specialist"), + (frozenset({"Agent: Coaching", "Agent: Coaching (URL)"}), "Coach"), + (frozenset({"Running a Workflow", "Task Management"}), "Automation Engineer"), + (frozenset({"Domain-Specific Agent", "Cross-Org Agent"}), "Domain Expert"), + (frozenset({"Email Drafting"}), "Communications Specialist"), + (frozenset({"Email Triage", "Meeting Scheduling", "Email Summarising", "Email Thread Summary"}), "Executive Assistant"), + (frozenset({"Document Drafting", "Agent: Content Generation", "Note Taking", "Document Summarising"}), "Content Writer"), + (frozenset({"Presentation Creation", "Presentation Summarising"}), "Presentation Designer"), + (frozenset({"Image Generation", "Image/Media Analysis", "Image / Media Analysis", "Agent: Ideation & Creative"}), "Visual Designer"), + (frozenset({"Meeting Prep", "Video Summarising"}), "Meeting Coordinator"), + (frozenset({"Web Searching", "PDF Analysis", "Agent: Knowledge Base"}), "Researcher"), + (frozenset({"Enterprise Searching", "SharePoint Access", "File Retrieval", "People Lookup"}), "Knowledge Navigator"), + (frozenset({"Spreadsheet Review", "Excel Assistance"}), "Spreadsheet Specialist"), + (frozenset({"Real-time Collaboration", "Form / Survey Work", "Form/Survey Work", "Teams Messaging"}), "Collaboration Lead"), +) + + +@functools.lru_cache(maxsize=None) +def compute_expertise_role(behavior_enriched_full: str) -> str: + for members, label in _EXPERTISE_RULES: + if behavior_enriched_full in members: + return label + return "" # AIBV returns BLANK() + + +# Verbatim port of AIBV `Efficiency_Breakdown` (behavior cascade, then Behavior_Category fallback). +_EFF_RULES: tuple[tuple[frozenset[str], str], ...] = ( + (frozenset({"Email Summarising", "Email Triage", "Email Thread Summary", "Email Drafting"}), "Email"), + (frozenset({"Document Summarising", "Note Taking", "Document Drafting", "Agent: Content Generation"}), "Document Assistance"), + (frozenset({"Presentation Summarising", "Presentation Creation"}), "Presentations"), + (frozenset({"Meeting Prep", "Video Summarising", "Meeting Scheduling"}), "Meetings"), + (frozenset({"Web Searching", "Enterprise Searching", "PDF Analysis", "SharePoint Access", "File Retrieval", "People Lookup", "Agent: Knowledge Base", "Agent: Research & Analysis"}), "Search & Research"), + (frozenset({"Spreadsheet Review", "Excel Assistance", "Spreadsheet Analysis", "Data Querying", "Agent: Data & Reporting"}), "Data & Spreadsheets"), + (frozenset({"Image Generation", "Image / Media Analysis", "Image/Media Analysis", "Agent: Ideation & Creative", "Code Writing", "Code Analysis", "Code Analysis (URL)"}), "Creative & Technical"), + (frozenset({"Teams Messaging", "Real-time Collaboration", "Form / Survey Work", "Task Management", "Running a Workflow"}), "Collaboration & Workflows"), + (frozenset({"Agent: Sales & Customer", "Agent: IT & Service Desk", "Agent: HR & People", "Agent: Compliance & Policy", "Agent: Coaching", "Agent: Coaching (URL)", "Domain-Specific Agent", "Cross-Org Agent"}), "Specialist Agents"), +) + + +@functools.lru_cache(maxsize=None) +def compute_efficiency_breakdown(behavior_enriched_full: str, behavior_category: str) -> str: + for members, label in _EFF_RULES: + if behavior_enriched_full in members: + return label + if behavior_category == "Teams Q&A": + return "Teams Chat" + if behavior_category == "M365 Chat Q&A": + return "BizChat Q&A" + if behavior_category == "Browser Q&A": + return "BizChat Q&A" + return "General Q&A" + + +# --------------------------------------------------------------------------- +# Embedded static value map (offloads the ROI baseline join). +# +# `Human Time Estimates` is a static lookup (Behavior -> Human Baseline (min)). +# The AIBV measure `Human Equivalent Hours` does: +# SUMX(FILTER(fact, isPrompt="TRUE"), DIVIDE(RELATED(HTE[Human Baseline]),60,0)) +# which traverses fact -> Behavior Value Map (BVM) -> Human Time Estimates (HTE) +# per row. We pre-join `Human_Baseline_Min` onto every fact row so the PBIT +# measure collapses to SUM(fact[Human_Baseline_Min])/60 (no SUMX, no RELATED). +# +# FIDELITY: HTE is reachable in the model ONLY through the BVM bridge, so a +# baseline is emitted ONLY when Behavior_Enriched_Full is present in BVM (then +# looked up in HTE; BVM is a strict subset of HTE, so the lookup always hits). +# Behaviors not in BVM contribute 0 hours in the current AIBV — we emit "" for +# them, which SUMs to 0 identically. Both maps are transcribed verbatim from +# the AIBV `.pbit` static #table literals (see temp/_offload_test/_parse_maps.py). +# --------------------------------------------------------------------------- + +_HUMAN_BASELINE_MIN: dict[str, int] = { + "Agent: Coaching": 45, + "Agent: Coaching (URL)": 25, + "Agent: Compliance & Policy": 25, + "Agent: Content Generation": 25, + "Agent: Data & Reporting": 35, + "Agent: General Purpose": 15, + "Agent: HR & People": 35, + "Agent: IT & Service Desk": 20, + "Agent: Ideation & Creative": 40, + "Agent: Knowledge Base": 12, + "Agent: Research & Analysis": 45, + "Agent: Sales & Customer": 35, + "Browser Q&A": 10, + "Code Analysis": 30, + "Code Analysis (URL)": 15, + "Code Writing": 45, + "Cross-Org Agent": 30, + "Data Querying": 30, + "Document Drafting": 60, + "Document Summarising": 20, + "Domain-Specific Agent": 25, + "Email Drafting": 8, + "Email Summarising": 4, + "Email Thread Summary": 5, + "Email Triage": 10, + "Enterprise Searching": 18, + "Excel Assistance": 30, + "File Retrieval": 15, + "Form / Survey Work": 25, + "Form/Survey Work": 25, + "General Chat": 10, + "General Q&A": 10, + "Image / Media Analysis": 8, + "Image Generation": 60, + "Image/Media Analysis": 8, + "M365 Chat Q&A": 10, + "Meeting Prep": 15, + "Meeting Scheduling": 12, + "Note Taking": 20, + "PDF Analysis": 35, + "People Lookup": 10, + "Presentation Creation": 90, + "Presentation Summarising": 12, + "Real-time Collaboration": 30, + "Running a Workflow": 15, + "Sensitive Content Interaction": 20, + "SharePoint Access": 12, + "Spreadsheet Analysis": 40, + "Spreadsheet Review": 25, + "Task Management": 20, + "Teams Messaging": 8, + "Teams Q&A": 10, + "Video Summarising": 30, + "Web Searching": 22, +} + +# Behaviors present in the Behavior Value Map bridge (gates HTE reachability). +_BVM_BEHAVIORS: frozenset = frozenset({ + "Agent: Coaching", + "Agent: Compliance & Policy", + "Agent: Content Generation", + "Agent: Data & Reporting", + "Agent: General Purpose", + "Agent: HR & People", + "Agent: IT & Service Desk", + "Agent: Ideation & Creative", + "Agent: Knowledge Base", + "Agent: Research & Analysis", + "Agent: Sales & Customer", + "Code Analysis", + "Code Writing", + "Data Querying", + "Document Drafting", + "Document Summarising", + "Domain-Specific Agent", + "Email Drafting", + "Email Summarising", + "Enterprise Searching", + "Excel Assistance", + "File Retrieval", + "Form / Survey Work", + "General Chat", + "Image / Media Analysis", + "Image Generation", + "Meeting Prep", + "Meeting Scheduling", + "Note Taking", + "PDF Analysis", + "People Lookup", + "Presentation Creation", + "Presentation Summarising", + "Real-time Collaboration", + "Running a Workflow", + "SharePoint Access", + "Spreadsheet Review", + "Task Management", + "Teams Messaging", + "Video Summarising", + "Web Searching", +}) + + +@functools.lru_cache(maxsize=None) +def compute_human_baseline_min(behavior_enriched_full: str) -> str: + # Emit the baseline only when reachable via the BVM bridge (AIBV topology). + if behavior_enriched_full in _BVM_BEHAVIORS: + return str(_HUMAN_BASELINE_MIN[behavior_enriched_full]) + return "" + + +# --------------------------------------------------------------------------- +# Remaining row-level calc cols (offloaded from AIBV DAX). +# --------------------------------------------------------------------------- + +# Verbatim port of AIBV `Behavior_Plausible`. +_UNLICENSED_PLAUSIBLE = frozenset({ + "General Chat", "Web Searching", "PDF Analysis", "Document Summarising", + "Image / Media Analysis", "Image Generation", "Code Analysis", "Translation", +}) +_BP_WORKAROUND_EMAIL = frozenset({"Email Summarising", "Email Drafting"}) +_BP_WORKAROUND_SHEET = frozenset({"Excel Assistance", "Spreadsheet Review", "Data Querying"}) +_BP_WORKAROUND_MEET = frozenset({"Meeting Prep", "Meeting Scheduling"}) +_BP_WORKAROUND_ENT = frozenset({"Enterprise Searching", "People Lookup"}) +_BP_WORKAROUND_WORKFLOW = frozenset({"Running a Workflow", "Task Management"}) + + +@functools.lru_cache(maxsize=None) +def compute_behavior_plausible(license_status: str, behavior_category: str) -> str: + lic = license_status + beh = behavior_category + if lic == "M365 Copilot Licensed" or beh in _UNLICENSED_PLAUSIBLE: + return beh + if beh in _BP_WORKAROUND_EMAIL: + return "Free Chat Workaround (pasting Email)" + if beh in _BP_WORKAROUND_SHEET: + return "Free Chat Workaround (pasting Spreadsheet/Data)" + if beh in _BP_WORKAROUND_MEET: + return "Free Chat Workaround (pasting Meeting info)" + if beh == "Teams Messaging": + return "Free Chat Workaround (pasting Teams content)" + if beh in _BP_WORKAROUND_ENT: + return "Free Chat Workaround (pasting Enterprise data)" + if beh in _BP_WORKAROUND_WORKFLOW: + return "Free Chat Workaround (pasting Workflow)" + if beh == "Real-time Collaboration": + return "Free Chat Workaround (pasting Loop content)" + if beh == "Code Writing": + return "Free Chat Workaround (pasting Code)" + if beh == "Video Summarising": + return "Free Chat Workaround (uploading Video)" + return "Free Chat Workaround (Other)" + + +# Verbatim port of AIBV `Workflow_Action`. +@functools.lru_cache(maxsize=None) +def compute_workflow_action(behavior_enriched_full: str, res_action: str, app_host: str) -> str: + if behavior_enriched_full != "Running a Workflow": + return "" + ra = (res_action or "").lower() + host = (app_host or "").lower() + if "send" in ra or "post" in ra or "notify" in ra: + return "Sending / Notifying" + if "create" in ra or "draft" in ra or "write" in ra or "add" in ra: + return "Creating Content" + if "invoke" in ra or "execute" in ra or "trigger" in ra or "run" in ra: + return "Invoking / Triggering" + if "update" in ra or "patch" in ra or "modify" in ra or "set" in ra: + return "Updating Records" + if "read" in ra or "get" in ra or "list" in ra or "fetch" in ra: + return "Reading Data" + if "delete" in ra or "remove" in ra: + return "Deleting / Removing" + if host == "autonomous": + return "Autonomous Run (no action logged)" + if host == "logic app": + return "Logic App Run (no action logged)" + return "Workflow (other)" + + +def compute_delegation_event_key( + audit_user_id: str, + interaction_date_str: str, + agent_name: str, + workflow_action: str, + app_host: str, +) -> str: + # Verbatim port of AIBV `Delegation_Event_Key`: + # Audit_UserId & "|" & FORMAT(InteractionDate,"yyyy-mm-dd") & "|" & + # COALESCE(AgentName, Workflow_Action, AppHost, "unknown-workflow") + # NOTE: DAX FORMAT "mm" resolves to MONTH here (not minute) because it does + # not follow h/hh — so interaction_date_str ("%Y-%m-%d") matches exactly. + # COALESCE: we treat empty/whitespace as blank (skip it). The blank-vs-empty + # nuance + rollup-vs-fanout grain interaction are flagged for Phase 4 live + # parity (measure: [Delegation Events] = DISTINCTCOUNT, filtered Usage_Mode + # = "5 - Delegating"). + tail = "unknown-workflow" + for candidate in (agent_name, workflow_action, app_host): + if candidate and candidate.strip(): + tail = candidate + break + return f"{audit_user_id}|{interaction_date_str}|{tail}" + + +def compute_user_month_key(audit_user_id: str, month_start_str: str) -> str: + if not audit_user_id or not month_start_str: + return "" + # MonthStart is YYYY-MM-DD; format key as YYYY-MM (mirrors DAX FORMAT(...,"yyyy-MM")) + return f"{audit_user_id}|{month_start_str[:7]}" + + +# Verbatim port of AIBV calc col `Agent Publish Status`. +@functools.lru_cache(maxsize=None) +def compute_agent_publish_status(agent_id: str, agent_name: str) -> str: + has_agent_id = bool((agent_id or "").strip()) + if not has_agent_id: + return "Not an Agent Row" + if "draft as 1p" in (agent_name or "").lower(): + return "Unpublished" + return "Published" + + +# Verbatim port of AIBV calc col `Is_Agent_Activity` (emitted as TRUE/FALSE text, +# mirroring the Is_Sensitive / Message_isPrompt convention; the PBIT types it +# logical in Phase 3). res_type is the per-resource AccessedResource_Type. +@functools.lru_cache(maxsize=None) +def compute_is_agent_activity(agent_name: str, agent_id: str, app_host: str, res_type: str) -> str: + has_agent = bool((agent_name or "").strip()) + has_agent_id = bool((agent_id or "").strip()) + host = (app_host or "").lower() + rt = (res_type or "").lower() + is_autonomous = host in {"autonomous", "logic app"} or rt in {"flow", "connector"} + return "TRUE" if (has_agent or has_agent_id or is_autonomous) else "FALSE" + + +# Verbatim port of AIBV calc col `Web_Grounded_Signal`. +@functools.lru_cache(maxsize=None) +def compute_web_grounded_signal(res_type: str, site_url: str) -> str: + rt = (res_type or "").lower() + su = (site_url or "").lower() + is_internal = "sharepoint.com" in su or ".onmicrosoft.com" in su + if ( + rt == "websearchquery" + or rt in {"external", "http"} + or (rt == "http://schema.skype.com/hyperlink" and not is_internal) + ): + return "Web Grounded" + return "Not Web Grounded" + + +# --------------------------------------------------------------------------- +# Entra loader / Users dim CSV writer +# --------------------------------------------------------------------------- + + +def _normalize_col_name(name: str) -> str: + return re.sub(r"[\s_\-.()\[\]]", "", (name or "").lower()) + + +def detect_has_license_column(headers: list[str]) -> str | None: + for variant in HAS_LICENSE_VARIANTS: + if variant in headers: + return variant + return None + + +def detect_upn_column(headers: list[str]) -> str | None: + for h in headers: + if _normalize_col_name(h) in UPN_VARIANTS_NORMALIZED: + return h + return None + + +def detect_department_column(headers: list[str]) -> str | None: + for h in headers: + if _normalize_col_name(h) in {DEPARTMENT_VARIANT_NORMALIZED, "organization", "organisation"}: + return h + return None + + +def detect_jobtitle_column(headers: list[str]) -> str | None: + for h in headers: + if _normalize_col_name(h) == "jobtitle": + return h + return None + + +def load_licensing_map(licensing_csv: str) -> tuple[dict[str, str], int]: + """Read a standalone licensing CSV (UPN + 'Has License' columns) and return + (normalized_upn -> raw license value, data_row_count). + + Used by the 3-file input mode to merge license status into a users-only + Entra file. UPN + license columns are detected with the same alias lists as + the Entra loader. Duplicate UPNs are last-write-wins (mirrors the M-code + Table.Distinct behavior on the licensed-users path). + """ + with open(licensing_csv, "r", encoding="utf-8-sig", newline="") as fin: + reader = csv.DictReader(fin) + headers = reader.fieldnames or [] + if not headers: + raise ValueError(f"Licensing CSV has no header row: {licensing_csv}") + upn_col = detect_upn_column(headers) + if upn_col is None: + raise ValueError( + f"Licensing CSV has no recognized UPN column " + f"(expected one of {sorted(UPN_VARIANTS_NORMALIZED)}): {licensing_csv}" + ) + lic_col = detect_has_license_column(headers) + if lic_col is None: + raise ValueError( + f"Licensing CSV has no recognized license column " + f"(expected one of {list(HAS_LICENSE_VARIANTS)}): {licensing_csv}" + ) + license_map: dict[str, str] = {} + row_count = 0 + for row in reader: + row_count += 1 + upn_norm = (row.get(upn_col) or "").strip().lower() + if not upn_norm: + continue + license_map[upn_norm] = row.get(lic_col) or "" + return license_map, row_count + + +# --------------------------------------------------------------------------- +# Org / manager hierarchy (Users dim enrichment; AIO + AIBV; always on). +# Built from the Entra manager links (id <-> manager_id, UPN fallback) that PAX +# already pulls. Structure is keyed on the INT UserKey surrogate, so every +# *_UserKey / OrgLevel / HierarchyPath column is identical with or without +# --deidentify; the *_Name columns use the display name as written (hashed when +# --deidentify is on). Filler for level slots deeper than a user is controlled +# by --hierarchy-fill (none|self|manager|fixed); 'fixed' uses --hierarchy-fill-label. +# --------------------------------------------------------------------------- +_HIER_LEVELS = 15 # Level0..Level14 denormalized top-down columns +_HIER_WALK_CAP = 1000 # safety backstop for manager-chain walks (cycle guard also applies) +_HIER_FILL_MODE = "none" # none | self | manager | fixed (set in main() from --hierarchy-fill) +_HIER_FILL_LABEL = "" # literal label for 'fixed' (set in main() from --hierarchy-fill-label) + + +def _hier_columns() -> list[str]: + cols = [ + "Manager_UserKey", "OrgLevel", "HierarchyPath", "TopOfChain_UserKey", + "IsManager", "DirectReports", "TotalReports", + ] + for i in range(_HIER_LEVELS): + cols.append(f"Level{i}_UserKey") + cols.append(f"Level{i}_Name") + return cols + + +_HIER_COLUMNS: list[str] = _hier_columns() + + +def _hier_filler(uk: int, mgr, name_by_uk: dict, mode: str, label: str) -> tuple[str, str]: + """(UserKey, Name) to place in a level slot DEEPER than the user's own level.""" + if mode == "self": + return str(uk), name_by_uk.get(uk, "") + if mode == "manager": + ref = mgr if mgr is not None else uk + return str(ref), name_by_uk.get(ref, "") + if mode == "fixed": + return "", label + return "", "" # none + + +def _build_org_hierarchy(uk_by_id, uk_by_upn, mgr_ptr, name_by_uk) -> dict: + """Return {UserKey -> {hier_col: value}}. + + uk_by_id : normalized Entra id -> UserKey + uk_by_upn : normalized UPN -> UserKey + mgr_ptr : UserKey -> (manager_id_norm, manager_upn_norm) + name_by_uk: UserKey -> display name (as written; deid'd when applicable) + """ + # Immediate manager UserKey for each user (id link first, UPN fallback). + direct_mgr: dict = {} + for uk, (mid, mupn) in mgr_ptr.items(): + m = uk_by_id.get(mid) if mid else None + if m is None and mupn: + m = uk_by_upn.get(mupn) + if m == uk: + m = None # ignore self-management + direct_mgr[uk] = m + + direct_reports: dict = {} + for uk, m in direct_mgr.items(): + if m is not None: + direct_reports[m] = direct_reports.get(m, 0) + 1 + + all_uks = set(uk_by_upn.values()) | set(name_by_uk.keys()) | set(direct_mgr.keys()) + total_reports: dict = {} + result: dict = {} + mode = _HIER_FILL_MODE + label = _HIER_FILL_LABEL + + for uk in all_uks: + chain = [] + seen = set() + cur = uk + while cur is not None and cur not in seen and len(chain) < _HIER_WALK_CAP: + seen.add(cur) + chain.append(cur) + cur = direct_mgr.get(cur) + # every ancestor of uk gains one report (cycle-safe via `seen`) + for anc in chain[1:]: + total_reports[anc] = total_reports.get(anc, 0) + 1 + chain.reverse() # top .. user + depth = len(chain) - 1 + top = chain[0] + mgr = direct_mgr.get(uk) + rec = { + "Manager_UserKey": str(mgr) if mgr is not None else "", + "OrgLevel": str(depth), + "HierarchyPath": "/".join(str(x) for x in chain), + "TopOfChain_UserKey": str(top), + } + n = len(chain) + for i in range(_HIER_LEVELS): + if i < n: + node = chain[i] + rec[f"Level{i}_UserKey"] = str(node) + rec[f"Level{i}_Name"] = name_by_uk.get(node, "") + else: + fk, fn = _hier_filler(uk, mgr, name_by_uk, mode, label) + rec[f"Level{i}_UserKey"] = fk + rec[f"Level{i}_Name"] = fn + result[uk] = rec + + for uk in all_uks: + dr = direct_reports.get(uk, 0) + result[uk]["DirectReports"] = str(dr) + result[uk]["IsManager"] = "TRUE" if dr > 0 else "FALSE" + result[uk]["TotalReports"] = str(total_reports.get(uk, 0)) + + return result + + +def load_entra_and_write_users( + entra_csv: str, + users_out_csv: str, + user_key_map: dict[str, int], + licensing_csv: str | None = None, + quiet: bool = False, +) -> dict[str, dict[str, str]]: + """ + Read the Entra CSV, write the Users dim CSV (with PBIP-compatible renames + + precomputed License Status + UserKey INT surrogate), and return a dict + keyed on PersonId_Normalized -> {"Has license": ..., "License Status": ...} + for fact-row lookup. + + Mutates `user_key_map` (normalized_upn -> int) in place — every Entra row + with a non-empty PersonId_Normalized is assigned a UserKey (1-based, in + Entra-file order). The same map is reused by the fact path so any audit + user already in Entra resolves to the same INT. + + Mirrors the rename/normalization logic in the existing PBIP M-code: + userPrincipalName/upn/personid -> PersonId + department -> Organization + jobTitle -> JobTitle + Has license variants -> "Has license" + adds PersonId_Normalized (lower+trim of PersonId) + adds License Status (precomputed) + adds TotalEmployees (row count, repeated per row) + + 3-file input mode: when `licensing_csv` is provided, `entra_csv` is treated + as a users-only file and the per-user license value is merged in from the + separate licensing CSV (keyed on normalized UPN). When `licensing_csv` is + None the function behaves exactly as before (license read from the combined + Entra row), so the legacy 2-file output is byte-identical. + """ + license_map: dict[str, str] | None = None + licensing_rows = 0 + if licensing_csv: + license_map, licensing_rows = load_licensing_map(licensing_csv) + + with open(entra_csv, "r", encoding="utf-8-sig", newline="") as fin: + # Sniff via a generous quote-aware reader; encoding="utf-8-sig" eats BOM if present. + reader = csv.DictReader(fin) + original_headers = reader.fieldnames or [] + if not original_headers: + raise ValueError(f"Entra CSV has no header row: {entra_csv}") + + upn_col = detect_upn_column(original_headers) + dept_col = detect_department_column(original_headers) + has_license_col = detect_has_license_column(original_headers) + jobtitle_col = detect_jobtitle_column(original_headers) + + # Build rename map: source_header -> target_header + rename_map: dict[str, str] = {} + if upn_col and upn_col != "PersonId": + rename_map[upn_col] = "PersonId" + if dept_col and dept_col != "Organization": + rename_map[dept_col] = "Organization" + if jobtitle_col and jobtitle_col != "JobTitle": + rename_map[jobtitle_col] = "JobTitle" + if has_license_col and has_license_col != "Has license": + rename_map[has_license_col] = "Has license" + + # Final header list for users CSV — preserve original order, apply renames, + # then append injected columns. UserKey is the INT surrogate that joins + # to the fact table. + renamed_headers = [rename_map.get(h, h) for h in original_headers] + injected = ["UserKey", "PersonId_Normalized", "License Status", "TotalEmployees"] + if "Has license" not in renamed_headers: + renamed_headers.append("Has license") + for inj in injected: + if inj not in renamed_headers: + renamed_headers.append(inj) + # Org/manager hierarchy columns (always appended; AIO/AIBV Users dim). + for hc in _HIER_COLUMNS: + if hc not in renamed_headers: + renamed_headers.append(hc) + + rows = list(reader) + + total_rows = len(rows) + user_lookup: dict[str, dict[str, str]] = {} + + # --- Org/manager hierarchy pre-pass: assign UserKeys in Entra-file order and + # build the link maps from the SAME rename + deid the write loop applies, then + # resolve the hierarchy. Always on for the AIO/AIBV Users dim. UserKeys assigned + # here are reused by the write loop (identical to the prior lazy assignment). --- + uk_by_id: dict[str, int] = {} + uk_by_upn: dict[str, int] = {} + mgr_ptr: dict[int, tuple[str, str]] = {} + name_by_uk: dict[int, str] = {} + for src_row in rows: + pid = src_row.get(upn_col, "") if (upn_col and upn_col != "PersonId") else src_row.get("PersonId", "") + pid = "" if pid is None else str(pid) + if _DEIDENTIFY: + pid = deid_upn(pid) + pid_norm = pid.strip().lower() + if not pid_norm: + continue + uk = user_key_map.get(pid_norm) + if uk is None: + uk = len(user_key_map) + 1 + user_key_map[pid_norm] = uk + uk_by_upn[pid_norm] = uk + rid = src_row.get("id", "") + rid = "" if rid is None else str(rid) + if _DEIDENTIFY: + rid = deid_guid(rid) + rid_norm = rid.strip().lower() + if rid_norm: + uk_by_id[rid_norm] = uk + mid = src_row.get("manager_id", "") + mid = "" if mid is None else str(mid) + if _DEIDENTIFY: + mid = deid_guid(mid) + mupn = src_row.get("manager_userPrincipalName", "") + mupn = "" if mupn is None else str(mupn) + if _DEIDENTIFY: + mupn = deid_upn(mupn) + mgr_ptr[uk] = (mid.strip().lower(), mupn.strip().lower()) + dn = src_row.get("displayName", "") + dn = "" if dn is None else str(dn) + if _DEIDENTIFY: + dn = deid_name(dn) + name_by_uk[uk] = dn + hier_by_uk = _build_org_hierarchy(uk_by_id, uk_by_upn, mgr_ptr, name_by_uk) + + out_dir = Path(users_out_csv).parent + out_dir.mkdir(parents=True, exist_ok=True) + + pax_licensed = 0 + pax_unlicensed = 0 + no_license_col = 0 + matched_in_licensing = 0 + seen_normalized_keys: set[str] = set() + + with open(users_out_csv, "w", encoding="utf-8", newline="") as fout: + writer = csv.DictWriter(fout, fieldnames=renamed_headers, lineterminator="\n") + writer.writeheader() + + for src_row in rows: + # Apply renames + ensure all renamed_headers keys exist in the out row. + out_row: dict[str, str] = {h: "" for h in renamed_headers} + for src_h, value in src_row.items(): + tgt_h = rename_map.get(src_h, src_h) + if tgt_h in out_row: + out_row[tgt_h] = "" if value is None else str(value) + + # Deidentification (no-op unless --deidentify): transform identity columns + # IN PLACE before PersonId_Normalized / UserKey derive from PersonId, so the + # fact<->Users join and manager links stay consistent across the hash. + if _DEIDENTIFY: + if "PersonId" in out_row: + out_row["PersonId"] = deid_upn(out_row["PersonId"]) + if "displayName" in out_row: + out_row["displayName"] = deid_name(out_row["displayName"]) + if "mail" in out_row: + out_row["mail"] = deid_upn(out_row["mail"]) + if "givenName" in out_row: + out_row["givenName"] = deid_name(out_row["givenName"]) + if "surname" in out_row: + out_row["surname"] = deid_name(out_row["surname"]) + if "UserName" in out_row: + out_row["UserName"] = deid_upn(out_row["UserName"]) + if "employeeId" in out_row: + out_row["employeeId"] = deid_token(out_row["employeeId"]) + if "onPremisesImmutableId" in out_row: + out_row["onPremisesImmutableId"] = deid_token(out_row["onPremisesImmutableId"]) + if "proxyAddresses_Primary" in out_row: + out_row["proxyAddresses_Primary"] = deid_proxy(out_row["proxyAddresses_Primary"]) + if "proxyAddresses_All" in out_row: + out_row["proxyAddresses_All"] = deid_proxy(out_row["proxyAddresses_All"]) + if "id" in out_row: + out_row["id"] = deid_guid(out_row["id"]) + if "manager_id" in out_row: + out_row["manager_id"] = deid_guid(out_row["manager_id"]) + if "manager_userPrincipalName" in out_row: + out_row["manager_userPrincipalName"] = deid_upn(out_row["manager_userPrincipalName"]) + if "manager_displayName" in out_row: + out_row["manager_displayName"] = deid_name(out_row["manager_displayName"]) + if "manager_mail" in out_row: + out_row["manager_mail"] = deid_upn(out_row["manager_mail"]) + if "ManagerID" in out_row: + out_row["ManagerID"] = deid_guid(out_row["ManagerID"]) + + # PersonId_Normalized + person_id = out_row.get("PersonId", "") + person_id_norm = person_id.strip().lower() if person_id else "" + out_row["PersonId_Normalized"] = person_id_norm + + # UserKey (INT surrogate; assigned in Entra-file order) + if person_id_norm: + user_key = user_key_map.get(person_id_norm) + if user_key is None: + user_key = len(user_key_map) + 1 + user_key_map[person_id_norm] = user_key + out_row["UserKey"] = str(user_key) + else: + out_row["UserKey"] = "" + + # License Status (mirrors PBIP DAX exactly). + # We also normalize Has license to canonical TRUE/FALSE so existing + # measures that filter `[Has license] = "FALSE"` match regardless + # of source casing. + # + # 3-file input mode: when a separate licensing file is supplied, the + # per-user license value comes from that file (keyed on normalized + # UPN). When it is NOT supplied, the value is read from the + # (combined) Entra row exactly as before -> byte-identical 2-file + # output. + if license_map is not None: + has_license_raw = license_map.get(person_id_norm, "") + else: + has_license_raw = out_row.get("Has license", "") + normalized_has_license = normalize_has_license(has_license_raw) + if license_map is not None: + if person_id_norm and person_id_norm in license_map: + matched_in_licensing += 1 + if (has_license_raw or "").strip().upper() in _LICENSE_TRUTHY: + pax_licensed += 1 + else: + pax_unlicensed += 1 + elif has_license_col is None: + no_license_col += 1 + elif (has_license_raw or "").strip().upper() in _LICENSE_TRUTHY: + pax_licensed += 1 + else: + pax_unlicensed += 1 + out_row["Has license"] = normalized_has_license + out_row["License Status"] = compute_license_status(normalized_has_license) + + # TotalEmployees (matches M-code: row count repeated per row) + out_row["TotalEmployees"] = str(total_rows) + + # Org/manager hierarchy columns (always emitted; blank if no UserKey). + uk_str = out_row.get("UserKey", "") + if uk_str: + hrec = hier_by_uk.get(int(uk_str)) + if hrec: + for hc in _HIER_COLUMNS: + out_row[hc] = hrec.get(hc, "") + + writer.writerow(out_row) + + # Build fact-lookup dict (dedupe on normalized key, last-wins matches + # M-code Table.Distinct behavior on the licensed-users path). + if person_id_norm: + user_lookup[person_id_norm] = { + "Has license": normalized_has_license, + "License Status": out_row["License Status"], + } + seen_normalized_keys.add(person_id_norm) + + if not quiet: + print(f" Entra rows: {total_rows:,}") + print(f" Unique users (norm): {len(seen_normalized_keys):,}") + if license_map is not None: + print(f" Licensing file: {licensing_csv}") + print(f" Licensing rows: {licensing_rows:,}") + print(f" Matched to users: {matched_in_licensing:,}") + print(f" Licensed: {pax_licensed:,}") + print(f" Unlicensed: {pax_unlicensed:,}") + elif has_license_col: + print(f" License col detected: '{has_license_col}'") + print(f" Licensed (PAX): {pax_licensed:,}") + print(f" Unlicensed (PAX): {pax_unlicensed:,}") + else: + print(f" License col detected: NO RECOGNIZED LICENSE COLUMN FOUND IN ENTRA CSV") + print(f" Fallback: every user will be tagged 'Unlicensed' until a recognized column is present.") + + return user_lookup + + +# --------------------------------------------------------------------------- +# Fact row explosion + output +# --------------------------------------------------------------------------- + + +def explode_record( + audit_data: dict[str, Any], + user_lookup: dict[str, dict[str, str]], + user_key_map: dict[str, int], + thread_key_map: dict[str, int], + profile: str, +) -> list[dict[str, Any]]: + creation_time_raw = audit_data.get("CreationTime") + creation_time_raw_str = to_text(creation_time_raw).strip() + # Cached bundle: 4 derived date strings in one shot, keyed on the raw + # timestamp string (~K distinct values across N records). + creation_date_str, interaction_date_str, week_start_str, month_start_str = ( + _date_strings_for_raw(creation_time_raw_str) + ) + app_identity_app_id, app_identity_display = app_identity_values(audit_data) + agent_id = to_text(audit_data.get("AgentId")) + agent_name = derive_agent_name(audit_data.get("AgentName"), app_identity_display, app_identity_app_id) + + ced = audit_data.get("CopilotEventData") + if not isinstance(ced, dict): + return [] + + prompts = prompt_messages(ced) + if not prompts: + return [] + + resources = resource_rows(ced) + real_resource_count = sum(1 for item in get_array(ced, "AccessedResources") if isinstance(item, dict)) + resource_count_value = real_resource_count if real_resource_count > 0 else 1 + first_context = first_dict_item(get_array(ced, "Contexts")) + first_plugin = first_dict_item(get_array(ced, "AISystemPlugin")) + first_model = first_dict_item(get_array(ced, "ModelTransparencyDetails")) + + audit_user_id_raw = to_text(audit_data.get("UserId")) + if not _is_human_upn(audit_user_id_raw): + return [] + # Deidentify (no-op unless --deidentify) AFTER the human-UPN filter so the filter + # sees the original; every downstream UserKey/Audit_UserId/join derives from the + # hashed value, keeping it consistent with the (also-hashed) Users dim. + audit_user_id_raw = deid_upn(audit_user_id_raw) + audit_user_id_norm = normalize_user_id(audit_user_id_raw) + # UserKey INT surrogate. If this audit user wasn't in Entra, mint a new + # INT and stash so subsequent rows for the same user reuse it. The + # caller tracks unmatched-vs-Entra via the lookup membership check. + if audit_user_id_norm: + user_key = user_key_map.get(audit_user_id_norm) + if user_key is None: + user_key = len(user_key_map) + 1 + user_key_map[audit_user_id_norm] = user_key + else: + user_key = "" + # ThreadId INT surrogate. deid_guid is a no-op unless --deidentify; under + # --deidentify it returns a deterministic, format-preserving token (same raw + # ThreadId -> same token across runs) so the INT-surrogate keying, the + # ThreadId_Raw output column, and cross-run append dedup stay consistent. + thread_id_raw = deid_guid(to_text(ced.get("ThreadId"))) + if thread_id_raw: + thread_key = thread_key_map.get(thread_id_raw) + if thread_key is None: + thread_key = len(thread_key_map) + 1 + thread_key_map[thread_id_raw] = thread_key + else: + thread_key = "" + app_host_str = to_text(ced.get("AppHost")) + sens_label_str = to_text(ced.get("SensitivityLabelId")) + ctx_type_str = to_text(first_context.get("Type")) if first_context else "" + plugin_id_str = to_text(first_plugin.get("Id")) if first_plugin else "" + model_name_str = to_text(first_model.get("ModelName")) if first_model else "" + + # User-level lookups (constant per record) + user_rec = user_lookup.get(audit_user_id_norm) or {} + has_license_raw = user_rec.get("Has license", "") + license_status = user_rec.get("License Status") or compute_license_status(has_license_raw) + environment = compute_environment(profile, has_license_raw, agent_name, agent_id, app_host_str) + ai_model = compute_ai_model(model_name_str) + user_month_key = compute_user_month_key(audit_user_id_raw, month_start_str) + + is_aibv = profile != "aio" + + # Per-record constants hoisted out of the (prompt x resource) inner loop. + # All grain values are pre-stringified via to_text() exactly once so the + # rollup loop can use the tuple directly as the dict key. + user_key_text = to_text(user_key) + thread_key_text = to_text(thread_key) + agent_title_id = derive_agent_title_id(agent_id) + aisystem_plugin_name_str = to_text(first_plugin.get("Name")) if first_plugin else "" + in_entra = (audit_user_id_norm in user_lookup) if audit_user_id_norm else True + # AIBV-only per-record constants. + agent_publish_status = compute_agent_publish_status(agent_id, agent_name) if is_aibv else "" + # has_agent gate for the Behavior_Category "logic app" branch (AIBV). + has_agent_ctx = bool(agent_name.strip()) or bool(agent_id.strip()) + + # Stable portion of the nongrain dict (everything that does NOT depend on + # the per-resource fields). Built once per record; copied per emitted row + # and updated with the resource-varying keys. Common keys first; AIBV-only + # keys appended only for the aibv profile so the AIO output stays the exact + # v3.1.0 column set. + base_nongrain: dict[str, Any] = { + "CreationDate": creation_date_str, + "WeekStart": week_start_str, + "MonthStart": month_start_str, + "UserMonthKey": user_month_key, + "Has license": has_license_raw, + "Resource_Count": resource_count_value, + "SensitivityLabelId": sens_label_str, + # AccessedResource_* injected per-resource below. + "AccessedResource_Type": "", + "AccessedResource_Action": "", + "AccessedResource_SiteUrl": "", + "AccessedResource_SensitivityLabelId": "", + "AppIdentity_DisplayName": app_identity_display, + "AISystemPlugin_Id": plugin_id_str, + "ModelTransparencyDetails_ModelName": model_name_str, + "Agent_TitleID": agent_title_id, + "Message_isPrompt": "TRUE", + # Behavior_Source / Value_Outcome injected per-resource below. + "Behavior_Source": "", + "Value_Outcome": "", + "ActivityDate": interaction_date_str, + # Stable, deid-consistent user identity (AIO parity with the + # AIBV [Audit_UserId_Normalized] value). Emitted only for the AIO profile + # (AIBV's header carries Audit_UserId_Normalized instead), so for AIBV this + # key is a harmless extra that its fact-header selection ignores. + "User_Id_Normalized": audit_user_id_norm, + # Cross-run append reconciliation key (trailing). Constant per record; + # Message_Id_Raw (per message) is injected in the emit loop below. + "ThreadId_Raw": thread_id_raw, + } + if is_aibv: + base_nongrain.update({ + # M1: UPN passthrough (raw mirrors AIBV [Audit_UserId]; normalized for joins). + "Audit_UserId": audit_user_id_raw, + "Audit_UserId_Normalized": audit_user_id_norm, + # Agent Filter injected per-resource; Agent Publish Status constant per record. + "Agent Filter": "", + "Agent Publish Status": agent_publish_status, + # Downstream chain + ROI baseline + remaining calc cols (per-resource). + "Behavior_Enriched_Full": "", + "Usage_Mode": "", + "Expertise_Role": "", + "Efficiency_Breakdown": "", + "Human_Baseline_Min": "", + "Behavior_Plausible": "", + "Delegation_Event_Key": "", + }) + + # Output schema: list of tuples + # (grain_tuple, message_id_str, nongrain_dict, in_entra, audit_user_id_norm) + # consumed directly by run_processor's rollup loop. The grain arity differs + # by profile (AIO 16 keys; AIBV 19 — the 3 promoted sliceable flags). + rows: list[tuple[tuple[str, ...], str, dict[str, Any], bool, str]] = [] + for message in prompts: + # deid_guid is a no-op unless --deidentify (then deterministic + format- + # preserving), so message_id doubles as the raw Message_Id_Raw dedup key + # AND the stable mid_to_int surrogate key that aligns with --seed-mid-map + # across runs. + message_id = deid_guid(to_text(message.get("Id"))) + for resource in resources: + res_type_str = to_text(resource.get("Type")) + res_action_str = to_text(resource.get("Action")) + res_site_str = to_text(resource.get("SiteUrl")) + res_sens_label_str = to_text(resource.get("SensitivityLabelId")) + behavior_category = compute_behavior_category( + profile, app_host_str, ctx_type_str, res_type_str, res_action_str, + res_site_str, plugin_id_str, has_agent_ctx, + ) + behavior_enriched = compute_behavior_enriched( + profile, behavior_category, agent_name, environment + ) + is_sensitive_str = compute_is_sensitive(sens_label_str, res_sens_label_str) + behavior_source = compute_behavior_source( + profile, behavior_category, environment, agent_name, + aisystem_plugin_name_str, app_host_str, + ) + value_outcome = compute_value_outcome( + profile, behavior_enriched, environment, is_sensitive_str, + ) + + nongrain = dict(base_nongrain) + nongrain["Message_Id_Raw"] = message_id + nongrain["AccessedResource_Type"] = res_type_str + nongrain["AccessedResource_Action"] = res_action_str + nongrain["AccessedResource_SiteUrl"] = deid_resource(res_site_str) + nongrain["AccessedResource_SensitivityLabelId"] = res_sens_label_str + nongrain["Behavior_Source"] = behavior_source + nongrain["Value_Outcome"] = value_outcome + + common_grain = ( + user_key_text, + interaction_date_str, + agent_id, + agent_name, + app_host_str, + environment, + license_status, + ctx_type_str, + behavior_category, + behavior_enriched, + ai_model, + is_sensitive_str, + ) + + if is_aibv: + # AIBV-faithful per-resource flags (3 are grain keys). + is_agent_activity_str = compute_is_agent_activity( + agent_name, agent_id, app_host_str, res_type_str + ) + web_grounded_str = compute_web_grounded_signal(res_type_str, res_site_str) + # Autonomy_Pattern depends on per-resource Is_Agent_Activity (AIBV). + autonomy_pattern = compute_autonomy_pattern( + profile, environment, is_agent_activity_str + ) + # Downstream chain (faithful without Agents 365 per F2). + behavior_enriched_full = compute_behavior_enriched_full(behavior_enriched) + usage_mode = compute_usage_mode(behavior_enriched_full, environment, app_host_str) + expertise_role = compute_expertise_role(behavior_enriched_full) + efficiency_breakdown = compute_efficiency_breakdown( + behavior_enriched_full, behavior_category + ) + human_baseline_min = compute_human_baseline_min(behavior_enriched_full) + behavior_plausible = compute_behavior_plausible(license_status, behavior_category) + workflow_action = compute_workflow_action( + behavior_enriched_full, res_action_str, app_host_str + ) + delegation_event_key = compute_delegation_event_key( + audit_user_id_raw, interaction_date_str, agent_name, + workflow_action, app_host_str, + ) + grain_tuple = common_grain + ( + autonomy_pattern, + app_identity_app_id, + aisystem_plugin_name_str, + thread_key_text, + is_agent_activity_str, + web_grounded_str, + workflow_action, + ) + nongrain["Agent Filter"] = "Agents" if is_agent_activity_str == "TRUE" else "" + nongrain["Behavior_Enriched_Full"] = behavior_enriched_full + nongrain["Usage_Mode"] = usage_mode + nongrain["Expertise_Role"] = expertise_role + nongrain["Efficiency_Breakdown"] = efficiency_breakdown + nongrain["Human_Baseline_Min"] = human_baseline_min + nongrain["Behavior_Plausible"] = behavior_plausible + nongrain["Delegation_Event_Key"] = delegation_event_key + else: + # AIO (v3.1.0): autonomy keyed purely off Environment; 16-key grain. + autonomy_pattern = compute_autonomy_pattern(profile, environment, "") + grain_tuple = common_grain + ( + autonomy_pattern, + app_identity_app_id, + aisystem_plugin_name_str, + thread_key_text, + ) + + rows.append((grain_tuple, message_id, nongrain, in_entra, audit_user_id_norm)) + + return rows + + +# --------------------------------------------------------------------------- +# Pre-aggregated tables (AIBV only) — offload the DAX calculated tables. +# +# Each of these replaces a DAX calc table that SUMMARIZEs the ENTIRE fact on +# every refresh. We compute them ONCE here from the same rollup that produces +# the fact CSV, so they equal exactly what the existing DAX would compute when +# evaluated over the rolled-up fact (internally consistent — a reviewer can +# keep the DAX calc tables over the rollup fact and get the same numbers). +# +# Grain-key index map (AIBV profile): see GRAIN_KEYS_AIBV. +# [1]=InteractionDate [3]=AgentName [6]=License Status +# The remaining inputs come from the nongrain dict +# (MonthStart, WeekStart, Audit_UserId, Behavior_Enriched_Full, Usage_Mode). +# --------------------------------------------------------------------------- + +_VALUEFOCUS_MODES = frozenset({"4 - Producing", "5 - Delegating"}) + + +def _percentile_inc(sorted_vals: list[float], p: float) -> float: + """PERCENTILE.INC / PERCENTILEX.INC — linear interpolation, p in [0, 1].""" + n = len(sorted_vals) + if n == 0: + return 0.0 + if n == 1: + return sorted_vals[0] + rank = p * (n - 1) + lo = int(rank) # floor (rank is non-negative) + if lo + 1 >= n: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[lo + 1] - sorted_vals[lo]) * frac + + +def _usage_rank(avg_ppw: float, p90: float, p75: float, p50: float, p25: float) -> str: + if avg_ppw == 0: + return "0. No Usage" + if avg_ppw >= p90: + return "5. Top 10% Users" + if avg_ppw >= p75: + return "4. 75-90% Users" + if avg_ppw >= p50: + return "3. 50-75% Users" + if avg_ppw >= p25: + return "2. 25-50% Users" + return "1. Bottom 25% Users" + + +def _user_stage(active_days: int, behavior_count: int, value_focus_share: float, has_agent: bool) -> str: + if active_days >= 15 or (active_days >= 10 and value_focus_share >= 0.30 and has_agent): + return "4 - Power" + if active_days >= 8 and behavior_count >= 5: + return "3 - Habitual" + if active_days >= 3 and behavior_count >= 3: + return "2 - Developing" + return "1 - Beginner" + + +def _activity_segment(avg_days: float) -> str: + if avg_days == 0: + return "0. No Activity" + if avg_days <= 5: + return "1. 1-5 Chat Days/Month - 'Infrequent'" + if avg_days <= 10: + return "2. 6-10 Chat Days/Month - 'Moderate'" + if avg_days <= 19: + return "3. 11-19 Chat Days/Month - 'Frequent'" + return "4. 20+ Chat Days/Month - 'Daily'" + + +def _fmt_float(x: float) -> str: + """Shortest round-trippable float, integers without trailing '.0'.""" + if x == int(x): + return str(int(x)) + return repr(x) + + +def compute_and_write_aggregates( + rollup: dict[tuple[Any, ...], dict[str, Any]], + agg_paths: dict[str, str], + quiet: bool = False, +) -> dict[str, int]: + """Build the 5 AIBV pre-aggregated tables from the rollup and write them. + + Returns {table_name: row_count}. `agg_paths` keys: + active_days, user_month_metrics, licensed_rankings, + unlicensed_rankings, licensed_summary. + """ + # Per (Audit_UserId, MonthStart) accumulators. + um: dict[tuple[str, str], dict[str, Any]] = {} + # Per Audit_UserId accumulators (for the rankings). + ua: dict[str, dict[str, Any]] = {} + + for grain_key, nongrain in rollup.items(): + gk = grain_key[0] # rollup key is ((grain_tuple), mid_int) + mid = grain_key[1] + interaction_date = gk[1] + agent_name = gk[3] + license_status = gk[6] + uid = nongrain["Audit_UserId"] + month = nongrain["MonthStart"] + week = nongrain["WeekStart"] + bef = nongrain["Behavior_Enriched_Full"] + usage_mode = nongrain["Usage_Mode"] + + mk = (uid, month) + a = um.get(mk) + if a is None: + a = um[mk] = { + "idates": set(), "mids": set(), "behaviors": set(), + "has_agent": False, "rows": 0, "valuefocus": 0, "license": license_status, + } + a["idates"].add(interaction_date) + a["mids"].add(mid) + a["behaviors"].add(bef) + if agent_name.strip(): + a["has_agent"] = True + a["rows"] += 1 + if usage_mode in _VALUEFOCUS_MODES: + a["valuefocus"] += 1 + if license_status < a["license"]: # MIN(License Status), lexicographic (matches DAX MIN) + a["license"] = license_status + + u = ua.get(uid) + if u is None: + u = ua[uid] = {"rows": 0, "weeks": set(), "license": license_status} + u["rows"] += 1 + u["weeks"].add(week) + if license_status < u["license"]: + u["license"] = license_status + + # ---- ActiveDaysSummary (filter ChatActiveDays > 0; always true here) ---- + ads_rows: list[tuple[str, str, int, int, str]] = [] + for (uid, month), a in um.items(): + chat_active_days = len(a["idates"]) + if chat_active_days <= 0: + continue + ads_rows.append((uid, month, chat_active_days, len(a["mids"]), a["license"])) + ads_rows.sort(key=lambda r: (r[0], r[1])) + + # ---- UserMonthMetrics ---- + umm_rows: list[tuple] = [] + for (uid, month), a in um.items(): + active_days = len(a["idates"]) + behavior_count = len(a["behaviors"]) + value_focus_share = (a["valuefocus"] / a["rows"]) if a["rows"] else 0.0 + has_agent = a["has_agent"] + user_month_key = f"{uid}|{month[:7]}" if (uid and month) else "" + stage = _user_stage(active_days, behavior_count, value_focus_share, has_agent) + umm_rows.append(( + uid, month, behavior_count, "True" if has_agent else "False", + active_days, user_month_key, stage, value_focus_share, + )) + umm_rows.sort(key=lambda r: (r[0], r[1])) + + # ---- Rankings (per user, partitioned by license) ---- + def _build_rankings(target_license: str) -> list[tuple]: + summary = [] # (uid, total_prompts, total_weeks, avg_ppw) + for uid, u in ua.items(): + if u["license"] != target_license: + continue + total_prompts = u["rows"] + total_weeks = len(u["weeks"]) + avg_ppw = (total_prompts / total_weeks) if total_weeks else 0.0 + summary.append((uid, total_prompts, total_weeks, avg_ppw)) + avgs = sorted(s[3] for s in summary) + p90 = _percentile_inc(avgs, 0.90) + p75 = _percentile_inc(avgs, 0.75) + p50 = _percentile_inc(avgs, 0.50) + p25 = _percentile_inc(avgs, 0.25) + out = [] + for uid, tp, tw, avg in summary: + out.append((uid, _usage_rank(avg, p90, p75, p50, p25), tp, tw, avg)) + out.sort(key=lambda r: r[0]) + return out + + licensed_rank_rows = _build_rankings("M365 Copilot Licensed") + unlicensed_rank_rows = _build_rankings("Unlicensed") + + # ---- Licensed Chat User Summary (from ActiveDaysSummary, licensed only) ---- + lsum: dict[str, dict[str, int]] = {} + for uid, month, chat_active_days, prompt_count, lic in ads_rows: + if lic != "M365 Copilot Licensed": + continue + s = lsum.get(uid) + if s is None: + s = lsum[uid] = {"days": 0, "months": 0, "prompts": 0} + s["days"] += chat_active_days + s["months"] += 1 # every ADS row already has ChatActiveDays > 0 + s["prompts"] += prompt_count + summary_rows: list[tuple] = [] + for uid, s in lsum.items(): + total_days = s["days"] + total_months = s["months"] + total_prompts = s["prompts"] + avg_days = (total_days / total_months) if total_months else 0.0 + summary_rows.append(( + uid, _activity_segment(avg_days), total_days, total_months, + total_prompts, avg_days, + )) + summary_rows.sort(key=lambda r: r[0]) + + # ---- write all 5 ---- + def _write(path: str, header: list[str], rows: list[tuple], float_cols: set[int]) -> int: + with open(path, "w", encoding="utf-8", newline="") as f: + w = csv.writer(f, lineterminator="\n") + w.writerow(header) + for r in rows: + w.writerow([_fmt_float(v) if i in float_cols else v for i, v in enumerate(r)]) + return len(rows) + + counts = {} + counts["active_days"] = _write( + agg_paths["active_days"], + ["Audit_UserId", "MonthStart", "ChatActiveDays", "PromptCount", "LicenseStatus"], + ads_rows, set(), + ) + counts["user_month_metrics"] = _write( + agg_paths["user_month_metrics"], + ["Audit_UserId", "MonthStart", "BehaviorCount", "HasAgent", "ActiveDays", + "UserMonthKey", "UserStage", "ValueFocusShare"], + umm_rows, {7}, + ) + counts["licensed_rankings"] = _write( + agg_paths["licensed_rankings"], + ["Audit_UserId", "Usage Rank", "TotalPrompts", "TotalWeeks", "AvgPromptsPerWeek"], + licensed_rank_rows, {4}, + ) + counts["unlicensed_rankings"] = _write( + agg_paths["unlicensed_rankings"], + ["Audit_UserId", "Usage Rank", "TotalPrompts", "TotalWeeks", "AvgPromptsPerWeek"], + unlicensed_rank_rows, {4}, + ) + counts["licensed_summary"] = _write( + agg_paths["licensed_summary"], + ["Audit_UserId", "Activity Segment", "TotalActiveDays", "TotalMonths", + "TotalPrompts", "AvgActiveDaysPerMonth"], + summary_rows, {5}, + ) + + if not quiet: + print(" Pre-aggregated tables (AIBV):") + print(f" ActiveDaysSummary: {counts['active_days']:,} rows") + print(f" UserMonthMetrics: {counts['user_month_metrics']:,} rows") + print(f" Licensed User Rankings: {counts['licensed_rankings']:,} rows") + print(f" Unlicensed User Rankings: {counts['unlicensed_rankings']:,} rows") + print(f" Licensed User Summary: {counts['licensed_summary']:,} rows") + + return counts + + +def run_processor( + purview_csv: str, + entra_csv: str, + fact_out_csv: str, + users_out_csv: str, + profile: str = "aibv", + agg_paths: dict[str, str] | None = None, + quiet: bool = False, + licensing_csv: str | None = None, + seed_mid_map_path: str | None = None, + seed_thread_map_path: str | None = None, + seed_userkey_map_path: str | None = None, +) -> dict[str, Any]: + start_time = time.perf_counter() + stats: dict[str, Any] = { + "input_records": 0, + "skipped_non_copilot": 0, + "output_rows": 0, + "errors": 0, + "unmatched_users": 0, + } + + if not quiet: + print(f"Purview CopilotInteraction Processor v{SCRIPT_VERSION}") + print(f" Profile: {profile}") + print(f" JSON engine: {_JSON_ENGINE}") + print(f" Purview input: {purview_csv}") + print(f" Entra input: {entra_csv}") + if licensing_csv: + print(f" Licensing: {licensing_csv}") + print(f" Purview output: {fact_out_csv}") + print(f" Entra output: {users_out_csv}") + print() + print("Loading Entra users + writing Users dim CSV...") + + # Shared INT-surrogate maps. UserKey is populated first by the Entra + # loader (so Entra-known users get the lowest INTs / lowest dictionary + # offsets in VertiPaq); the fact path then reuses + extends the map. + # mid_to_int is declared here (alongside the other two) so all three can be + # pre-seeded from the PAX append-target maps before the Entra load + fact loop. + user_key_map: dict[str, int] = {} + thread_key_map: dict[str, int] = {} + mid_to_int: dict[str, int] = {} + + # PAX append support (not in the standalone v4.0.0 reference): pre-seed the + # three INT-surrogate maps from JSON snapshots of the append-target's existing + # surrogates so retained rows keep stable Message_Id / ThreadKey / UserKey + # values across runs. No-op when the seed paths are None (the standalone + # 2/3-file runs and any non-append PAX run). + def _load_int_seed(path: str, target: dict[str, int]) -> None: + with open(path, "r", encoding="utf-8") as f: + data = json_loads(f.read()) + if not isinstance(data, dict): + return + for k, v in data.items(): + try: + target[str(k)] = int(v) + except (TypeError, ValueError): + continue + + if seed_userkey_map_path: + _load_int_seed(seed_userkey_map_path, user_key_map) + if seed_thread_map_path: + _load_int_seed(seed_thread_map_path, thread_key_map) + if seed_mid_map_path: + _load_int_seed(seed_mid_map_path, mid_to_int) + + user_lookup = load_entra_and_write_users( + entra_csv, users_out_csv, user_key_map, licensing_csv=licensing_csv, quiet=quiet + ) + + if not quiet: + print() + print("Flattening CopilotInteraction records...") + + # One row per (grain x distinct Message_Id). Per-resource accumulation + # is intentionally avoided here so counts are not inflated ~2.25x by + # per (prompt x AccessedResource) iteration. Downstream measures use + # DISTINCTCOUNT(Message_Id) for exact parity with the semantic-model + # definitions. + # + # Message_Id is INT-surrogated (1-based, encounter order) for CSV size + # and parse-time win on the highest-cardinality column. + # + # Key: (grain_tuple, message_id_int) + # Value: dict of non-grain attrs (last-write-wins on a per-resource + # basis for AccessedResource_* / SensitivityLabelId — same + # semantic as the prior dict-overwrite behavior). + rollup: dict[tuple[Any, ...], dict[str, Any]] = {} + unmatched: set[str] = set() + + with open(purview_csv, "r", encoding="utf-8-sig", newline="") as fin: + reader = csv.DictReader(fin) + + for raw_row in reader: + stats["input_records"] += 1 + + audit_raw = raw_row.get("AuditData", "") or "" + try: + audit_data = json_loads(audit_raw) if audit_raw.strip() else {} + except Exception: + stats["errors"] += 1 + continue + + if not isinstance(audit_data, dict): + stats["errors"] += 1 + continue + + if not is_copilot_interaction(audit_data, raw_row): + stats["skipped_non_copilot"] += 1 + continue + + try: + rows = explode_record(audit_data, user_lookup, user_key_map, thread_key_map, profile) + except Exception: + stats["errors"] += 1 + continue + + for grain_key, message_id_str, nongrain, in_entra, audit_user_norm in rows: + stats["output_rows"] += 1 + if not in_entra and audit_user_norm: + unmatched.add(audit_user_norm) + mid_int = mid_to_int.get(message_id_str) + if mid_int is None: + mid_int = len(mid_to_int) + 1 + mid_to_int[message_id_str] = mid_int + rollup[(grain_key, mid_int)] = nongrain + + if not quiet: + print(f" Input records: {stats['input_records']:,}") + print(f" Skipped (non-Copilot): {stats['skipped_non_copilot']:,}") + print(f" Raw prompt rows: {stats['output_rows']:,}") + print(f" Errors: {stats['errors']:,}") + print() + print("Writing rolled-up fact CSV...") + + # Profile-specific output schema (AIO = v3.1.0 36-col; AIBV = 50-col superset). + grain_keys, nongrain_attrs_sel, fact_header = schema_for(profile) + with open(fact_out_csv, "w", encoding="utf-8", newline="") as fout: + writer = csv.writer(fout, lineterminator="\n") + writer.writerow(fact_header) + # Pre-compute the index of Message_Id within FACT_HEADER so we can + # splice the INT surrogate into a list-of-attrs in one shot. The + # list-based csv.writer.writerow path is materially faster than + # DictWriter (skips dict-to-list translation + per-row genexpr). + nongrain_attrs = nongrain_attrs_sel # local rebind + grain_len = len(grain_keys) + for (grain_key, mid_int), attrs in rollup.items(): + # fact_header = grain_keys + ("Message_Id",) + nongrain_attrs + row_out = list(grain_key) + row_out.append(mid_int) + row_out.extend(attrs[k] for k in nongrain_attrs) + writer.writerow(row_out) + + stats["output_rows_rollup"] = len(rollup) + stats["distinct_message_ids"] = len(mid_to_int) + stats["distinct_thread_ids"] = len(thread_key_map) + stats["distinct_user_keys"] = len(user_key_map) + stats["unmatched_users"] = len(unmatched) + + # Pre-aggregated tables (AIBV profile only). These offload the DAX + # calculated tables (ActiveDaysSummary / UserMonthMetrics / rankings / + # summary), each of which otherwise SUMMARIZEs the whole fact on refresh. + if profile != "aio" and agg_paths: + if not quiet: + print() + print("Writing pre-aggregated tables...") + compute_and_write_aggregates(rollup, agg_paths, quiet=quiet) + elapsed = time.perf_counter() - start_time + if not quiet: + reduction_pct = (1 - len(rollup) / stats["output_rows"]) * 100 if stats["output_rows"] else 0 + print(f" Rollup rows: {len(rollup):,} ({reduction_pct:.1f}% reduction)") + print(f" Distinct Message_Ids: {len(mid_to_int):,}") + print(f" Distinct ThreadIds: {len(thread_key_map):,}") + print(f" Distinct UserKeys: {len(user_key_map):,}") + print(f" Unmatched users: {stats['unmatched_users']:,}") + print(f" Elapsed: {elapsed:.2f}s") + + return stats + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + f"Purview CopilotInteraction Processor v{SCRIPT_VERSION} - " + "Two/three-input, two-output preprocessor that produces a rolled-up " + "Interactions fact CSV (~85% row reduction via PromptCount grain) " + "and a Users dim CSV for the AI Business Value Dashboard PBIP." + ) + ) + parser.add_argument( + "--purview", + required=True, + help="Path to the raw Purview audit log CSV (must contain AuditData column).", + ) + parser.add_argument( + "--entra", + required=True, + help=( + "Path to the Entra users CSV (UPN + org columns). The license column " + "is optional: supply it separately via --licensing (recommended for " + "standalone use), or omit --licensing to use a single combined " + "users+licensing file (license column auto-detected)." + ), + ) + parser.add_argument( + "--licensing", + default=None, + help=( + "Path to a separate licensing CSV (UPN + 'Has License' columns), e.g. " + "the Microsoft Admin Center Copilot user export. When provided, " + "--entra is treated as a users-only file and license status is merged " + "in from this file (the 3-file workflow). Omit to use a single " + "combined Entra file (license column auto-detected). Applies to both " + "--profile aibv and --profile aio." + ), + ) + parser.add_argument( + "--combined-entra", + action="store_true", + default=False, + help=( + "Legacy 2-file mode: assert that --entra is a single combined " + "users+licensing file (as produced by the PAX script). Mutually " + "exclusive with --licensing. Optional - even without this flag a " + "combined file still works (the license column is auto-detected)." + ), + ) + parser.add_argument( + "--out-dir", + "-o", + default=None, + help="Directory for output files. Default: same directory as the Purview file.", + ) + parser.add_argument( + "--profile", + "-p", + choices=("aibv", "aio"), + default="aibv", + help=( + "Output profile. 'aibv' (default) = AI Business Value Dashboard " + "superset (50-col fact, 3-value Environment). 'aio' = AI-in-One " + "Dashboard (36-col fact, 5-value Environment) — reproduces the " + "v3.1.0 AIO output exactly." + ), + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + default=False, + help="Suppress progress output.", + ) + parser.add_argument( + "--with-aggregates", + action="store_true", + default=False, + help=( + "Also write the AIBV pre-aggregated tables (ActiveDaysSummary, " + "UserMonthMetrics, Licensed/Unlicensed user rankings, Licensed user " + "summary). OFF by default — the AIBV template only needs the two core " + "rollup files (Interactions + Users). No effect for --profile aio." + ), + ) + parser.add_argument( + # Deprecated no-op: aggregates are now OFF by default, so this flag does + # nothing. Kept so older command lines don't error. + "--no-aggregates", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--deidentify", + action="store_true", + default=False, + help=( + "One-way hash all identifying values (UPNs, names, manager fields, " + "Entra/mailbox GUIDs and SIDs, resource URLs) for anonymous reporting. " + "Deterministic and format-preserving, so manager links and " + "UserKey/Users joins are kept; irreversible (no decode map)." + ), + ) + parser.add_argument( + # PAX append support (not in the standalone v4.0.0 reference). + "--seed-mid-map", + default=None, + help=( + "Optional JSON file mapping {Message_Id_Raw -> existing INT surrogate} " + "extracted from the target Fact CSV. Pre-seeds mid_to_int so cross-run " + "appends preserve Message_Id INTs and dedup source rows on Message_Id_Raw." + ), + ) + parser.add_argument( + "--seed-thread-map", + default=None, + help=( + "Optional JSON file mapping {ThreadId_Raw -> existing INT surrogate} " + "extracted from the target Fact CSV. Pre-seeds thread_key_map so cross-run " + "appends preserve ThreadId INTs." + ), + ) + parser.add_argument( + "--seed-userkey-map", + default=None, + help=( + "Optional JSON file mapping {PersonId_Normalized -> existing UserKey INT} " + "extracted from the merged Users CSV. Pre-seeds user_key_map so Entra users " + "carried forward from prior runs keep their UserKey across the append." + ), + ) + parser.add_argument( + "--hierarchy-fill", + choices=("none", "self", "manager", "fixed"), + default="none", + help=( + "Filler for org-hierarchy level slots deeper than a user's own level " + "(Users dim, AIO/AIBV). 'none' (default) leaves them blank; 'self' " + "repeats the user; 'manager' repeats the user's manager; 'fixed' uses " + "--hierarchy-fill-label. The hierarchy columns are always emitted." + ), + ) + parser.add_argument( + "--hierarchy-fill-label", + default="", + help="Literal label used when '--hierarchy-fill fixed' is selected.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {SCRIPT_VERSION}", + ) + + args = parser.parse_args() + + global _DEIDENTIFY + _DEIDENTIFY = bool(args.deidentify) + + global _HIER_FILL_MODE, _HIER_FILL_LABEL + _HIER_FILL_MODE = args.hierarchy_fill + _HIER_FILL_LABEL = args.hierarchy_fill_label or "" + if _HIER_FILL_MODE == "fixed" and not _HIER_FILL_LABEL: + print( + 'ERROR: --hierarchy-fill fixed requires --hierarchy-fill-label "".', + file=sys.stderr, + ) + sys.exit(1) + if _HIER_FILL_MODE != "fixed" and _HIER_FILL_LABEL: + print( + "ERROR: --hierarchy-fill-label is only valid with --hierarchy-fill fixed.", + file=sys.stderr, + ) + sys.exit(1) + + if args.licensing and args.combined_entra: + print( + "ERROR: --licensing and --combined-entra are mutually exclusive. Use " + "--licensing for the 3-file (separate licensing) workflow, or " + "--combined-entra (or neither) for a single combined Entra file.", + file=sys.stderr, + ) + sys.exit(1) + + purview_path = os.path.abspath(args.purview) + entra_path = os.path.abspath(args.entra) + for label, p in (("Purview", purview_path), ("Entra", entra_path)): + if not os.path.isfile(p): + print(f"ERROR: {label} input file not found: {p}", file=sys.stderr) + sys.exit(1) + + licensing_path = os.path.abspath(args.licensing) if args.licensing else None + if licensing_path and not os.path.isfile(licensing_path): + print(f"ERROR: Licensing input file not found: {licensing_path}", file=sys.stderr) + sys.exit(1) + + out_dir = Path(os.path.abspath(args.out_dir)) if args.out_dir else Path(purview_path).parent + out_dir.mkdir(parents=True, exist_ok=True) + + purview_stem = Path(purview_path).stem + entra_stem = Path(entra_path).stem + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + # PAX embed (differs from the standalone v4.0.0, which timestamps these two): + # the core rollup outputs use NON-timestamped names so PAX's post-run + # Merge-FactCsv / Merge-UsersCsv resolve them by exact name. The run timestamp + # is already baked into the input filenames (Purview_Audit_*_.csv, + # EntraUsers_MAClicensing_.csv), so it is not duplicated on the output. + fact_out = str(out_dir / f"{purview_stem}_Interactions.csv") + users_out = str(out_dir / f"{entra_stem}_Users.csv") + + # Pre-aggregated table paths (AIBV only, opt-in via --with-aggregates). + # Default: not written — the AIBV template consumes only the Interactions + # + Users rollups. The aggregates remain available for the future + # calc-table offload (point the 5 DAX calc tables at these CSVs). + agg_paths: dict[str, str] | None = None + if args.profile != "aio" and args.with_aggregates: + agg_paths = { + "active_days": str(out_dir / f"{purview_stem}_ActiveDaysSummary_{run_ts}.csv"), + "user_month_metrics": str(out_dir / f"{purview_stem}_UserMonthMetrics_{run_ts}.csv"), + "licensed_rankings": str(out_dir / f"{purview_stem}_LicensedUserRankings_{run_ts}.csv"), + "unlicensed_rankings": str(out_dir / f"{purview_stem}_UnlicensedUserRankings_{run_ts}.csv"), + "licensed_summary": str(out_dir / f"{purview_stem}_LicensedUserSummary_{run_ts}.csv"), + } + + stats = run_processor( + purview_csv=purview_path, + entra_csv=entra_path, + fact_out_csv=fact_out, + users_out_csv=users_out, + profile=args.profile, + agg_paths=agg_paths, + quiet=args.quiet, + licensing_csv=licensing_path, + seed_mid_map_path=args.seed_mid_map, + seed_thread_map_path=args.seed_thread_map, + seed_userkey_map_path=args.seed_userkey_map, + ) + sys.exit(1 if stats["errors"] > 0 else 0) + + +if __name__ == "__main__": + main() +'@ +# <<< END-EMBEDDED-COPILOT-PROCESSOR + +# >>> BEGIN-EMBEDDED-M365-PROCESSOR +$Script:EMBEDDED_PROCESSOR_M365 = @' +#!/usr/bin/env python3 +""" +Purview M365 Usage Bundle Explosion Processor v2.6.1 +===================================================== +Two-mode processor for Purview audit log CSV exports: + + ROLLUP MODE (default): Aggregates exploded events into rolled-up rows keyed by + (UserId, CreationDate, Operation, Workload, SourceFileExtension, AppHost, + AgentId, AgentName, ContextType) + with EventCount, MIN(CreationTime), MAX(CreationTime), IsAgentInteraction. + Targets 80%+ row reduction for Power BI ingestion. + Streaming — no exploded rows held in memory. + + v2.3.0 CHANGES (validated against MS Learn audit-log schema + DAX TMDL fingerprint): + • Canonical Operation names enforced. Three legacy/wrong names auto-renamed at + intake (OP_RENAME), preserving historical data while emitting canonical values: + FileViewed → FileAccessed + MeetingParticipantJoined → MeetingParticipantDetail + ConnectedAIAppInteraction → AIAppInteraction + • TEAMS_OPS / FILE_OPS / COPILOT_OPS updated to the 14 DAX-required ops only. + • Rollup CSV header extended with 4 agent telemetry columns (AgentId, AgentName, + ContextType, IsAgentInteraction) so M365Usage.tmdl fingerprint check passes + without Power Query post-processing. + • Multi-file input supported: pass --input/-i multiple paths (or repeat the flag) + to combine N Purview exports into one rollup + UserStats + SessionCohort bundle. + Validated 4-pull strategy: Teams + Outlook + Files + Copilot in a single run. + • is_copilot() now also recognises AIAppInteraction (agent/connected-app events). + • All performance characteristics preserved: streaming intake, orjson, no buffered + explosion, per-record cap. Goal is to move processing OUT of Power BI INTO Python + so the .pbix loads/refreshes faster without losing fidelity. + + After the rollup CSV is written, a second pass streams through it to produce + two additional analytics files (unless --no-userstats is specified): + - UserStats: One row per user with 66 columns of pre-computed metrics + (Copilot/M365 event counts, tier classifications, priority + scores, usage ranks, active-day counts, activity segments). + - SessionCohort: One row per (UserId, App) pair with a session-count bucket + (1-5, 6-10, 11-20, 21-40, 41-60, 61-80, 81+). + + These files allow Power Query to join pre-computed results instead of + recalculating expensive DAX/M expressions, cutting dashboard load times. + + EVENT-LEVEL MODE (--mode event-level): v1-compatible 153-column explosion output. + Identical behavior to v1.0.0 for debugging and reconciliation. + UserStats and SessionCohort files are NOT generated in this mode. + +Requirements: + Python 3.9+ + pip install orjson (OPTIONAL - 5-10x faster JSON parsing; falls back to stdlib json) + +Usage: + # (A) Single PAX / PowerShell export: + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py --pax + + # (B) Manual 4-pull export from Purview Audit: + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py \ + --teams --outlook --files --copilot + + Common optional flags: + --output-dir Where to write outputs (default: input folder) + --skip-precompute Skip UserStats + SessionCohort + --reconcile Sample-based correctness check + --prompt-filter Prompt|Response|Both|Null + --debug-events v1-compatible 153-column event-level CSV + --quiet Suppress progress output + +Output files (rollup mode — all share the same timestamp): + _Rollup_.csv 13 columns — aggregated events + agent fields + _UserStats_.csv 66 columns — per-user metrics + _SessionCohort_.csv 3 columns — (UserId, App, Bucket) + _SessionStats_.csv 8 columns — (UserId, Date, AppHost, + SessionCount, PromptCount, + AgentPromptCount, + ResponseCount, AgentSessionCount) + matches AI in One DISTINCTCOUNT(ThreadId); + AgentPromptCount = prompts on agent-flagged + threads (v2.6.1) + ( = input file's stem for single input, or '_Combined' for multi-input. + Rename the output file or use --output-dir if you want a tenant-specific name.) + +Output file (event-level mode): + _Exploded_.csv 153 columns — one row per event + +Arguments: + --input, -i Path to Purview audit log CSV (required). + --output-dir, -o Directory for output files (default: input file's directory). + --mode, -m Processing mode: rollup (default) or event-level. + --reconcile Run sample-based reconciliation after rollup processing. + --prompt-filter Filter Copilot messages: Prompt|Response|Both|Null. + --no-userstats Skip UserStats and SessionCohort generation (rollup only). + --quiet, -q Suppress progress output (only errors are printed). + --version Show version and exit. + +Examples: + # Default rollup (13-column output + UserStats + SessionCohort) + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv + + # Combine the validated 4-pull bundle (Teams + Outlook + Files + Copilot) in one run + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py \ + -i Teams_Export.csv Outlook_Export.csv Files_Export.csv Copilot_Export.csv \ + --combined-stem ZavaCorp_2025_11 + + # Rollup with output in a different directory + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --output-dir ./output + + # Rollup only — skip UserStats and SessionCohort generation + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --no-userstats + + # v1-compatible event-level explosion (153-column output) + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --mode event-level + + # Rollup with sample-based reconciliation check + python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --reconcile + +Validated 4-pull strategy (Purview Audit → Activities filter, type+click each chip): + Teams (7d): MessageSent, MessageRead, ChatCreated, TeamsSessionStarted, + MeetingParticipantDetail + Outlook (30d): MailItemsAccessed, Send, MailboxLogin + Files (60d): FileAccessed, FileModified, FileDownloaded, FileUploaded + Copilot (30d): CopilotInteraction, AIAppInteraction (filter by record type) + +Author: Microsoft Copilot Growth ROI Advisory Team (copilot-roi-advisory-team-gh@microsoft.com) +Version: 2.6.1 +""" + +from __future__ import annotations + +import argparse +import bisect +import csv +import hashlib +import hmac +import os +import random +import re +import sys +import time +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed +from datetime import datetime, timezone, date, timedelta +from pathlib import Path +from typing import Any + +# ─── Fast JSON: prefer orjson, fall back to stdlib ─────────────────────────── +try: + import orjson + + def json_loads(s: str | bytes) -> Any: + if isinstance(s, str): + s = s.encode("utf-8") + return orjson.loads(s) + + def json_dumps_compact(obj: Any) -> str: + return orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS).decode("utf-8") + + _JSON_ENGINE = "orjson" +except ImportError: + import json as _json + + def json_loads(s: str | bytes) -> Any: # type: ignore[misc] + if isinstance(s, bytes): + s = s.decode("utf-8") + return _json.loads(s) + + def json_dumps_compact(obj: Any) -> str: # type: ignore[misc] + return _json.dumps(obj, separators=(",", ":"), default=str) + + _JSON_ENGINE = "json (stdlib)" + +# ═════════════════════════════════════════════════════════════════════════════ +# CONSTANTS +# ═════════════════════════════════════════════════════════════════════════════ + +SCRIPT_VERSION = "2.6.1" + +EXPLOSION_PER_RECORD_ROW_CAP = 1000 +STREAMING_CHUNK_SIZE = 5000 + +# Unified 153-column header matching Power BI M code schema exactly. +# Order matches the #"Changed Type" step in M365Usage.tmdl. +# AuditData is intentionally excluded — raw JSON is never written to output. +M365_UNIFIED_HEADER: list[str] = [ + "RecordId", "CreationDate", "RecordType", "Operation", "UserId", + "AssociatedAdminUnits", "AssociatedAdminUnitsNames", + "@odata.type", "CreationTime", "Id", "OrganizationId", + "ResultStatus", "UserKey", "UserType", "Version", "Workload", + "ClientIP", "ObjectId", "AzureActiveDirectoryEventType", + "ActorContextId", "ActorIpAddress", "InterSystemsId", "IntraSystemId", + "SupportTicketId", "TargetContextId", "ApplicationId", + "DeviceProperties.OS", "DeviceProperties.BrowserType", + "ErrorNumber", + "SiteUrl", "SourceRelativeUrl", "SourceFileName", "SourceFileExtension", + "ListId", "ListItemUniqueId", "WebId", "ApplicationDisplayName", "EventSource", + "ItemType", "SiteSensitivityLabelId", "GeoLocation", "IsManagedDevice", + "DeviceDisplayName", "ListBaseType", "ListServerTemplate", + "AuthenticationType", "Site", "DoNotDistributeEvent", "HighPriorityMediaProcessing", + "BrowserName", "BrowserVersion", "CorrelationId", "Platform", "UserAgent", + "ActorInfoString", "AppId", "AuthType", "ClientAppId", "ClientIPAddress", + "ClientInfoString", "ExternalAccess", "InternalLogonType", "LogonType", + "LogonUserSid", "MailboxGuid", "MailboxOwnerSid", "MailboxOwnerUPN", + "OrganizationName", "OriginatingServer", "SessionId", + "TokenObjectId", "TokenTenantId", "TokenType", "SaveToSentItems", + "OperationCount", "FileSizeBytes", + "MeetingId", "MeetingType", "EventSignature", "EventData", + "Permission", "SensitivityLabelId", "SharingLinkScope", + "TargetUserOrGroupType", "TargetUserOrGroupName", + "MeetingURL", "ChatId", "MessageId", "MessageSizeInBytes", "MessageType", + "FormId", "FormName", "VideoId", "VideoName", "ChannelId", "ViewDuration", + "ClientRegion", "CopilotLogVersion", "TargetId", + "TeamName", "TeamGuid", "ResponseId", "IsAnonymous", "DeviceType", + "ChannelName", "ChannelGuid", "ChannelType", "AppName", "EnvironmentName", + "PlanId", "PlanName", "TaskId", "TaskName", "PercentComplete", + "CrossMailboxOperation", + "RecordTypeNum", "ResultStatus_Audit", + "ModelId", "ModelProvider", "ModelFamily", + "TokensTotal", "TokensInput", "TokensOutput", "DurationMs", "OutcomeStatus", + "ConversationId", "TurnNumber", "RetryCount", "ClientVersion", "ClientPlatform", + "AgentId", "AgentName", "AgentVersion", "AgentCategory", "ApplicationName", + "AppHost", "ThreadId", + "Context_Id", "Context_Type", + "Message_Id", "Message_isPrompt", + "AccessedResource_Action", "AccessedResource_PolicyDetails", "AccessedResource_SiteUrl", + "AISystemPlugin_Id", "AISystemPlugin_Name", + "ModelTransparencyDetails_ModelName", "MessageIds", + "AccessedResource_Name", "AccessedResource_SensitivityLabel", + "AccessedResource_ResourceType", "SensitivityLabel", "Context_Item", +] + +# Rollup output header (13 columns) — matches M365Usage.tmdl fingerprint: +# required keys + EventCount + temporal MIN/MAX + agent telemetry. +ROLLUP_HEADER: list[str] = [ + "UserId", "CreationDate", "Operation", "Workload", + "SourceFileExtension", "AppHost", + "EventCount", "ItemsAccessedCount", "CreationTime", "MaxCreationTime", + "AgentId", "AgentName", "ContextType", "IsAgentInteraction", +] + +# Reconciliation sample size +RECONCILE_SAMPLE_SIZE = 10_000 + +# ── Operation canonicalization (v2.3.0) ────────────────────────────────────── +# Legacy/wrong names that have appeared in older exports or older DAX models. +# Renamed at intake so historical data merges cleanly with current canonical pulls. +OP_RENAME: dict[str, str] = { + "FileViewed": "FileAccessed", + "MeetingParticipantJoined": "MeetingParticipantDetail", + "ConnectedAIAppInteraction": "AIAppInteraction", +} + +# ── UserStats classification sets (match Power Query logic exactly) ────────── +WORD_EXTS: set[str] = {"docx", "doc", "dotx"} +EXCEL_EXTS: set[str] = {"xlsx", "xls", "xlsm", "csv"} +PPT_EXTS: set[str] = {"pptx", "ppt", "ppsx"} +OFFICE_EXTS: set[str] = WORD_EXTS | EXCEL_EXTS | PPT_EXTS + +# Canonical 14 ops required by the CLO TMDL DAX measures, validated against MS Learn. +FILE_OPS: set[str] = { + "FileAccessed", # canonical (was FileViewed in legacy) + "FileModified", + "FileDownloaded", + "FileUploaded", +} +OUTLOOK_OPS: set[str] = {"Send", "MailItemsAccessed", "MailboxLogin"} # active-DAY + COUNT +TEAMS_OPS: set[str] = { + "MessageSent", # Msgs Sent + "MessageRead", "ChatCreated", # Msgs Read (Graph-API tenants emit ChatCreated) + "MeetingParticipantDetail", # canonical (was MeetingParticipantJoined) + "TeamsSessionStarted", # Meetings/calls fallback +} +COPILOT_OPS: set[str] = {"CopilotInteraction", "AIAppInteraction"} # AIAppInteraction = agents/connected apps + +# AppHost values that indicate an agent / connected-app interaction. +AGENT_APPHOSTS: set[str] = {"agent", "copilotstudio", "declarativeagent", "customengineagent"} + +# ── DAX-aligned op/ext sets (for the CE/LP precomputed columns added in v2.4.0) ── +# These mirror the exact filters in the PBIT measures Word/Excel/PowerPoint/Outlook/ +# Teams Activity *V2, Copilot All Apps Total, and CE Copilot Percentile. +# Important: ops are matched AFTER OP_RENAME canonicalization, so the legacy names +# ("FileViewed", "MeetingParticipantJoined") are listed under their canonical aliases. +DAX_FILE_OPS: set[str] = { + "FileAccessed", # canonical of legacy FileViewed (DAX checks both) + "FilePreviewed", + "FileModified", + "FileDownloaded", + "FileUploaded", +} +DAX_OUTLOOK_OPS: set[str] = {"Send", "MailItemsAccessed"} # MailboxLogin intentionally excluded (matches DAX) +DAX_TEAMS_OPS: set[str] = { + "MessageSent", "MessageRead", "MessagesListed", "ChatRetrieved", + "MeetingParticipantDetail", # canonical of MeetingParticipantJoined + "MeetingStarted", "MeetingEnded", "TeamsSessionStarted", +} + +USERSTATS_HEADER: list[str] = [ + "UserId", + "CopilotEC", "M365EC", "ExCopEC", "ExM365EC", + "IsCopilotUser", "CopilotTierColumn", "M365TierColumn", + "PriorityScatterColumn", "ExcelPriority", + "CopilotUsageRankColumn", "M365UsageRankColumn", + "TeamsActiveDays", "OutlookActiveDays", "WordActiveDays", + "ExcelActiveDays", "PowerPointActiveDays", + "TeamsActivityCount", "OutlookActivityCount", "OfficeFilesActivityCount", + "TeamsActivitySegment", "OutlookActivitySegment", "WordActivitySegment", + "ExcelActivitySegment", "PowerPointActivitySegment", + "OfficeFilesActivitySegment", "OverallM365ActivitySegment", + # ── v2.5.0: precomputed raw activity counts + CE percentile ranks per window. + # Windows: _L30 = trailing 30 days ending at max(CreationDate); _L60 = trailing 60; + # _Full = entire data range. Filters match the corresponding DAX measures exactly + # (post-canonicalization). CE ranks are integer 0-100; blank when raw is 0. + "TeamsRaw_L30", "TeamsRaw_L60", "TeamsRaw_Full", + "OutlookRaw_L30", "OutlookRaw_L60", "OutlookRaw_Full", + "WordRaw_L30", "WordRaw_L60", "WordRaw_Full", + "ExcelRaw_L30", "ExcelRaw_L60", "ExcelRaw_Full", + "PowerPointRaw_L30", "PowerPointRaw_L60", "PowerPointRaw_Full", + "CopilotChatRaw_L30", "CopilotChatRaw_L60", "CopilotChatRaw_Full", + "CERank_Teams_L30", "CERank_Teams_L60", "CERank_Teams_Full", + "CERank_Outlook_L30", "CERank_Outlook_L60", "CERank_Outlook_Full", + "CERank_Word_L30", "CERank_Word_L60", "CERank_Word_Full", + "CERank_Excel_L30", "CERank_Excel_L60", "CERank_Excel_Full", + "CERank_PowerPoint_L30", "CERank_PowerPoint_L60", "CERank_PowerPoint_Full", + "CERank_M365AllApps_L30", "CERank_M365AllApps_L60", "CERank_M365AllApps_Full", + "CECopilotPercentile_L30", "CECopilotPercentile_L60", "CECopilotPercentile_Full", +] + +# v2.5.0: percentile window codes used in column names. Order matters for writer. +RANK_WINDOWS: tuple[str, ...] = ("L30", "L60", "Full") + +SESSIONCOHORT_HEADER: list[str] = ["UserId", "AppColumn", "SessionCohort"] + +# v2.6.0: SessionStats — AI in One parity. Per (UserId, CreationDate, AppHost) we count +# DISTINCT ThreadIds (matches Microsoft AI in One `Sessions` measure), plus prompt / +# response counts and an agent-only thread count. License filtering happens downstream +# in DAX via the EntraUsers relationship; this CSV stays license-agnostic. +# v2.6.1: Added AgentPromptCount — exact chat vs agent split at message-tally time +# (uses the same is_agent flag the rollup already determines per record). +SESSIONSTATS_HEADER: list[str] = [ + "UserId", "CreationDate", "AppHost", + "SessionCount", "PromptCount", "AgentPromptCount", + "ResponseCount", "AgentSessionCount", +] + +# Date formats accepted for CreationDate normalization (broadest to narrowest) +_CREATION_DATE_FORMATS: tuple[str, ...] = ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", + "%Y-%m-%d", + "%m/%d/%Y", +) + +# GroupKey type (v2.3.0): adds agent_id, agent_name, context_type so multi-agent users +# don't collapse rows together. IsAgentInteraction is derived on write from AgentId. +# (user_id_lower, creation_date_normalized, operation, workload, sfe_lower, app_host, +# agent_id, agent_name, context_type) +GroupKey = tuple[str, str, str, str, str, str, str, str, str] + + +class RollupAccum: + """Lightweight accumulator for one rollup group — avoids dataclass import overhead.""" + __slots__ = ( + "event_count", "items_accessed_count", + "min_creation_time", "max_creation_time", + "original_user_id", + "is_agent_interaction", + ) + + def __init__( + self, + event_count: int, + items_accessed: int, + min_ct: str, + max_ct: str, + original_uid: str, + is_agent: bool = False, + ) -> None: + self.event_count = event_count + self.items_accessed_count = items_accessed + self.min_creation_time = min_ct + self.max_creation_time = max_ct + self.original_user_id = original_uid # first-seen casing for output + self.is_agent_interaction = is_agent + + +# SessionStats group key: (uid_lower, creation_date, app_host). +SessionKey = tuple[str, str, str] + + +class SessionAccum: + """Per-(user, date, app_host) Copilot session accumulator (v2.6.0). + + Mirrors the AI in One `Sessions` measure: DISTINCTCOUNT(ThreadId) where at least + one message in the thread is a user prompt (isPrompt=True). Threads with only + AI responses (no user prompt) are excluded — same as the AI in One filter. + """ + __slots__ = ( + "thread_ids", "agent_thread_ids", + "prompt_count", "agent_prompt_count", "response_count", + "original_user_id", + ) + + def __init__(self, original_uid: str) -> None: + self.thread_ids: set[str] = set() + self.agent_thread_ids: set[str] = set() + self.prompt_count: int = 0 + self.agent_prompt_count: int = 0 # v2.6.1: prompts on agent-flagged threads + self.response_count: int = 0 + self.original_user_id = original_uid + + +def normalize_creation_date(raw: str) -> str: + """Parse any Purview date format → 'YYYY-MM-DDT00:00:00.000Z' (midnight UTC).""" + if not raw or not isinstance(raw, str): + return "" + raw = raw.strip() + if not raw: + return "" + for fmt in _CREATION_DATE_FORMATS: + try: + dt = datetime.strptime(raw, fmt) + return dt.strftime("%Y-%m-%d") + "T00:00:00.000Z" + except ValueError: + continue + # Fallback: try extracting date portion from ISO-like string + if len(raw) >= 10 and raw[4:5] == "-": + return raw[:10] + "T00:00:00.000Z" + return raw # unparseable — pass through + + +def _norm_key_str(val: Any) -> str: + """Normalize a string value for use as a rollup key: strip whitespace, empty if None.""" + if val is None: + return "" + if not isinstance(val, str): + val = str(val) + val = val.strip() + if val.lower() in ("null", "none"): + return "" + return val + + +# ═════════════════════════════════════════════════════════════════════════════ +# UTILITY FUNCTIONS +# ═════════════════════════════════════════════════════════════════════════════ + +def safe_get(obj: Any, key: str) -> Any: + """Safely retrieve a property from a dict-like object.""" + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def select_first_non_null(values: list[Any]) -> Any: + """Return the first non-None, non-empty-string value.""" + for v in values: + if v is not None and v != "": + return v + return None + + +def to_num(val: Any) -> float | None: + """Convert to number, return None on failure.""" + if val is None: + return None + if isinstance(val, (int, float)): + return float(val) + if isinstance(val, str): + val = val.strip() + if not val: + return None + try: + return float(val) + except (ValueError, TypeError): + return None + return None + + +def format_date_purview(val: Any) -> str: + """Format a date value to ISO 8601 UTC string.""" + if val is None: + return "" + if isinstance(val, str): + val = val.strip() + if not val: + return "" + # Try common Purview date formats + for fmt in ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", + ): + try: + dt = datetime.strptime(val, fmt) + return dt.replace(tzinfo=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + except ValueError: + continue + return val # Return as-is if no format matches + return str(val) + + +def to_json_if_object(val: Any) -> str: + """Serialize non-scalars to compact JSON, pass scalars through as strings.""" + if val is None: + return "" + if isinstance(val, (str, int, float, bool)): + return str(val) + try: + return json_dumps_compact(val) + except Exception: + return "" + + +def bool_tf(val: Any) -> str: + """Convert bool-like value to 'TRUE'/'FALSE' string.""" + if val is None: + return "" + if isinstance(val, bool): + return "TRUE" if val else "FALSE" + if isinstance(val, str): + low = val.strip().lower() + if low in ("true", "1", "yes"): + return "TRUE" + if low in ("false", "0", "no"): + return "FALSE" + return str(val) + + +def get_array_fast(obj: Any, key: str) -> list: + """Extract an array property, always returning a list.""" + if obj is None: + return [] + val = safe_get(obj, key) + if val is None: + return [] + if isinstance(val, list): + return val + if isinstance(val, (str, int, float, bool)): + return [] + try: + return list(val) + except (TypeError, ValueError): + return [] + + +# ═════════════════════════════════════════════════════════════════════════════ +# AGENT CATEGORIZATION +# ═════════════════════════════════════════════════════════════════════════════ + +def categorize_agent(agent_id: Any) -> str: + """Categorize agent based on AgentId pattern.""" + if not agent_id or not isinstance(agent_id, str): + return "" + if agent_id.startswith("CopilotStudio.Declarative."): + return "Declarative Agent" + if agent_id.startswith("CopilotStudio.CustomEngine."): + return "Custom Engine Agent" + if agent_id.startswith("P_"): + return "Declarative Agent (Purview)" + return "Other Agent" + + +# ═════════════════════════════════════════════════════════════════════════════ +# USERSTATS CLASSIFICATION & COMPUTATION HELPERS +# ═════════════════════════════════════════════════════════════════════════════ + +def is_copilot(op: str, wl: str) -> bool: + """True if the row represents a Copilot or agent / connected-app event.""" + return wl == "Copilot" or op in COPILOT_OPS + +def is_excel_file_op(ext: str, op: str) -> bool: + """True if the row is a file operation on an Excel-family extension.""" + return (ext or "").lower() in EXCEL_EXTS and op in FILE_OPS + + +def app_column(ext: str, op: str, wl: str) -> str: + """Classify a row into an application column for session cohort grouping.""" + e = (ext or "").lower() + if e in WORD_EXTS and op in FILE_OPS: + return "Word" + if e in EXCEL_EXTS and op in FILE_OPS: + return "Excel" + if e in PPT_EXTS and op in FILE_OPS: + return "PowerPoint" + if wl == "Exchange" and op in OUTLOOK_OPS: + return "Outlook" + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + return "Teams" + if wl == "Copilot" or op == "CopilotInteraction": + return "Copilot" + return "M365 All Apps" + + +def percentile_inc(values: list[float], p: float) -> float: + """Inclusive linear interpolation — matches PQ List.Percentile and numpy 'linear'.""" + if not values: + return 0.0 + sorted_vals = sorted(values) + n = len(sorted_vals) + idx = p * (n - 1) + lo, hi = int(idx), min(int(idx) + 1, n - 1) + return sorted_vals[lo] + (idx - lo) * (sorted_vals[hi] - sorted_vals[lo]) + + +def tier_fn(cnt: float, p90: float, p75: float, p50: float, zero_is_bottom: bool) -> str: + """Assign a percentile-tier label.""" + if zero_is_bottom and cnt == 0: + return "Bottom 50%" + if cnt >= p90: + return "Top 10%" + if cnt >= p75: + return "10-25%" + if cnt >= p50: + return "25-50%" + return "Bottom 50%" + + +def priority_fn(m365_tier: str, cop_tier: str) -> str: + """Map (M365 tier, Copilot tier) pair to a priority label.""" + top2 = {"Top 10%", "10-25%"} + if m365_tier in top2 and cop_tier in top2: + return "Promoter" + if m365_tier == "Top 10%" and cop_tier == "25-50%": + return "High" + if m365_tier == "Top 10%" and cop_tier == "Bottom 50%": + return "Critical" + if m365_tier == "10-25%" and cop_tier == "25-50%": + return "Medium" + if m365_tier == "10-25%" and cop_tier == "Bottom 50%": + return "High" + if m365_tier in {"25-50%", "Bottom 50%"} and cop_tier in top2: + return "Promoter" + if m365_tier == "25-50%" and cop_tier == "25-50%": + return "Medium" + if m365_tier == "25-50%" and cop_tier == "Bottom 50%": + return "Medium" + return "Low" + + +def seg_fn(days: int, window_days: int) -> str: + """Map active-day count to a per-week engagement-segment label. + + Normalizes to active days per week so labels mean the same thing regardless + of pull length: <1, 1-2, 3-4, 5+ days/week. Window_days is the calendar + span (max date - min date + 1) of the rolled-up data; 0 yields No Usage. + """ + if window_days <= 0 or days <= 0: + return "0. No Usage" + rate = days * 7.0 / window_days + if rate < 1.0: + return "1. <1 Day/Week (Light)" + if rate < 3.0: + return "2. 1-2 Days/Week (Moderate)" + if rate < 5.0: + return "3. 3-4 Days/Week (Frequent)" + return "4. 5+ Days/Week (Daily)" + + +def compute_ranks(values_by_uid: dict[str, float]) -> dict[str, int]: + """ + 0-based ascending rank via stable sort. Ties get different sequential + indices (matches PQ Table.Sort + Table.AddIndexColumn behaviour). + """ + sorted_uids = sorted(values_by_uid.items(), key=lambda x: x[1]) + return {uid: i for i, (uid, _) in enumerate(sorted_uids)} + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROLLUP KEY EXTRACTION (lightweight — no row dicts built) +# ═════════════════════════════════════════════════════════════════════════════ + +def _compute_copilot_event_count( + ced: dict, + operation: str, + prompt_filter: str | None, +) -> int: + """ + Compute the number of exploded rows a Copilot record would produce, + using the same array-length logic as v1 (explode_copilot_record), + but WITHOUT materializing any row dicts. + Returns 0 if prompt_filter eliminates all messages (record skipped). + """ + messages = get_array_fast(ced, "Messages") + contexts = get_array_fast(ced, "Contexts") + resources = get_array_fast(ced, "AccessedResources") + plugins_raw = get_array_fast(ced, "AISystemPlugin") + model_det_raw = get_array_fast(ced, "ModelTransparencyDetails") + sensitivity_labels = get_array_fast(ced, "SensitivityLabels") + + # Prompt filtering (same logic as v1 lines 571-581) + if prompt_filter: + pf_lower = prompt_filter.lower() + if pf_lower == "null": + messages = [m for m in messages if safe_get(m, "isPrompt") is None] + elif pf_lower == "both": + messages = [m for m in messages if safe_get(m, "isPrompt") is not None] + elif pf_lower == "prompt": + messages = [m for m in messages if safe_get(m, "isPrompt") is True] + elif pf_lower == "response": + messages = [m for m in messages if safe_get(m, "isPrompt") is False] + if not messages: + return 0 # record filtered out entirely + + # Context items max for CopilotInteraction + context_items_max: int = 0 + if operation == "CopilotInteraction" and contexts: + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and len(items) > context_items_max: + context_items_max = len(items) + + if prompt_filter: + row_count = max(1, len(messages)) + else: + array_counts = [ + 1, len(messages), len(contexts), len(resources), + len(sensitivity_labels), len(plugins_raw), len(model_det_raw), + ] + if context_items_max > 0: + array_counts.append(context_items_max) + row_count = max(array_counts) + + return min(max(row_count, 1), EXPLOSION_PER_RECORD_ROW_CAP) + + +def _count_mail_items_accessed(audit_data: dict) -> int: + """Items represented by one MailItemsAccessed event. + Sums len(Folders[].FolderItems[]) when present; falls back to 1 (Bind-style).""" + folders = audit_data.get("Folders") + if isinstance(folders, list): + total = 0 + for fld in folders: + if not isinstance(fld, dict): + continue + fi = fld.get("FolderItems") + if isinstance(fi, list): + total += len(fi) + if total > 0: + return total + return 1 + + +# Non-human/system identities found in Purview audit logs (Teams Sync, SharePoint app, +# SupervisoryReview bots, ServicePrincipals, NT-style accounts, SIDs, bare GUIDs, etc.). +# These have no matching userPrincipalName in EntraUsers and would render as blank +# User/Department rows in license-recommendation visuals. Filter out at the rollup stage. +_UPN_LOCAL_RE = re.compile(r"^[^\s\\@]+$") +_BARE_GUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + + +def _is_human_upn(uid: str) -> bool: + """True iff uid is a syntactically valid human UPN (local@domain.tld), excluding + well-known service/bot patterns (SupervisoryReview{...}@..., bare GUIDs).""" + if not uid: + return False + s = uid.strip() + if _BARE_GUID_RE.match(s): + return False + if s.lower().startswith("supervisoryreview{"): + return False + if "@" not in s or s.count("@") != 1: + return False + local, domain = s.split("@", 1) + if not _UPN_LOCAL_RE.match(local): + return False + if "." not in domain or not domain or domain.startswith(".") or domain.endswith("."): + return False + return True + + +# --------------------------------------------------------------------------- +# Deidentification (--deidentify): one-way, salted, format-preserving. +# OFF by default; enabled by main() setting the module flag from --deidentify +# (and propagated to explosion worker processes via _deid_init_worker). Every +# PII value becomes a deterministic token so relationships (user joins, +# distinct-resource counts) are preserved while identities are removed. +# Irreversible (no decode map). The SAME salt + algorithm + formats MUST exist +# verbatim in the PowerShell raw-path deidentifier and the CopilotInteraction +# processor (PAX deidentify spec) so tokens match across engines. +# --------------------------------------------------------------------------- +_DEIDENTIFY: bool = False +_DEID_SALT = b"PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3" +_DEID_DOMAIN = "deidentified.domain" +_deid_cache: dict[str, str] = {} + + +def _deid_init_worker(flag: bool) -> None: + """ProcessPoolExecutor initializer: propagate the deidentify flag to workers.""" + global _DEIDENTIFY + _DEIDENTIFY = flag + + +def _deid_hex(value: str, length: int) -> str: + return hmac.new( + _DEID_SALT, value.strip().lower().encode("utf-8"), hashlib.sha256 + ).hexdigest()[:length] + + +def deid_upn(value: str) -> str: + """UPN / email -> <12hex>@deidentified.domain. No-op when off or value empty.""" + if not _DEIDENTIFY or not value: + return value + k = "upn\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + "@" + _DEID_DOMAIN + _deid_cache[k] = v + return v + + +def deid_name(value: str) -> str: + """Person/device display name -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "name\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_guid(value: str) -> str: + """GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.""" + if not _DEIDENTIFY or not value: + return value + k = "guid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}" + _deid_cache[k] = v + return v + + +def deid_sid(value: str) -> str: + """SID -> deterministic S-1-5-21---- shape.""" + if not _DEIDENTIFY or not value: + return value + k = "sid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = "S-1-5-21-{0}-{1}-{2}-{3}".format( + int(h[0:8], 16), int(h[8:16], 16), int(h[16:24], 16), int(h[24:32], 16) + ) + _deid_cache[k] = v + return v + + +def deid_token(value: str) -> str: + """Opaque id (employeeId, immutableId) -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "tok\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_resource(value: str) -> str: + """Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count).""" + if not _DEIDENTIFY or not value: + return value + k = "res\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "site_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_file(value: str) -> str: + """File / document name -> file_<12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "file\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "file_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_proxy(value: str) -> str: + """proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + Handles ';'-delimited multi-value fields.""" + if not _DEIDENTIFY or not value: + return value + out = [] + for entry in value.split(";"): + if not entry: + out.append(entry) + elif ":" in entry: + prefix, addr = entry.split(":", 1) + out.append(prefix + ":" + deid_upn(addr)) + else: + out.append(deid_upn(entry)) + return ";".join(out) + + +def _extract_rollup_keys( + record: dict, + audit_data: dict, + ced: dict | None, + prompt_filter: str | None = None, +) -> tuple[GroupKey, int, int, str, str, bool] | None: + """ + Extract rollup group key + event count + items-accessed count + creation_time + + original UserId + is_agent_interaction flag. + + Returns None if the record is filtered out (e.g. prompt_filter eliminates all messages). + Returns: + (group_key, event_count, items_accessed_count, creation_time_iso, + original_user_id, is_agent_interaction) + where group_key uses lowercased UserId for case-insensitive grouping and + includes (agent_id, agent_name, context_type) so multi-agent users don't collapse. + """ + # UserId: original casing preserved for output; lowered for grouping key + raw_uid = _norm_key_str(safe_get(audit_data, "UserId") or record.get("UserId", "")) + # Filter non-human/system identities (Teams Sync, ServicePrincipals, SIDs, bots, etc.) + if not _is_human_upn(raw_uid): + return None + uid_lower = raw_uid.lower() + + # CreationDate: from CSV, normalized to midnight + creation_date = normalize_creation_date(record.get("CreationDate", "")) + + # Operation: from audit_data → CSV fallback, preserve case, then canonicalize via OP_RENAME + operation = _norm_key_str( + safe_get(audit_data, "Operation") or record.get("Operation", "") or record.get("Operations", "") + ) + operation = OP_RENAME.get(operation, operation) + + # Workload: from audit_data, preserve case + workload = _norm_key_str(safe_get(audit_data, "Workload")) + + # SourceFileExtension: lowercased for DAX LOWER() compatibility + sfe = _norm_key_str(safe_get(audit_data, "SourceFileExtension")).lower() + + # AppHost: from CED for Copilot, empty otherwise, preserve case + if ced: + app_host = _norm_key_str( + safe_get(ced, "AppHost") or safe_get(audit_data, "AppHost") + ) + else: + app_host = _norm_key_str(safe_get(audit_data, "AppHost")) + + # Agent telemetry: from CopilotEventData when present, fall back to top-level AuditData fields + agent_id = _norm_key_str( + (safe_get(ced, "AgentId") if ced else None) + or safe_get(audit_data, "AgentId") + ) + agent_name = _norm_key_str( + (safe_get(ced, "AgentName") if ced else None) + or safe_get(audit_data, "AgentName") + ) + context_type = "" + if ced: + contexts = get_array_fast(ced, "Contexts") + if contexts: + # First context wins for the key; multi-context records still collapse + # cleanly because event_count already reflects context-array length. + first_ctx = contexts[0] + if isinstance(first_ctx, dict): + context_type = _norm_key_str(safe_get(first_ctx, "Type")) + + # CreationTime: from audit_data, ISO formatted for lexicographic MIN/MAX + creation_time = format_date_purview(safe_get(audit_data, "CreationTime")) + + # Event count + if ced: + event_count = _compute_copilot_event_count(ced, operation, prompt_filter) + if event_count == 0: + return None # filtered out + else: + event_count = 1 # Non-Copilot: always 1:1 + + # Items accessed count: only meaningful for MailItemsAccessed (Exchange). + items_accessed_count = 0 + if operation == "MailItemsAccessed": + items_accessed_count = _count_mail_items_accessed(audit_data) + + # IsAgentInteraction: TRUE iff AgentId present, AppHost is an agent surface, + # or Operation is an agent op (AIAppInteraction). + is_agent_interaction = bool( + agent_id + or app_host.lower() in AGENT_APPHOSTS + or operation == "AIAppInteraction" + ) + + group_key: GroupKey = ( + uid_lower, creation_date, operation, workload, sfe, app_host, + agent_id, agent_name, context_type, + ) + return ( + group_key, + event_count, + items_accessed_count, + creation_time, + raw_uid, + is_agent_interaction, + ) + + +# ═════════════════════════════════════════════════════════════════════════════ +# PATH A: NON-COPILOT M365 EXTRACTION +# ═════════════════════════════════════════════════════════════════════════════ + + +def _get_nv_prop(nv_list: Any, prop_name: str) -> Any: + """Extract a value from a Name/Value pair list by Name — matches M code's GetNVProp.""" + if not nv_list or not isinstance(nv_list, list): + return None + for item in nv_list: + if isinstance(item, dict) and safe_get(item, "Name") == prop_name: + return safe_get(item, "Value") + return None + + +def _build_unified_row(record: dict, audit_data: dict) -> dict: + """ + Build a complete row dict with all 153 M code columns populated. + Extracts fields from the CSV record and AuditData JSON. + DeviceProperties uses NV-pivot for .OS and .BrowserType only (matches M code). + RecordTypeNum and ResultStatus_Audit are computed aliases. + """ + # CSV-level fields + record_id = ( + record.get("RecordId") + or record.get("Identity") + or record.get("Id") + or safe_get(audit_data, "Id") + or "" + ) + # Read from singular first, fall back to plural for backwards-compatible input + op_val = safe_get(audit_data, "Operation") or record.get("Operation") or record.get("Operations", "") + uid_val = safe_get(audit_data, "UserId") or record.get("UserId") or record.get("UserIds", "") + record_type = record.get("RecordType", "") + result_status = safe_get(audit_data, "ResultStatus") or "" + + # CreationTime formatting + creation_time_raw = safe_get(audit_data, "CreationTime") + creation_time = format_date_purview(creation_time_raw) if creation_time_raw else "" + + # DeviceProperties NV-pivot (matches M code's GetNVProp — only .OS and .BrowserType) + dev_props = safe_get(audit_data, "DeviceProperties") + dp_os = _get_nv_prop(dev_props, "OS") or "" + dp_browser = _get_nv_prop(dev_props, "BrowserType") or "" + + # Computed alias: RecordTypeNum = int(RecordType) + try: + record_type_num = int(record_type) if record_type else "" + except (ValueError, TypeError): + record_type_num = "" + + # ApplicationId with fallback chain + app_id_resolved = select_first_non_null([ + safe_get(audit_data, "ApplicationId"), + safe_get(audit_data, "AppId"), + safe_get(audit_data, "ClientAppId"), + ]) or "" + + # AgentCategory is computed + agent_id_val = safe_get(audit_data, "AgentId") or "" + agent_category = categorize_agent(agent_id_val) if agent_id_val else "" + + row = { + "RecordId": record_id, + "CreationDate": record.get("CreationDate", ""), + "RecordType": record_type, + "Operation": op_val, + "UserId": uid_val, + "AssociatedAdminUnits": record.get("AssociatedAdminUnits", "") or safe_get(audit_data, "AssociatedAdminUnits") or "", + "AssociatedAdminUnitsNames": record.get("AssociatedAdminUnitsNames", "") or safe_get(audit_data, "AssociatedAdminUnitsNames") or "", + "@odata.type": safe_get(audit_data, "@odata.type") or "", + "CreationTime": creation_time, + "Id": safe_get(audit_data, "Id") or "", + "OrganizationId": safe_get(audit_data, "OrganizationId") or "", + "ResultStatus": result_status, + "UserKey": safe_get(audit_data, "UserKey") or "", + "UserType": safe_get(audit_data, "UserType") or "", + "Version": safe_get(audit_data, "Version") or "", + "Workload": safe_get(audit_data, "Workload") or "", + "ClientIP": safe_get(audit_data, "ClientIP") or "", + "ObjectId": safe_get(audit_data, "ObjectId") or "", + "AzureActiveDirectoryEventType": safe_get(audit_data, "AzureActiveDirectoryEventType") or "", + "ActorContextId": safe_get(audit_data, "ActorContextId") or "", + "ActorIpAddress": safe_get(audit_data, "ActorIpAddress") or "", + "InterSystemsId": safe_get(audit_data, "InterSystemsId") or "", + "IntraSystemId": safe_get(audit_data, "IntraSystemId") or "", + "SupportTicketId": safe_get(audit_data, "SupportTicketId") or "", + "TargetContextId": safe_get(audit_data, "TargetContextId") or "", + "ApplicationId": app_id_resolved, + "DeviceProperties.OS": dp_os, + "DeviceProperties.BrowserType": dp_browser, + "ErrorNumber": safe_get(audit_data, "ErrorNumber") or "", + "SiteUrl": safe_get(audit_data, "SiteUrl") or "", + "SourceRelativeUrl": safe_get(audit_data, "SourceRelativeUrl") or "", + "SourceFileName": safe_get(audit_data, "SourceFileName") or "", + "SourceFileExtension": safe_get(audit_data, "SourceFileExtension") or "", + "ListId": safe_get(audit_data, "ListId") or "", + "ListItemUniqueId": safe_get(audit_data, "ListItemUniqueId") or "", + "WebId": safe_get(audit_data, "WebId") or "", + "ApplicationDisplayName": safe_get(audit_data, "ApplicationDisplayName") or "", + "EventSource": safe_get(audit_data, "EventSource") or "", + "ItemType": safe_get(audit_data, "ItemType") or "", + "SiteSensitivityLabelId": safe_get(audit_data, "SiteSensitivityLabelId") or "", + "GeoLocation": safe_get(audit_data, "GeoLocation") or "", + "IsManagedDevice": safe_get(audit_data, "IsManagedDevice") or "", + "DeviceDisplayName": safe_get(audit_data, "DeviceDisplayName") or "", + "ListBaseType": safe_get(audit_data, "ListBaseType") or "", + "ListServerTemplate": safe_get(audit_data, "ListServerTemplate") or "", + "AuthenticationType": safe_get(audit_data, "AuthenticationType") or "", + "Site": safe_get(audit_data, "Site") or "", + "DoNotDistributeEvent": safe_get(audit_data, "DoNotDistributeEvent") or "", + "HighPriorityMediaProcessing": safe_get(audit_data, "HighPriorityMediaProcessing") or "", + "BrowserName": safe_get(audit_data, "BrowserName") or "", + "BrowserVersion": safe_get(audit_data, "BrowserVersion") or "", + "CorrelationId": safe_get(audit_data, "CorrelationId") or "", + "Platform": safe_get(audit_data, "Platform") or "", + "UserAgent": safe_get(audit_data, "UserAgent") or "", + "ActorInfoString": safe_get(audit_data, "ActorInfoString") or "", + "AppId": safe_get(audit_data, "AppId") or "", + "AuthType": safe_get(audit_data, "AuthType") or "", + "ClientAppId": safe_get(audit_data, "ClientAppId") or "", + "ClientIPAddress": safe_get(audit_data, "ClientIPAddress") or "", + "ClientInfoString": safe_get(audit_data, "ClientInfoString") or "", + "ExternalAccess": safe_get(audit_data, "ExternalAccess") or "", + "InternalLogonType": safe_get(audit_data, "InternalLogonType") or "", + "LogonType": safe_get(audit_data, "LogonType") or "", + "LogonUserSid": safe_get(audit_data, "LogonUserSid") or "", + "MailboxGuid": safe_get(audit_data, "MailboxGuid") or "", + "MailboxOwnerSid": safe_get(audit_data, "MailboxOwnerSid") or "", + "MailboxOwnerUPN": safe_get(audit_data, "MailboxOwnerUPN") or "", + "OrganizationName": safe_get(audit_data, "OrganizationName") or "", + "OriginatingServer": safe_get(audit_data, "OriginatingServer") or "", + "SessionId": safe_get(audit_data, "SessionId") or "", + "TokenObjectId": safe_get(audit_data, "TokenObjectId") or "", + "TokenTenantId": safe_get(audit_data, "TokenTenantId") or "", + "TokenType": safe_get(audit_data, "TokenType") or "", + "SaveToSentItems": safe_get(audit_data, "SaveToSentItems") or "", + "OperationCount": safe_get(audit_data, "OperationCount") or "", + "FileSizeBytes": safe_get(audit_data, "FileSizeBytes") or "", + # Teams / Meetings / Chat + "MeetingId": safe_get(audit_data, "MeetingId") or "", + "MeetingType": safe_get(audit_data, "MeetingType") or "", + "EventSignature": safe_get(audit_data, "EventSignature") or "", + "EventData": safe_get(audit_data, "EventData") or "", + "Permission": safe_get(audit_data, "Permission") or "", + "SensitivityLabelId": safe_get(audit_data, "SensitivityLabelId") or "", + "SharingLinkScope": safe_get(audit_data, "SharingLinkScope") or "", + "TargetUserOrGroupType": safe_get(audit_data, "TargetUserOrGroupType") or "", + "TargetUserOrGroupName": safe_get(audit_data, "TargetUserOrGroupName") or "", + "MeetingURL": safe_get(audit_data, "MeetingURL") or "", + "ChatId": safe_get(audit_data, "ChatId") or "", + "MessageId": safe_get(audit_data, "MessageId") or "", + "MessageSizeInBytes": safe_get(audit_data, "MessageSizeInBytes") or "", + "MessageType": safe_get(audit_data, "MessageType") or "", + # Forms + "FormId": safe_get(audit_data, "FormId") or "", + "FormName": safe_get(audit_data, "FormName") or "", + # Video / Stream + "VideoId": safe_get(audit_data, "VideoId") or "", + "VideoName": safe_get(audit_data, "VideoName") or "", + "ChannelId": safe_get(audit_data, "ChannelId") or "", + "ViewDuration": safe_get(audit_data, "ViewDuration") or "", + "ClientRegion": safe_get(audit_data, "ClientRegion") or "", + "CopilotLogVersion": safe_get(audit_data, "CopilotLogVersion") or "", + "TargetId": safe_get(audit_data, "TargetId") or "", + # Teams details + "TeamName": safe_get(audit_data, "TeamName") or "", + "TeamGuid": safe_get(audit_data, "TeamGuid") or "", + "ResponseId": safe_get(audit_data, "ResponseId") or "", + "IsAnonymous": safe_get(audit_data, "IsAnonymous") or "", + "DeviceType": safe_get(audit_data, "DeviceType") or "", + "ChannelName": safe_get(audit_data, "ChannelName") or "", + "ChannelGuid": safe_get(audit_data, "ChannelGuid") or "", + "ChannelType": safe_get(audit_data, "ChannelType") or "", + "AppName": safe_get(audit_data, "AppName") or "", + "EnvironmentName": safe_get(audit_data, "EnvironmentName") or "", + # Planner + "PlanId": safe_get(audit_data, "PlanId") or "", + "PlanName": safe_get(audit_data, "PlanName") or "", + "TaskId": safe_get(audit_data, "TaskId") or "", + "TaskName": safe_get(audit_data, "TaskName") or "", + "PercentComplete": safe_get(audit_data, "PercentComplete") or "", + "CrossMailboxOperation": safe_get(audit_data, "CrossMailboxOperation") or "", + # Computed aliases + "RecordTypeNum": record_type_num, + "ResultStatus_Audit": result_status, + # Copilot model/token fields (populated from CED for Copilot records; from root for non-Copilot) + "ModelId": safe_get(audit_data, "ModelId") or "", + "ModelProvider": safe_get(audit_data, "ModelProvider") or "", + "ModelFamily": safe_get(audit_data, "ModelFamily") or "", + "TokensTotal": safe_get(audit_data, "TokensTotal") or "", + "TokensInput": safe_get(audit_data, "TokensInput") or "", + "TokensOutput": safe_get(audit_data, "TokensOutput") or "", + "DurationMs": safe_get(audit_data, "DurationMs") or "", + "OutcomeStatus": safe_get(audit_data, "OutcomeStatus") or "", + "ConversationId": safe_get(audit_data, "ConversationId") or "", + "TurnNumber": safe_get(audit_data, "TurnNumber") or "", + "RetryCount": safe_get(audit_data, "RetryCount") or "", + "ClientVersion": safe_get(audit_data, "ClientVersion") or "", + "ClientPlatform": safe_get(audit_data, "ClientPlatform") or "", + "AgentId": agent_id_val, + "AgentName": safe_get(audit_data, "AgentName") or "", + "AgentVersion": safe_get(audit_data, "AgentVersion") or "", + "AgentCategory": agent_category, + "ApplicationName": safe_get(audit_data, "ApplicationName") or "", + "SensitivityLabel": safe_get(audit_data, "SensitivityLabel") or "", + # CED sub-fields — empty for non-Copilot records, populated by Copilot path + "AppHost": "", + "ThreadId": "", + "Context_Id": "", + "Context_Type": "", + "Message_Id": "", + "Message_isPrompt": "", + "AccessedResource_Action": "", + "AccessedResource_PolicyDetails": "", + "AccessedResource_SiteUrl": "", + "AISystemPlugin_Id": "", + "AISystemPlugin_Name": "", + "ModelTransparencyDetails_ModelName": "", + "MessageIds": "", + "AccessedResource_Name": "", + "AccessedResource_SensitivityLabel": "", + "AccessedResource_ResourceType": "", + "Context_Item": "", + } + if _DEIDENTIFY: + # Exploded (153-col) identity + resource fields. AccessedResource_SiteUrl/Name on + # Copilot rows are hashed in explode_copilot_record (set there after this returns). + row["UserId"] = deid_upn(row["UserId"]) + row["MailboxOwnerUPN"] = deid_upn(row["MailboxOwnerUPN"]) + row["MailboxGuid"] = deid_guid(row["MailboxGuid"]) + row["LogonUserSid"] = deid_sid(row["LogonUserSid"]) + row["MailboxOwnerSid"] = deid_sid(row["MailboxOwnerSid"]) + row["DeviceDisplayName"] = deid_name(row["DeviceDisplayName"]) + row["SiteUrl"] = deid_resource(row["SiteUrl"]) + row["SourceRelativeUrl"] = deid_resource(row["SourceRelativeUrl"]) + row["SourceFileName"] = deid_file(row["SourceFileName"]) + return row + + +def explode_m365_record(record: dict, audit_data: dict) -> list[dict]: + """ + Extract a non-Copilot M365 record (Path A). + Produces exactly 1 row per record with all 153 M code columns. + No array explosion — M code does not explode non-Copilot arrays. + """ + return [_build_unified_row(record, audit_data)] + + +# ═════════════════════════════════════════════════════════════════════════════ +# PATH B: COPILOT EXPLOSION +# ═════════════════════════════════════════════════════════════════════════════ + +def explode_copilot_record( + record: dict, + audit_data: dict, + ced: dict, + prompt_filter: str | None = None, +) -> list[dict]: + """ + Explode a Copilot record (Path B). + Starts from the unified 153-column base row, then overrides CED-specific fields. + Extracts Messages, Contexts, AccessedResources, AISystemPlugin, + ModelTransparencyDetails, SensitivityLabels and builds N parallel-indexed rows. + """ + # Extract array fields from CopilotEventData + messages = get_array_fast(ced, "Messages") + contexts = get_array_fast(ced, "Contexts") + resources = get_array_fast(ced, "AccessedResources") + plugins_raw = get_array_fast(ced, "AISystemPlugin") + model_det_raw = get_array_fast(ced, "ModelTransparencyDetails") + message_ids = get_array_fast(ced, "MessageIds") + sensitivity_labels = get_array_fast(ced, "SensitivityLabels") + + # Prompt filtering + if prompt_filter: + filtered: list = [] + pf_lower = prompt_filter.lower() + if pf_lower == "null": + filtered = [m for m in messages if safe_get(m, "isPrompt") is None] + elif pf_lower == "both": + filtered = [m for m in messages if safe_get(m, "isPrompt") is not None] + elif pf_lower == "prompt": + filtered = [m for m in messages if safe_get(m, "isPrompt") is True] + elif pf_lower == "response": + filtered = [m for m in messages if safe_get(m, "isPrompt") is False] + messages = filtered + if not messages: + return [] + + # Detect activity type for 2-level explosion + activity_type = safe_get(audit_data, "Operation") or "" + + # Context items max for CopilotInteraction + context_items_max: int = 0 + if activity_type == "CopilotInteraction" and contexts: + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and len(items) > context_items_max: + context_items_max = len(items) + + # Calculate row count + if prompt_filter: + row_count = max(1, len(messages)) + else: + array_counts = [ + 1, len(messages), len(contexts), len(resources), + len(sensitivity_labels), len(plugins_raw), len(model_det_raw), + ] + if context_items_max > 0: + array_counts.append(context_items_max) + row_count = max(array_counts) + + row_count = min(row_count, EXPLOSION_PER_RECORD_ROW_CAP) + if row_count < 1: + row_count = 1 + + # ── Build unified base row with all 153 M code columns ─────────────── + base = _build_unified_row(record, audit_data) + + # ── Override CED-specific scalar fields with deep CED extraction ───── + # AppHost: prefer CED → audit_data → Workload + base["AppHost"] = select_first_non_null([ + safe_get(ced, "AppHost"), + safe_get(audit_data, "AppHost"), + safe_get(audit_data, "Workload"), + ]) or "" + + base["ThreadId"] = safe_get(ced, "ThreadId") or "" + + # AgentVersion: prefer audit_data → CED fallbacks + base["AgentVersion"] = select_first_non_null([ + safe_get(audit_data, "AgentVersion"), + safe_get(ced, "AgentVersion"), + safe_get(ced, "Version"), + ]) or "" + + # ApplicationName: prefer audit_data → CED fallbacks + base["ApplicationName"] = select_first_non_null([ + safe_get(audit_data, "ApplicationName"), + safe_get(ced, "HostAppName"), + safe_get(ced, "ClientAppName"), + ]) or "" + + # Model fields from CED with fallbacks + base["ModelId"] = select_first_non_null([ + safe_get(ced, "ModelId"), safe_get(ced, "ModelID"), safe_get(audit_data, "ModelId"), + ]) or "" + base["ModelProvider"] = select_first_non_null([ + safe_get(ced, "ModelProvider"), safe_get(ced, "Provider"), safe_get(ced, "ModelVendor"), + ]) or "" + base["ModelFamily"] = select_first_non_null([ + safe_get(ced, "ModelFamily"), safe_get(ced, "ModelType"), + ]) or "" + + # Token usage from CED + usage_node = select_first_non_null([ + safe_get(ced, "Usage"), safe_get(ced, "TokenUsage"), + safe_get(ced, "Tokens"), safe_get(audit_data, "Usage"), + ]) + tokens_total: Any = None + tokens_input: Any = None + tokens_output: Any = None + if usage_node and isinstance(usage_node, dict): + tokens_total = to_num(select_first_non_null([ + safe_get(usage_node, "Total"), safe_get(usage_node, "TotalTokens"), + safe_get(usage_node, "TokensTotal"), + ])) + tokens_input = to_num(select_first_non_null([ + safe_get(usage_node, "Input"), safe_get(usage_node, "Prompt"), + safe_get(usage_node, "InputTokens"), safe_get(usage_node, "TokensInput"), + ])) + tokens_output = to_num(select_first_non_null([ + safe_get(usage_node, "Output"), safe_get(usage_node, "Completion"), + safe_get(usage_node, "OutputTokens"), safe_get(usage_node, "TokensOutput"), + ])) + if not tokens_total and (tokens_input or tokens_output): + try: + tokens_total = (tokens_input or 0) + (tokens_output or 0) + except Exception: + pass + base["TokensTotal"] = tokens_total if tokens_total is not None else "" + base["TokensInput"] = tokens_input if tokens_input is not None else "" + base["TokensOutput"] = tokens_output if tokens_output is not None else "" + + # Duration, outcome, conversation from CED + duration_ms = to_num(select_first_non_null([ + safe_get(ced, "DurationMs"), safe_get(ced, "ElapsedMs"), + safe_get(ced, "ProcessingTimeMs"), safe_get(ced, "LatencyMs"), + ])) + base["DurationMs"] = duration_ms if duration_ms is not None else "" + + outcome_status: Any = select_first_non_null([ + safe_get(ced, "OutcomeStatus"), safe_get(ced, "Outcome"), + safe_get(ced, "Result"), safe_get(ced, "Status"), + ]) + if isinstance(outcome_status, bool): + outcome_status = "Success" if outcome_status else "Failure" + base["OutcomeStatus"] = outcome_status or "" + + base["ConversationId"] = select_first_non_null([ + safe_get(ced, "ConversationId"), safe_get(ced, "ConversationID"), + safe_get(ced, "SessionId"), + ]) or "" + + turn_number = to_num(select_first_non_null([ + safe_get(ced, "TurnNumber"), safe_get(ced, "TurnIndex"), + safe_get(ced, "MessageIndex"), + ])) + base["TurnNumber"] = turn_number if turn_number is not None else "" + + retry_count = to_num(select_first_non_null([ + safe_get(ced, "RetryCount"), safe_get(ced, "Retries"), + ])) + base["RetryCount"] = retry_count if retry_count is not None else "" + + base["ClientVersion"] = select_first_non_null([ + safe_get(ced, "ClientVersion"), safe_get(ced, "Version"), safe_get(ced, "Build"), + ]) or "" + base["ClientPlatform"] = select_first_non_null([ + safe_get(ced, "ClientPlatform"), safe_get(ced, "Platform"), safe_get(ced, "OS"), + ]) or "" + + # MessageIds: semicolon-joined (matches M code's Text.Combine) + base["MessageIds"] = ";".join(str(m) for m in message_ids) if message_ids else "" + + # ── Build rows with indexed array access ───────────────────────────── + rows: list[dict] = [] + for i in range(row_count): + row = dict(base) # shallow copy of all 153 columns + + # Indexed array access — Contexts + if i < len(contexts) and contexts[i]: + row["Context_Id"] = safe_get(contexts[i], "Id") or "" + row["Context_Type"] = safe_get(contexts[i], "Type") or "" + else: + row["Context_Id"] = "" + row["Context_Type"] = "" + + # Messages + if i < len(messages): + msg = messages[i] + if isinstance(msg, dict): + row["Message_Id"] = safe_get(msg, "Id") or "" + row["Message_isPrompt"] = bool_tf(safe_get(msg, "isPrompt")) + else: + row["Message_Id"] = str(msg) if msg is not None else "" + row["Message_isPrompt"] = "" + else: + row["Message_Id"] = "" + row["Message_isPrompt"] = "" + + # AccessedResources + if i < len(resources) and resources[i]: + res = resources[i] + row["AccessedResource_Action"] = safe_get(res, "Action") or "" + row["AccessedResource_PolicyDetails"] = to_json_if_object(safe_get(res, "PolicyDetails")) + row["AccessedResource_SiteUrl"] = deid_resource(safe_get(res, "SiteUrl") or "") + row["AccessedResource_Name"] = deid_file(safe_get(res, "Name") or "") + row["AccessedResource_SensitivityLabel"] = safe_get(res, "SensitivityLabel") or "" + row["AccessedResource_ResourceType"] = safe_get(res, "ResourceType") or "" + else: + row["AccessedResource_Action"] = "" + row["AccessedResource_PolicyDetails"] = "" + row["AccessedResource_SiteUrl"] = "" + row["AccessedResource_Name"] = "" + row["AccessedResource_SensitivityLabel"] = "" + row["AccessedResource_ResourceType"] = "" + + # AISystemPlugin + if i < len(plugins_raw) and plugins_raw[i]: + row["AISystemPlugin_Id"] = safe_get(plugins_raw[i], "Id") or "" + row["AISystemPlugin_Name"] = safe_get(plugins_raw[i], "Name") or "" + else: + row["AISystemPlugin_Id"] = "" + row["AISystemPlugin_Name"] = "" + + # ModelTransparencyDetails + if i < len(model_det_raw) and model_det_raw[i]: + row["ModelTransparencyDetails_ModelName"] = safe_get(model_det_raw[i], "ModelName") or "" + else: + row["ModelTransparencyDetails_ModelName"] = "" + + # SensitivityLabel (from CED SensitivityLabels array) + if i < len(sensitivity_labels): + row["SensitivityLabel"] = str(sensitivity_labels[i]) if sensitivity_labels[i] is not None else "" + + # Context_Item — full mode: one item per row across all contexts + if activity_type == "CopilotInteraction": + found_item = None + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and i < len(items): + found_item = items[i] + break + row["Context_Item"] = to_json_if_object(found_item) if found_item else "" + else: + row["Context_Item"] = "" + + rows.append(row) + + return rows + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROUTER: Dispatch to Path A or Path B +# ═════════════════════════════════════════════════════════════════════════════ + +def explode_record( + record: dict, + prompt_filter: str | None = None, +) -> list[dict]: + """ + Parse AuditData and route to appropriate explosion path. + Returns list of flattened row dicts, or empty list on error. + """ + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + return [] + + try: + audit_data = json_loads(audit_data_raw) + except Exception: + return [] + + if not isinstance(audit_data, dict): + return [] + + ced = safe_get(audit_data, "CopilotEventData") + if ced and isinstance(ced, dict): + return explode_copilot_record(record, audit_data, ced, prompt_filter=prompt_filter) + else: + return explode_m365_record(record, audit_data) + + +# ═════════════════════════════════════════════════════════════════════════════ +# HEADER — Fixed schema, no dynamic discovery needed +# ═════════════════════════════════════════════════════════════════════════════ +# Output columns are exactly M365_UNIFIED_HEADER (153 columns in M code order). +# No schema discovery pass is needed because both Path A and Path B produce +# row dicts that contain exactly these keys. + + +# ═════════════════════════════════════════════════════════════════════════════ +# CHUNK PROCESSOR (unit of parallel work) +# ═════════════════════════════════════════════════════════════════════════════ + +def _process_chunk(args: tuple) -> tuple[list[dict], int, int]: + """ + Process a chunk of CSV rows → exploded row dicts. + Returns (exploded_rows, input_count, error_count). + """ + chunk, prompt_filter = args + results: list[dict] = [] + errors = 0 + + for record in chunk: + try: + rows = explode_record(record, prompt_filter=prompt_filter) + results.extend(rows) + except Exception: + errors += 1 + + return results, len(chunk), errors + + +# ═════════════════════════════════════════════════════════════════════════════ +# MAIN EXPLOSION ORCHESTRATOR +# ═════════════════════════════════════════════════════════════════════════════ + +def run_explosion( + input_csv: str, + output_csv: str, + prompt_filter: str | None = None, + workers: int = 0, + chunk_size: int = STREAMING_CHUNK_SIZE, + quiet: bool = False, +) -> dict[str, Any]: + """ + Main entry point: reads input CSV, explodes all records, writes output CSV. + Uses multiprocessing for large files, single-process for small ones. + + Returns a stats dict with counts and timing. + """ + if not os.path.isfile(input_csv): + print(f"ERROR: Input file not found: {input_csv}", file=sys.stderr) + sys.exit(1) + + if workers <= 0: + workers = min(os.cpu_count() or 1, 8) + + t_start = time.perf_counter() + stats = { + "input_records": 0, + "output_rows": 0, + "errors": 0, + "chunks_processed": 0, + } + + if not quiet: + print(f"Purview M365 Usage Bundle Explosion Processor v{SCRIPT_VERSION}") + print(f" JSON engine: {_JSON_ENGINE}") + print(f" Input: {input_csv}") + print(f" Output: {output_csv}") + print(f" Prompt filter: {prompt_filter or 'None'}") + print(f" Workers: {workers}") + print(f" Chunk size: {chunk_size}") + print() + + # ── Phase 1: Fixed schema ───────────────────────────────────────────── + final_header = list(M365_UNIFIED_HEADER) # 153 columns in M code order + if not quiet: + print(f"Phase 1: Using fixed {len(final_header)}-column M code schema") + + # ── Phase 2: Process chunks ────────────────────────────────────────── + if not quiet: + print("Phase 2: Processing records...") + + # Accumulate all exploded rows (we need dynamic columns before writing header) + all_rows: list[dict] = [] + + # Read CSV in chunks + chunks: list[list[dict]] = [] + current_chunk: list[dict] = [] + + with open(input_csv, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + current_chunk.append(row) + if len(current_chunk) >= chunk_size: + chunks.append(current_chunk) + current_chunk = [] + if current_chunk: + chunks.append(current_chunk) + + total_input = sum(len(c) for c in chunks) + stats["input_records"] = total_input + + if not quiet: + print(f" Loaded {total_input:,} input records in {len(chunks)} chunk(s)") + + # Determine whether to use multiprocessing + use_parallel = workers > 1 and len(chunks) > 1 + + if use_parallel: + chunk_args = [(chunk, prompt_filter) for chunk in chunks] + with ProcessPoolExecutor(max_workers=workers, initializer=_deid_init_worker, initargs=(_DEIDENTIFY,)) as executor: + futures = {executor.submit(_process_chunk, arg): idx for idx, arg in enumerate(chunk_args)} + for future in as_completed(futures): + try: + exploded, _in_count, err_count = future.result() + all_rows.extend(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}/{len(chunks)}") + except Exception as exc: + stats["errors"] += 1 + if not quiet: + print(f" Chunk failed: {exc}", file=sys.stderr) + else: + for chunk in chunks: + exploded, _in_count, err_count = _process_chunk((chunk, prompt_filter)) + all_rows.extend(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}/{len(chunks)}") + + stats["output_rows"] = len(all_rows) + + # ── Phase 3: Write output CSV with fixed header ────────────────────── + if not quiet: + print("Phase 3: Writing output CSV...") + + # Write output using fixed M code-ordered header + os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) + with open(output_csv, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=final_header, extrasaction="ignore", lineterminator="\n") + writer.writeheader() + for row in all_rows: + writer.writerow(row) + + t_elapsed = time.perf_counter() - t_start + + # ── Summary ────────────────────────────────────────────────────────── + if not quiet: + print() + print("=== EXPLOSION SUMMARY ===") + print(f" Input records: {stats['input_records']:,}") + print(f" Output rows: {stats['output_rows']:,}") + if stats["output_rows"] > stats["input_records"] and stats["input_records"] > 0: + ratio = round(stats["output_rows"] / stats["input_records"], 2) + extra = stats["output_rows"] - stats["input_records"] + print(f" Expansion: {ratio}x ({extra:,} additional rows from array explosion)") + elif stats["output_rows"] == stats["input_records"]: + print(" Expansion: 1:1 (no arrays exploded)") + elif stats["input_records"] > 0: + filtered = stats["input_records"] - stats["output_rows"] + print(f" Reduction: {filtered:,} records filtered out") + if stats["errors"] > 0: + print(f" Errors: {stats['errors']:,} record(s) failed to process") + print(f" Columns: {len(final_header):,}") + print(f" Elapsed: {t_elapsed:.2f}s") + if stats["output_rows"] > 0 and t_elapsed > 0: + print(f" Throughput: {stats['output_rows'] / t_elapsed:,.0f} rows/sec") + print(f" Output file: {output_csv}") + print() + + return stats + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROLLUP ORCHESTRATOR (streaming — no exploded rows in memory) +# ═════════════════════════════════════════════════════════════════════════════ + +def run_rollup( + input_csv: str | list[str], + output_csv: str, + prompt_filter: str | None = None, + quiet: bool = False, + session_stats_csv: str | None = None, +) -> dict[str, Any]: + """ + Streaming rollup: read one or more CSVs row-by-row → parse AuditData → extract + 9 group keys + CreationTime + agent flag → accumulate into + dict[GroupKey, RollupAccum] → write 13-column CSV. + + `input_csv` accepts a single path (PAX/PowerShell single-file export) or a + list of paths (manual 4-pull export from Purview Audit). Output schema is + identical either way — the same PBIT template ingests both modes. + + When `session_stats_csv` is provided, a parallel pass over CopilotEventData + accumulates per-(UserId, CreationDate, AppHost) DISTINCTCOUNT(ThreadId) + + prompt/response counts and writes an 8-column SessionStats CSV. This matches + the AI in One `Sessions` measure unit (one thread = one session). + + No exploded row dicts are ever stored in memory. + """ + if isinstance(input_csv, (str, Path)): + input_paths: list[str] = [str(input_csv)] + else: + input_paths = [str(p) for p in input_csv] + + for p in input_paths: + if not os.path.isfile(p): + print(f"ERROR: Input file not found: {p}", file=sys.stderr) + sys.exit(1) + + t_start = time.perf_counter() + rollup: dict[GroupKey, RollupAccum] = {} + sessions: dict[SessionKey, SessionAccum] = {} if session_stats_csv else {} + track_sessions: bool = bool(session_stats_csv) + stats: dict[str, Any] = { + "input_records": 0, + "virtual_exploded_event_count": 0, + "output_rows": 0, + "parse_errors": 0, + "session_rows": 0, + "session_threads": 0, + "session_prompts": 0, + } + + if not quiet: + print(f"Purview M365 Usage Bundle Explosion Processor v{SCRIPT_VERSION} [ROLLUP MODE]") + print(f" JSON engine: {_JSON_ENGINE}") + if len(input_paths) == 1: + print(f" Input: {input_paths[0]}") + else: + print(f" Inputs ({len(input_paths)}):") + for p in input_paths: + print(f" {p}") + print(f" Output: {output_csv}") + if session_stats_csv: + print(f" Session stats: {session_stats_csv}") + print(f" Prompt filter: {prompt_filter or 'None'}") + print() + print("Processing records (streaming rollup)...") + + # ── Streaming read + accumulate (across one OR many input files) ───── + for input_csv_path in input_paths: + with open(input_csv_path, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for record in reader: + stats["input_records"] += 1 + + # Progress indicator + if not quiet and stats["input_records"] % 500_000 == 0: + print(f" {stats['input_records']:>12,} records processed, " + f"{len(rollup):,} groups...") + + # Parse AuditData JSON + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + stats["parse_errors"] += 1 + continue + try: + audit_data = json_loads(audit_data_raw) + except Exception: + stats["parse_errors"] += 1 + continue + if not isinstance(audit_data, dict): + stats["parse_errors"] += 1 + continue + + ced = safe_get(audit_data, "CopilotEventData") + if ced and not isinstance(ced, dict): + ced = None + + # Extract rollup keys (lightweight — no row dict built) + result = _extract_rollup_keys(record, audit_data, ced, prompt_filter) + if result is None: + continue # filtered out by prompt_filter or non-human UPN + + (group_key, event_count, items_accessed_count, + creation_time, original_uid, is_agent) = result + stats["virtual_exploded_event_count"] += event_count + + # Accumulate into rollup dict + if group_key in rollup: + acc = rollup[group_key] + acc.event_count += event_count + acc.items_accessed_count += items_accessed_count + if is_agent: + acc.is_agent_interaction = True + if creation_time: + if not acc.min_creation_time or creation_time < acc.min_creation_time: + acc.min_creation_time = creation_time + if not acc.max_creation_time or creation_time > acc.max_creation_time: + acc.max_creation_time = creation_time + else: + rollup[group_key] = RollupAccum( + event_count=event_count, + items_accessed=items_accessed_count, + min_ct=creation_time, + max_ct=creation_time, + original_uid=original_uid, + is_agent=is_agent, + ) + + # ── SessionStats accumulation (v2.6.0 — AI in One parity) ─── + # Only records with a CopilotEventData payload contribute. Threads + # without at least one user prompt are excluded (matches AI in One + # `Message_isPrompt = TRUE` filter). + if track_sessions and ced: + msgs = get_array_fast(ced, "Messages") + prompts_here = 0 + responses_here = 0 + for m in msgs: + ip = safe_get(m, "isPrompt") + if ip is True: + prompts_here += 1 + elif ip is False: + responses_here += 1 + thread_id = _norm_key_str(safe_get(ced, "ThreadId")) + # group_key layout: (uid_lower, creation_date, op, wl, sfe, + # app_host, agent_id, agent_name, context_type) + skey: SessionKey = (group_key[0], group_key[1], group_key[5]) + sacc = sessions.get(skey) + if sacc is None: + sacc = SessionAccum(original_uid=original_uid) + sessions[skey] = sacc + sacc.prompt_count += prompts_here + sacc.response_count += responses_here + if is_agent: + sacc.agent_prompt_count += prompts_here # v2.6.1: exact chat/agent split + if thread_id and prompts_here > 0: + sacc.thread_ids.add(thread_id) + if is_agent: + sacc.agent_thread_ids.add(thread_id) + stats["session_prompts"] += prompts_here + + # ── Write rollup output CSV ────────────────────────────────────────── + stats["output_rows"] = len(rollup) + + if not quiet: + print(f" {stats['input_records']:>12,} records processed (done)") + print(f" Writing {stats['output_rows']:,} rollup rows...") + + os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) + with open(output_csv, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(ROLLUP_HEADER) + for (uid_lower, cdate, op, wl, sfe, ah, agent_id, agent_name, ctx_type), acc in rollup.items(): + writer.writerow([ + deid_upn(acc.original_user_id), # output original casing, NOT lowered key + cdate, + op, + wl, + sfe, + ah, + acc.event_count, + acc.items_accessed_count, + acc.min_creation_time, # CreationTime = MIN + acc.max_creation_time, # MaxCreationTime = MAX + agent_id, + agent_name, + ctx_type, + "TRUE" if acc.is_agent_interaction else "FALSE", + ]) + + # ── SessionStats CSV (v2.6.0 — AI in One parity) ───────────────────── + if track_sessions: + os.makedirs(os.path.dirname(os.path.abspath(session_stats_csv)), exist_ok=True) + with open(session_stats_csv, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(SESSIONSTATS_HEADER) + for (uid_lower, cdate, ah), sacc in sessions.items(): + session_count = len(sacc.thread_ids) + if session_count == 0 and sacc.prompt_count == 0: + continue # no signal — skip + writer.writerow([ + deid_upn(sacc.original_user_id), + cdate, + ah, + session_count, + sacc.prompt_count, + sacc.agent_prompt_count, # v2.6.1 + sacc.response_count, + len(sacc.agent_thread_ids), + ]) + stats["session_rows"] += 1 + stats["session_threads"] += session_count + + t_elapsed = time.perf_counter() - t_start + + # ── Summary report ─────────────────────────────────────────────────── + if not quiet: + pct = 0.0 + if stats["virtual_exploded_event_count"] > 0: + pct = (1 - stats["output_rows"] / stats["virtual_exploded_event_count"]) * 100 + print() + print("=== ROLLUP SUMMARY ===") + print(f" Input records: {stats['input_records']:>14,}") + print(f" Virtual exploded events: {stats['virtual_exploded_event_count']:>14,}") + print(f" Rollup output rows: {stats['output_rows']:>14,}") + print(f" Row reduction: {pct:>13.1f}%" + f" ({stats['virtual_exploded_event_count']:,} -> {stats['output_rows']:,})") + if stats["parse_errors"] > 0: + print(f" Parse errors: {stats['parse_errors']:>14,}") + print(f" Columns: {len(ROLLUP_HEADER):>14}") + print(f" Elapsed: {t_elapsed:>13.2f}s") + if stats["input_records"] > 0 and t_elapsed > 0: + print(f" Throughput: {stats['input_records'] / t_elapsed:>12,.0f} input records/sec") + print(f" Output file: {output_csv}") + if track_sessions: + print() + print("=== SESSIONSTATS SUMMARY (AI in One parity) ===") + print(f" SessionStats rows: {stats['session_rows']:>14,}") + print(f" Distinct Copilot sessions: {stats['session_threads']:>14,}") + print(f" User prompts counted: {stats['session_prompts']:>14,}") + print(f" Output file: {session_stats_csv}") + print() + + return stats + + +# ═════════════════════════════════════════════════════════════════════════════ +# RECONCILIATION (sample-based validation of rollup correctness) +# ═════════════════════════════════════════════════════════════════════════════ + +def run_reconcile( + input_csv: str, + prompt_filter: str | None = None, + sample_size: int = RECONCILE_SAMPLE_SIZE, + quiet: bool = False, +) -> bool: + """ + Sample-based reconciliation: read a sample of records, run both rollup-key + extraction and full event-level explosion, compare total and filtered counts. + + Returns True if all checks pass, False otherwise. + """ + if not os.path.isfile(input_csv): + print(f"ERROR: Input file not found: {input_csv}", file=sys.stderr) + return False + + if not quiet: + print(f"\n=== RECONCILIATION CHECK (sample {sample_size:,} records) ===\n") + + # ── Read sample ────────────────────────────────────────────────────── + all_records: list[dict] = [] + with open(input_csv, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for record in reader: + all_records.append(record) + + if len(all_records) > sample_size: + sample = random.sample(all_records, sample_size) + else: + sample = all_records + sample_size = len(sample) + + if not quiet: + print(f" Total input records: {len(all_records):,}") + print(f" Sample size: {sample_size:,}") + + # ── Run event-level explosion on sample ────────────────────────────── + event_rows: list[dict] = [] + event_errors = 0 + for record in sample: + try: + rows = explode_record(record, prompt_filter=prompt_filter) + # Apply same non-human UPN filter as rollup path so totals reconcile + rows = [r for r in rows if _is_human_upn(r.get("UserId", ""))] + event_rows.extend(rows) + except Exception: + event_errors += 1 + + # ── Run rollup-key extraction on same sample ───────────────────────── + rollup_sample: dict[GroupKey, RollupAccum] = {} + rollup_errors = 0 + for record in sample: + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + rollup_errors += 1 + continue + try: + audit_data = json_loads(audit_data_raw) + except Exception: + rollup_errors += 1 + continue + if not isinstance(audit_data, dict): + rollup_errors += 1 + continue + + ced = safe_get(audit_data, "CopilotEventData") + if ced and not isinstance(ced, dict): + ced = None + + result = _extract_rollup_keys(record, audit_data, ced, prompt_filter) + if result is None: + continue + (group_key, event_count, items_accessed_count, + creation_time, original_uid, is_agent) = result + + if group_key in rollup_sample: + acc = rollup_sample[group_key] + acc.event_count += event_count + acc.items_accessed_count += items_accessed_count + if is_agent: + acc.is_agent_interaction = True + if creation_time: + if not acc.min_creation_time or creation_time < acc.min_creation_time: + acc.min_creation_time = creation_time + if not acc.max_creation_time or creation_time > acc.max_creation_time: + acc.max_creation_time = creation_time + else: + rollup_sample[group_key] = RollupAccum( + event_count=event_count, + items_accessed=items_accessed_count, + min_ct=creation_time, + max_ct=creation_time, + original_uid=original_uid, + is_agent=is_agent, + ) + + # ── Compare totals ─────────────────────────────────────────────────── + rollup_total = sum(acc.event_count for acc in rollup_sample.values()) + event_total = len(event_rows) + all_pass = True + + def _check(label: str, rollup_val: Any, event_val: Any) -> bool: + nonlocal all_pass + match = rollup_val == event_val + symbol = "PASS" if match else "FAIL" + if not quiet: + print(f" {label}") + print(f" Rollup: {rollup_val} Event-level: {event_val} [{symbol}]") + if not match: + all_pass = False + return match + + _check("Total event count (SUM(EventCount) vs COUNTROWS)", + rollup_total, event_total) + + # ── Filter-specific checks ─────────────────────────────────────────── + # Helper: count event-level rows matching a filter + def _ev_count(**filters: str | set) -> int: + count = 0 + for row in event_rows: + match = True + for col, val in filters.items(): + row_val = row.get(col, "") + if isinstance(val, set): + if row_val.lower() not in val: + match = False + break + else: + if row_val != val: + match = False + break + if match: + count += 1 + return count + + # Helper: sum EventCount from rollup for matching groups + def _ru_count(**filters: str | set) -> int: + total = 0 + for (uid_l, cdate, op, wl, sfe, ah, agent_id, agent_name, ctx_type), acc in rollup_sample.items(): + match = True + key_map = {"Operation": op, "Workload": wl, + "SourceFileExtension": sfe, "AppHost": ah, + "AgentId": agent_id, "AgentName": agent_name, + "ContextType": ctx_type} + for col, val in filters.items(): + key_val = key_map.get(col, "") + if isinstance(val, set): + if key_val.lower() not in val: + match = False + break + else: + if key_val != val: + match = False + break + if match: + total += acc.event_count + return total + + # Check 1: Teams MessageSent + _check("Teams MessageSent (Workload=MicrosoftTeams, Operation=MessageSent)", + _ru_count(Workload="MicrosoftTeams", Operation="MessageSent"), + _ev_count(Workload="MicrosoftTeams", Operation="MessageSent")) + + # Check 2: Exchange Send + _check("Exchange Send (Workload=Exchange, Operation=Send)", + _ru_count(Workload="Exchange", Operation="Send"), + _ev_count(Workload="Exchange", Operation="Send")) + + # Check 3: CopilotInteraction + AppHost=Teams + _check("Copilot Teams (Operation=CopilotInteraction, AppHost=Teams)", + _ru_count(Operation="CopilotInteraction", AppHost="Teams"), + _ev_count(Operation="CopilotInteraction", AppHost="Teams")) + + # Check 4: Excel FileAccessed (was FileViewed in pre-v2.3 — renamed via OP_RENAME) + _check("Excel FileAccessed (Operation=FileAccessed, SourceFileExtension in xlsx/xls/xlsm/csv)", + _ru_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"}), + _ev_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"})) + + # ── Temporal checks ────────────────────────────────────────────────── + rollup_min_ct = min((acc.min_creation_time for acc in rollup_sample.values() if acc.min_creation_time), default="") + rollup_max_ct = max((acc.max_creation_time for acc in rollup_sample.values() if acc.max_creation_time), default="") + event_times = [r.get("CreationTime", "") for r in event_rows if r.get("CreationTime")] + event_min_ct = min(event_times) if event_times else "" + event_max_ct = max(event_times) if event_times else "" + + _check("MIN(CreationTime)", rollup_min_ct, event_min_ct) + _check("MAX(CreationTime)", rollup_max_ct, event_max_ct) + + if not quiet: + print() + reduction_pct = 0.0 + if event_total > 0: + reduction_pct = (1 - len(rollup_sample) / event_total) * 100 + print(f" Event-level rows: {event_total:,} -> Rollup groups: {len(rollup_sample):,}" + f" ({reduction_pct:.1f}% reduction)") + print(f" Overall: {'ALL CHECKS PASSED' if all_pass else 'SOME CHECKS FAILED'}") + print() + + return all_pass + + +# ═════════════════════════════════════════════════════════════════════════════ +# USERSTATS & SESSION COHORT WRITER +# ═════════════════════════════════════════════════════════════════════════════ + +def write_userstats_files( + aggregated_csv_path: str | Path, + userstats_csv_path: str | Path, + session_csv_path: str | Path, + quiet: bool, + session_stats_csv_path: str | Path | None = None, +) -> tuple[int, int]: + """ + Read the just-written aggregated rollup CSV and produce two additional files: + *_UserStats.csv — one row per unique UserId with pre-computed metrics + *_SessionCohort.csv — one row per (UserId, AppColumn) with session cohort label + + When `session_stats_csv_path` is provided (v2.6.0+), the CECopilotPercentile_* + columns are computed from per-user PromptCount (human interactions) instead of + raw audit-event counts. This matches the AI in One semantics and prevents + service-principal / plugin-chain inflation from skewing the CE Quadrant. + + Returns (user_count, session_cohort_row_count). + """ + agg_path = Path(aggregated_csv_path) + if not agg_path.is_file(): + if not quiet: + print(f"[UserStats] WARNING: Aggregated CSV not found: {agg_path} — skipping.", + file=sys.stderr) + return 0, 0 + + userstats_path = Path(userstats_csv_path) + session_path = Path(session_csv_path) + + t_start = time.perf_counter() + + # ── Per-user accumulators ──────────────────────────────────────────── + uid_original: dict[str, str] = {} # uid_lower → first-seen casing + cop_ec: dict[str, int] = defaultdict(int) + m365_ec: dict[str, int] = defaultdict(int) + ex_cop_ec: dict[str, int] = defaultdict(int) + ex_m365_ec: dict[str, int] = defaultdict(int) + + t_days: dict[str, set[str]] = defaultdict(set) + o_days: dict[str, set[str]] = defaultdict(set) + w_days: dict[str, set[str]] = defaultdict(set) + x_days: dict[str, set[str]] = defaultdict(set) + p_days: dict[str, set[str]] = defaultdict(set) + + t_ec: dict[str, int] = defaultdict(int) + o_ec: dict[str, int] = defaultdict(int) + off_ec: dict[str, int] = defaultdict(int) + + # ── v2.5.0: DAX-aligned per-user raw activity counts, computed per window. + # Three windows (L30, L60, Full) feed both the LP Weighted measures and the + # CE percentile ranks. Each is a dict keyed by window code → {uid: count}. + def _wbuckets() -> dict[str, dict[str, int]]: + return {w: defaultdict(int) for w in RANK_WINDOWS} + teams_raw = _wbuckets() + outlook_raw = _wbuckets() + word_raw = _wbuckets() + excel_raw = _wbuckets() + ppt_raw = _wbuckets() + copilot_chat_raw = _wbuckets() # Operation = "CopilotInteraction" (LP) + ce_copilot_raw = _wbuckets() # broad: Workload="Copilot" OR Op contains "CopilotInteraction" (CE) + + session_ops: dict[tuple[str, str], set[str]] = defaultdict(set) + + # Track every distinct CreationDate seen in the rollup so we can + # derive the data-window span (max - min + 1 calendar days) and + # normalize the engagement segmentation to active-days-per-week. + all_dates: set[str] = set() + + # ── v2.5.0: Pass 1 — determine the trailing-window cutoffs ────────── + # Scan CreationDate only to find the most-recent date in the rollup. Cutoffs + # are inclusive lower bounds; a row qualifies for window W iff date_key >= cutoff[W]. + # The "Full" window has no cutoff and always qualifies. + _d_max_str = "" + with open(agg_path, "r", encoding="utf-8-sig", newline="") as _f: + _r = csv.DictReader(_f) + for _row in _r: + _dk = (_row.get("CreationDate", "") or "")[:10] + if _dk and _dk > _d_max_str: + _d_max_str = _dk + if _d_max_str: + try: + _d_max = date.fromisoformat(_d_max_str) + cutoff_l30 = (_d_max - timedelta(days=29)).isoformat() + cutoff_l60 = (_d_max - timedelta(days=59)).isoformat() + except ValueError: + # Bad date — fall back to "everything qualifies" + cutoff_l30 = "" + cutoff_l60 = "" + else: + # No data — sentinel that nothing qualifies for L30/L60 (Full still does) + cutoff_l30 = "9999-12-31" + cutoff_l60 = "9999-12-31" + if not quiet: + print(f"[UserStats] Window cutoffs: L30 >= {cutoff_l30 or '(all)'}, " + f"L60 >= {cutoff_l60 or '(all)'}, max date = {_d_max_str or '(none)'}") + + # ── Stream through aggregated CSV ──────────────────────────────────── + row_count = 0 + with open(agg_path, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + row_count += 1 + + user_id = row.get("UserId", "") + uid_lower = user_id.lower() + if uid_lower not in uid_original: + uid_original[uid_lower] = user_id + + date_key = row.get("CreationDate", "")[:10] # YYYY-MM-DD + op = row.get("Operation", "") + wl = row.get("Workload", "") + ext = (row.get("SourceFileExtension", "") or "").lower() + app_host = (row.get("AppHost", "") or "").lower() + + if date_key: + all_dates.add(date_key) + + try: + event_count = int(row.get("EventCount", "1") or "1") + except (ValueError, TypeError): + event_count = 1 + + copilot = is_copilot(op, wl) + excel_file = is_excel_file_op(ext, op) + + # Core event counts + if copilot: + cop_ec[uid_lower] += event_count + else: + m365_ec[uid_lower] += event_count + + # ExCopEC: Copilot interactions in Excel (via AppHost); ExM365EC: Excel file ops by non-Copilot + if copilot and app_host == "excel": + ex_cop_ec[uid_lower] += 1 # row count, not EventCount + if excel_file and not copilot: + ex_m365_ec[uid_lower] += 1 # row count, not EventCount + + # Active days (distinct CreationDate values) + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + t_days[uid_lower].add(date_key) + if wl == "Exchange" and op in OUTLOOK_OPS: + o_days[uid_lower].add(date_key) + if ext in WORD_EXTS and op in FILE_OPS: + w_days[uid_lower].add(date_key) + if ext in EXCEL_EXTS and op in FILE_OPS: + x_days[uid_lower].add(date_key) + if ext in PPT_EXTS and op in FILE_OPS: + p_days[uid_lower].add(date_key) + + # Activity event counts + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + t_ec[uid_lower] += event_count + if wl == "Exchange" and op in OUTLOOK_OPS: + o_ec[uid_lower] += event_count + if ext in OFFICE_EXTS and op in FILE_OPS: + off_ec[uid_lower] += event_count + + # ── v2.5.0: DAX-aligned raw counts, accumulated per window. + # Helper closure: write to Full always; to L60/L30 only if the row's + # date_key satisfies the trailing-window cutoff. + def _bump(buckets: dict[str, dict[str, int]], n: int) -> None: + buckets["Full"][uid_lower] += n + if date_key >= cutoff_l60: + buckets["L60"][uid_lower] += n + if date_key >= cutoff_l30: + buckets["L30"][uid_lower] += n + + if wl == "MicrosoftTeams" and op in DAX_TEAMS_OPS: + _bump(teams_raw, event_count) + if wl == "Exchange" and op in DAX_OUTLOOK_OPS: + _bump(outlook_raw, event_count) + if op in DAX_FILE_OPS: + if ext in WORD_EXTS: + _bump(word_raw, event_count) + elif ext in EXCEL_EXTS: + _bump(excel_raw, event_count) + elif ext in PPT_EXTS: + _bump(ppt_raw, event_count) + if op == "CopilotInteraction": + _bump(copilot_chat_raw, event_count) + # CE Copilot Percentile filter: Workload="Copilot" OR Operation contains "CopilotInteraction" + if wl == "Copilot" or "CopilotInteraction" in op: + _bump(ce_copilot_raw, event_count) + + # Session cohort: distinct active dates per (user, app) + app = app_column(ext, op, wl) + if app != "M365 All Apps": + session_ops[(uid_lower, app)].add(date_key) + + if row_count == 0: + if not quiet: + print("[UserStats] Aggregated CSV has 0 rows — skipping.") + return 0, 0 + + # ── Data-window span ──────────────────────────────────────────────── + # Calendar-day span between the earliest and latest CreationDate in the + # rollup, inclusive. Used to normalize per-app engagement segments to + # active-days-per-week, so labels mean the same thing whether the pull + # covers 8 days, 30 days, or 6 months. + if all_dates: + try: + d_min = min(all_dates) + d_max = max(all_dates) + window_days = ( + date.fromisoformat(d_max) - date.fromisoformat(d_min) + ).days + 1 + except ValueError: + window_days = max(len(all_dates), 1) + else: + window_days = 1 + if not quiet: + print(f"[UserStats] Data window: {window_days} calendar day(s) " + f"({d_min if all_dates else '?'} -> {d_max if all_dates else '?'})") + + # ── Percentile thresholds ──────────────────────────────────────────── + all_uids = sorted(uid_original.keys()) + total_users = len(all_uids) + + cop_vals = [cop_ec.get(u, 0) for u in all_uids] + m365_vals = [m365_ec.get(u, 0) for u in all_uids] + ex_cop_vals = [ex_cop_ec.get(u, 0) for u in all_uids] + ex_m365_vals = [ex_m365_ec.get(u, 0) for u in all_uids] + + cop_p90 = percentile_inc(cop_vals, 0.90) + cop_p75 = percentile_inc(cop_vals, 0.75) + cop_p50 = percentile_inc(cop_vals, 0.50) + + m365_p90 = percentile_inc(m365_vals, 0.90) + m365_p75 = percentile_inc(m365_vals, 0.75) + m365_p50 = percentile_inc(m365_vals, 0.50) + + exc_p90 = percentile_inc(ex_cop_vals, 0.90) + exc_p75 = percentile_inc(ex_cop_vals, 0.75) + exc_p50 = percentile_inc(ex_cop_vals, 0.50) + + exm_p90 = percentile_inc(ex_m365_vals, 0.90) + exm_p75 = percentile_inc(ex_m365_vals, 0.75) + exm_p50 = percentile_inc(ex_m365_vals, 0.50) + + # ── Ranks ──────────────────────────────────────────────────────────── + # Copilot rank: computed only among Copilot users so the range maps to [0, 1] + # within that group. Non-Copilot users are hardcoded to 0.0 downstream. + copilot_uids = [u for u in all_uids if cop_ec.get(u, 0) > 0] + copilot_user_count = len(copilot_uids) + cop_rank = compute_ranks({u: cop_ec.get(u, 0) for u in copilot_uids}) + m365_rank = compute_ranks({u: m365_ec.get(u, 0) for u in all_uids}) + + # ── v2.5.0: CE percentile ranks per window (integer 0–100, match DAX exactly) ── + # DAX formula: ROUND( COUNTROWS(users with score <= mine) / COUNTROWS(users with score > 0) * 100 , 0) + # Users with score 0 / no activity → BLANK (we emit empty string). + def _ce_rank_pct(scores: dict[str, int]) -> dict[str, str]: + """Return DAX-exact CE percentile rank per user, as a string ('' for BLANK).""" + positives = sorted(v for v in scores.values() if v > 0) + total = len(positives) + out: dict[str, str] = {} + if total == 0: + return {u: "" for u in scores} + for u, v in scores.items(): + if v <= 0: + out[u] = "" + else: + below = bisect.bisect_right(positives, v) + out[u] = str(round(below / total * 100)) + return out + + # M365 All Apps raw is derived per window (sum of 5 app raws). LP has no M365-AllApps + # measure, so we only compute the rank — not stored as a column. + m365_all_apps_raw = { + w: { + u: teams_raw[w].get(u, 0) + outlook_raw[w].get(u, 0) + word_raw[w].get(u, 0) + + excel_raw[w].get(u, 0) + ppt_raw[w].get(u, 0) + for u in all_uids + } + for w in RANK_WINDOWS + } + ce_rank_teams = {w: _ce_rank_pct({u: teams_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_outlook = {w: _ce_rank_pct({u: outlook_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_word = {w: _ce_rank_pct({u: word_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_excel = {w: _ce_rank_pct({u: excel_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_ppt = {w: _ce_rank_pct({u: ppt_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_all = {w: _ce_rank_pct(m365_all_apps_raw[w]) for w in RANK_WINDOWS} + + # ── v2.6.0: CE Copilot Percentile based on PROMPT COUNT (human interactions) ── + # Read the SessionStats CSV (if produced by run_rollup) and tally PromptCount per + # user per window. This is the AI in One semantics: one count per `isPrompt=TRUE` + # message — resistant to AI-response fanout, plugin chains, retries, and most + # service-principal noise. Falls back to audit-event tally if SessionStats is + # missing (older script invocations). + prompt_raw = _wbuckets() + if session_stats_csv_path: + _ss = Path(session_stats_csv_path) + if _ss.is_file(): + with open(_ss, "r", encoding="utf-8-sig", newline="") as _f: + _r = csv.DictReader(_f) + for _row in _r: + _uid = (_row.get("UserId") or "").strip().lower() + if not _uid: + continue + _date_key = (_row.get("CreationDate") or "")[:10] + try: + _pc = int(_row.get("PromptCount") or 0) + except ValueError: + _pc = 0 + if _pc <= 0: + continue + prompt_raw["Full"][_uid] += _pc + if cutoff_l60 and _date_key >= cutoff_l60: + prompt_raw["L60"][_uid] += _pc + if cutoff_l30 and _date_key >= cutoff_l30: + prompt_raw["L30"][_uid] += _pc + if not quiet: + _tot = sum(prompt_raw["Full"].values()) + print(f"[UserStats] CE Copilot Percentile source: PromptCount " + f"({_tot:,} prompts across {len(prompt_raw['Full']):,} users)") + else: + if not quiet: + print(f"[UserStats] WARNING: SessionStats CSV not found: {_ss} — " + f"falling back to audit-event count for CE Copilot Percentile.", + file=sys.stderr) + prompt_raw = ce_copilot_raw # fallback to legacy event-based percentile + else: + prompt_raw = ce_copilot_raw # legacy mode (script invoked without SessionStats) + + ce_copilot_pct = {w: _ce_rank_pct({u: prompt_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + + # ── Write *_UserStats.csv ──────────────────────────────────────────── + with open(userstats_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(USERSTATS_HEADER) + + for uid in all_uids: + c_ec = cop_ec.get(uid, 0) + m_ec = m365_ec.get(uid, 0) + exc_ec = ex_cop_ec.get(uid, 0) + exm_ec = ex_m365_ec.get(uid, 0) + + is_cop_user = "Copilot User" if c_ec > 0 else "Non-Copilot User" + cop_tier = tier_fn(c_ec, cop_p90, cop_p75, cop_p50, zero_is_bottom=True) + m365_tier = tier_fn(m_ec, m365_p90, m365_p75, m365_p50, zero_is_bottom=False) + priority = priority_fn(m365_tier, cop_tier) + + ex_m365_tier = tier_fn(exm_ec, exm_p90, exm_p75, exm_p50, zero_is_bottom=False) + ex_cop_tier = tier_fn(exc_ec, exc_p90, exc_p75, exc_p50, zero_is_bottom=False) + excel_pri = priority_fn(ex_m365_tier, ex_cop_tier) + + cop_rank_val = 0.0 if c_ec == 0 else cop_rank[uid] / max(copilot_user_count, 1) + m365_rank_val = m365_rank[uid] / total_users + + td = len(t_days.get(uid, set())) + od = len(o_days.get(uid, set())) + wd = len(w_days.get(uid, set())) + xd = len(x_days.get(uid, set())) + pd_ = len(p_days.get(uid, set())) + + t_act = t_ec.get(uid, 0) + o_act = o_ec.get(uid, 0) + off_act = off_ec.get(uid, 0) + + t_seg = "0. No Usage" if td == 0 else seg_fn(td, window_days) + o_seg = "0. No Usage" if od == 0 else seg_fn(od, window_days) + w_seg = "0. No Usage" if wd == 0 else seg_fn(wd, window_days) + x_seg = "0. No Usage" if xd == 0 else seg_fn(xd, window_days) + p_seg = "0. No Usage" if pd_ == 0 else seg_fn(pd_, window_days) + + office_days = wd + xd + pd_ + off_seg = "0. No Usage" if office_days == 0 else seg_fn(office_days, window_days) + + overall_days = len( + t_days.get(uid, set()) | o_days.get(uid, set()) | + w_days.get(uid, set()) | x_days.get(uid, set()) | + p_days.get(uid, set()) + ) + overall_seg = "0. No Usage" if overall_days == 0 else seg_fn(overall_days, window_days) + + writer.writerow([ + uid_original[uid], + c_ec, m_ec, exc_ec, exm_ec, + is_cop_user, cop_tier, m365_tier, + priority, excel_pri, + f"{cop_rank_val:.6f}", f"{m365_rank_val:.6f}", + td, od, wd, xd, pd_, + t_act, o_act, off_act, + t_seg, o_seg, w_seg, x_seg, p_seg, + off_seg, overall_seg, + # v2.5.0: precomputed raw + CE rank columns per window (order must + # match USERSTATS_HEADER: 6 raws × 3 windows, then 7 ranks × 3 windows) + teams_raw["L30"].get(uid, 0), teams_raw["L60"].get(uid, 0), teams_raw["Full"].get(uid, 0), + outlook_raw["L30"].get(uid, 0), outlook_raw["L60"].get(uid, 0), outlook_raw["Full"].get(uid, 0), + word_raw["L30"].get(uid, 0), word_raw["L60"].get(uid, 0), word_raw["Full"].get(uid, 0), + excel_raw["L30"].get(uid, 0), excel_raw["L60"].get(uid, 0), excel_raw["Full"].get(uid, 0), + ppt_raw["L30"].get(uid, 0), ppt_raw["L60"].get(uid, 0), ppt_raw["Full"].get(uid, 0), + copilot_chat_raw["L30"].get(uid, 0), copilot_chat_raw["L60"].get(uid, 0), copilot_chat_raw["Full"].get(uid, 0), + ce_rank_teams["L30"][uid], ce_rank_teams["L60"][uid], ce_rank_teams["Full"][uid], + ce_rank_outlook["L30"][uid], ce_rank_outlook["L60"][uid], ce_rank_outlook["Full"][uid], + ce_rank_word["L30"][uid], ce_rank_word["L60"][uid], ce_rank_word["Full"][uid], + ce_rank_excel["L30"][uid], ce_rank_excel["L60"][uid], ce_rank_excel["Full"][uid], + ce_rank_ppt["L30"][uid], ce_rank_ppt["L60"][uid], ce_rank_ppt["Full"][uid], + ce_rank_all["L30"][uid], ce_rank_all["L60"][uid], ce_rank_all["Full"][uid], + ce_copilot_pct["L30"][uid], ce_copilot_pct["L60"][uid], ce_copilot_pct["Full"][uid], + ]) + + # ── Write *_SessionCohort.csv ──────────────────────────────────────── + session_count = 0 + with open(session_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(SESSIONCOHORT_HEADER) + + for (uid, app), ops in sorted(session_ops.items()): + n = len(ops) + if n == 0: + continue + if n <= 5: + cohort = "1-5 sessions" + elif n <= 10: + cohort = "6-10 sessions" + elif n <= 20: + cohort = "11-20 sessions" + elif n <= 40: + cohort = "21-40 sessions" + elif n <= 60: + cohort = "41-60 sessions" + elif n <= 80: + cohort = "61-80 sessions" + else: + cohort = "81+ sessions" + writer.writerow([uid_original[uid], app, cohort]) + session_count += 1 + + t_elapsed = time.perf_counter() - t_start + + if not quiet: + print(f"[UserStats] {total_users:,} users \u2192 {userstats_path.name} " + f"({len(USERSTATS_HEADER)} columns)") + print(f"[SessionCohort] {session_count:,} (user, app) pairs \u2192 {session_path.name}") + print(f"[UserStats] Elapsed: {t_elapsed:.2f}s") + + return total_users, session_count + + +# ═════════════════════════════════════════════════════════════════════════════ +# CLI ENTRY POINT +# ═════════════════════════════════════════════════════════════════════════════ + +def main() -> None: + parser = argparse.ArgumentParser( + prog="purview_m365_processor", + description=( + f"Purview M365 Usage Bundle Processor v{SCRIPT_VERSION}\n" + "Pre-computes the M365 Usage rollup + UserStats + SessionCohort CSVs\n" + "consumed by the Power BI template. Accepts either layout:\n" + " (A) ONE PAX / PowerShell export ............ use --pax\n" + " (B) FOUR manual Purview Audit exports ...... use --teams --outlook --files --copilot\n" + "Output schema is IDENTICAL for both layouts — same PBIT template ingests either." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +EXAMPLES +======== + +(A) Single PAX export (PAX tool or PowerShell Search-UnifiedAuditLog): + + python %(prog)s --pax Purview_Export.csv + + python %(prog)s --pax Purview_Export.csv --output-dir ./output + + +(B) Manual 4-pull export from Purview Audit (validated chip strategy): + + Teams (7d): MessageSent, MessageRead, ChatCreated, TeamsSessionStarted, + MeetingParticipantDetail + Outlook (30d): MailItemsAccessed, Send, MailboxLogin + Files (60d): FileAccessed, FileModified, FileDownloaded, FileUploaded + Copilot (30d): CopilotInteraction, AIAppInteraction (filter by record type) + + python %(prog)s ^ + --teams Teams_Export.csv ^ + --outlook Outlook_Export.csv ^ + --files Files_Export.csv ^ + --copilot Copilot_Export.csv ^ + --output-dir .\output + + +OUTPUT (rollup mode, both layouts produce the same three files): + + _Rollup_.csv 13 cols -> M365Usage table + _UserStats_.csv 40 cols -> UserStats table + _SessionCohort_.csv 3 cols -> SessionCohort table + + defaults to the input file's name (single input) or '_Combined' + (multi-input). Rename the file or use --output-dir if you want a tenant-named folder. + + +ADVANCED +======== + --reconcile Sample-based correctness check vs full event-level explosion. + --debug-events v1-compatible 153-column event-level CSV (single input only). + --skip-precompute Skip UserStats and SessionCohort generation. + --prompt-filter Copilot message filter: Prompt | Response | Both | Null. + --input/-i Power-user / scripted fallback for one or more CSVs. +""", + ) + + # ── Input layout (mutually exclusive, exactly one required) ────────── + layout = parser.add_argument_group( + "INPUT LAYOUT (choose ONE shape that matches how you exported the data)" + ) + layout.add_argument( + "--pax", + metavar="CSV", + help="(A) Single CSV from PAX or PowerShell Search-UnifiedAuditLog.", + ) + layout.add_argument( + "--teams", + metavar="CSV", + help="(B) Teams workload pull from Purview Audit.", + ) + layout.add_argument( + "--outlook", + metavar="CSV", + help="(B) Outlook / Exchange workload pull from Purview Audit.", + ) + layout.add_argument( + "--files", + metavar="CSV", + help="(B) Files (SharePoint + OneDrive) workload pull from Purview Audit.", + ) + layout.add_argument( + "--copilot", + metavar="CSV", + help="(B) Copilot record-type pull (CopilotInteraction + AIAppInteraction).", + ) + layout.add_argument( + "--input", "-i", + nargs="+", + metavar="CSV", + help="Power-user fallback: one or more CSV paths (any combination).", + ) + + # ── Output naming & location ───────────────────────────────────────── + output = parser.add_argument_group("OUTPUT") + output.add_argument( + "--output-dir", "-o", + metavar="DIR", + default=None, + help="Directory for output files. Default: same folder as the (first) input.", + ) + + # ── Optional behaviour flags ───────────────────────────────────────── + advanced = parser.add_argument_group("ADVANCED") + advanced.add_argument( + "--skip-precompute", + action="store_true", + default=False, + help="Skip *_UserStats.csv and *_SessionCohort.csv (only the Rollup is written).", + ) + advanced.add_argument( + "--no-session-stats", + action="store_true", + default=False, + help="Skip *_SessionStats.csv (the AI in One DISTINCTCOUNT(ThreadId) output).", + ) + advanced.add_argument( + "--debug-events", + action="store_true", + default=False, + help="Emit v1-compatible 153-column event-level CSV instead of the rollup (single input only).", + ) + advanced.add_argument( + "--reconcile", + action="store_true", + default=False, + help="Run sample-based reconciliation against the first input.", + ) + advanced.add_argument( + "--prompt-filter", + choices=["Prompt", "Response", "Both", "Null"], + default=None, + help="Filter Copilot messages by isPrompt value.", + ) + advanced.add_argument( + "--quiet", "-q", + action="store_true", + default=False, + help="Suppress progress output (only errors are printed).", + ) + # Hidden legacy alias (kept for older scripts that referenced --no-userstats). + advanced.add_argument( + "--no-userstats", + dest="skip_precompute", + action="store_true", + help=argparse.SUPPRESS, + ) + advanced.add_argument( + "--deidentify", + action="store_true", + default=False, + help=( + "One-way hash all identifying values (UserId, mailbox UPN/GUID/SIDs, " + "device name, resource URLs/file names) for anonymous reporting. " + "Deterministic and format-preserving; irreversible (no decode map)." + ), + ) + advanced.add_argument( + "--rebuild-sidecars-from-rollup", + metavar="ROLLUP_CSV", + default=None, + help=( + "Regenerate UserStats and SessionCohort sidecars from an existing rollup CSV " + "(no Purview input required). Sidecars are written to --output-dir (default: " + "the rollup's parent directory) using the rollup's base stem." + ), + ) + advanced.add_argument( + "--session-stats-for-rebuild", + metavar="SESSIONSTATS_CSV", + default=None, + help=( + "Optional companion to --rebuild-sidecars-from-rollup: when supplied, the " + "CECopilotPercentile_* columns are computed from this SessionStats CSV's " + "PromptCount (AI in One semantics). If omitted, the sidecar rebuild falls " + "back to the audit-event count (legacy behaviour)." + ), + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {SCRIPT_VERSION}", + ) + + args = parser.parse_args() + + global _DEIDENTIFY + _DEIDENTIFY = bool(args.deidentify) + + # ── Standalone sidecar regeneration mode ───────────────────────────── + # When --rebuild-sidecars-from-rollup is supplied, ignore every other + # input/dispatch flag and rebuild UserStats + SessionCohort sidecars + # from the given rollup CSV. Used by the PAX append-merge workflow + # after the PowerShell side unions the current run's rollup with a + # customer-supplied target. + if args.rebuild_sidecars_from_rollup: + rollup_in = os.path.abspath(args.rebuild_sidecars_from_rollup) + if not os.path.isfile(rollup_in): + print(f"ERROR: Rollup CSV not found: {rollup_in}", file=sys.stderr) + sys.exit(1) + session_stats_in: str | None = None + if args.session_stats_for_rebuild: + session_stats_in = os.path.abspath(args.session_stats_for_rebuild) + if not os.path.isfile(session_stats_in): + print( + f"ERROR: SessionStats CSV not found: {session_stats_in}", + file=sys.stderr, + ) + sys.exit(1) + out_dir = ( + Path(os.path.abspath(args.output_dir)) + if args.output_dir + else Path(rollup_in).parent + ) + os.makedirs(out_dir, exist_ok=True) + rollup_stem = Path(rollup_in).stem + m = re.match(r"^(.*?)(?:_Rollup(?:_\d{8}_\d{6})?)$", rollup_stem) + base_stem = m.group(1) if m else rollup_stem + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + userstats_path = str(out_dir / f"{base_stem}_UserStats_{run_ts}.csv") + session_path = str(out_dir / f"{base_stem}_SessionCohort_{run_ts}.csv") + write_userstats_files( + rollup_in, userstats_path, session_path, args.quiet, + session_stats_csv_path=session_stats_in, + ) + sys.exit(0) + + # ── Resolve the input layout into a flat list ──────────────────────── + pax_inputs: list[str] = [args.pax] if args.pax else [] + workload_inputs: list[tuple[str, str]] = [] # [(label, path), ...] preserves order + for label in ("teams", "outlook", "files", "copilot"): + path = getattr(args, label) + if path: + workload_inputs.append((label, path)) + legacy_inputs: list[str] = list(args.input) if args.input else [] + + if pax_inputs and workload_inputs: + parser.error("--pax cannot be combined with --teams/--outlook/--files/--copilot. " + "Pick the shape that matches your export.") + if (pax_inputs or workload_inputs) and legacy_inputs: + parser.error("--input/-i cannot be combined with --pax or the workload flags.") + + if pax_inputs: + input_paths = [os.path.abspath(pax_inputs[0])] + layout_label = "pax" + elif workload_inputs: + input_paths = [os.path.abspath(p) for _, p in workload_inputs] + layout_label = "manual_4pull" + elif legacy_inputs: + input_paths = [os.path.abspath(p) for p in legacy_inputs] + layout_label = "legacy_input" + else: + parser.error( + "No input given. Use ONE of:\n" + " --pax (single PAX export)\n" + " --teams --outlook --files --copilot (manual 4-pull export)\n" + " --input/-i [ ...] (power-user fallback)" + ) + + for p in input_paths: + if not os.path.isfile(p): + print(f"ERROR: Input file not found: {p}", file=sys.stderr) + sys.exit(1) + + # ── Determine output directory & filenames ─────────────────────────── + first_stem = Path(input_paths[0]).stem + stem = first_stem if len(input_paths) == 1 else f"{first_stem}_Combined" + + output_dir = Path(os.path.abspath(args.output_dir)) if args.output_dir else Path(input_paths[0]).parent + os.makedirs(output_dir, exist_ok=True) + + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + event_level = args.debug_events + + if not event_level: + rollup_path = str(output_dir / f"{stem}_Rollup_{run_ts}.csv") + userstats_path = str(output_dir / f"{stem}_UserStats_{run_ts}.csv") + session_path = str(output_dir / f"{stem}_SessionCohort_{run_ts}.csv") + session_stats_path: str | None = ( + None if args.no_session_stats + else str(output_dir / f"{stem}_SessionStats_{run_ts}.csv") + ) + else: + if len(input_paths) > 1: + print("ERROR: --debug-events accepts only one input CSV.", file=sys.stderr) + sys.exit(1) + rollup_path = str(output_dir / f"{stem}_Exploded_{run_ts}.csv") + session_stats_path = None + + # ── Dispatch ───────────────────────────────────────────────────────── + if not event_level: + stats = run_rollup( + input_csv=input_paths if len(input_paths) > 1 else input_paths[0], + output_csv=rollup_path, + prompt_filter=args.prompt_filter, + quiet=args.quiet, + session_stats_csv=session_stats_path, + ) + exit_code = 1 if stats["parse_errors"] > stats["input_records"] * 0.1 else 0 + + if not args.skip_precompute: + write_userstats_files( + rollup_path, userstats_path, session_path, args.quiet, + session_stats_csv_path=session_stats_path, + ) + else: + stats = run_explosion( + input_csv=input_paths[0], + output_csv=rollup_path, + prompt_filter=args.prompt_filter, + quiet=args.quiet, + ) + exit_code = 1 if stats["errors"] > 0 else 0 + + if args.reconcile: + reconcile_passed = run_reconcile( + input_csv=input_paths[0], + prompt_filter=args.prompt_filter, + quiet=args.quiet, + ) + if not reconcile_passed: + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() +'@ +# <<< END-EMBEDDED-M365-PROCESSOR + +# ============================================================================ +# ROLLUP POST-PROCESSOR HELPER FUNCTIONS +# ============================================================================ +# Resolve-PythonExe -> locates a Python 3.10+ interpreter (auto-installs via winget, +# falls back to the python.org installer if winget unavailable). +# Install-OrjsonIfMissing -> installs the optional 'orjson' package; warn-and-continue +# on failure (both processors auto-fallback to stdlib json). +# Invoke-EmbeddedProcessor -> materializes the embedded .py to .pax_incremental, +# runs it with the provided argument list, captures stderr, +# and deletes the temp file in finally. +# All three functions log via Write-LogHost / Write-Log (defined earlier in the script). +# These functions are only invoked when -Rollup or -RollupPlusRaw is used. +# Minimum required Python version is 3.10 (the embedded processors use PEP 604 unions). +# ============================================================================ + +$Script:ROLLUP_PYTHON_MIN_MAJOR = 3 +$Script:ROLLUP_PYTHON_MIN_MINOR = 10 + +function Resolve-PythonExe { + [CmdletBinding()] + param([switch]$AllowAutoInstall) + + # Returns: hashtable @{ Path=; Args=; Version='3.13.1' } + # Throws on hard failure (no usable interpreter even after install attempts). + + $candidates = @( + @{ Cmd = 'python'; LauncherArgs = @() } + @{ Cmd = 'py'; LauncherArgs = @('-3.13') } + @{ Cmd = 'py'; LauncherArgs = @('-3.12') } + @{ Cmd = 'py'; LauncherArgs = @('-3.11') } + @{ Cmd = 'py'; LauncherArgs = @('-3.10') } + @{ Cmd = 'python3'; LauncherArgs = @() } + ) + + foreach ($c in $candidates) { + try { + $cmdInfo = Get-Command -Name $c.Cmd -ErrorAction SilentlyContinue + if (-not $cmdInfo) { continue } + $probeArgs = @($c.LauncherArgs) + @('-c', 'import sys; print("{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro))') + $verOutput = & $c.Cmd @probeArgs 2>$null + if ($LASTEXITCODE -ne 0 -or -not $verOutput) { continue } + $verText = ($verOutput | Select-Object -First 1).Trim() + if ($verText -notmatch '^(\d+)\.(\d+)\.(\d+)') { continue } + $maj = [int]$Matches[1] + $min = [int]$Matches[2] + if ($maj -lt $Script:ROLLUP_PYTHON_MIN_MAJOR) { continue } + if ($maj -eq $Script:ROLLUP_PYTHON_MIN_MAJOR -and $min -lt $Script:ROLLUP_PYTHON_MIN_MINOR) { continue } + return @{ Path = $c.Cmd; Args = $c.LauncherArgs; Version = $verText } + } + catch { continue } + } + + if (-not $AllowAutoInstall) { + throw "No Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ interpreter found on PATH." + } + + # Auto-install path: try winget first, then python.org installer. + Write-LogHost "Rollup: no Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ found. Attempting auto-install..." -ForegroundColor Yellow + + $winget = Get-Command -Name 'winget' -ErrorAction SilentlyContinue + $installed = $false + if ($winget) { + Write-LogHost "Rollup: installing Python 3.13 via winget (Python.Python.3.13)..." -ForegroundColor Cyan + try { + $wingetArgs = @('install', '--id', 'Python.Python.3.13', '-e', '--accept-source-agreements', '--accept-package-agreements', '--scope', 'user', '--silent') + & winget @wingetArgs | Out-Host + if ($LASTEXITCODE -eq 0) { $installed = $true } + } + catch { + Write-LogHost "Rollup: winget install failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + else { + Write-LogHost "Rollup: winget is not available on this host; falling back to python.org installer." -ForegroundColor Yellow + } + + if (-not $installed) { + # python.org per-user silent install + try { + $tmpDir = Join-Path ([IO.Path]::GetTempPath()) ("PAX_PythonInstaller_" + [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $tmpDir -Force | Out-Null + $installerUrl = 'https://www.python.org/ftp/python/3.13.1/python-3.13.1-amd64.exe' + $installerPath = Join-Path $tmpDir 'python-3.13.1-amd64.exe' + Write-LogHost "Rollup: downloading $installerUrl ..." -ForegroundColor Cyan + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + Write-LogHost "Rollup: running silent per-user install (PrependPath=1)..." -ForegroundColor Cyan + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/quiet', 'InstallAllUsers=0', 'PrependPath=1', 'Include_launcher=1', 'Include_pip=1') -Wait -PassThru + if ($proc.ExitCode -eq 0) { $installed = $true } + else { Write-LogHost "Rollup: python.org installer exited with code $($proc.ExitCode)." -ForegroundColor Yellow } + } + catch { + Write-LogHost "Rollup: python.org install failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + if (-not $installed) { + throw "Rollup: failed to auto-install Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+. Install Python manually from https://www.python.org/downloads/ and re-run." + } + + # Refresh PATH from process + machine + user scopes so new install is visible without restart. + $paths = @( + [Environment]::GetEnvironmentVariable('Path', 'Machine'), + [Environment]::GetEnvironmentVariable('Path', 'User'), + $env:Path + ) | Where-Object { $_ } + $env:Path = ($paths -join ';') + + # Recurse once with auto-install disabled to prevent loops. + return Resolve-PythonExe -AllowAutoInstall:$false +} + +function Install-OrjsonIfMissing { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + + # Best-effort install. Both embedded processors fall back to stdlib json on import failure, + # so we warn-and-continue on any error here rather than throwing. + $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'orjson') + & $PythonExe @showArgs 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { return $true } + + Write-LogHost "Rollup: 'orjson' not installed; installing for faster JSON parsing (~5-10x). Falls back to stdlib json on failure." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'orjson') + try { + & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } + if ($LASTEXITCODE -eq 0) { return $true } + Write-LogHost "Rollup: 'orjson' install returned exit code $LASTEXITCODE. Continuing with stdlib json." -ForegroundColor Yellow + return $false + } + catch { + Write-LogHost "Rollup: 'orjson' install threw: $($_.Exception.Message). Continuing with stdlib json." -ForegroundColor Yellow + return $false + } +} + +function Install-DeltalakeIfMissing { + <# + .SYNOPSIS + Ensures the Python 'deltalake' package is installed for Fabric Lakehouse Delta-table writes. + + .DESCRIPTION + Invoked only when the run will write to a Fabric Lakehouse target (Tables/ namespace). + Performs a quiet per-user install on first use. On failure, returns $false with a clear + actionable message — the caller must abort the Delta write path rather than continue. + + Unlike Install-OrjsonIfMissing, the deltalake package is NOT optional: the Fabric Tables + write path cannot fall back to a pure-Python implementation. Callers must check the + return value and abort the run cleanly if the install fails. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + + $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'deltalake') + & $PythonExe @showArgs 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { return $true } + + Write-LogHost "Fabric: 'deltalake' Python package not installed; installing (required for Lakehouse Tables/ writes)." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'deltalake>=0.15') + try { + & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } + if ($LASTEXITCODE -eq 0) { + Write-LogHost "Fabric: 'deltalake' installed successfully." -ForegroundColor Green + return $true + } + Write-LogHost "Fabric: 'deltalake' install returned exit code $LASTEXITCODE. Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } + catch { + Write-LogHost "Fabric: 'deltalake' install threw: $($_.Exception.Message). Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } +} + +function ConvertTo-UsersSeedMap { + <# + .SYNOPSIS + Extract {PersonId_Normalized -> UserKey (INT)} from a target Users CSV and write it + as a compact JSON file the embedded CopilotInteraction processor consumes via + `--seed-userkey-map`. + + .DESCRIPTION + Pre-flight helper for -AppendUserInfo. Read the existing target Users CSV produced by a + prior run, build a dict of PersonId_Normalized -> UserKey, and write JSON. Skips rows + with empty PersonId_Normalized or non-integer UserKey. First occurrence of any + PersonId_Normalized wins (target is the source of truth). + + .OUTPUTS + Int count of entries written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUsersCsv, + [Parameter(Mandatory)] [string] $OutJsonPath + ) + if (-not (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { + throw "ConvertTo-UsersSeedMap: target Users CSV not found: '$TargetUsersCsv'" + } + $rows = @(Import-Csv -LiteralPath $TargetUsersCsv -Encoding UTF8) + $map = [ordered]@{} + foreach ($r in $rows) { + $personId = $r.PersonId_Normalized + $uk = $r.UserKey + if ([string]::IsNullOrWhiteSpace($personId)) { continue } + if ([string]::IsNullOrWhiteSpace($uk)) { continue } + $ukInt = 0 + if (-not [int]::TryParse([string]$uk, [ref]$ukInt)) { continue } + if (-not $map.Contains($personId)) { $map[$personId] = $ukInt } + } + $json = if ($map.Count -gt 0) { ($map | ConvertTo-Json -Compress -Depth 2) } else { '{}' } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + $outDir = Split-Path -Parent $OutJsonPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + [System.IO.File]::WriteAllText($OutJsonPath, $json, $utf8NoBom) + return $map.Count +} + +function ConvertTo-FactSeedMaps { + <# + .SYNOPSIS + Extract {Message_Id_Raw -> Message_Id (INT)} and {ThreadId_Raw -> ThreadKey (INT)} + from a target CopilotInteraction Fact CSV and write each as a compact JSON file + the embedded processor consumes via `--seed-mid-map` / `--seed-thread-map`. + + .DESCRIPTION + Pre-flight helper for -AppendFile (CopilotInteraction rollup mode). Reads the + existing target Fact CSV produced by a prior run, builds two dicts keyed by the + raw GUID columns, and writes both JSON files. First occurrence wins. Rows with + blank raw GUID or non-integer surrogate are skipped. + + .OUTPUTS + PSCustomObject with .MidCount and .ThreadCount integer fields. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutMidJsonPath, + [Parameter(Mandatory)] [string] $OutThreadJsonPath + ) + if (-not (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + throw "ConvertTo-FactSeedMaps: target Fact CSV not found: '$TargetFactCsv'" + } + $rows = @(Import-Csv -LiteralPath $TargetFactCsv -Encoding UTF8) + $midMap = [ordered]@{} + $threadMap = [ordered]@{} + foreach ($r in $rows) { + $midRaw = $r.Message_Id_Raw + $threadRaw = $r.ThreadId_Raw + $midInt = 0 + $threadInt = 0 + if (-not [string]::IsNullOrWhiteSpace($midRaw)) { + if ([int]::TryParse([string]$r.Message_Id, [ref]$midInt)) { + if (-not $midMap.Contains($midRaw)) { $midMap[$midRaw] = $midInt } + } + } + if (-not [string]::IsNullOrWhiteSpace($threadRaw)) { + $threadVal = if ($r.PSObject.Properties.Match('ThreadKey').Count) { $r.ThreadKey } else { $r.ThreadId } + if ([int]::TryParse([string]$threadVal, [ref]$threadInt)) { + if (-not $threadMap.Contains($threadRaw)) { $threadMap[$threadRaw] = $threadInt } + } + } + } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + foreach ($pair in @(@($OutMidJsonPath, $midMap), @($OutThreadJsonPath, $threadMap))) { + $path = $pair[0] + $dict = $pair[1] + $outDir = Split-Path -Parent $path + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + $json = if ($dict.Count -gt 0) { ($dict | ConvertTo-Json -Compress -Depth 2) } else { '{}' } + [System.IO.File]::WriteAllText($path, $json, $utf8NoBom) + } + return [PSCustomObject]@{ + MidCount = $midMap.Count + ThreadCount = $threadMap.Count + } +} + +function ConvertTo-RecordIdExclusion { + <# + .SYNOPSIS + Extract the RecordId column from a target exploded/event-level CSV and write a + newline-delimited file the embedded M365Bundle processor consumes via + `--exclude-record-ids` for cross-run dedup. + + .DESCRIPTION + Pre-flight helper for -AppendFile (M365Bundle event-level mode / -RollupPlusRaw + raw-CSV append). Streams the target CSV without loading the full row set, finds + the RecordId column index from the header, and writes one RecordId per line. + Blank / duplicate RecordIds are skipped. + + .OUTPUTS + Int count of unique RecordIds written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetCsv, + [Parameter(Mandatory)] [string] $OutListPath + ) + if (-not (Test-Path -LiteralPath $TargetCsv -PathType Leaf)) { + throw "ConvertTo-RecordIdExclusion: target CSV not found: '$TargetCsv'" + } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + $outDir = Split-Path -Parent $OutListPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + $reader = $null + $writer = $null + $seen = [System.Collections.Generic.HashSet[string]]::new() + try { + $reader = [System.IO.StreamReader]::new($TargetCsv, [System.Text.Encoding]::UTF8) + $headerLine = $reader.ReadLine() + if (-not $headerLine) { return 0 } + # Tolerate UTF-8 BOM on header + if ($headerLine.Length -gt 0 -and [int][char]$headerLine[0] -eq 0xFEFF) { + $headerLine = $headerLine.Substring(1) + } + $cols = $headerLine.Split(',') + $ridIdx = -1 + for ($i = 0; $i -lt $cols.Length; $i++) { + $name = $cols[$i].Trim().Trim('"') + if ($name -eq 'RecordId') { $ridIdx = $i; break } + } + if ($ridIdx -lt 0) { + throw "ConvertTo-RecordIdExclusion: 'RecordId' column not found in '$TargetCsv'." + } + $writer = [System.IO.StreamWriter]::new($OutListPath, $false, $utf8NoBom) + while (-not $reader.EndOfStream) { + $line = $reader.ReadLine() + if ([string]::IsNullOrEmpty($line)) { continue } + # Lightweight CSV split that respects double-quoted fields containing commas. + $fields = [System.Collections.Generic.List[string]]::new() + $buf = [System.Text.StringBuilder]::new() + $inQ = $false + for ($j = 0; $j -lt $line.Length; $j++) { + $ch = $line[$j] + if ($ch -eq '"') { $inQ = -not $inQ; continue } + if ($ch -eq ',' -and -not $inQ) { + [void]$fields.Add($buf.ToString()); [void]$buf.Clear(); continue + } + [void]$buf.Append($ch) + } + [void]$fields.Add($buf.ToString()) + if ($ridIdx -ge $fields.Count) { continue } + $rid = $fields[$ridIdx].Trim() + if ([string]::IsNullOrEmpty($rid)) { continue } + if ($seen.Add($rid)) { + $writer.WriteLine($rid) + } + } + } + finally { + if ($writer) { $writer.Dispose() } + if ($reader) { $reader.Dispose() } + } + return $seen.Count +} + +function script:Import-CsvDeduped { + <# + .SYNOPSIS + Import-Csv wrapper that tolerates duplicate / BOM-prefixed / whitespace-padded + header columns. Required when reading Users CSVs that may have round-tripped + through the embedded Python processor and acquired duplicate header tokens + (e.g. both 'userPrincipalName' and 'PersonId' present pre-rename). + + .DESCRIPTION + PowerShell's Import-Csv calls Add-Member internally for each header cell, which + throws 'The member "X" is already present' when two header cells normalize to + the same name. This helper rewrites the header line with later duplicates + suffixed (_dup2, _dup3, …) before parsing, so the file always loads. The first + occurrence keeps its original name; callers that care can drop the *_dup* extras. + + Also strips a leading BOM from the header line and trims whitespace from each + header cell. The data rows are passed through unchanged. + + .OUTPUTS + Array of [pscustomobject]. Empty array if file is missing or contains only a + header (or only whitespace). + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$LiteralPath) + if (-not (Test-Path -LiteralPath $LiteralPath -PathType Leaf)) { return @() } + $lines = @(Get-Content -LiteralPath $LiteralPath -Encoding UTF8) + if ($lines.Count -eq 0) { return @() } + $hdrLine = $lines[0] -replace '^\uFEFF','' + # Parse header honoring quoted commas using TextFieldParser. + try { Add-Type -AssemblyName Microsoft.VisualBasic -ErrorAction Stop } catch { } + $sr = [System.IO.StringReader]::new($hdrLine) + $tfp = New-Object Microsoft.VisualBasic.FileIO.TextFieldParser($sr) + $tfp.SetDelimiters(',') + $tfp.HasFieldsEnclosedInQuotes = $true + $hdrCells = @() + try { $hdrCells = @($tfp.ReadFields()) } catch { $hdrCells = @($hdrLine -split ',') } + finally { $tfp.Dispose(); $sr.Dispose() } + $seen = [System.Collections.Generic.Dictionary[string,int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $newHdr = New-Object System.Collections.Generic.List[string] + foreach ($c in $hdrCells) { + $n = ($c -as [string]) + if ($null -eq $n) { $n = '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { + $seen[$n] = $seen[$n] + 1 + $newHdr.Add(('{0}_dup{1}' -f $n, $seen[$n])) + } else { + $seen[$n] = 1 + $newHdr.Add($n) + } + } + $newHdrLine = ($newHdr | ForEach-Object { '"' + ($_ -replace '"','""') + '"' }) -join ',' + if ($lines.Count -eq 1) { return @() } + $rest = $lines | Select-Object -Skip 1 + $rewritten = @($newHdrLine) + @($rest) + return @($rewritten | ConvertFrom-Csv) +} + +function Merge-UsersCsv { + <# + .SYNOPSIS + Union-merge a target Users CSV with the current run's freshly-emitted Users CSV. + + .DESCRIPTION + Post-Python helper for -AppendUserInfo. Reads both files and writes a single union + CSV. By default the union is written in place of the current-run CSV (legacy + behavior). Pass -OutputPath to direct the union to a third path so the current-run + CSV remains pristine (the pristine-raw separation pattern used by -AppendUserInfo). + Semantics: + - Retained users (in BOTH): keep target's UserKey + Date_Added; In_Latest_Append=TRUE. + - New users (only in current run): mint Date_Added=today; In_Latest_Append=TRUE. + - Departed users (only in target): carry target row forward; In_Latest_Append=FALSE. + - Latest_Append_Date stamped today on every row. + - TotalEmployees recomputed across the union and rewritten on every row. + + Atomic rewrite: writes to '.merging' then renames over the output path. + + .OUTPUTS + Hashtable with stats (Retained / New / Departed / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUsersCsv, + [Parameter(Mandatory)] [string] $CurrentUsersCsv, + [Parameter()] [string] $OutputPath, + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd') + ) + if (-not (Test-Path -LiteralPath $CurrentUsersCsv -PathType Leaf)) { + throw "Merge-UsersCsv: current Users CSV not found: '$CurrentUsersCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentUsersCsv } + # Use script:Import-CsvDeduped instead of Import-Csv: a round-tripped Users CSV + # can contain duplicate header columns (e.g. both pre-rename 'userPrincipalName' + # and post-rename 'PersonId') which crashes Import-Csv with "member already + # present". The helper rewrites the header to dedupe (suffixing later dupes + # with _dup2/_dup3) before parsing. + $targetRows = @() + if (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf) { + $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetUsersCsv) + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetUsersCsv).Length -gt 0)) { + throw "Merge-UsersCsv: target '$TargetUsersCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + # Schema-narrowing warning. The only header we strictly require on the target + # is the dedup key (PersonId_Normalized) — without it the union cannot + # correctly classify Retained vs New vs Departed. Display / enrichment + # columns (userPrincipalName, mail, id, displayName, UserKey) are intentionally + # renamed or surrogate-replaced by the rollup processor (e.g. userPrincipalName + # → PersonId on the rolled-up Users CSV), so checking for them here would + # produce false-positive warnings on every rollup-shape seed. Header union + # handles legitimately absent display columns gracefully (target rows get + # blanks for current-only fields; current rows get blanks for target-only + # fields). + # + # Derive PersonId_Normalized on-the-fly when absent or empty: the raw + # 47-column EntraUsers CSV emitted by the M365 bundle path does NOT carry + # this column (it is added only by the CopilotInteraction Python rollup at + # L4651). Without derivation, both target and current dedup indexes stay + # empty on M365-bundle append runs and the seed content is silently dropped + # from the union. Fallback chain: PersonId_Normalized -> userPrincipalName + # -> PersonId, each lower+trim (matches the Python normalization exactly). + $_normPid = { + param($row) + $v = $row.PSObject.Properties['PersonId_Normalized'] + if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } + $v = $row.PSObject.Properties['userPrincipalName'] + if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } + $v = $row.PSObject.Properties['PersonId'] + if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } + return '' + } + $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentUsersCsv) + + $targetByPid = @{} + $targetDerivedCnt = 0 + foreach ($r in $targetRows) { + $personId = & $_normPid $r + if ([string]::IsNullOrWhiteSpace($personId)) { continue } + $existing = $r.PSObject.Properties['PersonId_Normalized'] + if (-not $existing -or [string]::IsNullOrWhiteSpace([string]$existing.Value)) { + $r | Add-Member -NotePropertyName 'PersonId_Normalized' -NotePropertyValue $personId -Force + $targetDerivedCnt++ + } + if (-not $targetByPid.ContainsKey($personId)) { $targetByPid[$personId] = $r } + } + $currentByPid = @{} + $currentDerivedCnt = 0 + foreach ($r in $currentRows) { + $personId = & $_normPid $r + if ([string]::IsNullOrWhiteSpace($personId)) { continue } + $existing = $r.PSObject.Properties['PersonId_Normalized'] + if (-not $existing -or [string]::IsNullOrWhiteSpace([string]$existing.Value)) { + $r | Add-Member -NotePropertyName 'PersonId_Normalized' -NotePropertyValue $personId -Force + $currentDerivedCnt++ + } + if (-not $currentByPid.ContainsKey($personId)) { $currentByPid[$personId] = $r } + } + # Surface the derivation result so the run log is auditable. The warning + # only fires now when derivation truly failed for every target row (no + # usable dedup field anywhere on the seed) — in that case the union really + # does fall back to current-run-only and the operator needs to know. + if ($targetRows.Count -gt 0 -and $targetByPid.Count -eq 0) { + Write-LogHost ( + ("WARNING: Merge-UsersCsv: target Users CSV has no derivable dedup key (PersonId_Normalized / userPrincipalName / PersonId are all empty on every row). " + + "Cannot classify Retained / New / Departed; the union will contain only the current-run rows. " + + "Target: {0}") -f $TargetUsersCsv + ) -ForegroundColor Yellow + } + elseif ($targetDerivedCnt -gt 0 -or $currentDerivedCnt -gt 0) { + Write-LogHost ( + "Merge-UsersCsv: derived PersonId_Normalized on the fly (target rows: {0}, current rows: {1}) — column was absent on the seed/raw shape (expected for M365-bundle EntraUsers CSVs)." -f $targetDerivedCnt, $currentDerivedCnt + ) -ForegroundColor Cyan + } + + # Header union: preserve current-run order, then append target-only columns, then provenance. + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + if ($currentRows.Count -gt 0) { + foreach ($p in $currentRows[0].PSObject.Properties) { + if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + } + } + if ($targetRows.Count -gt 0) { + foreach ($p in $targetRows[0].PSObject.Properties) { + if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + } + } + foreach ($c in @('Date_Added','Latest_Append_Date','In_Latest_Append')) { + if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } + } + + $merged = New-Object System.Collections.Generic.List[psobject] + + # 1. Current-run rows (retained + new). + foreach ($r in $currentRows) { + $personId = $r.PersonId_Normalized + $obj = [ordered]@{} + foreach ($c in $hdrOrder) { $obj[$c] = '' } + foreach ($p in $r.PSObject.Properties) { $obj[$p.Name] = $p.Value } + + if ($personId -and $targetByPid.ContainsKey($personId)) { + $tr = $targetByPid[$personId] + # Target's UserKey wins (continuity across runs). + if ($tr.UserKey -and -not [string]::IsNullOrWhiteSpace([string]$tr.UserKey)) { + $obj['UserKey'] = $tr.UserKey + } + $tda = $tr.PSObject.Properties['Date_Added'] + if ($tda -and -not [string]::IsNullOrWhiteSpace([string]$tda.Value)) { + $obj['Date_Added'] = $tda.Value + } else { + # Target lacked Date_Added — first-seen today. + $obj['Date_Added'] = $RunDate + } + } else { + $obj['Date_Added'] = $RunDate + } + $obj['Latest_Append_Date'] = $RunDate + $obj['In_Latest_Append'] = 'TRUE' + $merged.Add([pscustomobject]$obj) + } + + # 2. Departed users (in target, not in current). + $departedCount = 0 + foreach ($personId in $targetByPid.Keys) { + if ($currentByPid.ContainsKey($personId)) { continue } + $tr = $targetByPid[$personId] + $obj = [ordered]@{} + foreach ($c in $hdrOrder) { $obj[$c] = '' } + foreach ($p in $tr.PSObject.Properties) { $obj[$p.Name] = $p.Value } + $tda = $tr.PSObject.Properties['Date_Added'] + if (-not $tda -or [string]::IsNullOrWhiteSpace([string]$tda.Value)) { + $obj['Date_Added'] = $RunDate + } + $obj['Latest_Append_Date'] = $RunDate + $obj['In_Latest_Append'] = 'FALSE' + $merged.Add([pscustomobject]$obj) + $departedCount++ + } + + # 3. Recompute TotalEmployees over the union. + $unionCount = $merged.Count + if ($hdrSet.Contains('TotalEmployees')) { + foreach ($r in $merged) { $r.TotalEmployees = "$unionCount" } + } + + # 4. Atomic rewrite. + $tmpPath = "$OutputPath.merging" + $hdrArray = [string[]]$hdrOrder.ToArray() + $merged | Select-Object -Property $hdrArray | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force + + $retainedCount = 0 + $newCount = 0 + foreach ($personId in $currentByPid.Keys) { + if ($targetByPid.ContainsKey($personId)) { $retainedCount++ } else { $newCount++ } + } + + return @{ + Retained = $retainedCount + New = $newCount + Departed = $departedCount + Union = $unionCount + } +} + +function Get-FactCompositeKeyColumns { + <# + .SYNOPSIS + Return the grain-composite append dedup key column list for a rolled-up + Interactions Fact CSV, derived from its header columns. + .DESCRIPTION + FIX 1 (v1.11.12). Rolled-up fact rows FAN OUT: many rows share one + Message_Id_Raw, one per distinct grain (e.g. per AccessedResource). Cross-run + append dedup must therefore key on the FULL grain PLUS Message_Id_Raw; keying + on Message_Id_Raw alone collapses every fan-out row for a message to one and + silently discards the rest on merge. + + Grain columns use their stable, cross-run-comparable forms: the per-run INT + surrogates 'UserKey' and 'ThreadId' are replaced by the deid-consistent + normalized user identity (AIO: 'User_Id_Normalized' (FIX 5); AIBV: the + pre-existing 'Audit_UserId_Normalized') and by 'ThreadId_Raw' respectively. + The remaining names mirror the embedded processor grain (GRAIN_KEYS_AIO / + GRAIN_KEYS_AIBV) verbatim. Profile is detected from the header (AIBV carries + 'Is_Agent_Activity'). Returns @() when the header is not a recognizable fact + header (no Message_Id_Raw) so the caller falls back to its own guard. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string[]] $HeaderColumns) + + if (-not ($HeaderColumns -contains 'Message_Id_Raw')) { return @() } + $isAibv = ($HeaderColumns -contains 'Is_Agent_Activity') + $userIdCol = if ($isAibv) { 'Audit_UserId_Normalized' } else { 'User_Id_Normalized' } + $cols = [System.Collections.Generic.List[string]]::new() + $cols.Add($userIdCol) # replaces the per-run 'UserKey' INT surrogate + $cols.Add('InteractionDate') + $cols.Add('AgentId') + $cols.Add('AgentName') + $cols.Add('AppHost') + $cols.Add('Environment') + $cols.Add('License Status') + $cols.Add('Context_Type') + $cols.Add('Behavior_Category') + $cols.Add('Behavior_Enriched') + $cols.Add('AI_Model') + $cols.Add('Is_Sensitive') + $cols.Add('Autonomy_Pattern') + $cols.Add('AppIdentity_AppId') + $cols.Add('AISystemPlugin_Name') + $cols.Add('ThreadId_Raw') # replaces the per-run 'ThreadId' INT surrogate + if ($isAibv) { + $cols.Add('Is_Agent_Activity') + $cols.Add('Web_Grounded_Signal') + $cols.Add('Workflow_Action') + } + $cols.Add('Message_Id_Raw') + # Emit as a flat string[]; callers collect with @(...) (an empty return above + # unrolls to nothing, which @(...) normalizes to an empty array). + return $cols.ToArray() +} + +function Merge-FactCsv { + <# + .SYNOPSIS + Union-merge a target Interactions/Fact CSV with the current run's freshly-emitted Fact CSV. + + .DESCRIPTION + Post-Python helper for -AppendFile. Mirrors Merge-UsersCsv semantics but for the + fact CSV. Keyed on -KeyColumn (default 'Message_Id_Raw' for the rollup Fact CSV; + pass 'RecordId' for the raw audit CSV used by non-rollup -AppendFile). By default + the union is written in place of the current-run CSV (rollup semantics). Pass + -OutputPath to direct the union to a third path so the current-run CSV remains + pristine (the pristine-raw separation pattern used by non-rollup -AppendFile). + + When -KeyColumn is 'Message_Id_Raw' the target's Message_Id INT is carried + forward on retained rows (continuity across runs). For any other key column the + current-run row's values are taken verbatim (target serves only as the source + of departed rows). + + Atomic rewrite: writes to '.merging' then renames over the output path. + + .OUTPUTS + Hashtable with stats (Retained / New / Departed / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $CurrentFactCsv, + [Parameter()] [string] $OutputPath, + [Parameter()] [string] $KeyColumn = 'Message_Id_Raw', + [Parameter()] [string[]] $CompositeKeyColumn = @(), + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd') + ) + if (-not (Test-Path -LiteralPath $CurrentFactCsv -PathType Leaf)) { + throw "Merge-FactCsv: current Fact CSV not found: '$CurrentFactCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentFactCsv } + # Grain-composite dedup key. When -CompositeKeyColumn is + # supplied, a row's key is the U+001F-joined values of those columns (ASCII Unit + # Separator 0x1F is a control char that never appears in audit field data), so + # fan-out fact rows (many per Message_Id_Raw) are each keyed distinctly instead of + # collapsing to one. Otherwise the single -KeyColumn is used (RecordId raw-audit + # path, unchanged). $getRowKey is invoked for every target/current row below. + $useCompositeKey = ($null -ne $CompositeKeyColumn -and @($CompositeKeyColumn).Count -gt 0) + $compositeSep = [char]0x1F + $getRowKey = { + param($row) + if ($useCompositeKey) { + $parts = foreach ($c in $CompositeKeyColumn) { + $pp = $row.PSObject.Properties[$c] + if ($pp) { [string]$pp.Value } else { '' } + } + return ($parts -join $compositeSep) + } + $kp = $row.PSObject.Properties[$KeyColumn] + if ($kp) { [string]$kp.Value } else { '' } + } + # Use script:Import-CsvDeduped for header-dupe safety (parallel to Merge-UsersCsv). + $targetRows = @() + if (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) { + $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetFactCsv) + } + # APPEND SAFETY (data-loss guard): if the target file exists with content but parsed to + # zero rows, the read failed (parse/encoding/memory) or the key cannot be matched. + # Overwriting would replace the existing target file with current-run rows only (a + # row-count shrink). Abort instead so the existing target is left untouched; the caller keeps the + # fresh current-run CSV on disk for a manual merge. + if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetFactCsv).Length -gt 0)) { + throw "Merge-FactCsv: target '$TargetFactCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data). Verify the file's schema/key column ('$KeyColumn')." + } + # Schema-narrowing warning. The only header we strictly require on the target + # is the dedup key (-KeyColumn) — without it the union cannot correctly + # classify Retained vs New vs Departed. Display / enrichment columns + # (PersonId_Normalized, ThreadId, Message_Id, etc.) are intentionally renamed + # or surrogate-replaced by the rollup processor on the rolled-up Fact CSV, so + # checking for them here would produce false-positive warnings on every + # rollup-shape seed. Header union handles legitimately absent display columns + # gracefully (target rows get blanks for current-only fields; current rows + # get blanks for target-only fields). + if ($targetRows.Count -gt 0) { + $targetHeaders = @($targetRows[0].PSObject.Properties.Name) + $requiredKeyCols = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + $missingKeyCols = @($requiredKeyCols | Where-Object { $_ -notin $targetHeaders }) + if ($missingKeyCols.Count -gt 0) { + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column(s) '{0}'. " + + "Cannot classify Retained / New / Departed rows reliably; treating ALL current-run rows as New. " + + "Target: {1}") -f ($missingKeyCols -join ', '), $TargetFactCsv + ) -ForegroundColor Yellow + } + } + $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentFactCsv) + + $targetByKey = @{} + foreach ($r in $targetRows) { + $k = [string](& $getRowKey $r) + if (-not [string]::IsNullOrWhiteSpace($k) -and -not $targetByKey.ContainsKey($k)) { + $targetByKey[$k] = $r + } + } + $currentByKey = @{} + foreach ($r in $currentRows) { + $k = [string](& $getRowKey $r) + if (-not [string]::IsNullOrWhiteSpace($k) -and -not $currentByKey.ContainsKey($k)) { + $currentByKey[$k] = $r + } + } + + # Header union: preserve current-run order, then append target-only columns, then provenance. + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + if ($currentRows.Count -gt 0) { + foreach ($p in $currentRows[0].PSObject.Properties) { + if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + } + } + if ($targetRows.Count -gt 0) { + foreach ($p in $targetRows[0].PSObject.Properties) { + if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + } + } + foreach ($c in @('Date_Added','Latest_Append_Date','In_Latest_Append')) { + if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } + } + + $merged = New-Object System.Collections.Generic.List[psobject] + + # 1. Current-run rows (retained + new). + foreach ($r in $currentRows) { + $k = [string](& $getRowKey $r) + $obj = [ordered]@{} + foreach ($c in $hdrOrder) { $obj[$c] = '' } + foreach ($p in $r.PSObject.Properties) { $obj[$p.Name] = $p.Value } + + if ($k -and $targetByKey.ContainsKey($k)) { + $tr = $targetByKey[$k] + # When dedup-keyed on Message_Id_Raw (single or as part of the grain- + # composite key), target's Message_Id INT wins (continuity across runs; + # embedded Python seed-mid-map normally aligns this, but enforce here so a + # seed-prep failure still produces a continuous union). For other single key + # columns (e.g. RecordId on the raw audit CSV) there is no surrogate-INT + # continuity contract. + if ($KeyColumn -eq 'Message_Id_Raw' -or ($useCompositeKey -and (@($CompositeKeyColumn) -contains 'Message_Id_Raw'))) { + $trMid = $tr.PSObject.Properties['Message_Id'] + if ($trMid -and -not [string]::IsNullOrWhiteSpace([string]$trMid.Value)) { + $obj['Message_Id'] = $trMid.Value + } + } + $tda = $tr.PSObject.Properties['Date_Added'] + if ($tda -and -not [string]::IsNullOrWhiteSpace([string]$tda.Value)) { + $obj['Date_Added'] = $tda.Value + } else { + $obj['Date_Added'] = $RunDate + } + } else { + $obj['Date_Added'] = $RunDate + } + $obj['Latest_Append_Date'] = $RunDate + $obj['In_Latest_Append'] = 'TRUE' + $merged.Add([pscustomobject]$obj) + } + + # 2. Departed rows (in target, not in current). + $departedCount = 0 + foreach ($k in $targetByKey.Keys) { + if ($currentByKey.ContainsKey($k)) { continue } + $tr = $targetByKey[$k] + $obj = [ordered]@{} + foreach ($c in $hdrOrder) { $obj[$c] = '' } + foreach ($p in $tr.PSObject.Properties) { $obj[$p.Name] = $p.Value } + $tda = $tr.PSObject.Properties['Date_Added'] + if (-not $tda -or [string]::IsNullOrWhiteSpace([string]$tda.Value)) { + $obj['Date_Added'] = $RunDate + } + $obj['Latest_Append_Date'] = $RunDate + $obj['In_Latest_Append'] = 'FALSE' + $merged.Add([pscustomobject]$obj) + $departedCount++ + } + + # 3. Atomic rewrite. + $unionCount = $merged.Count + # APPEND SAFETY (shrink guard): a union must never be smaller than the existing target. + # If the target keyed but the dedup column did not match (Retained+Departed=0 while the + # target had rows), the union collapses to current-only rows and the file shrinks. Abort + # before any write so the target is preserved. + if ($targetRows.Count -gt 0 -and $unionCount -lt $targetRows.Count) { + throw ("Merge-FactCsv: refusing to write a smaller file than the target (union={0} < target={1}); key '{2}' likely did not match. Target left unchanged." -f $unionCount, $targetRows.Count, $KeyColumn) + } + $tmpPath = "$OutputPath.merging" + $hdrArray = [string[]]$hdrOrder.ToArray() + $merged | Select-Object -Property $hdrArray | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force + + $retainedCount = 0 + $newCount = 0 + foreach ($k in $currentByKey.Keys) { + if ($targetByKey.ContainsKey($k)) { $retainedCount++ } else { $newCount++ } + } + + return @{ + Retained = $retainedCount + New = $newCount + Departed = $departedCount + Union = $unionCount + } +} + +function Merge-M365RollupCsv { + <# + .SYNOPSIS + Union-merge a target M365Bundle rollup CSV with the current run's freshly-emitted rollup. + + .DESCRIPTION + Post-Python helper for -AppendFile in M365Bundle mode. Performs a 9-tuple-keyed + union-merge with additive counter semantics: + + Composite key : (lower(UserId), CreationDate, Operation, Workload, + lower(SourceFileExtension), AppHost, AgentId, + AgentName, ContextType) + EventCount : target + current + ItemsAccessedCount : target + current + CreationTime : min(target, current) (lexicographic on ISO-8601) + MaxCreationTime : max(target, current) + IsAgentInteraction : OR (any TRUE -> TRUE) + UserId : target's casing wins on retained groups (first-seen) + + Schema tolerance: target may be a legacy 9- or 10-column rollup. Missing + columns are padded with empty strings; IsAgentInteraction defaults to FALSE. + + Atomic rewrite: writes to '.merging' and renames over -OutputPath. + -OutputPath defaults to -TargetRollupCsv (in-place append semantics). + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetRollupCsv, + [Parameter(Mandatory)] [string] $CurrentRollupCsv, + [Parameter()] [string] $OutputPath + ) + + if (-not (Test-Path -LiteralPath $CurrentRollupCsv -PathType Leaf)) { + throw "Merge-M365RollupCsv: current rollup CSV not found: '$CurrentRollupCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $TargetRollupCsv } + + # Canonical 14-column rollup header (order is contractual for downstream sidecar regen). + $rollupHeader = @( + 'UserId','CreationDate','Operation','Workload','SourceFileExtension', + 'AppHost','EventCount','ItemsAccessedCount','CreationTime','MaxCreationTime', + 'AgentId','AgentName','ContextType','IsAgentInteraction' + ) + + # Project an incoming row onto the 14 canonical columns. Missing columns + # become '' (except IsAgentInteraction which defaults to 'FALSE' so legacy + # 9/10-col targets contribute correctly to the OR aggregate). + $projectRow = { + param($row) + $obj = [ordered]@{} + foreach ($c in $rollupHeader) { + $p = $row.PSObject.Properties[$c] + $obj[$c] = if ($p) { [string]$p.Value } else { '' } + } + if ([string]::IsNullOrWhiteSpace([string]$obj['IsAgentInteraction'])) { + $obj['IsAgentInteraction'] = 'FALSE' + } + return $obj + } + + # Composite 9-tuple key (case-folded UserId + SourceFileExtension). + $makeKey = { + param($obj) + $userLc = ([string]$obj['UserId']).ToLowerInvariant() + $extLc = ([string]$obj['SourceFileExtension']).ToLowerInvariant() + '{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}' -f ` + $userLc, $obj['CreationDate'], $obj['Operation'], $obj['Workload'], ` + $extLc, $obj['AppHost'], $obj['AgentId'], $obj['AgentName'], $obj['ContextType'] + } + + # Accumulate one (already-projected) row into an existing bucket entry. + $accumulateInto = { + param($acc, $obj) + $accEC = 0; [void][int]::TryParse([string]$acc['EventCount'], [ref]$accEC) + $rEC = 0; [void][int]::TryParse([string]$obj['EventCount'], [ref]$rEC) + $accIA = 0; [void][int]::TryParse([string]$acc['ItemsAccessedCount'], [ref]$accIA) + $rIA = 0; [void][int]::TryParse([string]$obj['ItemsAccessedCount'], [ref]$rIA) + $acc['EventCount'] = ($accEC + $rEC).ToString() + $acc['ItemsAccessedCount'] = ($accIA + $rIA).ToString() + $rCT = [string]$obj['CreationTime'] + $rMCT = [string]$obj['MaxCreationTime'] + if (-not [string]::IsNullOrWhiteSpace($rCT)) { + $accCT = [string]$acc['CreationTime'] + if ([string]::IsNullOrWhiteSpace($accCT) -or ($rCT -lt $accCT)) { $acc['CreationTime'] = $rCT } + } + if (-not [string]::IsNullOrWhiteSpace($rMCT)) { + $accMCT = [string]$acc['MaxCreationTime'] + if ([string]::IsNullOrWhiteSpace($accMCT) -or ($rMCT -gt $accMCT)) { $acc['MaxCreationTime'] = $rMCT } + } + if (([string]$obj['IsAgentInteraction']).Equals('TRUE', [System.StringComparison]::OrdinalIgnoreCase)) { + $acc['IsAgentInteraction'] = 'TRUE' + } + } + + # Load inputs. + $targetRows = @() + if (Test-Path -LiteralPath $TargetRollupCsv -PathType Leaf) { + $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetRollupCsv) + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetRollupCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetRollupCsv).Length -gt 0)) { + throw "Merge-M365RollupCsv: target '$TargetRollupCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentRollupCsv) + + # Schema-narrowing warning for legacy 9/10-col targets. + if ($targetRows.Count -gt 0) { + $targetHeaders = @($targetRows[0].PSObject.Properties.Name) + $missing = @($rollupHeader | Where-Object { $_ -notin $targetHeaders }) + if ($missing.Count -gt 0) { + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365RollupCsv: target rollup is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; IsAgentInteraction defaults to FALSE. " + + "Target: {3}") -f $missing.Count, $rollupHeader.Count, ($missing -join ','), $TargetRollupCsv + ) -ForegroundColor Yellow + } + } + + # Bucket: composite key -> accumulator ordered hashtable. + # Seed target rows first so the target's UserId casing wins on retained groups. + $bucket = [ordered]@{} + foreach ($r in $targetRows) { + $obj = & $projectRow $r + $k = & $makeKey $obj + if ($bucket.Contains($k)) { + & $accumulateInto $bucket[$k] $obj + } else { + $bucket[$k] = $obj + } + } + + # Snapshot keys present after target-seed (to classify Retained vs New for current rows). + $targetKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($k in $bucket.Keys) { [void]$targetKeys.Add($k) } + + # Merge in current-run rows. + $newCount = 0 + $retainedCount = 0 + $updatedCount = 0 + foreach ($r in $currentRows) { + $obj = & $projectRow $r + $k = & $makeKey $obj + if ($bucket.Contains($k)) { + & $accumulateInto $bucket[$k] $obj + $updatedCount++ + if ($targetKeys.Contains($k)) { $retainedCount++ } + } else { + $bucket[$k] = $obj + $newCount++ + } + } + + # Atomic rewrite: write to .merging, rename over $OutputPath. + $unionCount = $bucket.Count + $tmpPath = "$OutputPath.merging" + [pscustomobject[]]$mergedRows = $bucket.Values | ForEach-Object { [pscustomobject]$_ } + $mergedRows | Select-Object -Property $rollupHeader | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force + + return [pscustomobject]@{ + Retained = $retainedCount + New = $newCount + Updated = $updatedCount + Union = $unionCount + } +} + +function Merge-M365SessionStatsCsv { + <# + .SYNOPSIS + Union-merge a target M365Bundle SessionStats CSV with the current run's freshly-emitted SessionStats. + + .DESCRIPTION + Post-Python helper for -AppendFile in M365Bundle mode. Performs a 3-tuple-keyed + union-merge with additive counter semantics on the SessionStats sidecar (v2.6.0+): + + Composite key : (lower(UserId), CreationDate, AppHost) + SessionCount : target + current + PromptCount : target + current + AgentPromptCount : target + current + ResponseCount : target + current + AgentSessionCount : target + current + UserId : target's casing wins on retained groups (first-seen) + + Schema tolerance: target may be missing one or more counter columns (e.g. a + pre-2.6.0 placeholder). Missing counters are treated as 0 during accumulation + and emitted as the union value. + + Atomic rewrite: writes to '.merging' and renames over -OutputPath. + -OutputPath defaults to -TargetSessionStatsCsv (in-place append semantics). + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetSessionStatsCsv, + [Parameter(Mandatory)] [string] $CurrentSessionStatsCsv, + [Parameter()] [string] $OutputPath + ) + + if (-not (Test-Path -LiteralPath $CurrentSessionStatsCsv -PathType Leaf)) { + throw "Merge-M365SessionStatsCsv: current SessionStats CSV not found: '$CurrentSessionStatsCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $TargetSessionStatsCsv } + + # Canonical 8-column SessionStats header (matches SESSIONSTATS_HEADER in the embedded processor). + $ssHeader = @( + 'UserId','CreationDate','AppHost', + 'SessionCount','PromptCount','AgentPromptCount','ResponseCount','AgentSessionCount' + ) + + # Project an incoming row onto the 8 canonical columns. Missing columns become ''. + $projectRow = { + param($row) + $obj = [ordered]@{} + foreach ($c in $ssHeader) { + $p = $row.PSObject.Properties[$c] + $obj[$c] = if ($p) { [string]$p.Value } else { '' } + } + return $obj + } + + # Composite 3-tuple key (case-folded UserId). + $makeKey = { + param($obj) + $userLc = ([string]$obj['UserId']).ToLowerInvariant() + '{0}|{1}|{2}' -f $userLc, $obj['CreationDate'], $obj['AppHost'] + } + + # Accumulate one (already-projected) row into an existing bucket entry. + $accumulateInto = { + param($acc, $obj) + foreach ($col in @('SessionCount','PromptCount','AgentPromptCount','ResponseCount','AgentSessionCount')) { + $accV = 0; [void][int]::TryParse([string]$acc[$col], [ref]$accV) + $rV = 0; [void][int]::TryParse([string]$obj[$col], [ref]$rV) + $acc[$col] = ($accV + $rV).ToString() + } + } + + # Load inputs. + $targetRows = @() + if (Test-Path -LiteralPath $TargetSessionStatsCsv -PathType Leaf) { + $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetSessionStatsCsv) + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetSessionStatsCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetSessionStatsCsv).Length -gt 0)) { + throw "Merge-M365SessionStatsCsv: target '$TargetSessionStatsCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentSessionStatsCsv) + + # Schema-narrowing warning for legacy targets missing canonical columns. + if ($targetRows.Count -gt 0) { + $targetHeaders = @($targetRows[0].PSObject.Properties.Name) + $missing = @($ssHeader | Where-Object { $_ -notin $targetHeaders }) + if ($missing.Count -gt 0) { + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365SessionStatsCsv: target SessionStats is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; missing counters treated as 0. Target: {3}") ` + -f $missing.Count, $ssHeader.Count, ($missing -join ','), $TargetSessionStatsCsv + ) -ForegroundColor Yellow + } + } + + # Bucket: composite key -> accumulator ordered hashtable. + # Seed target rows first so the target's UserId casing wins on retained groups. + $bucket = [ordered]@{} + foreach ($r in $targetRows) { + $obj = & $projectRow $r + $k = & $makeKey $obj + if ($bucket.Contains($k)) { + & $accumulateInto $bucket[$k] $obj + } else { + $bucket[$k] = $obj + } + } + + # Snapshot keys present after target-seed (to classify Retained vs New for current rows). + $targetKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($k in $bucket.Keys) { [void]$targetKeys.Add($k) } + + # Merge in current-run rows. + $newCount = 0 + $retainedCount = 0 + $updatedCount = 0 + foreach ($r in $currentRows) { + $obj = & $projectRow $r + $k = & $makeKey $obj + if ($bucket.Contains($k)) { + & $accumulateInto $bucket[$k] $obj + $updatedCount++ + if ($targetKeys.Contains($k)) { $retainedCount++ } + } else { + $bucket[$k] = $obj + $newCount++ + } + } + + # Atomic rewrite: write to .merging, rename over $OutputPath. + $unionCount = $bucket.Count + $tmpPath = "$OutputPath.merging" + [pscustomobject[]]$mergedRows = $bucket.Values | ForEach-Object { [pscustomobject]$_ } + $mergedRows | Select-Object -Property $ssHeader | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force + + return [pscustomobject]@{ + Retained = $retainedCount + New = $newCount + Updated = $updatedCount + Union = $unionCount + } +} + +function Convert-CsvToDelta { + <# + .SYNOPSIS + Convert a local CSV file to a Delta Lake table at a OneLake DFS URI (or local path). + + .DESCRIPTION + Invokes Python with the 'deltalake' package to read the CSV with pyarrow and write + it as a Delta table at the destination. Used by the rollup post-processor to land + Fabric Lakehouse Tables/ outputs after the embedded Python processor has written + the canonical CSV. + + Supports two destination forms: + • Local path (file:// or absolute path) — used for unit testing and for the + pre-publish stage when -OutputPath resolves to a Fabric OneLake URL. + • OneLake DFS URL (https://-onelake.dfs.fabric.microsoft.com/.../Tables/) + — used directly for Fabric Lakehouse Tables/ destinations. The caller supplies + a bearer token (OneLake storage audience). + + Schema evolution: write_deltalake is called with schema_mode='merge' so additive + column changes do not require a full overwrite. The mode itself is the caller's + choice (overwrite for first run, append for subsequent appends). + + .PARAMETER InputCsv + Local path to the source CSV. Must exist and be UTF-8 encoded. + + .PARAMETER TargetUri + Destination Delta table URI. Either a local absolute path or a OneLake DFS https URL. + + .PARAMETER Mode + Delta write mode: 'overwrite' or 'append'. Defaults to 'overwrite'. + + .PARAMETER BearerToken + Optional OneLake storage-audience access token. Required when TargetUri is a + Fabric DFS URL. Ignored for local destinations. + + .PARAMETER PythonExe + Resolved Python interpreter path (from Resolve-PythonExe). + + .PARAMETER LauncherArgs + Launcher arguments (e.g. py -3 prefix) for the resolved interpreter. + + .OUTPUTS + $true on success, $false on failure (errors are logged via Write-LogHost). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $InputCsv, + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter()] [ValidateSet('overwrite','append')] [string] $Mode = 'overwrite', + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $InputCsv -PathType Leaf)) { + Write-LogHost "Convert-CsvToDelta: input CSV not found: '$InputCsv'" -ForegroundColor Red + return $false + } + # Inline Python: pyarrow reads CSV, deltalake writes table. Storage options only + # when TargetUri is a remote OneLake URL. Column names are sanitized to satisfy + # Delta's forbidden-character set (' ,;{}()\n\t='); the source CSV on disk is + # left unchanged so PBIP semantic models that read the CSV directly continue + # to bind by the original column names. + $inlinePy = @" +import os, sys, json, re +in_csv = os.environ['PAX_DELTA_IN'] +out_uri = os.environ['PAX_DELTA_OUT'] +mode = os.environ['PAX_DELTA_MODE'] +token = os.environ.get('PAX_DELTA_TOKEN', '') +try: + import pyarrow.csv as pacsv + from deltalake import write_deltalake +except Exception as e: + print(f'Convert-CsvToDelta: import failed: {e}', file=sys.stderr) + sys.exit(2) +# Delta forbids these characters in column names; the Fabric SQL endpoint also +# rejects spaces in column names. Replace each with an underscore. Collisions +# after sanitization (e.g., two source columns differing only in a forbidden +# character) are disambiguated by appending _2, _3, ... . The disambiguated +# name is also recorded so a later raw column whose sanitized form literally +# matches an earlier disambiguated name cannot silently map to the same Delta +# column. +_DELTA_FORBIDDEN = re.compile(r'[ ,;\{\}\(\)\n\t=]') +def _sanitize_col_names(cols): + seen = {} + out = [] + for c in cols: + s = _DELTA_FORBIDDEN.sub('_', c) + if s in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + while candidate in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + s = candidate + seen[s] = 1 + else: + seen[s] = 1 + out.append(s) + return out +try: + # All-string read avoids per-column inference mismatches between runs; the + # Lakehouse semantic model casts at query time. + table = pacsv.read_csv( + in_csv, + convert_options=pacsv.ConvertOptions(strings_can_be_null=False), + ) + cols = table.column_names + safe_cols = _sanitize_col_names(cols) + import pyarrow as pa + table = pa.Table.from_arrays([table.column(c).cast(pa.string()) for c in cols], names=safe_cols) + storage_options = None + if out_uri.startswith('https://') or out_uri.startswith('abfss://'): + if not token: + print('Convert-CsvToDelta: remote target requires a bearer token', file=sys.stderr) + sys.exit(3) + storage_options = {'bearer_token': token, 'use_fabric_endpoint': 'true'} + write_deltalake( + out_uri, + table, + mode=mode, + schema_mode='merge', + storage_options=storage_options, + ) + print(f'Convert-CsvToDelta: wrote {table.num_rows} rows to {out_uri} (mode={mode})') +except Exception as e: + print(f'Convert-CsvToDelta: write failed: {e}', file=sys.stderr) + sys.exit(4) +"@ + $env:PAX_DELTA_IN = $InputCsv + $env:PAX_DELTA_OUT = $TargetUri + $env:PAX_DELTA_MODE = $Mode + $env:PAX_DELTA_TOKEN = $BearerToken + try { + $fullArgs = @($LauncherArgs) + @('-c', $inlinePy) + & $PythonExe @fullArgs 2>&1 | ForEach-Object { Write-Log $_ } + $exit = $LASTEXITCODE + } + finally { + Remove-Item Env:\PAX_DELTA_IN -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_OUT -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_MODE -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_TOKEN -ErrorAction SilentlyContinue + } + if ($exit -eq 0) { return $true } + Write-LogHost "Convert-CsvToDelta: Python exited $exit (input='$InputCsv', target='$TargetUri', mode=$Mode)." -ForegroundColor Red + return $false +} + +function Test-DeltaTableSchemaCompat { + <# + .SYNOPSIS + Pre-flight check: confirm an existing Delta table at TargetUri has a column set + compatible with the CSV the caller is about to convert. + + .DESCRIPTION + Reads the Delta table's schema (column names only — types are all strings in our + write path) and compares against the CSV header. Compatible = every existing-table + column is present in the new CSV (schema_mode='merge' will additively expand it). + Incompatible = at least one existing column is absent from the new CSV (would + require a destructive overwrite-with-different-schema). + + Returns a hashtable: { Compatible=, ExistingCols=[], NewCols=[], Missing=[] }. + When the Delta table does not yet exist, returns Compatible=$true with empty arrays. + + .PARAMETER TargetUri + Delta table URI (local path or OneLake DFS URL). + + .PARAMETER NewCsv + Local CSV whose header is the proposed new schema. + + .PARAMETER BearerToken + Optional OneLake storage-audience token for remote URIs. + + .PARAMETER PythonExe + Resolved Python interpreter. + + .PARAMETER LauncherArgs + Launcher prefix args. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter(Mandatory)] [string] $NewCsv, + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $NewCsv -PathType Leaf)) { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = "New CSV not found: '$NewCsv'" } + } + $inlinePy = @" +import os, sys, json, csv, re +target = os.environ['PAX_DELTA_OUT'] +new_csv = os.environ['PAX_DELTA_IN'] +token = os.environ.get('PAX_DELTA_TOKEN', '') +try: + from deltalake import DeltaTable +except Exception as e: + print(json.dumps({'error': f'import: {e}'})) + sys.exit(0) +# Must match the sanitization applied by Convert-CsvToDelta — otherwise the +# compat probe would compare sanitized stored names against unsanitized CSV +# headers and always report drift. +_DELTA_FORBIDDEN = re.compile(r'[ ,;\{\}\(\)\n\t=]') +def _sanitize_col_names(cols): + seen = {} + out = [] + for c in cols: + s = _DELTA_FORBIDDEN.sub('_', c) + if s in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + while candidate in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + s = candidate + seen[s] = 1 + else: + seen[s] = 1 + out.append(s) + return out +try: + with open(new_csv, 'r', encoding='utf-8-sig', newline='') as f: + rdr = csv.reader(f) + new_cols = _sanitize_col_names(next(rdr, [])) +except Exception as e: + print(json.dumps({'error': f'csv-header: {e}'})) + sys.exit(0) +storage_options = None +if target.startswith('https://') or target.startswith('abfss://'): + if token: + storage_options = {'bearer_token': token, 'use_fabric_endpoint': 'true'} +try: + dt = DeltaTable(target, storage_options=storage_options) + existing = [f.name for f in dt.schema().fields] +except Exception: + print(json.dumps({'existing': [], 'new': new_cols, 'exists': False})) + sys.exit(0) +print(json.dumps({'existing': existing, 'new': new_cols, 'exists': True})) +"@ + $env:PAX_DELTA_OUT = $TargetUri + $env:PAX_DELTA_IN = $NewCsv + $env:PAX_DELTA_TOKEN = $BearerToken + $outJson = '' + try { + $fullArgs = @($LauncherArgs) + @('-c', $inlinePy) + $outJson = (& $PythonExe @fullArgs 2>&1 | Out-String).Trim() + } + finally { + Remove-Item Env:\PAX_DELTA_OUT -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_IN -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_TOKEN -ErrorAction SilentlyContinue + } + try { + $parsed = $outJson | ConvertFrom-Json -ErrorAction Stop + } catch { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = "Could not parse schema probe output: $outJson" } + } + if ($parsed.PSObject.Properties['error']) { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = [string]$parsed.error } + } + $existing = @($parsed.existing) + $newCols = @($parsed.new) + if (-not $parsed.exists) { + return @{ Compatible = $true; ExistingCols = $existing; NewCols = $newCols; Missing = @() } + } + $newSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$newCols, [System.StringComparer]::OrdinalIgnoreCase) + $missing = @($existing | Where-Object { -not $newSet.Contains($_) }) + $compatible = ($missing.Count -eq 0) + return @{ Compatible = $compatible; ExistingCols = $existing; NewCols = $newCols; Missing = $missing } +} + +function Write-DeltaAppend { + <# + .SYNOPSIS + Append a local CSV to a Delta table (Fabric Lakehouse Tables/ or local), creating + the table if it does not yet exist. Hard-errors on destructive schema drift. + + .DESCRIPTION + Thin wrapper around Test-DeltaTableSchemaCompat + Convert-CsvToDelta: + 1. Probe the target Delta schema. If unreachable due to nonexistent table, + fall through to Convert-CsvToDelta with mode='append' (deltalake creates + the table on first write). The returned object's IsInit flag is set so + callers can label the banner '[append-init]'. + 2. If the table exists, refuse the write when any existing column is absent + from the new CSV header (column removal / type narrowing). Additive new + columns are absorbed via schema_mode='merge'. + 3. On compatible schema, invoke Convert-CsvToDelta -Mode append. + + .OUTPUTS + Hashtable: { Success=, IsInit=, AddedCols=[], Missing=[], Error= }. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $InputCsv, + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $InputCsv -PathType Leaf)) { + return @{ Success = $false; IsInit = $false; AddedCols = @(); Missing = @(); Error = "Input CSV not found: '$InputCsv'" } + } + $probe = Test-DeltaTableSchemaCompat -TargetUri $TargetUri -NewCsv $InputCsv -BearerToken $BearerToken -PythonExe $PythonExe -LauncherArgs $LauncherArgs + $isInit = (-not ($probe.ExistingCols -and @($probe.ExistingCols).Count -gt 0)) + if (-not $probe.Compatible -and -not $isInit) { + $missingList = ($probe.Missing -join ', ') + $msg = "Write-DeltaAppend: destructive schema drift rejected. Existing column(s) not present in source CSV: $missingList" + Write-LogHost $msg -ForegroundColor Red + return @{ Success = $false; IsInit = $false; AddedCols = @(); Missing = @($probe.Missing); Error = $msg } + } + $existingSet = if ($probe.ExistingCols) { [System.Collections.Generic.HashSet[string]]::new([string[]]@($probe.ExistingCols), [System.StringComparer]::OrdinalIgnoreCase) } else { [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) } + $addedCols = @(@($probe.NewCols) | Where-Object { -not $existingSet.Contains($_) }) + $ok = Convert-CsvToDelta -InputCsv $InputCsv -TargetUri $TargetUri -Mode 'append' -BearerToken $BearerToken -PythonExe $PythonExe -LauncherArgs $LauncherArgs + if (-not $ok) { + return @{ Success = $false; IsInit = $isInit; AddedCols = $addedCols; Missing = @(); Error = 'Convert-CsvToDelta returned failure.' } + } + $tag = if ($isInit) { '[append-init]' } else { '[append]' } + $colNote = if ($addedCols.Count -gt 0) { (" (added columns: {0})" -f ($addedCols -join ', ')) } else { '' } + Write-LogHost "Write-DeltaAppend: $tag '$TargetUri'$colNote" -ForegroundColor Green + return @{ Success = $true; IsInit = $isInit; AddedCols = $addedCols; Missing = @() } +} + +# ============================================================================ +# REMOTE OUTPUT HELPERS — SharePoint (Graph drives API) and Fabric (OneLake DFS) +# ---------------------------------------------------------------------------- +# All helpers are no-ops when $script:RemoteOutputMode -eq 'None'. They are +# invoked from a single dispatch point: Invoke-OutputUpload. +# +# SharePoint path: +# - Reuses the existing Connect-MgGraph token (Graph audience). +# - Resolves the SP URL to siteId/driveId/folder via Graph site lookup. +# - Files <= 250 MB: PUT /drives/{driveId}/root:/path:/content (Graph simple-upload max) +# - Larger files: createUploadSession + 5 MB chunked PUT (must be 320 KB-aligned). +# +# Fabric/OneLake path: +# - Uses a SEPARATE token with audience https://storage.azure.com/ (Az.Accounts). +# - DFS Create -> Append (chunked) -> Flush. Path = //.Lakehouse/Files/ +# ============================================================================ + +function Get-DisplayPath { + <# + .SYNOPSIS + Returns the user-visible path to display for an output artifact, accounting for + a SharePoint or OneLake -OutputPath* value. Local-output runs see the local path unchanged; + remote-output runs see the destination URL the file is uploaded to. + + .DESCRIPTION + Every "Output File: ...", "Output Directory: ...", "Log file: ..." line in the + console / log banner should be filtered through this helper. The local scratch + path is irrelevant to a customer who passed a SharePoint or OneLake -OutputPath*; + showing it implies the artifact lives there, which it does not (it is uploaded + then, on success, the local copy is reaped). + + File mode (default): appends the local file's leaf name to the remote root URL. + -Directory mode: returns the remote root URL with no leaf appended. + + When $script:RemoteOutputMode -eq 'None' (or remote URL not yet set), the input + path is returned verbatim — making this safe to call unconditionally at every + display site. + + .PARAMETER LocalPath + The local scratch path of the artifact. Required. Can be empty/null. + + .PARAMETER Directory + If set, treats LocalPath as a directory and returns the remote root URL alone + (no leaf appended). Use for "Output Directory: " lines. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [AllowNull()] [AllowEmptyString()] [string] $LocalPath, + [switch] $Directory + ) + if ($script:RemoteOutputMode -eq 'None' -or [string]::IsNullOrWhiteSpace($LocalPath) -or [string]::IsNullOrWhiteSpace($script:RemoteOutputUrl)) { + return $LocalPath + } + $base = $script:RemoteOutputUrl.TrimEnd('/') + if ($Directory) { return $base } + $leaf = [System.IO.Path]::GetFileName($LocalPath.TrimEnd('\','/')) + if ([string]::IsNullOrWhiteSpace($leaf)) { return $base } + # Guard against URL doubling: $RemoteOutputUrl can be either a folder URL + # (the normal -OutputPath case) or already a full file URL (when promoted + # from -AppendFile or when -OutputPath was given as a file URL). The + # upstream normalizer in the param-validation block strips the file leaf + # from RemoteOutputUrl when it can recognize the extension, but this + # helper is called from many sites and must remain robust if a caller + # leaves $base as a file URL. + # + # Two doubling cases to defend against: + # 1) Same name ('.../foo.csv' + 'foo.csv' -> '.../foo.csv' ) + # 2) Sibling ('.../foo.csv' + 'foo.log' -> '.../foo.log' ) + # When $baseLeaf carries a data-artifact extension, strip it before + # appending $leaf so siblings land in the correct parent folder. + # .Lakehouse is a folder marker; do NOT strip. + $baseLeaf = ($base -split '/') | Select-Object -Last 1 + if ($baseLeaf -ieq $leaf) { return $base } + if ($baseLeaf -match '\.(csv|log|xlsx|json|parquet|txt)$') { + $base = ($base -replace '/[^/]+$','') + } + return ($base + '/' + $leaf) +} + +function Resolve-SharePointTarget { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Url) + + # Accepts any of: + # https:///sites//[/] + # https:///teams//[/] + # https:///personal//[/] + # https:///[/] (root site collection) + # where can be any sharepoint.* tenant variant (sharepoint.com, + # sharepoint-df.com, sharepoint.us, sharepoint.de, sharepoint.cn, + # sharepoint-mil.us, ...). + # + # The first one or two URL segments identify the site collection; the remainder + # is library + (optional) folder path. Graph's site-lookup endpoint is the + # authoritative resolver -- we just hand it the server-relative site path. + $u = [Uri]$Url + $hostName = $u.Host + $segments = ($u.AbsolutePath.TrimStart('/') -split '/') | Where-Object { $_ -ne '' } + if (-not $segments -or $segments.Count -lt 1) { + throw "SharePoint URL has no path; expected https://///[/] : '$Url'" + } + + # Determine where the site collection ends and the library begins. + $prefixed = @('sites','teams','personal') -contains $segments[0].ToLower() + if ($prefixed) { + if ($segments.Count -lt 3) { + throw "SharePoint URL is missing a library/folder after the site name: '$Url'" + } + $sitePathSegs = $segments[0..1] # e.g. sites/Analytics or teams/MyTeam + $libAndFolder = $segments[2..($segments.Count-1)] + $siteName = $segments[1] + } else { + # Root site collection: no /sites/ or /teams/ prefix; first segment is the library. + $sitePathSegs = @() + $libAndFolder = $segments + $siteName = '(root)' + } + $libraryName = $libAndFolder[0] + $folderInLibrary = if ($libAndFolder.Count -gt 1) { ($libAndFolder[1..($libAndFolder.Count-1)]) -join '/' } else { '' } + + # Resolve site via Graph. Site path is server-relative; if root, omit the colon-path. + if ($sitePathSegs.Count -gt 0) { + $sitePath = ($sitePathSegs -join '/') + $siteLookupUri = "https://graph.microsoft.com/v1.0/sites/${hostName}:/${sitePath}" + } else { + $siteLookupUri = "https://graph.microsoft.com/v1.0/sites/${hostName}" + } + try { + $site = Invoke-MgGraphRequest -Method GET -Uri $siteLookupUri -ErrorAction Stop + } catch { + # Classify the failure so customers see actionable guidance instead of raw Graph plumbing. + # We inspect the HTTP status, current Graph context state, and granted scopes to decide + # the most likely cause. The structured message is consumed by Test-RemoteDestination's + # error block (which formats the banner with permissions-table back-reference). + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + + $mgCtx = $null + try { $mgCtx = Get-MgContext -ErrorAction SilentlyContinue } catch {} + $hasToken = [bool]$mgCtx + $grantedScopes = @() + if ($mgCtx -and $mgCtx.Scopes) { $grantedScopes = @($mgCtx.Scopes) } + $isAppOnly = $false + if ($mgCtx) { $isAppOnly = ([string]::IsNullOrWhiteSpace($mgCtx.Account)) -or ($mgCtx.AuthType -eq 'AppOnly') } + + # Surface the raw response body for context (truncated; appended to the classified message). + $rawDetail = $_.Exception.Message + try { + if ($_.ErrorDetails -and $_.ErrorDetails.Message) { $rawDetail = $_.ErrorDetails.Message } + elseif ($_.Exception.Response) { + $resp = $_.Exception.Response + $reader = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $body = $reader.ReadToEnd(); $reader.Close() + if ($body) { $rawDetail = "HTTP $status $body" } + } + } catch {} + if ($rawDetail.Length -gt 600) { $rawDetail = $rawDetail.Substring(0, 600) + '...(truncated)' } + + # Build the classified diagnosis. Lines are joined with newline so Test-RemoteDestination + # can split and indent them under its banner. + $diagLines = New-Object System.Collections.Generic.List[string] + $diagLines.Add("SharePoint site lookup failed (host '$hostName', site path '$($sitePathSegs -join '/')').") + + if (-not $hasToken) { + $diagLines.Add('Cause: Microsoft Graph is not connected (no MgContext present).') + $diagLines.Add('Action: This is unexpected at this point in the script — please report it as a bug.') + } + elseif ($status -eq 401) { + if (-not $isAppOnly -and ($grantedScopes -notcontains 'Sites.ReadWrite.All')) { + $diagLines.Add('Cause: The Graph access token is missing the Sites.ReadWrite.All scope.') + $diagLines.Add('Action: Re-run after granting/consenting Sites.ReadWrite.All. See the') + $diagLines.Add(' "Permissions Required for THIS run" table earlier in this output') + $diagLines.Add(' for the full list of required scopes for SharePoint output.') + } else { + $diagLines.Add('Cause: Graph rejected the token (401). For app-only auth this usually means') + $diagLines.Add(' application permissions (Sites.ReadWrite.All / Files.ReadWrite.All)') + $diagLines.Add(' have not been ADMIN-CONSENTED on the app registration / managed identity.') + $diagLines.Add('Action: Have a Global Administrator grant admin consent to the application') + $diagLines.Add(' permissions listed in the "Permissions Required for THIS run" table above.') + } + } + elseif ($status -eq 403) { + $signedIn = if ($mgCtx -and $mgCtx.Account) { $mgCtx.Account } else { '(app identity)' } + $diagLines.Add("Cause: The signed-in identity ('$signedIn') is authenticated but does not have") + $diagLines.Add(' access to the target SharePoint site (HTTP 403 Forbidden).') + $diagLines.Add('Action: In SharePoint, add this identity to the target site as at least Member,') + $diagLines.Add(' OR re-run with an identity that already has access to the site.') + } + elseif ($status -eq 404) { + $diagLines.Add("Cause: The site was not found on host '$hostName' (HTTP 404).") + $diagLines.Add('Action: Verify the URL was copied via SharePoint Details -> Path -> Copy.') + $diagLines.Add(' Check spelling of the /sites/ or /teams/ segment.') + } + elseif ($status -ge 500) { + $diagLines.Add("Cause: Microsoft Graph returned a server error (HTTP $status). This is usually transient.") + $diagLines.Add('Action: Wait a minute and retry. If it persists, check https://status.cloud.microsoft.') + } + else { + # Includes name-resolution failures (no HTTP status) — typically dogfood/PPE hostnames. + if ($hostName -match 'sharepoint-(df|ppe)\.com$') { + $diagLines.Add("Cause: Host '$hostName' is a Microsoft-internal dogfood/PPE SharePoint endpoint") + $diagLines.Add(' and is not reachable from the public Microsoft Graph service. This script') + $diagLines.Add(' does not currently support dogfood/PPE SharePoint targets.') + $diagLines.Add('Action: Use a production SharePoint URL (*.sharepoint.com) instead.') + } else { + $diagLines.Add("Cause: Graph site lookup failed (HTTP status: $status).") + $diagLines.Add('Action: Verify the URL is correct and reachable, and that the signed-in identity') + $diagLines.Add(' has the required Graph scopes and SharePoint site access.') + } + } + $diagLines.Add('') + $diagLines.Add("Graph response: $rawDetail") + + throw ($diagLines -join "`n") + } + $siteId = $site.id + if (-not $siteId) { throw "Unable to resolve SharePoint site '$Url' (Graph returned no id)." } + + # Resolve drive (document library) by name + $drivesUri = "https://graph.microsoft.com/v1.0/sites/${siteId}/drives" + $drives = Invoke-MgGraphRequest -Method GET -Uri $drivesUri -ErrorAction Stop + $drive = $drives.value | Where-Object { $_.name -eq $libraryName -or $_.webUrl -match "/$libraryName(/|$)" } | Select-Object -First 1 + if (-not $drive) { + # Fallback: default drive + $drive = Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/sites/${siteId}/drive" -ErrorAction Stop + # If the URL named a non-default library, treat the entire path-after-site as folder under default drive + $folderInLibrary = ($libAndFolder -join '/') + } + + return [pscustomobject]@{ + HostName = $hostName + SiteName = $siteName + SiteId = $siteId + DriveId = $drive.id + LibraryName = $libraryName + FolderPath = $folderInLibrary.TrimEnd('/') + WebUrl = $drive.webUrl + } +} + +function Resolve-FabricTarget { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Url) + + # Accepted URL shapes (all return enough metadata for callers to route correctly): + # 1. https://onelake.dfs.fabric.microsoft.com//.Lakehouse + # (root mode - recommended; operational artifacts auto-routed under Files/, + # Delta tables under Tables/) + # 2. https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Files[/] + # (explicit Files; FilesPath = ) + # 3. https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables[/] + # (explicit Tables for table-typed destinations; TablesPath = ) + # All three shapes are accepted here; the call-site helpers + # (Send-FileToOneLake / Get-RemoteFile-OneLake) route operational artifacts + # under /Files/ regardless of TablesPath, so an -OutputPath that points at + # the lakehouse root or at an explicit /Tables/ still lands + # log / checkpoint / mirror artifacts in the correct /Files/ subtree. + $u = [Uri]$Url + $accountUrl = "{0}://{1}" -f $u.Scheme, $u.Host + $segments = ($u.AbsolutePath.TrimStart('/') -split '/' | Where-Object { $_ -ne '' }) + # Force into an array shape so .Count is reliable even for 1- or 0-element results. + $segments = @($segments) + if ($segments.Count -lt 2) { + throw "Fabric URL must include workspace and item: '$Url'" + } + $workspace = $segments[0] + $itemFull = $segments[1] # e.g. MyLakehouse.Lakehouse OR an item GUID + $itemGuidPattern = '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + if ($itemFull -match '\.(Lakehouse|Warehouse)$') { + $itemType = $Matches[1] + $itemName = $itemFull.Substring(0, $itemFull.Length - ($itemType.Length + 1)) + } + elseif ($itemFull -match $itemGuidPattern) { + # GUID-addressed item: OneLake references the item by its GUID with no type + # suffix (https://onelake.dfs.fabric.microsoft.com///...). + # The DFS path is type-agnostic; PAX only emits Lakehouse-shaped output, so + # record the type as Lakehouse for display/diagnostics. ItemFull stays the + # GUID verbatim so every downstream path ("$FilesystemBase/$ItemFull/...") + # builds the correct GUID-form DFS URL. + $itemType = 'Lakehouse' + $itemName = $itemFull + } + else { + throw "Fabric item must end with .Lakehouse/.Warehouse, or be an item GUID: '$itemFull'" + } + + $filesPath = '' + $tablesPath = '' + $rootMode = $false + if ($segments.Count -eq 2) { + $rootMode = $true + } + elseif ($segments[2] -eq 'Files') { + $filesPath = if ($segments.Count -gt 3) { ($segments | Select-Object -Skip 3) -join '/' } else { '' } + } + elseif ($segments[2] -eq 'Tables') { + $tablesPath = if ($segments.Count -gt 3) { ($segments | Select-Object -Skip 3) -join '/' } else { '' } + } + else { + throw "Fabric URL third segment must be 'Files', 'Tables', or absent (lakehouse root): '$Url'" + } + + return [pscustomobject]@{ + AccountUrl = $accountUrl + Workspace = $workspace + ItemName = $itemName + ItemType = $itemType + ItemFull = $itemFull + FilesPath = $filesPath.TrimEnd('/') + TablesPath = $tablesPath.TrimEnd('/') + RootMode = $rootMode + FilesystemBase = "$accountUrl/$workspace" + } +} + +# ============================================== +# AZURE (FABRIC/ONELAKE) TOKEN INFRASTRUCTURE +# ============================================== +# Mirrors the Graph token refresh design ($script:SharedAuthState + +# Refresh-GraphTokenIfNeeded + Invoke-TokenRefresh) but for the OneLake DFS +# audience (https://storage.azure.com/), which is acquired through Az.Accounts. +# +# Layers: +# Get-FabricStorageTokenRaw - private; runs Connect-AzAccount/Get-AzAccessToken; +# normalizes ExpiresOn to UTC DateTime; tags AuthMethod. +# Invoke-AzTokenAcquire - private; calls Raw + stale-token rejection + +# updates $script:AzAuthState + emits log line. +# Refresh-FabricTokenIfNeeded - public; proactive (5-min buffer + 50-min age cap), +# cooldown (auth-mode aware), -Force for 401 reactive. +# Get-FabricStorageToken - public; backward-compatible string-returning wrapper +# (always returns a refresh-checked token). +# Invoke-FabricWebRequest - public; wraps Invoke-WebRequest for OneLake calls. +# Pre-flight refresh + transparent 401 retry once. +# Reactive triggers: 401 detected by Invoke-FabricWebRequest -> -Force refresh + retry. +# Proactive triggers: BufferMinutes (5) or token age >50 min, whichever comes first. +# Cooldown: 45s for app-only/MI (silent client_credentials/IMDS); 5min for interactive +# (avoids re-prompt spam on long upload phases). +function Get-FabricStorageTokenRaw { + [CmdletBinding()] + param() + $resource = 'https://storage.azure.com/' + if (-not (Get-Module -Name Az.Accounts -ListAvailable -ErrorAction SilentlyContinue)) { + throw "Az.Accounts module not installed. Install with: Install-Module Az.Accounts -Scope CurrentUser" + } + Import-Module Az.Accounts -ErrorAction Stop | Out-Null + + $authMethodTag = 'Interactive' + $ctx = Get-AzContext -ErrorAction SilentlyContinue + if (-not $ctx) { + if ($Auth -eq 'ManagedIdentity') { + $authMethodTag = 'ManagedIdentity' + if ($env:AZURE_CLIENT_ID) { + Connect-AzAccount -Identity -AccountId $env:AZURE_CLIENT_ID -ErrorAction Stop | Out-Null + } else { + Connect-AzAccount -Identity -ErrorAction Stop | Out-Null + } + } + elseif ($Auth -eq 'AppRegistration' -and $script:TenantId -and $script:ClientId -and $script:ClientSecret) { + $authMethodTag = 'AppRegistration' + $secureSecret = ConvertTo-SecureString -String $script:ClientSecret -AsPlainText -Force + $cred = New-Object System.Management.Automation.PSCredential($script:ClientId, $secureSecret) + Connect-AzAccount -ServicePrincipal -Tenant $script:TenantId -Credential $cred -ErrorAction Stop | Out-Null + } + else { + Connect-AzAccount -ErrorAction Stop | Out-Null + } + } else { + # Reuse existing context; classify by account type so cooldown/log lines stay accurate. + if ($ctx.Account -and $ctx.Account.Type -eq 'ManagedService') { $authMethodTag = 'ManagedIdentity' } + elseif ($ctx.Account -and $ctx.Account.Type -eq 'ServicePrincipal') { $authMethodTag = 'AppRegistration' } + } + + $tokenObj = Get-AzAccessToken -ResourceUrl $resource -ErrorAction Stop + # Token shape: newer Az returns SecureString; older returns plain string. + $tokenStr = if ($tokenObj.Token -is [System.Security.SecureString]) { + [System.Net.NetworkCredential]::new('', $tokenObj.Token).Password + } else { + [string]$tokenObj.Token + } + # ExpiresOn shape: newer Az returns DateTimeOffset; older returns DateTime; defensive parse for string. + $expiresUtc = if ($tokenObj.ExpiresOn -is [System.DateTimeOffset]) { + $tokenObj.ExpiresOn.UtcDateTime + } elseif ($tokenObj.ExpiresOn -is [datetime]) { + if ($tokenObj.ExpiresOn.Kind -eq [System.DateTimeKind]::Utc) { $tokenObj.ExpiresOn } else { $tokenObj.ExpiresOn.ToUniversalTime() } + } else { + ([datetime]$tokenObj.ExpiresOn).ToUniversalTime() + } + + return [pscustomobject]@{ + Token = $tokenStr + ExpiresOn = $expiresUtc + AuthMethod = $authMethodTag + } +} + +function Invoke-AzTokenAcquire { + [CmdletBinding()] + param([string] $Reason = 'initial') + try { + $tokenObj = Get-FabricStorageTokenRaw + # Stale-token rejection: refuse a token that is already at/near expiry. Protects + # against MSAL cache returning a stale entry after process suspension or system sleep. + $nowUtc = (Get-Date).ToUniversalTime() + $minutesValid = ($tokenObj.ExpiresOn - $nowUtc).TotalMinutes + if ($minutesValid -le 2) { + throw ("Acquired OneLake storage token is already expired or near-expiry (expires in {0:N1} min)." -f $minutesValid) + } + + $script:AzAuthState.Token = $tokenObj.Token + $script:AzAuthState.ExpiresOn = $tokenObj.ExpiresOn + $script:AzAuthState.AcquiredAt = Get-Date + $script:AzAuthState.LastRefresh = Get-Date + $script:AzAuthState.RefreshCount++ + $script:AzAuthState.AuthMethod = $tokenObj.AuthMethod + # Mirror to legacy $script:FabricToken so any cached references stay in sync. + $script:FabricToken = $tokenObj.Token + + $verb = if ($script:AzAuthState.RefreshCount -eq 1) { 'acquired' } else { 'refreshed' } + Write-LogHost (" [AZ-TOKEN] OneLake storage token {0} ({1}); auth: {2}; expires {3} UTC; refresh #{4}" -f $verb, $Reason, $tokenObj.AuthMethod, $tokenObj.ExpiresOn.ToString('yyyy-MM-dd HH:mm:ss'), $script:AzAuthState.RefreshCount) -ForegroundColor Gray + return $true + } + catch { + Write-LogHost (" [AZ-TOKEN] [!] Failed to acquire OneLake storage token ({0}): {1}" -f $Reason, $_.Exception.Message) -ForegroundColor Red + return $false + } +} + +function Refresh-FabricTokenIfNeeded { + <# + .SYNOPSIS + Proactively refreshes the Fabric/OneLake storage token if nearing expiry. + .PARAMETER BufferMinutes + Refresh if token expires within this many minutes. Default: 5. + .PARAMETER Force + Bypass cooldown and force re-acquisition (used by 401 reactive retry). + .OUTPUTS + $true - token is valid (either still-valid or freshly refreshed) + $false - acquisition/refresh failed; caller must surface the error + #> + [CmdletBinding()] + param( + [int] $BufferMinutes = 5, + [switch] $Force + ) + $now = (Get-Date).ToUniversalTime() + + # First-time acquisition: no state yet. + if (-not $script:AzAuthState.Token -or -not $script:AzAuthState.ExpiresOn) { + return (Invoke-AzTokenAcquire -Reason 'initial') + } + + $minutesRemaining = ($script:AzAuthState.ExpiresOn - $now).TotalMinutes + $tokenAge = if ($script:AzAuthState.AcquiredAt) { + ((Get-Date) - $script:AzAuthState.AcquiredAt).TotalMinutes + } else { 999 } + + # Trigger conditions: + # - Forced (typically 401 reactive) + # - Within BufferMinutes of expiry (proactive) + # - Token age > 50 min (belt-and-suspenders for any auth mode where ExpiresOn might be optimistic) + $needRefresh = $Force.IsPresent -or ($minutesRemaining -le $BufferMinutes) -or ($tokenAge -gt 50) + if (-not $needRefresh) { return $true } + + # Cooldown: app-only / MI refresh silently via client_credentials or IMDS (cheap), + # so 45s is enough. Interactive may prompt the user, so 5 min keeps that bearable. + $isAppOnly = ($script:AzAuthState.AuthMethod -in @('ManagedIdentity', 'AppRegistration')) + $cooldownMinutes = if ($isAppOnly) { 0.75 } else { 5.0 } + if (-not $Force -and $script:AzAuthState.LastRefreshAttempt) { + $sinceLast = ((Get-Date) - $script:AzAuthState.LastRefreshAttempt).TotalMinutes + if ($sinceLast -lt $cooldownMinutes) { + # Still in cooldown; report current token as valid only if it actually is. + return ($minutesRemaining -gt 0) + } + } + $script:AzAuthState.LastRefreshAttempt = Get-Date + + $reason = if ($Force.IsPresent) { + 'forced (401 retry or explicit)' + } elseif ($minutesRemaining -le $BufferMinutes) { + ("near-expiry ({0:N1} min remaining)" -f $minutesRemaining) + } else { + ("age cap ({0:N1} min)" -f $tokenAge) + } + return (Invoke-AzTokenAcquire -Reason $reason) +} + +function Get-FabricStorageToken { + <# + .SYNOPSIS + Returns a valid OneLake storage-audience bearer token (string) for OneLake DFS calls. + .DESCRIPTION + Backward-compatible wrapper around the Az token state machine. Always invokes + Refresh-FabricTokenIfNeeded first so callers receive a token guaranteed to have at + least BufferMinutes of validity (or a fresh one). Throws if acquisition fails. + #> + [CmdletBinding()] param() + $ok = Refresh-FabricTokenIfNeeded + if (-not $ok -or -not $script:AzAuthState.Token) { + throw "Failed to acquire Fabric/OneLake storage token (see prior [AZ-TOKEN] messages)." + } + return $script:AzAuthState.Token +} + +function Invoke-FabricWebRequest { + <# + .SYNOPSIS + Invoke-WebRequest wrapper for OneLake DFS calls with proactive token refresh + and transparent 401 reactive retry. + .DESCRIPTION + Before each call: ensures token is fresh via Refresh-FabricTokenIfNeeded. + During the call: if a 401 Unauthorized is returned, forces a refresh and + retries the request once with the new token. Any other status (429, 5xx, + 403, 404, etc.) is re-thrown unchanged so the caller's existing retry/error + handling (e.g., chunk-upload 429/5xx backoff loops) continues to work. + .PARAMETER Headers + Caller-supplied headers. The Authorization header is ALWAYS overwritten with + the current $script:AzAuthState.Token. x-ms-version defaults to '2021-06-08' + if the caller does not supply it. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Uri, + [Parameter(Mandatory)] [string] $Method, + [hashtable] $Headers, + $Body, + [string] $ContentType, + [string] $OutFile + ) + # Pre-flight refresh. + $null = Refresh-FabricTokenIfNeeded + if (-not $script:AzAuthState.Token) { + throw "Invoke-FabricWebRequest: no OneLake storage token available." + } + + # Build effective headers; always set Authorization from current state. + $h = @{} + if ($Headers) { + foreach ($k in $Headers.Keys) { + if ($k -ne 'Authorization') { $h[$k] = $Headers[$k] } + } + } + $h['Authorization'] = "Bearer $($script:AzAuthState.Token)" + if (-not $h.ContainsKey('x-ms-version')) { $h['x-ms-version'] = '2021-06-08' } + + $invokeParams = @{ + Uri = $Uri + Method = $Method + Headers = $h + UseBasicParsing = $true + ErrorAction = 'Stop' + } + if ($PSBoundParameters.ContainsKey('Body')) { $invokeParams.Body = $Body } + if ($ContentType) { $invokeParams.ContentType = $ContentType } + if ($OutFile) { $invokeParams.OutFile = $OutFile } + + try { + return Invoke-WebRequest @invokeParams + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 401) { + Write-LogHost " [AZ-TOKEN] OneLake returned 401 Unauthorized - forcing token refresh and retrying once..." -ForegroundColor Yellow + $refreshed = Refresh-FabricTokenIfNeeded -Force + if (-not $refreshed) { + throw ("OneLake 401 Unauthorized and token refresh failed: {0}" -f $_.Exception.Message) + } + $h['Authorization'] = "Bearer $($script:AzAuthState.Token)" + $invokeParams.Headers = $h + return Invoke-WebRequest @invokeParams + } + throw + } +} + +function Test-RemoteDestination { + [CmdletBinding()] param() + if ($script:RemoteOutputMode -eq 'None') { return } + if ($script:RemoteOutputMode -eq 'SharePoint') { + $resolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl + $script:SPResolved = $resolved + # Ensure folder exists (create if missing) + if ($resolved.FolderPath) { + $folderUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/$($resolved.FolderPath)" + try { + Invoke-MgGraphRequest -Method GET -Uri $folderUri -ErrorAction Stop | Out-Null + } catch { + # Create folder hierarchy + $parts = $resolved.FolderPath -split '/' + $accumulated = '' + foreach ($p in $parts) { + $parentUri = if ($accumulated) { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${accumulated}:/children" } else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root/children" } + # FolderPath segments come from [Uri].AbsolutePath, which percent-encodes spaces + # and other reserved characters (e.g. 'PAX Exports' -> 'PAX%20Exports'). The + # parent addressing above (root:/...:/children) is colon-path syntax that Graph + # DECODES, but the 'name' field below is a literal display name that Graph does + # NOT decode — so passing the raw segment created a folder literally named + # 'PAX%20Exports' alongside the decoded 'PAX Exports' that the upload path + # (Send-FileToSharePoint, also colon-path) resolves to. Decode the segment for + # the display name so both mechanisms agree and only one folder is created. + $folderDisplayName = [System.Uri]::UnescapeDataString($p) + $body = @{ name = $folderDisplayName; folder = @{}; '@microsoft.graph.conflictBehavior' = 'replace' } | ConvertTo-Json + try { Invoke-MgGraphRequest -Method POST -Uri $parentUri -Body $body -ContentType 'application/json' -ErrorAction Stop | Out-Null } catch {} + $accumulated = if ($accumulated) { "$accumulated/$p" } else { $p } + } + } + } + } + elseif ($script:RemoteOutputMode -eq 'Fabric') { + $resolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl + $script:FabricResolved = $resolved + # Initial token acquisition (populates $script:AzAuthState). Classify Az/auth + # failures here so customers see Cause/Action lines instead of raw exceptions. + try { + $null = Get-FabricStorageToken + } catch { + $tokenErr = $_.Exception.Message + $tokenDiag = New-Object System.Collections.Generic.List[string] + $tokenDiag.Add("OneLake storage token acquisition failed for workspace '$($resolved.Workspace)'.") + if ($tokenErr -match 'Az\.Accounts module not installed') { + $tokenDiag.Add('Cause: The Az.Accounts PowerShell module is not installed.') + $tokenDiag.Add('Action: Install-Module Az.Accounts -Scope CurrentUser') + } elseif ($tokenErr -match 'AADSTS' -or $tokenErr -match 'consent' -or $tokenErr -match 'admin') { + $tokenDiag.Add('Cause: Azure AD rejected the sign-in for the storage.azure.com audience.') + $tokenDiag.Add('Action: Verify the identity has Azure AD sign-in to your tenant, and that') + $tokenDiag.Add(' any conditional access / MFA requirements are satisfied.') + } elseif ($tokenErr -match 'IDENTITY|managed identity|MSI|IMDS') { + $tokenDiag.Add('Cause: Managed identity token acquisition failed (no IMDS endpoint reachable,') + $tokenDiag.Add(' or AZURE_CLIENT_ID points to an identity not assigned to this host).') + $tokenDiag.Add('Action: Re-run from a host with an assigned managed identity, or use') + $tokenDiag.Add(' -Auth Interactive / -Auth AppRegistration instead.') + } else { + $tokenDiag.Add('Cause: Az.Accounts could not acquire a token for https://storage.azure.com/.') + $tokenDiag.Add('Action: Verify your authentication mode (-Auth) and that you can sign in via') + $tokenDiag.Add(' Connect-AzAccount manually.') + } + $tokenDiag.Add('') + $tokenDiag.Add("Az response: $tokenErr") + throw ($tokenDiag -join "`n") + } + # HEAD the filesystem (workspace) to verify access. Invoke-FabricWebRequest sets + # Authorization + x-ms-version automatically and handles 401 reactive refresh. + $probeUri = "$($resolved.FilesystemBase)?resource=filesystem" + try { + $null = Invoke-FabricWebRequest -Uri $probeUri -Method HEAD + } catch { + # Classify OneLake DFS responses. Unlike Graph, OneLake rarely returns a JSON + # body on auth failures — the HTTP status is the primary signal. + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + $rawDetail = $_.Exception.Message + if ($rawDetail.Length -gt 600) { $rawDetail = $rawDetail.Substring(0, 600) + '...(truncated)' } + + $diagLines = New-Object System.Collections.Generic.List[string] + $diagLines.Add("OneLake filesystem probe failed (workspace '$($resolved.Workspace)', item '$($resolved.ItemFull)').") + + if ($status -eq 401) { + $diagLines.Add('Cause: OneLake rejected the storage token (401). The token audience is correct') + $diagLines.Add(' but the identity is not recognized by this Fabric workspace.') + $diagLines.Add('Action: Verify the identity is signed in to the same tenant that owns the Fabric') + $diagLines.Add(' workspace, and that the storage.azure.com token is not expired/blocked.') + } + elseif ($status -eq 403) { + $diagLines.Add('Cause: The identity is authenticated but lacks permissions on the Fabric workspace') + $diagLines.Add(' (HTTP 403 Forbidden from OneLake DFS).') + $diagLines.Add('Action: In the Fabric portal -> Workspace settings -> Manage access, grant the') + $diagLines.Add(' identity at least Contributor. See the "Permissions Required for THIS run"') + $diagLines.Add(' table earlier in this output for the exact Azure RBAC roles required.') + } + elseif ($status -eq 404) { + $diagLines.Add("Cause: Workspace or item not found (HTTP 404). One of these is wrong:") + $diagLines.Add(" workspace = '$($resolved.Workspace)'") + $diagLines.Add(" item = '$($resolved.ItemFull)' (must exist as a Lakehouse/Warehouse)") + $diagLines.Add('Action: Verify the URL by opening the lakehouse in the Fabric portal and using') + $diagLines.Add(' the OneLake "Copy ABFS path" option, then converting to https:// form.') + } + elseif ($status -ge 500) { + $diagLines.Add("Cause: OneLake returned a server error (HTTP $status). This is usually transient.") + $diagLines.Add('Action: Wait a minute and retry. If it persists, check Fabric service health.') + } + else { + $diagLines.Add("Cause: OneLake DFS probe failed (HTTP status: $status).") + $diagLines.Add('Action: Verify the Fabric workspace URL is correct and that the identity has') + $diagLines.Add(' the Azure RBAC roles listed in the "Permissions Required for THIS run" table.') + } + $diagLines.Add('') + $diagLines.Add("OneLake response: $rawDetail") + + throw ($diagLines -join "`n") + } + } +} + +function Get-GraphErrorDetail { + # Best-effort extraction of the Microsoft Graph error body from a caught error. + # PowerShell attaches the raw HTTP response body to $_.ErrorDetails.Message for both + # Invoke-MgGraphRequest and Invoke-WebRequest; fall back to the response stream, then + # to the plain exception message. Returns a short single-line string for logging. + [CmdletBinding()] + param([Parameter(Mandatory)] $ErrorRecord) + $raw = $null + if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { + $raw = $ErrorRecord.ErrorDetails.Message + } + elseif ($ErrorRecord.Exception -and $ErrorRecord.Exception.Response) { + try { + $stream = $ErrorRecord.Exception.Response.GetResponseStream() + if ($stream) { + $reader = New-Object System.IO.StreamReader($stream) + $raw = $reader.ReadToEnd() + $reader.Dispose() + } + } catch { } + } + if (-not $raw) { return $ErrorRecord.Exception.Message } + # Try to pull error.code / error.message out of the JSON body. + try { + $j = $raw | ConvertFrom-Json -ErrorAction Stop + if ($j.error) { + $code = $j.error.code + $msg = $j.error.message + return ("{0}: {1}" -f $code, $msg).Trim(': ') + } + } catch { } + # Non-JSON body - return a trimmed single line. + return ($raw -replace '\s+', ' ').Trim() +} + +function Send-FileToSharePoint { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + [string] $RemoteFileName, + # Optional per-data-type SharePoint parent URL. When supplied, the destination + # is resolved against $ParentOverride instead of the script-wide Purview + # RemoteOutputUrl. Used by per-data-type append surfaces (UserInfo / Agent365 / + # Log) when their -OutputPath* points at a different SharePoint folder than + # the primary Purview output. + [string] $ParentOverride + ) + if (-not (Test-Path -LiteralPath $LocalPath)) { throw "Send-FileToSharePoint: source not found: $LocalPath" } + if ($ParentOverride) { + $resolved = Resolve-SharePointTarget -Url $ParentOverride + } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $fileName = if ($RemoteFileName) { $RemoteFileName } else { Split-Path -Leaf $LocalPath } + # Per-segment URI escaping for the filename. The auto-defaulted basenames used by + # this script are safe ASCII (timestamps + underscores), but routing custom basenames + # through the same encoding pipeline avoids any per-call-site special casing. + $fileNameEnc = [System.Uri]::EscapeDataString($fileName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$fileNameEnc" } else { $fileNameEnc } + $fileInfo = Get-Item -LiteralPath $LocalPath + $sizeMb = $fileInfo.Length / 1MB + + # Graph simple upload (PUT .../content) supports up to 250 MB; use it for everything + # up to that ceiling so as many files as possible avoid the resumable-upload-session + # API (which some locked-down libraries / egress proxies reject even when a simple PUT + # is allowed). Files above 250 MB still use createUploadSession + chunked PUT. + $simpleUploadMax = 250MB + if ($fileInfo.Length -le $simpleUploadMax) { + $putUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/content" + $bytes = [System.IO.File]::ReadAllBytes($LocalPath) + try { + Invoke-MgGraphRequest -Method PUT -Uri $putUri -Body $bytes -ContentType 'application/octet-stream' -ErrorAction Stop | Out-Null + } + catch { + $detail = Get-GraphErrorDetail -ErrorRecord $_ + throw "SharePoint simple upload (PUT) failed for '$relPath': $detail" + } + } + else { + # Create upload session + $sessUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/createUploadSession" + $sessBody = @{ item = @{ '@microsoft.graph.conflictBehavior' = 'replace'; name = $fileName } } | ConvertTo-Json -Depth 4 + try { + $session = Invoke-MgGraphRequest -Method POST -Uri $sessUri -Body $sessBody -ContentType 'application/json' -ErrorAction Stop + } + catch { + $detail = Get-GraphErrorDetail -ErrorRecord $_ + throw "createUploadSession failed for '$relPath': $detail" + } + $uploadUrl = $session.uploadUrl + if (-not $uploadUrl) { throw "SharePoint createUploadSession returned no uploadUrl for '$relPath'." } + + $chunkSize = 5MB # multiple of 320 KB + $fs = [System.IO.File]::OpenRead($LocalPath) + try { + $buffer = New-Object byte[] $chunkSize + $offset = 0L + while ($offset -lt $fileInfo.Length) { + $toRead = [int][Math]::Min([int64]$chunkSize, $fileInfo.Length - $offset) + # FileStream.Read may return fewer bytes than requested; loop until the + # fragment is fully populated (or EOF) so every Content-Range fragment + # carries exactly the byte count its header declares. + $read = 0 + while ($read -lt $toRead) { + $n = $fs.Read($buffer, $read, $toRead - $read) + if ($n -le 0) { break } + $read += $n + } + if ($read -le 0) { break } + $end = $offset + $read - 1 + $range = "bytes $offset-$end/$($fileInfo.Length)" + # Send a true byte[] body. PowerShell range-indexing ($buffer[0..N]) + # returns an Object[] of boxed bytes, which Invoke-WebRequest stringifies + # instead of sending as raw bytes — producing a body whose length does not + # match the declared Content-Range and a 400 (Bad Request) from Graph on + # the final partial fragment. Copy into a right-sized byte[] instead. + if ($read -eq $chunkSize) { + $slice = $buffer + } + else { + $slice = New-Object byte[] $read + [System.Array]::Copy($buffer, 0, $slice, 0, $read) + } + $attempt = 0 + while ($true) { + try { + $null = Invoke-WebRequest -Uri $uploadUrl -Method PUT -Body $slice -Headers @{ 'Content-Range' = $range } -ContentType 'application/octet-stream' -UseBasicParsing -ErrorAction Stop + break + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } + $offset += $read + } + } + finally { $fs.Dispose() } + } + Write-LogHost (" -> SharePoint upload OK: {0} ({1:N2} MB)" -f $relPath, $sizeMb) -ForegroundColor DarkGray +} + +function Send-FileToOneLake { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + [string] $RemoteFileName, + # Optional per-data-type Fabric/OneLake parent URL. See Send-FileToSharePoint + # for the rationale. When omitted, falls back to the script-wide Purview URL. + [string] $ParentOverride + ) + if (-not (Test-Path -LiteralPath $LocalPath)) { throw "Send-FileToOneLake: source not found: $LocalPath" } + if ($ParentOverride) { + $resolved = Resolve-FabricTarget -Url $ParentOverride + } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + $fileName = if ($RemoteFileName) { $RemoteFileName } else { Split-Path -Leaf $LocalPath } + # Per-SEGMENT URI escaping. $RemoteFileName may contain '/' separators + # (e.g. ".pax_resume//" from Sync-FabricResumeMirror); escaping the + # whole string as one data segment would percent-encode the slashes ('%2F'), + # malforming the DFS path so PUT (Create File) and PATCH (Append) target + # different resolved files, surfacing as "400 (position not equal to length)". + # Split-then-escape-per-segment preserves the hierarchical path. + $fileNameEnc = ($fileName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$fileNameEnc" } else { $fileNameEnc } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + $fileInfo = Get-Item -LiteralPath $LocalPath + $sizeMb = $fileInfo.Length / 1MB + + # Authorization + x-ms-version are set per-request by Invoke-FabricWebRequest using the + # current $script:AzAuthState. Do not cache headers locally - the token may rotate. + + # Step 0: Pre-create any parent directories. OneLake Lakehouse Files/ uses a + # hierarchical namespace; PUT ?resource=file does not auto-create intermediate + # directories. Without this, mirroring artifacts to e.g. ".pax_resume//" + # can land the leaf file in an inconsistent state that surfaces downstream as + # a 400 (position not equal to length) on the first append. Idempotent: a 409 + # response means the directory already exists and is treated as success. + $parentRel = $null + if ($relInItem -match '/') { $parentRel = $relInItem.Substring(0, $relInItem.LastIndexOf('/')) } + if ($parentRel) { + $dirUri = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$parentRel`?resource=directory" + try { + $null = Invoke-FabricWebRequest -Uri $dirUri -Method PUT + } + catch { + $dirStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($dirStatus -ne 409) { throw } + } + } + + # Step 1: Create (PUT ?resource=file) - overwrites any existing file. + $createUri = "$dfsPath`?resource=file" + $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT + + # Step 2: Append in chunks (PATCH ?action=append&position=N). + # Inner retry loop handles 429/5xx with exponential backoff. 401 is handled transparently + # inside Invoke-FabricWebRequest (force-refresh + retry once); any 401 that escapes here + # means refresh itself failed, which is fatal for this upload and propagates up. + $chunkSize = 4MB + $fs = [System.IO.File]::OpenRead($LocalPath) + try { + $buffer = New-Object byte[] $chunkSize + $position = 0L + while ($position -lt $fileInfo.Length) { + $toRead = [Math]::Min([int64]$chunkSize, $fileInfo.Length - $position) + $read = $fs.Read($buffer, 0, [int]$toRead) + if ($read -le 0) { break } + $slice = if ($read -eq $chunkSize) { $buffer } else { $buffer[0..($read-1)] } + $appendUri = "$dfsPath`?action=append&position=$position" + $attempt = 0 + while ($true) { + try { + $null = Invoke-FabricWebRequest -Uri $appendUri -Method PATCH -Body $slice -ContentType 'application/octet-stream' + break + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } + $position += $read + } + } + finally { $fs.Dispose() } + + # Step 3: Flush (PATCH ?action=flush&position=) + $flushUri = "$dfsPath`?action=flush&position=$($fileInfo.Length)" + $null = Invoke-FabricWebRequest -Uri $flushUri -Method PATCH + + Write-LogHost (" -> OneLake upload OK: {0} ({1:N2} MB)" -f $relInItem, $sizeMb) -ForegroundColor DarkGray +} + +function Get-RemoteFile-SharePoint { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RelativeName, + [Parameter(Mandatory)] [string] $DestinationPath, + # Optional per-data-type SharePoint parent URL (see Send-FileToSharePoint). + [string] $ParentOverride + ) + if ($ParentOverride) { + $resolved = Resolve-SharePointTarget -Url $ParentOverride + } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $relName = [System.Uri]::EscapeDataString($RelativeName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$relName" } else { $relName } + $dlUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/content" + Invoke-MgGraphRequest -Method GET -Uri $dlUri -OutputFilePath $DestinationPath -ErrorAction Stop | Out-Null +} + +function Get-RemoteFile-OneLake { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RelativeName, + [Parameter(Mandatory)] [string] $DestinationPath, + # Optional per-data-type Fabric/OneLake parent URL (see Send-FileToSharePoint). + [string] $ParentOverride + ) + if ($ParentOverride) { + $resolved = Resolve-FabricTarget -Url $ParentOverride + } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + # Per-SEGMENT URI escaping. $RelativeName may contain '/' separators (e.g. + # ".pax_resume//" from Restore-FabricResumeMirror); escaping the + # whole string as one data segment would percent-encode the slashes to '%2F' + # and resolve to a different (or non-existent) DFS path than the uploader + # wrote to. Mirrors the Send-FileToOneLake fix. + $relName = ($RelativeName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$relName" } else { $relName } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + # Invoke-FabricWebRequest handles Authorization, x-ms-version, proactive refresh, 401 retry. + $null = Invoke-FabricWebRequest -Uri $dfsPath -Method GET -OutFile $DestinationPath +} + +function Get-DataTypeForOutputFile { + # Map a customer-visible artifact basename to its data-type key. Used by the + # upload sweep to route each file to its per-data-type destination (-OutputPath*). + # Returns one of: 'Purview' | 'UserInfo' | 'Agent365Info' | 'Log'. + # Unknown / unmatched names fall back to 'Purview' (primary audit destination). + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $FileName) + $n = [System.IO.Path]::GetFileName($FileName) + if ($n -like 'EntraUsers_*' -or $n -like '*_Users.csv' -or $n -like '*_Users_*.csv') { return 'UserInfo' } + if ($n -like 'Agent365_*' -or $n -like 'Agent365.csv') { return 'Agent365Info' } + if ($n -like '*.log' -or $n -like '*.partial.log') { return 'Log' } + return 'Purview' +} + +function Invoke-OutputUpload { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + # Optional per-data-type parent URL. When supplied, the upload targets this + # URL instead of the script-wide Purview destination. Used to honor + # -OutputPathUserInfo / -OutputPathAgent365Info / -OutputPathLog routing. + [string] $ParentOverride + ) + if ($script:RemoteOutputMode -eq 'None') { return } + if (-not (Test-Path -LiteralPath $LocalPath)) { + Write-Verbose "Invoke-OutputUpload: skipping (file not found): $LocalPath" + return + } + # NOTE: No catch wrapper here — every caller either (a) has its own try/catch with a + # context-specific WARNING (upload sweep / metrics / log file) or (b) wants silent + # best-effort behavior with Write-Verbose only (checkpoint mirror). An inner WARNING + # would always be either redundant or unwanted. Let exceptions propagate untouched. + switch ($script:RemoteOutputMode) { + 'SharePoint' { Send-FileToSharePoint -LocalPath $LocalPath -ParentOverride $ParentOverride } + 'Fabric' { Send-FileToOneLake -LocalPath $LocalPath -ParentOverride $ParentOverride } + } +} + +function Invoke-EmbeddedProcessor { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateSet('CopilotInteraction', 'M365Bundle')] + [string] $ProcessorMode, + + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs, + [Parameter(Mandatory)] [string[]] $ProcessorArgs, + [Parameter(Mandatory)] [string] $IncrementalDir + ) + + # Materialize the appropriate embedded source as a temp .py inside .pax_incremental, + # named with the global run timestamp so it is auto-reaped by the end-of-run cleanup + # AND the finally-block safety-net even if we crash. Returns the processor's exit code. + + if (-not (Test-Path -LiteralPath $IncrementalDir -PathType Container)) { + New-Item -ItemType Directory -Path $IncrementalDir -Force | Out-Null + } + + switch ($ProcessorMode) { + 'CopilotInteraction' { + $label = 'CopilotInteractionProcessor' + $source = $Script:EMBEDDED_PROCESSOR_COPILOT + $ver = $Script:EMBEDDED_PROCESSOR_COPILOT_VERSION + } + 'M365Bundle' { + $label = 'M365BundleProcessor' + $source = $Script:EMBEDDED_PROCESSOR_M365 + $ver = $Script:EMBEDDED_PROCESSOR_M365_VERSION + } + } + + $tempPyName = "PAX_${label}_$($global:ScriptRunTimestamp).py" + $tempPyPath = Join-Path $IncrementalDir $tempPyName + + $utf8NoBom = [Text.UTF8Encoding]::new($false) + [IO.File]::WriteAllText($tempPyPath, $source, $utf8NoBom) + + Write-LogHost "Rollup: invoking embedded $label v$ver ($PythonExe)" -ForegroundColor Cyan + Write-LogFile "Rollup: temp script -> $tempPyPath" + Write-LogFile "Rollup: args -> $($ProcessorArgs -join ' ')" + + $exitCode = 1 + # Force UTF-8 across the Python subprocess boundary so Unicode glyphs the + # processor emits to stdout / stderr (right-arrow, section separators, + # bullets) are not mojibaked when captured through PowerShell on hosts + # whose console output encoding defaults to a legacy OEM code page. + # Both sides matter: + # * [Console]::OutputEncoding controls how PowerShell decodes the bytes + # it reads back from the subprocess. + # * $env:PYTHONIOENCODING tells the Python interpreter what to encode + # its stdout / stderr text streams as. + # Outer values are captured here and restored in the finally block so the + # host's session-wide encoding is not perturbed. + $prevConsoleOutEnc = [Console]::OutputEncoding + $prevPyIoEnc = $env:PYTHONIOENCODING + try { + [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false) + $env:PYTHONIOENCODING = 'utf-8' + $fullArgs = @($LauncherArgs) + @($tempPyPath) + @($ProcessorArgs) + # In remote-output mode, suppress the Python processors' echoed input/output PATH + # lines (e.g. "Purview input:", "Output file:"). Those values are LOCAL scratch + # paths; surfacing them to the customer when a remote -OutputPath* is + # in effect is misleading. The remote-aware path summary is emitted by the + # PowerShell wrapper after the processor returns. All other Python output + # (header, counts, timing, errors) is preserved verbatim. + $pathLineRegex = if ($script:RemoteOutputMode -ne 'None') { + '^\s*(Purview input:|Entra input:|Purview output:|Entra output:|Input:|Output:|Output file:)\s' + } else { $null } + & $PythonExe @fullArgs 2>&1 | ForEach-Object { + $line = [string]$_ + if ($pathLineRegex -and $line -match $pathLineRegex) { return } + # Stream stdout+stderr through Write-Log (which mirrors to host AND log file). + Write-Log $line + } + $exitCode = $LASTEXITCODE + } + catch { + Write-LogHost "Rollup: $label threw an exception: $($_.Exception.Message)" -ForegroundColor Red + $exitCode = 1 + } + finally { + # Restore the outer console output encoding and PYTHONIOENCODING regardless + # of outcome so subsequent host writes (and any later subprocess) see the + # same environment they would have without this call. + try { [Console]::OutputEncoding = $prevConsoleOutEnc } catch { } + if ($null -eq $prevPyIoEnc) { + Remove-Item Env:PYTHONIOENCODING -ErrorAction SilentlyContinue + } else { + $env:PYTHONIOENCODING = $prevPyIoEnc + } + try { + if (Test-Path -LiteralPath $tempPyPath) { Remove-Item -LiteralPath $tempPyPath -Force -ErrorAction SilentlyContinue } + } + catch { } + } + + return $exitCode +} + +# Validate MaxConcurrency range (Microsoft Purview enforces 10 concurrent search job limit per user account) +# ============================================================================ +# APPENDFILE COLUMN VALIDATION FUNCTIONS +# ============================================================================ +# Early validation to prevent wasting time on Graph API queries when explosion +# parameters don't match between existing file and new data parameters. +# Note: For explosion modes, actual column schemas are dynamic and vary by data, +# so we validate explosion parameter compatibility rather than exact columns. +# ============================================================================ + +function Get-LikelyExplosionParams { + param([string[]]$Columns) + + # Check for deep explosion indicators (CopilotEventData.* columns) + $hasDeepColumns = $Columns | Where-Object { $_ -match '^CopilotEventData\.' } + if ($hasDeepColumns) { + return @{ Mode = "ExplodeDeep"; DisplayName = "-ExplodeDeep" } + } + + # Check for array explosion indicators (exploded field names like Message_, Context_, AgentId, etc.) + $hasArrayColumns = $Columns | Where-Object { $_ -match '^(Message_|Context_|Interaction_|AgentId|AgentName|AgentVersion|AccessedResource_|AISystemPlugin_)' } + if ($hasArrayColumns) { + return @{ Mode = "ExplodeArrays"; DisplayName = "-ExplodeArrays" } + } + + # Check for standard mode indicators (AuditData JSON column present) + $hasAuditData = $Columns -contains 'AuditData' + if ($hasAuditData) { + return @{ Mode = "Standard"; DisplayName = "Standard (no explosion)" } + } + + # Unable to determine + return @{ Mode = "Unknown"; DisplayName = "Unknown mode" } +} + +function Get-AppendFileRecordIds { + <# + .SYNOPSIS + Stream the RecordId column from an existing AppendFile (CSV or XLSX) into a HashSet. + + .DESCRIPTION + Cross-run dedup helper for -AppendFile. Reads only the RecordId column from the + existing target file and returns a HashSet[string] of identifiers. Used to seed + Merge-IncrementalSaves-Streaming's -ExcludeRecordIds parameter so this run skips + records already present in the target. + + CSV path: streams via StreamReader + manual header parse — never loads the entire + file into memory (the target may be hundreds of MB). + Excel path: uses Import-Excel (RecordId column only). The Excel branch is bounded + by ImportExcel's own behavior; very large workbooks should use CSV. + + .PARAMETER Path + Path to the AppendFile. Missing file returns an empty set (first-run semantic). + + .OUTPUTS + HashSet[string] of RecordIds (empty when the file or column is absent). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Path + ) + $set = New-Object System.Collections.Generic.HashSet[string] + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return ,$set } + + $ext = [System.IO.Path]::GetExtension($Path).ToLowerInvariant() + if ($ext -eq '.xlsx') { + try { + if (-not (Get-Module -Name ImportExcel -ListAvailable)) { return ,$set } + Import-Module ImportExcel -ErrorAction Stop + $rows = Import-Excel -Path $Path -ErrorAction Stop + foreach ($r in $rows) { + $rid = $r.RecordId + if ($rid -and -not [string]::IsNullOrWhiteSpace([string]$rid)) { + [void]$set.Add([string]$rid) + } + } + } catch { + Write-LogHost "Get-AppendFileRecordIds: failed to read '$Path': $($_.Exception.Message). Cross-run dedup disabled for this file." -ForegroundColor Yellow + } + return ,$set + } + + # CSV path — manual streaming for memory safety on large files. + try { + $reader = [System.IO.StreamReader]::new($Path, [System.Text.Encoding]::UTF8) + try { + $header = $reader.ReadLine() + if (-not $header) { return ,$set } + $cols = $header.Split(',') | ForEach-Object { $_.Trim('"') } + $ridIdx = -1 + for ($i = 0; $i -lt $cols.Count; $i++) { + if ($cols[$i] -ieq 'RecordId') { $ridIdx = $i; break } + } + if ($ridIdx -lt 0) { return ,$set } + $line = $null + while ($null -ne ($line = $reader.ReadLine())) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + # Simple field split; AppendFiles use Export-Csv quoting so RecordId (a GUID) + # never contains commas or quotes. Fall back to ConvertFrom-Csv for the rare + # row that has embedded commas before the RecordId column. + $fields = $line.Split(',') + if ($fields.Count -gt $ridIdx) { + $raw = $fields[$ridIdx].Trim('"').Trim() + if ($raw) { [void]$set.Add($raw) } + } + } + } finally { + $reader.Dispose() + } + } catch { + Write-LogHost "Get-AppendFileRecordIds: failed to read '$Path': $($_.Exception.Message). Cross-run dedup disabled for this file." -ForegroundColor Yellow + } + return ,$set +} + +function Test-AppendFileCompatibility { + param( + [string]$FilePath, + [bool]$IsExcel, + [bool]$ExplodeArrays, + [bool]$ExplodeDeep, + [string]$TargetSheet = $null + ) + + $result = @{ + Compatible = $true + ExistingMode = $null + CurrentMode = $null + ExistingColumns = @() + ExistingCount = 0 + ErrorMessage = $null + } + + try { + # Determine current explosion mode + if ($ExplodeDeep) { + $result.CurrentMode = @{ Mode = "ExplodeDeep"; DisplayName = "-ExplodeDeep" } + } + elseif ($ExplodeArrays) { + $result.CurrentMode = @{ Mode = "ExplodeArrays"; DisplayName = "-ExplodeArrays" } + } + else { + $result.CurrentMode = @{ Mode = "Standard"; DisplayName = "Standard (no explosion)" } + } + + # Read existing file columns + if ($IsExcel) { + # Validate Excel file and read columns + if (-not (Get-Module -Name ImportExcel -ListAvailable)) { + $result.ErrorMessage = "ImportExcel module not available for validation" + $result.Compatible = $false + return $result + } + + Import-Module ImportExcel -ErrorAction Stop + + # Get sheet info + $sheets = Get-ExcelSheetInfo -Path $FilePath -ErrorAction Stop + + if ($TargetSheet) { + # Validate specific sheet + $sheet = $sheets | Where-Object { $_.Name -eq $TargetSheet } + if (-not $sheet) { + $result.ErrorMessage = "Target sheet '$TargetSheet' not found in workbook" + $result.Compatible = $false + return $result + } + } + else { + # Use first sheet + $sheet = $sheets | Select-Object -First 1 + } + + # Read header row from Excel + $headerData = Import-Excel -Path $FilePath -WorksheetName $sheet.Name -StartRow 1 -EndRow 1 -NoHeader -ErrorAction Stop + $existingCols = $headerData[0].PSObject.Properties.Value | Where-Object { $_ } + } + else { + # CSV: Read first line (header) + $firstLine = Get-Content -Path $FilePath -First 1 -Encoding UTF8 -ErrorAction Stop + $existingCols = ($firstLine -split ',') | ForEach-Object { $_.Trim('"') } + } + + $result.ExistingColumns = $existingCols + $result.ExistingCount = $existingCols.Count + + # Detect explosion mode of existing file + $result.ExistingMode = Get-LikelyExplosionParams -Columns $existingCols + + # Check if explosion modes match + if ($result.ExistingMode.Mode -ne $result.CurrentMode.Mode) { + $result.Compatible = $false + $result.ErrorMessage = "Explosion parameter mismatch: existing file is '$($result.ExistingMode.DisplayName)' but current command uses '$($result.CurrentMode.DisplayName)'" + } + else { + # Modes match - compatible + # Note: We don't validate exact columns because explosion schemas are dynamic + # and vary based on actual data content. As long as explosion params match, + # the append will work correctly. + $result.Compatible = $true + } + } + catch { + $result.ErrorMessage = $_.Exception.Message + $result.Compatible = $false + } + + return $result +} + +# ============================================================================ +# END APPENDFILE VALIDATION FUNCTIONS +# ============================================================================ + +if ($MaxConcurrency -lt 1 -or $MaxConcurrency -gt 10) { + Write-Host "ERROR: -MaxConcurrency must be between 1 and 10." -ForegroundColor Red + Write-Host "Microsoft Purview enforces a maximum of 10 concurrent search jobs per user account." -ForegroundColor Yellow + Write-Host "Current value: $MaxConcurrency" -ForegroundColor Yellow + Write-Host "Please specify a value between 1 and 10 and re-run." -ForegroundColor Yellow + exit 1 +} + +# Establish date defaults / validation depending on mode. +if ($RAWInputCSV) { + $parsedStart = $null; $parsedEnd = $null + if ($PSBoundParameters.ContainsKey('StartDate')) { + try { $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } catch { Write-Host "ERROR: StartDate must be yyyy-MM-dd if provided." -ForegroundColor Red; exit 1 } + } + if ($PSBoundParameters.ContainsKey('EndDate')) { + try { $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } catch { Write-Host "ERROR: EndDate must be yyyy-MM-dd if provided." -ForegroundColor Red; exit 1 } + } + if ($parsedStart -and $parsedEnd -and $parsedEnd -lt $parsedStart) { Write-Host "ERROR: EndDate ($EndDate) is earlier than StartDate ($StartDate)." -ForegroundColor Red; exit 1 } + if (-not $PSBoundParameters.ContainsKey('StartDate')) { $StartDate = '*' } + if (-not $PSBoundParameters.ContainsKey('EndDate')) { $EndDate = '*' } +} +else { + if (-not $PSBoundParameters.ContainsKey('StartDate') -and -not $PSBoundParameters.ContainsKey('EndDate')) { + $yesterdayUtc = (Get-Date).ToUniversalTime().Date.AddDays(-1) + $StartDate = $yesterdayUtc.ToString('yyyy-MM-dd') + $EndDate = $yesterdayUtc.AddDays(1).ToString('yyyy-MM-dd') + } + elseif (-not $PSBoundParameters.ContainsKey('StartDate')) { + $StartDate = '*' + try { + $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } catch { Write-Host "ERROR: EndDate must be yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + } + elseif (-not $PSBoundParameters.ContainsKey('EndDate')) { + $EndDate = '*' + try { + $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } catch { Write-Host "ERROR: StartDate must be yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + } + else { + try { + $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { Write-Host "ERROR: StartDate/EndDate must be in yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + if ($parsedEnd -lt $parsedStart) { Write-Host "ERROR: EndDate ($EndDate) is earlier than StartDate ($StartDate)." -ForegroundColor Red; exit 1 } + } +} + +# Client-side date-range trim boundaries — Purview's partition-based indexing can +# return records outside the requested date range (observed up to ~10 h past EndDate). +# These UTC boundaries are used after dedup to trim any out-of-range records. +# SpecifyKind(Utc) is critical: ParseExact returns Kind=Unspecified, and .ToUniversalTime() +# on Unspecified assumes LOCAL time, shifting the boundary by the machine's UTC offset. +$script:TrimStartDateUTC = if ($StartDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } +$script:TrimEndDateUTC = if ($EndDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } +$script:DateTrimCount = 0 + +if ($BlockHours -le 0) { Write-Host "ERROR: BlockHours must be positive." -ForegroundColor Red; exit 1 } + +try { if ($PSVersionTable.PSEdition -eq 'Core' -and ($global:InformationPreference -in @('SilentlyContinue', 'Ignore'))) { $global:InformationPreference = 'Continue' } } catch {} + +if ($RAWInputCSV) { + $rawConflictParams = @('BlockHours', 'ResultSize', 'PacingMs', 'Auth', 'ParallelMode', 'MaxParallelGroups', 'MaxConcurrency', 'EnableParallel', 'GroupNames') + $specifiedConflicts = @() + foreach ($cp in $rawConflictParams) { if ($PSBoundParameters.ContainsKey($cp)) { $specifiedConflicts += $cp } } + if ($specifiedConflicts.Count -gt 0) { + Write-Host "ERROR: -RAWInputCSV cannot be combined with live query parameter(s): $($specifiedConflicts -join ', ')" -ForegroundColor Red + Write-Host "Remove those conflicting parameters and re-run. Allowed with RAWInputCSV: StartDate, EndDate, ActivityTypes, AgentId, AgentsOnly, UserIds, OutputFile, AppendFile, explosion switches." -ForegroundColor Yellow + Write-Host "Note: -GroupNames requires authentication and cannot be used in replay mode. Use -UserIds with explicit email addresses instead." -ForegroundColor Yellow + exit 1 + } +} + +# Validate -UseEOM compatibility with parallel processing +if ($UseEOM) { + $parallelConflicts = @() + + # Check for explicit parallel mode settings + if ($PSBoundParameters.ContainsKey('EnableParallel') -and $EnableParallel) { + $parallelConflicts += '-EnableParallel' + } + + if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Off') { + $parallelConflicts += "-ParallelMode $ParallelMode" + } + + if ($parallelConflicts.Count -gt 0) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: -UseEOM Incompatible with Parallel Processing" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host "Exchange Online Management mode (-UseEOM) only supports SERIAL processing." -ForegroundColor Yellow + Write-Host "The Search-UnifiedAuditLog cmdlet cannot be used in parallel ThreadJobs due to" -ForegroundColor Gray + Write-Host "implicit remoting architecture limitations in the EOM PowerShell module." -ForegroundColor Gray + Write-Host "" + Write-Host "CONFLICTING PARAMETERS DETECTED:" -ForegroundColor Yellow + foreach ($conflict in $parallelConflicts) { + Write-Host " • $conflict" -ForegroundColor Red + } + Write-Host "" + Write-Host "RESOLUTION OPTIONS:" -ForegroundColor Cyan + Write-Host " 1. Remove -UseEOM switch to enable Graph API mode (supports parallel processing)" -ForegroundColor White + Write-Host " 2. Remove parallel parameters and use serial-only processing with -UseEOM" -ForegroundColor White + Write-Host " 3. Set -ParallelMode Off explicitly: -UseEOM -ParallelMode Off" -ForegroundColor White + Write-Host "" + Write-Host "NOTE: Graph API mode (default, no -UseEOM) supports parallel processing in PowerShell 7+." -ForegroundColor Gray + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + + # Log to file if log initialized + if ($script:logFile -and (Test-Path $script:logFile)) { + $timestamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + Add-Content -Path $script:logFile -Value "[$timestamp] ERROR: -UseEOM incompatible with parallel processing parameters: $($parallelConflicts -join ', ')" + Add-Content -Path $script:logFile -Value "[$timestamp] Script terminated. Resolution: Remove -UseEOM or disable parallel mode." + } + + exit 1 + } + + # Force ParallelMode Off in EOM mode even if Auto is set + if ($ParallelMode -ne 'Off') { + Write-Host "" + Write-Host "NOTE: -UseEOM mode requires serial processing. Forcing -ParallelMode Off." -ForegroundColor Yellow + Write-Host "" + $ParallelMode = 'Off' + } +} + +$script:learnedActivityBlockSize = @{} +$script:globalLearnedBlockSize = $BlockHours +$script:subdivisionSequence = @(0.5, 0.25, 0.133333, 0.066667, 0.033333, 0.016667, 0.010417, 0.005556, 0.002778, 0.001389) # 12h, 6h, 3.2h, 1.6h, 48m, 24m, 15m, 8m, 4m, 2m +$script:Hit10KLimit = $false +$script:Hit1MLimit = $false # Graph API 1,000,000 record limit per query +$script:LimitTimeWindow = "" +$script:SubdividedPartitions = @{} # Track partitions that needed subdivision (key=original range, value=count) +$script:Connected = $false + +# ============================================================================ +# GRAPH API SECURITY AUDIT ENDPOINT VERSION CONFIGURATION +# ============================================================================ +# Manually configure these variables if Microsoft updates the API version +# PAX will try CURRENT version first, then fallback to PREVIOUS version +# ============================================================================ +$script:GraphAuditApiVersion_Current = 'v1.0' # Try this version first +$script:GraphAuditApiVersion_Previous = 'beta' # Fallback to this version if current unavailable +$script:GraphAuditApiVersion = $null # Runtime-detected version (do not edit) +# ============================================================================ + +# Suppress PowerShell's web request progress bar (prevents "Reading web response stream" noise) +$ProgressPreference = 'SilentlyContinue' + +# Telemetry tracking for Graph API parallel queries (per-slice lifecycle data) +$script:telemetryData = @() + +# ============================================================================ +# SHAPE CONTRACTS +# ---------------------------------------------------------------------------- +# These two helpers freeze the canonical shapes of $script:metrics and of each +# entry in $script:partitionStatus. They are intentionally cheap, fail-fast, +# and called only at well-known initialization points. They do NOT police every +# mutation; their purpose is to catch a future change that introduces an +# incompatible re-init (e.g. drops a required key) at script-start time rather +# than at consumer-time hours later. +# +# If you add a new REQUIRED field to either structure, add it here so the +# contract drift is caught early. +# ============================================================================ +$script:MetricsRequiredFields = @( + 'StartTime', 'QueryMs', 'ExplosionMs', 'ExportMs', + 'PagesFetched', 'TotalRecordsFetched', 'TotalStructuredRows', + 'ExplosionEvents', 'ExplosionRowsFromEvents', 'ExplosionMaxPerRecord', 'ExplosionTruncated', + 'Activities', + 'FilteringSkippedRecords', 'FilteringMissingAuditData', + 'FilteringPromptFiltered', 'FilteringParseFailures' +) +$script:PartitionStatusRequiredFields = @( + 'Partition', 'AttemptNumber', 'QueryId', 'QueryName', + 'Status', 'LastError', 'RecordCount' +) + +function Assert-MetricsShape { + <# + .SYNOPSIS + Fail-fast contract check for $script:metrics. + .DESCRIPTION + Verifies the script-scope metrics hashtable exists and contains every + required field. Throws a single descriptive error on first mismatch so + the failure surfaces at startup rather than via a NullReferenceException + deep in the summary writer. + #> + param([hashtable]$Metrics = $script:metrics) + if (-not $Metrics) { + throw "Assert-MetricsShape: \$script:metrics is null. A required initialization step was skipped." + } + foreach ($k in $script:MetricsRequiredFields) { + if (-not $Metrics.ContainsKey($k)) { + throw "Assert-MetricsShape: \$script:metrics is missing required field '$k'. This indicates a structural mismatch." + } + } +} + +function Assert-PartitionStatusEntry { + <# + .SYNOPSIS + Fail-fast contract check for a single $script:partitionStatus entry. + .DESCRIPTION + Verifies the supplied entry contains every field required by the + downstream STATUS display, retry logic, and summary writer. Optional + fields (ParentPartition, SubdivisionReason) are intentionally not + asserted. + #> + param( + [Parameter(Mandatory = $true)] $Entry, + [string]$Context = '' + ) + if ($null -eq $Entry) { + throw "Assert-PartitionStatusEntry [$Context]: entry is null." + } + foreach ($k in $script:PartitionStatusRequiredFields) { + if (-not $Entry.ContainsKey($k)) { + throw "Assert-PartitionStatusEntry [$Context]: missing required field '$k'. This indicates a structural mismatch." + } + } +} + +$script:metrics = @{ + StartTime = (Get-Date).ToUniversalTime() + QueryMs = 0 + ExplosionMs = 0 + ExportMs = 0 + PagesFetched = 0 + TotalRecordsFetched = 0 + TotalStructuredRows = 0 + ExplosionEvents = 0 + ExplosionRowsFromEvents = 0 + ExplosionMaxPerRecord = 0 + ExplosionTruncated = $false + ShrinkEvents = 0 + Activities = @{} + EffectiveChunkSize = 0 + ParallelBatchSizeFinal = 0 + ParallelThrottleFinal = 0 + AgentFilterApplied = $false + AgentFilterPreCount = 0 + AgentFilterPostCount = 0 + AgentFilterRemovedCount = 0 + AgentFilterElapsedSec = 0 + ExcludeAgentsApplied = $false + ExcludeAgentsPreCount = 0 + ExcludeAgentsPostCount = 0 + ExcludeAgentsRemoved = 0 + ExcludeAgentsElapsedSec = 0 + PromptFilterApplied = $false + PromptFilterType = '' + PromptFilterPreCount = 0 + PromptFilterPostCount = 0 + PromptFilterRemovedCount = 0 + PromptFilterElapsedSec = 0 + PromptFilterMsgBefore = 0 + PromptFilterMsgAfter = 0 + PromptFilterMsgRemoved = 0 + PromptFilterRecordsMixed = 0 + PromptFilterRecordsPromptOnly = 0 + PromptFilterRecordsResponseOnly = 0 + PromptFilterRecordsNoMessages = 0 + FilteringSkippedRecords = 0 + FilteringMissingAuditData = 0 + FilteringParseFailures = 0 + FilteringPromptFiltered = 0 + FilteringAgentFiltered = 0 + FilteringExcludeAgents = 0 + FilteringUserIds = 0 + FilteringGroupNames = 0 + FilteringOther = 0 + AdaptiveEvents = @() + AdaptiveMemoryReductions = 0 + AdaptiveLatencyReductions = 0 + AdaptiveLatencyIncreases = 0 + ThroughputBaselineRps = 0 + CircuitBreakerTrips = 0 + BackoffTotalDelaySeconds = 0 + PartitionCapsApplied = 0 + PartitionCapHighestRequested = 0 +} + +# v6 shape-contract guard: catches accidental drops of required metrics fields +# at startup rather than at summary-writer time. See Assert-MetricsShape above. +Assert-MetricsShape + +$script:summaryWritten = $false + +# Streaming dataset profiler (live & replay) +$script:profiler = @{ + Rows = 0 + Operations = @{} + RecordTypes = @{} + HasCopilot = 0 + MaxDepth = 0 + DepthCounts = @{} + MaxArrayLen = 0 +} + +$script:shapeCache = @{} + +function Get-RecordShapeKey { + param([object]$AuditData) + try { + $rt = $AuditData.RecordType + } catch { $rt = '' } + try { + $op = $AuditData.Operation + } catch { $op = '' } + try { + $hasCopilot = $AuditData.PSObject.Properties['CopilotEventData'] -ne $null + } catch { $hasCopilot = $false } + return "$rt|$op|$hasCopilot" +} + +function Get-RecordShape { + param([object]$AuditData) + if ($null -eq $AuditData) { return $null } + $key = Get-RecordShapeKey $AuditData + if ($script:shapeCache.ContainsKey($key)) { return $script:shapeCache[$key] } + $shape = @{} + try { + $shape.RecordType = $AuditData.RecordType + $shape.Operation = $AuditData.Operation + } catch {} + try { $shape.HasCopilot = $AuditData.PSObject.Properties['CopilotEventData'] -ne $null } catch { $shape.HasCopilot = $false } + try { $shape.Depth = Get-JsonDepth $AuditData 0 } catch { $shape.Depth = 0 } + $shape.Mode = if ($shape.HasCopilot) { 'Copilot' } else { 'AuditData' } + $script:shapeCache[$key] = $shape + return $shape +} + +function Reset-Profiler { + $script:profiler = @{ + Rows = 0 + Operations = @{} + RecordTypes = @{} + HasCopilot = 0 + MaxDepth = 0 + DepthCounts = @{} + MaxArrayLen = 0 + } +} + +function Get-JsonDepth([object]$node, [int]$d = 0) { + if ($null -eq $node -or (Test-ScalarValue $node)) { return $d } + if ($node -is [System.Collections.IDictionary]) { + $maxd = $d + foreach ($v in $node.Values) { $maxd = [math]::Max($maxd, (Get-JsonDepth $v ($d + 1))) } + return $maxd + } + if ($node -is [System.Collections.IEnumerable] -and -not ($node -is [string])) { + $maxd = $d + $i = 0 + foreach ($el in $node) { $maxd = [math]::Max($maxd, (Get-JsonDepth $el ($d + 1))); $i++ } + if ($i -gt $script:profiler.MaxArrayLen) { $script:profiler.MaxArrayLen = $i } + return $maxd + } + return $d +} + +function Profile-AuditData { + param([object]$AuditData) + if ($null -eq $AuditData) { return } + try { + $script:profiler.Rows++ + # Operation + try { + $op = $AuditData.Operation + if (-not [string]::IsNullOrWhiteSpace($op)) { + if (-not $script:profiler.Operations.ContainsKey($op)) { $script:profiler.Operations[$op] = 0 } + $script:profiler.Operations[$op] += 1 + } + } catch {} + # RecordType + try { + $rt = $AuditData.RecordType + if (-not [string]::IsNullOrWhiteSpace([string]$rt)) { + if (-not $script:profiler.RecordTypes.ContainsKey([string]$rt)) { $script:profiler.RecordTypes[[string]$rt] = 0 } + $script:profiler.RecordTypes[[string]$rt] += 1 + } + } catch {} + # CopilotEventData presence + try { if ($AuditData.PSObject.Properties['CopilotEventData']) { $script:profiler.HasCopilot++ } } catch {} + # Depth & arrays + $depth = Get-JsonDepth $AuditData 0 + if ($depth -gt $script:profiler.MaxDepth) { $script:profiler.MaxDepth = $depth } + if (-not $script:profiler.DepthCounts.ContainsKey($depth)) { $script:profiler.DepthCounts[$depth] = 0 } + $script:profiler.DepthCounts[$depth] += 1 + } catch {} +} + +function Write-ProfilerSummary { + param([int]$TopOps = 20, [int]$TopDepths = 10) + try { + Write-LogHost "Profiler: Rows=$($script:profiler.Rows), MaxDepth=$($script:profiler.MaxDepth), MaxArrayLen=$($script:profiler.MaxArrayLen), HasCopilot=$($script:profiler.HasCopilot)" -ForegroundColor Gray + if ($script:profiler.Operations.Count -gt 0) { + Write-LogHost "Profiler: Operations (top $TopOps):" -ForegroundColor Gray + $script:profiler.Operations.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First $TopOps | ForEach-Object { Write-LogHost " $($_.Key): $($_.Value)" -ForegroundColor Gray } + } + if ($script:profiler.DepthCounts.Count -gt 0) { + Write-LogHost "Profiler: Depth distribution (top $TopDepths):" -ForegroundColor Gray + $script:profiler.DepthCounts.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First $TopDepths | ForEach-Object { Write-LogHost " Depth $($_.Key): $($_.Value)" -ForegroundColor Gray } + } + } catch {} +} +$script:adaptiveThroughputBaseline = $null +$script:adaptiveLowLatencyStreak = 0 +$script:consecutiveBlockFailures = 0 +$script:circuitBreakerOpen = $false +$script:circuitBreakerOpenUntil = $null + +# ============================================== +# GRAPH API VERSION DETECTION HELPER +# ============================================== +# Automatically detects and uses configured current version or falls back to previous version +# Version configuration is at top of script for easy manual updates + +function Get-GraphAuditApiUri { + <# + .SYNOPSIS + Builds Graph API audit endpoint URI with automatic version detection. + + .DESCRIPTION + Attempts to use the configured current version first. If not available, + falls back to the previous version. Version detection is cached per session. + + Configure versions at top of script: + $script:GraphAuditApiVersion_Current = 'v1.0' (try first) + $script:GraphAuditApiVersion_Previous = 'beta' (fallback) + + .PARAMETER Path + The audit API path (e.g., "queries", "queries/{id}", "queries/{id}/records") + + .OUTPUTS + String - Full Graph API URI with appropriate version + + .EXAMPLE + $uri = Get-GraphAuditApiUri -Path "queries" + # Returns: https://graph.microsoft.com/v1.0/security/auditLog/queries + # or: https://graph.microsoft.com/beta/security/auditLog/queries (if v1.0 unavailable) + #> + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + # Auto-detect version on first use (cached for session) + if ($null -eq $script:GraphAuditApiVersion) { + $currentVer = $script:GraphAuditApiVersion_Current + $previousVer = $script:GraphAuditApiVersion_Previous + + try { + # Test if current version endpoint is available + $testUri = "https://graph.microsoft.com/$currentVer/security/auditLog/queries" + Invoke-MgGraphRequest -Method GET -Uri $testUri -ErrorAction Stop | Out-Null + $script:GraphAuditApiVersion = $currentVer + Write-LogHost "Graph API: security/auditLog endpoint using version $currentVer" -ForegroundColor Green + } catch { + # Current version not available, fallback to previous + $script:GraphAuditApiVersion = $previousVer + Write-LogHost "Graph API: security/auditLog endpoint using version $previousVer (fallback from $currentVer)" -ForegroundColor Yellow + } + } + + return "https://graph.microsoft.com/$($script:GraphAuditApiVersion)/security/auditLog/$Path" +} + +# ============================================== +# CTRL+C GRACEFUL EXIT HANDLER +# ============================================== +# Track Ctrl+C state for graceful exit messaging in finally block + +$script:CtrlCPressed = $false +$script:ScriptCompleted = $false +$script:EarlyExit = $false +# Track whether ANY file upload (SharePoint/Fabric output, metrics, or run log) failed +# this run. When true, local run files are preserved at end of run - treated the same as +# Ctrl+C / early-exit / crash - so nothing is lost when a destination upload did not complete. +$script:AnyUploadFailed = $false + +# Register exit handler that ALWAYS runs when PowerShell exits +# This works even when Ctrl+C is pressed before the try block (e.g., during module loading) +# Uses environment variable for cross-runspace communication since Register-EngineEvent runs in isolated scope +$env:PAX_GRACEFUL_EXIT_DONE = $null +$env:PAX_REPLAY_MODE = $null # Will be set to "1" when RAWInputCSV is used +Register-EngineEvent -SourceIdentifier PowerShell.Exiting -Action { + if (-not $env:PAX_GRACEFUL_EXIT_DONE) { + # Skip interrupt messaging in replay mode - no Graph connection to disconnect + if (-not $env:PAX_REPLAY_MODE) { + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host " Script Interrupted - Performing Graceful Cleanup" -ForegroundColor Yellow + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + Write-Host " Cleanup complete. Exiting..." -ForegroundColor Green + Write-Host "" + } + } +} | Out-Null + +# Define cleanup function (used by catch block for PipelineStoppedException) +function Invoke-GracefulExit { + param([string]$Reason = "Script interrupted") + + if ($script:CtrlCPressed) { return } # Prevent multiple invocations + $script:CtrlCPressed = $true + + # Signal to engine event handler that graceful exit is handling this + $env:PAX_GRACEFUL_EXIT_DONE = "1" + + # Skip interrupt messaging and Graph disconnect in replay mode - no connections to clean up + if ($env:PAX_REPLAY_MODE) { + exit 0 + } + + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host " Script Interrupted - Performing Graceful Cleanup" -ForegroundColor Yellow + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + + # Disconnect from Microsoft Graph - ALWAYS attempt disconnect + Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Cyan + try { + Disconnect-MgGraph -ErrorAction Stop | Out-Null + Write-Host " Microsoft Graph disconnected" -ForegroundColor Green + } + catch { + if ($_.Exception.Message -match 'No application to sign out from') { + Write-Host " (Not connected to Microsoft Graph)" -ForegroundColor DarkGray + } else { + Write-Host " Microsoft Graph session cleared" -ForegroundColor Green + } + } + + # Disconnect from Exchange Online (if connected via EOM mode) + try { + $eomSession = Get-PSSession | Where-Object { $_.ConfigurationName -eq 'Microsoft.Exchange' -and $_.State -eq 'Opened' } + if ($eomSession) { + Write-Host " Disconnecting from Exchange Online Management..." -ForegroundColor Cyan + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue | Out-Null + Write-Host " Exchange Online disconnected" -ForegroundColor Green + } + } + catch { + Write-Host " (Exchange Online cleanup completed)" -ForegroundColor Gray + } + + # Log the graceful exit + if ($LogFile -and (Test-Path $LogFile)) { + Write-Output "" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Script interrupted by user (Ctrl+C)" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Graceful cleanup completed" | Out-File -FilePath $LogFile -Append -Encoding utf8 + } + + # Show checkpoint resume message if checkpoint is enabled + if ($script:CheckpointEnabled -and $script:CheckpointPath -and (Test-Path $script:CheckpointPath)) { + Show-CheckpointExitMessage + } + + # Release the exclusive checkpoint lock so a future -Resume of THIS checkpoint + # (from any host) is not blocked by our stale lock. The lock file itself is removed + # by Release-CheckpointLock; the checkpoint JSON is preserved for resume. + try { script:Release-CheckpointLock } catch {} + + Write-Host "" + Write-Host " Cleanup complete. Exiting..." -ForegroundColor Green + Write-Host "" + + # Exit cleanly (env var PAX_GRACEFUL_EXIT_DONE already set at function start) + exit 0 +} + +# Trap for catching terminating errors (including Ctrl+C) +trap { + if ($_.Exception -is [System.Management.Automation.PipelineStoppedException]) { + Invoke-GracefulExit + break + } + # Re-throw other exceptions + throw $_ +} + +# ============================================== +# MODULE PREREQUISITES +# ============================================== +# Load required modules based on mode selection (-UseEOM vs Graph API default) + +# Emit PAX banner + sensitive-data warning to terminal/log BEFORE any other startup output +Write-LogHost "=== Portable Audit eXporter (PAX) - Purview Audit Log Exporter ===" -ForegroundColor Cyan +Write-LogHost ("Script Version: v$ScriptVersion") -ForegroundColor White +if (-not $SkipVersionCheck) { Invoke-PaxVersionCheck -CurrentVersion $ScriptVersion } +Write-LogHost "" +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost " !! SENSITIVE DATA WARNING - CUSTOMER RESPONSIBILITY !!" -ForegroundColor Yellow +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost "The audit data exported by this script is HIGHLY SENSITIVE. Output may" -ForegroundColor Yellow +Write-LogHost "contain user identifiers (UPN, email, GUID), file/site/resource paths," -ForegroundColor Yellow +Write-LogHost "conversation and message IDs, agent identifiers, prompt/response metadata" -ForegroundColor Yellow +Write-LogHost "(timestamps, lengths, classifications), and other personally identifiable" -ForegroundColor Yellow +Write-LogHost "information drawn directly from your tenant's Unified Audit Log." -ForegroundColor Yellow +Write-LogHost "" +Write-LogHost " * Data is NOT hashed, masked, redacted, anonymized, or de-identified." -ForegroundColor Yellow +Write-LogHost " Records are exported in raw, attributable form as Purview returns them." -ForegroundColor Yellow +Write-LogHost " * Outputs (CSV/Excel/JSON metrics, checkpoint files, logs) may contain" -ForegroundColor Yellow +Write-LogHost " confidential business content, regulated data (PII, PHI, financial," -ForegroundColor Yellow +Write-LogHost " IP), and end-user communications." -ForegroundColor Yellow +Write-LogHost " * The customer (you / your organization) is SOLELY RESPONSIBLE for the" -ForegroundColor Yellow +Write-LogHost " secure handling, storage, transmission, retention, disclosure, access" -ForegroundColor Yellow +Write-LogHost " control, and deletion of all data produced by this script, and for" -ForegroundColor Yellow +Write-LogHost " ensuring its use complies with all applicable laws, regulations," -ForegroundColor Yellow +Write-LogHost " contractual obligations, and internal policies - including but not" -ForegroundColor Yellow +Write-LogHost " limited to GDPR, HIPAA, CCPA, employee monitoring laws, works-council" -ForegroundColor Yellow +Write-LogHost " agreements, and data-residency requirements." -ForegroundColor Yellow +Write-LogHost " * Microsoft has no visibility into, control over, or responsibility" -ForegroundColor Yellow +Write-LogHost " for the data customers extract using this tool or how that data is" -ForegroundColor Yellow +Write-LogHost " subsequently used, shared, or stored. Microsoft disclaims any and all" -ForegroundColor Yellow +Write-LogHost " liability arising from or related to customer use of this script and" -ForegroundColor Yellow +Write-LogHost " its output." -ForegroundColor Yellow +Write-LogHost " * Treat all output files as HIGHLY CONFIDENTIAL. Restrict access to" -ForegroundColor Yellow +Write-LogHost " authorized personnel with a documented business need. Encrypt at rest" -ForegroundColor Yellow +Write-LogHost " and in transit. Apply tenant DLP / sensitivity labels as appropriate." -ForegroundColor Yellow +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost "" + +if ($RAWInputCSV) { + # Set replay mode flag for graceful exit handling (skip Graph disconnect messaging) + $env:PAX_REPLAY_MODE = "1" + Write-LogHost "`nReplay mode: Skipping module loading`n" -ForegroundColor Cyan +} +elseif (-not $UseEOM) { + # DEFAULT MODE: Microsoft Graph Security API + # Requires Microsoft.Graph.Authentication and Microsoft.Graph.Security modules + Write-LogHost "" + Write-LogHost "Loading Microsoft Graph modules..." -ForegroundColor Cyan + + try { + # ============================================ + # AUTO-UPDATE CHECK: Ensure latest SDK version + # ============================================ + # The Graph Security auditLog API has known issues with older SDK versions. + # Always check for and install the latest version to ensure compatibility. + + Write-LogHost " Checking for Microsoft Graph SDK updates..." -ForegroundColor Gray + + # Get currently installed version + $installedAuth = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + $installedAuthVersion = if ($installedAuth) { $installedAuth.Version } else { [Version]"0.0.0" } + + # Check PSGallery for latest version (with 15-second timeout to avoid hangs) + $latestAuthVersion = $null + try { + $updateCheckJob = Start-Job -ScriptBlock { Find-Module -Name Microsoft.Graph.Authentication -Repository PSGallery -ErrorAction Stop } + $jobCompleted = Wait-Job -Job $updateCheckJob -Timeout 15 + if ($jobCompleted) { + $galleryAuth = Receive-Job -Job $updateCheckJob -ErrorAction Stop + $latestAuthVersion = [Version]$galleryAuth.Version + } + else { + Write-LogHost " PSGallery check timed out (15s) - skipping update check" -ForegroundColor Yellow + } + Remove-Job -Job $updateCheckJob -Force -ErrorAction SilentlyContinue + } + catch { + Write-LogHost " Warning: Could not check PSGallery for updates: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Continuing with installed version..." -ForegroundColor Yellow + } + + # Update if newer version available + $updatePerformed = $false + if ($latestAuthVersion -and ($latestAuthVersion -gt $installedAuthVersion)) { + Write-LogHost " Update available: v$installedAuthVersion → v$latestAuthVersion" -ForegroundColor Yellow + Write-LogHost " Installing Microsoft.Graph.Authentication v$latestAuthVersion..." -ForegroundColor Yellow + + try { + # Install latest Authentication module + Install-Module -Name Microsoft.Graph.Authentication -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Authentication updated to v$latestAuthVersion" -ForegroundColor Green + + # Install matching Security module + Write-LogHost " Installing Microsoft.Graph.Security v$latestAuthVersion..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Security -RequiredVersion $latestAuthVersion -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Security updated to v$latestAuthVersion" -ForegroundColor Green + + $updatePerformed = $true + + # Refresh module info after update + $installedAuth = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + $installedAuthVersion = $installedAuth.Version + } + catch { + Write-LogHost " Warning: Update failed: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Continuing with existing version v$installedAuthVersion..." -ForegroundColor Yellow + } + } + elseif ($latestAuthVersion) { + Write-LogHost " Microsoft Graph SDK is up to date (v$installedAuthVersion)" -ForegroundColor Green + } + else { + Write-LogHost " Using installed version v$installedAuthVersion" -ForegroundColor Gray + } + + # ============================================ + # LOAD MODULES + # ============================================ + + $authModule = Get-Module -Name Microsoft.Graph.Authentication | Select-Object -First 1 + if (-not $authModule) { + $authModule = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + } + if (-not $authModule) { + Write-LogHost " Installing Microsoft.Graph.Authentication module (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Authentication -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + $authModule = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + } + $authVersion = $authModule.Version + Write-LogHost " Importing Microsoft.Graph.Authentication v$authVersion..." -ForegroundColor Gray + Import-Module Microsoft.Graph.Authentication -RequiredVersion $authVersion -Force -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Authentication v$authVersion loaded" -ForegroundColor Green + + # Load Microsoft.Graph.Security matching auth version (exact if possible, otherwise same major/minor) + $securityModule = Get-Module -Name Microsoft.Graph.Security | Where-Object { $_.Version -eq $authVersion } | Select-Object -First 1 + if (-not $securityModule) { + $securityModule = Get-Module -ListAvailable -Name Microsoft.Graph.Security | + Where-Object { $_.Version.Major -eq $authVersion.Major -and $_.Version.Minor -eq $authVersion.Minor } | + Sort-Object Version -Descending | + Select-Object -First 1 + } + if (-not $securityModule) { + Write-LogHost " Installing Microsoft.Graph.Security v$authVersion (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Security -RequiredVersion $authVersion -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + $securityModule = Get-Module -ListAvailable -Name Microsoft.Graph.Security | Where-Object { $_.Version -eq $authVersion } | Select-Object -First 1 + } + $secVersion = $securityModule.Version + Write-LogHost " Importing Microsoft.Graph.Security v$secVersion..." -ForegroundColor Gray + Import-Module Microsoft.Graph.Security -RequiredVersion $secVersion -Force -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Security v$secVersion loaded" -ForegroundColor Green + } + catch { + Write-LogHost " ERROR: Failed to load Microsoft Graph module: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "`nTroubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure PowerShell Gallery access is available" -ForegroundColor White + Write-LogHost " 2. Try manual installation: Install-Module -Name Microsoft.Graph -Force" -ForegroundColor White + Write-LogHost " 3. Use -UseEOM switch to fall back to Exchange Online Management mode" -ForegroundColor White + throw + } + + Write-LogHost "Microsoft Graph modules loaded successfully`n" -ForegroundColor Green +} +else { + # EOM MODE: Exchange Online Management + # Graph modules not required in EOM mode + Write-LogHost "`nEOM Mode: Skipping Microsoft Graph module loading`n" -ForegroundColor Cyan +} + +# ============================================== +# DUAL-MODE AUTHENTICATION FUNCTION +# ============================================== +# Unified authentication supporting both EOM and Graph API modes + +function Connect-PurviewAudit { + <# + .SYNOPSIS + Unified authentication for Purview audit log access via EOM or Graph API. + + .DESCRIPTION + Authenticates to Microsoft 365 using either Exchange Online Management (EOM) + or Microsoft Graph Security API based on the -UseEOM switch. + + EOM Mode (-UseEOM): + - Uses Connect-ExchangeOnline cmdlet + - Requires Exchange Online RBAC roles + - Serial processing only + + Graph API Mode (Default): + - Uses Connect-MgGraph with AuditLogsQuery.Read.All scope + - Requires Azure AD roles + Graph API permissions + - Supports parallel processing + + .PARAMETER AuthMethod + Authentication method: WebLogin, DeviceCode, Credential, Silent, AppRegistration + + .PARAMETER UseEOMMode + If true, use EOM mode. If false, use Graph API mode. + #> + + param( + [Parameter(Mandatory = $true)] + [ValidateSet('WebLogin', 'DeviceCode', 'Credential', 'Silent', 'AppRegistration', 'ManagedIdentity')] + [string]$AuthMethod, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Exchange Online Management + # ======================================== + + if ($script:Connected) { + Write-LogHost "Already connected to Exchange Online." -ForegroundColor Gray + return + } + + Write-LogHost "Connecting to Microsoft 365 Security & Compliance Center (EOM)..." -ForegroundColor Cyan + + # Ensure ExchangeOnlineManagement module is available + try { + $existingEOM = Get-Module -ListAvailable -Name ExchangeOnlineManagement | Sort-Object Version -Descending | Select-Object -First 1 + if (-not $existingEOM) { + Write-LogHost "Installing ExchangeOnlineManagement module (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + } + Import-Module ExchangeOnlineManagement -Force -ErrorAction Stop + + $eomVersion = (Get-Module ExchangeOnlineManagement).Version + Write-LogHost " ExchangeOnlineManagement v$eomVersion loaded" -ForegroundColor Green + } + catch { + Write-LogHost "ERROR: Module load/install failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } + + # Authenticate based on method + try { + switch ($AuthMethod.ToLower()) { + 'appregistration' { + Write-LogHost "AppRegistration authentication is not supported with -UseEOM. Remove -UseEOM to use Graph mode." -ForegroundColor Yellow + throw "AppRegistration authentication is only available in Graph API mode" + } + 'weblogin' { + $exoCmd = Get-Command Connect-ExchangeOnline -ErrorAction Stop + $hasUseWeb = $exoCmd.Parameters.ContainsKey('UseWebLogin') + + if ($hasUseWeb) { + Write-LogHost "Using Connect-ExchangeOnline -UseWebLogin..." -ForegroundColor Gray + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + else { + Write-LogHost "UseWebLogin parameter not available; using standard interactive auth..." -ForegroundColor Yellow + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + } + + 'devicecode' { + Write-LogHost "Using device code flow..." -ForegroundColor Gray + Connect-ExchangeOnline -ShowBanner:$false -Device -ErrorAction Stop | Out-Null + } + + 'credential' { + Write-LogHost "Using credential-based authentication..." -ForegroundColor Gray + $cred = Get-Credential -Message 'Enter admin credentials for Exchange Online' + Connect-ExchangeOnline -ShowBanner:$false -Credential $cred -ErrorAction Stop | Out-Null + } + + 'silent' { + Write-LogHost "Attempting silent authentication..." -ForegroundColor Gray + $silentOk = $true + try { + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + catch { + $silentOk = $false + } + + if (-not $silentOk) { + Write-LogHost "Silent auth failed, falling back to WebLogin..." -ForegroundColor Yellow + try { + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + catch { + Write-LogHost "ERROR: Silent + fallback auth failed: $($_.Exception.Message)" -ForegroundColor Red + throw + } + } + } + 'managedidentity' { + Write-LogHost "ERROR: -Auth ManagedIdentity is not supported with -UseEOM." -ForegroundColor Red + Write-LogHost " Exchange Online Management does not accept managed-identity auth." -ForegroundColor Yellow + Write-LogHost " Use -Auth ManagedIdentity without -UseEOM (Graph API path)." -ForegroundColor Yellow + throw "ManagedIdentity auth is not supported with -UseEOM" + } + } + + $script:Connected = $true + Write-LogHost "Successfully connected to Exchange Online" -ForegroundColor Green + + # Verify connection + try { + $connInfo = Get-ConnectionInformation -ErrorAction SilentlyContinue | Where-Object { $_.TokenStatus -ne 'Expired' } | Select-Object -First 1 + if ($connInfo) { + Write-LogHost " Tenant ID: $($connInfo.TenantId)" -ForegroundColor Gray + Write-LogHost " User: $($connInfo.UserPrincipalName)" -ForegroundColor Gray + } + } + catch { + # Connection info not critical, continue + } + } + catch { + Write-LogHost "ERROR: EOM authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "`nTroubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Verify you have required Exchange Online roles" -ForegroundColor White + Write-LogHost " 2. Check Multi-Factor Authentication requirements" -ForegroundColor White + Write-LogHost " 3. Try a different auth method (-Auth parameter)" -ForegroundColor White + throw + } + } + else { + # ======================================== + # GRAPH API MODE: Microsoft Graph Security + # ======================================== + + # Clear any stale Graph session from a previous script run or Ctrl+C in this terminal. + # This forces a fresh Connect-MgGraph with a new token, preventing issues where MSAL + # silently returns a cached expired token or a token from a different user account. + Write-LogHost "Clearing any previous Graph session..." -ForegroundColor Gray + try { Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null } catch { } + + Write-LogHost "Connecting to Microsoft Graph Security API..." -ForegroundColor Cyan + + # Define required scopes for Purview audit log access via the Microsoft Graph + # Security API. The exact scope set is built dynamically based on which features + # the caller has activated, so users only consent to what they actually use. + # + # Conditional (added only when the relevant switch is set): + # Audit query (skipped under -OnlyUserInfo since no audit calls are made): + # AuditLogsQuery.Read.All - umbrella scope for /security/auditLog/queries + # (covers CopilotInteraction and other workload- + # agnostic record types) + # -IncludeM365Usage: + # AuditLogsQuery-Exchange.Read.All + # AuditLogsQuery-OneDrive.Read.All + # AuditLogsQuery-SharePoint.Read.All + # -IncludeUserInfo / -OnlyUserInfo (Graph API mode only): + # User.Read.All (read /users for Entra directory + license map) + # Organization.Read.All (read /subscribedSkus for license SKU lookup) + # -GroupNames (Graph API mode only): + # GroupMember.Read.All (least-privilege scope per Microsoft Graph docs + # for GET /groups and GET /groups/{id}/members; + # used by Expand-PurviewGroupMembership) + $RequiredScopes = [System.Collections.Generic.List[string]]::new() + if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + [void]$RequiredScopes.Add('AuditLogsQuery.Read.All') + } + if ($IncludeM365Usage) { + [void]$RequiredScopes.Add('AuditLogsQuery-Exchange.Read.All') + [void]$RequiredScopes.Add('AuditLogsQuery-OneDrive.Read.All') + [void]$RequiredScopes.Add('AuditLogsQuery-SharePoint.Read.All') + } + if ($IncludeUserInfo -or $OnlyUserInfo) { + if ($RequiredScopes -notcontains 'User.Read.All') { [void]$RequiredScopes.Add('User.Read.All') } + if ($RequiredScopes -notcontains 'Organization.Read.All') { [void]$RequiredScopes.Add('Organization.Read.All') } + } + if ($GroupNames -and $GroupNames.Count -gt 0) { + if ($RequiredScopes -notcontains 'GroupMember.Read.All') { [void]$RequiredScopes.Add('GroupMember.Read.All') } + } + # Microsoft Agent 365 enrichment scopes - DELEGATED auth modes only. + # Delegated modes (WebLogin / DeviceCode / Credential / Silent) request + # CopilotPackages.Read.All (+ Application.Read.All) as DELEGATED scopes here at sign-in. + # App-only modes (AppRegistration certificate/secret, ManagedIdentity) do NOT request + # delegated scopes: the Agent 365 catalog is read with the matching APPLICATION app-roles + # granted + admin-consented on the app / managed-identity service principal out-of-band + # (see Connect-Agent365InteractiveContext). The $AuthMethod -notin guard below therefore + # correctly excludes both app-only modes from delegated scope requests. + if (($IncludeAgent365Info -or $OnlyAgent365Info) -and $AuthMethod -notin @('AppRegistration','ManagedIdentity')) { + if ($RequiredScopes -notcontains 'CopilotPackages.Read.All') { [void]$RequiredScopes.Add('CopilotPackages.Read.All') } + if ($RequiredScopes -notcontains 'Application.Read.All') { [void]$RequiredScopes.Add('Application.Read.All') } + } + # SharePoint remote-output destination needs delegated/app drive write scopes. + # Fabric/OneLake uses a separate storage-audience token (Az.Accounts), not a Graph scope. + if ($script:RemoteOutputMode -eq 'SharePoint') { + if ($RequiredScopes -notcontains 'Sites.ReadWrite.All') { [void]$RequiredScopes.Add('Sites.ReadWrite.All') } + if ($RequiredScopes -notcontains 'Files.ReadWrite.All') { [void]$RequiredScopes.Add('Files.ReadWrite.All') } + } + $RequiredScopes = $RequiredScopes.ToArray() + + try { + switch ($AuthMethod.ToLower()) { + 'weblogin' { + Write-LogHost "Using interactive browser authentication..." -ForegroundColor Gray + Connect-MgGraph -Scopes $RequiredScopes -NoWelcome -ErrorAction Stop + } + + 'devicecode' { + Write-LogHost "Using device code flow..." -ForegroundColor Gray + Write-LogHost "A browser window will open. Follow the instructions to authenticate." -ForegroundColor Yellow + Connect-MgGraph -Scopes $RequiredScopes -UseDeviceCode -NoWelcome -ErrorAction Stop + } + + 'credential' { + Write-LogHost "Using client secret credential..." -ForegroundColor Gray + + # Check for required environment variables + $tenantId = $env:GRAPH_TENANT_ID + $clientId = $env:GRAPH_CLIENT_ID + $clientSecret = $env:GRAPH_CLIENT_SECRET + + if (-not $tenantId -or -not $clientId -or -not $clientSecret) { + Write-LogHost "ERROR: Credential authentication requires environment variables:" -ForegroundColor Red + Write-LogHost " GRAPH_TENANT_ID : Your Azure AD Tenant ID" -ForegroundColor Yellow + Write-LogHost " GRAPH_CLIENT_ID : Your App Registration Client ID" -ForegroundColor Yellow + Write-LogHost " GRAPH_CLIENT_SECRET : Your App Registration Client Secret" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "Set these variables before running the script:" -ForegroundColor Yellow + Write-LogHost " `$env:GRAPH_TENANT_ID = 'your-tenant-id'" -ForegroundColor White + Write-LogHost " `$env:GRAPH_CLIENT_ID = 'your-client-id'" -ForegroundColor White + Write-LogHost " `$env:GRAPH_CLIENT_SECRET = 'your-client-secret'" -ForegroundColor White + throw "Missing required environment variables for credential authentication" + } + + $secureSecret = ConvertTo-SecureString -String $clientSecret -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential($clientId, $secureSecret) + + # Clear plain-text secret from memory + Clear-Variable -Name clientSecret -Force -ErrorAction SilentlyContinue + + Connect-MgGraph -TenantId $tenantId -ClientSecretCredential $credential -NoWelcome -ErrorAction Stop + } + + 'silent' { + Write-LogHost "Using managed identity or existing token..." -ForegroundColor Gray + Connect-MgGraph -Identity -NoWelcome -ErrorAction Stop + } + 'managedidentity' { + Write-LogHost "Using managed identity (system-assigned or user-assigned)..." -ForegroundColor Gray + if ($env:AZURE_CLIENT_ID) { + Write-LogHost (" -> User-assigned MI client id: {0}" -f $env:AZURE_CLIENT_ID) -ForegroundColor DarkGray + Connect-MgGraph -Identity -ClientId $env:AZURE_CLIENT_ID -NoWelcome -ErrorAction Stop + } else { + Connect-MgGraph -Identity -NoWelcome -ErrorAction Stop + } + } + 'appregistration' { + Write-LogHost "Using app registration authentication..." -ForegroundColor Gray + + $appTenantId = $script:TenantId + if ([string]::IsNullOrWhiteSpace($appTenantId)) { $appTenantId = $env:GRAPH_TENANT_ID } + if ([string]::IsNullOrWhiteSpace($appTenantId)) { + Write-LogHost "ERROR: -TenantId or GRAPH_TENANT_ID is required for AppRegistration auth." -ForegroundColor Red + throw "Missing TenantId for AppRegistration authentication" + } + + $appClientId = $script:ClientId + if ([string]::IsNullOrWhiteSpace($appClientId)) { $appClientId = $env:GRAPH_CLIENT_ID } + if ([string]::IsNullOrWhiteSpace($appClientId)) { + Write-LogHost "ERROR: -ClientId or GRAPH_CLIENT_ID is required for AppRegistration auth." -ForegroundColor Red + throw "Missing ClientId for AppRegistration authentication" + } + + # Store auth config for potential re-authentication during long-running operations + $script:AuthConfig.Method = 'AppRegistration' + $script:AuthConfig.TenantId = $appTenantId + $script:AuthConfig.ClientId = $appClientId + $script:AuthConfig.CertStoreLocation = $script:ClientCertificateStoreLocation + + $secretValue = $script:ClientSecret + if ([string]::IsNullOrWhiteSpace($secretValue)) { $secretValue = $env:GRAPH_CLIENT_SECRET } + + $certThumbprint = $script:ClientCertificateThumbprint + if ([string]::IsNullOrWhiteSpace($certThumbprint)) { $certThumbprint = $env:GRAPH_CLIENT_CERT_THUMBPRINT } + + $certPath = $script:ClientCertificatePath + if ([string]::IsNullOrWhiteSpace($certPath)) { $certPath = $env:GRAPH_CLIENT_CERT_PATH } + + $certPasswordSecure = $script:ClientCertificatePassword + if (-not $certPasswordSecure -and $env:GRAPH_CLIENT_CERT_PASSWORD) { + $certPasswordSecure = ConvertTo-SecureString $env:GRAPH_CLIENT_CERT_PASSWORD -AsPlainText -Force + } + + $certPasswordPlain = $null + if ($certPasswordSecure) { + $certPasswordPlain = [System.Net.NetworkCredential]::new('', $certPasswordSecure).Password + } + + if (-not [string]::IsNullOrWhiteSpace($secretValue)) { + Write-LogHost " -> Authenticating with client secret" -ForegroundColor Gray + $secureSecret = ConvertTo-SecureString -String $secretValue -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential($appClientId, $secureSecret) + # Store secret securely for re-authentication (keep a copy before clearing) + $script:AuthConfig.ClientSecret = $secureSecret.Copy() + $script:AuthConfig.CanReauthenticate = $true + Clear-Variable -Name secretValue -Force -ErrorAction SilentlyContinue + Connect-MgGraph -TenantId $appTenantId -ClientSecretCredential $credential -NoWelcome -ErrorAction Stop + Clear-Variable -Name secureSecret -Force -ErrorAction SilentlyContinue + Clear-Variable -Name credential -Force -ErrorAction SilentlyContinue + } + elseif (-not [string]::IsNullOrWhiteSpace($certThumbprint)) { + # Normalize: strip spaces and any non-hex paste artifacts (the Windows cert + # dialog can prepend a hidden left-to-right mark), then upper-case. + $certThumbprint = ($certThumbprint -replace '[^0-9A-Fa-f]', '').ToUpperInvariant() + Write-LogHost " -> Authenticating with certificate thumbprint $certThumbprint" -ForegroundColor Gray + # Search the requested store first, then the OTHER store. App-registration + # certs on servers are commonly in LocalMachine\My even when the run defaults + # to CurrentUser; this finds the cert in either location. + $requestedLoc = $script:ClientCertificateStoreLocation + $otherLoc = if ($requestedLoc -eq 'LocalMachine') { 'CurrentUser' } else { 'LocalMachine' } + $certificate = $null + $resolvedLoc = $null + foreach ($loc in @($requestedLoc, $otherLoc)) { + $storeLocation = [System.Security.Cryptography.X509Certificates.StoreLocation]::$loc + $store = New-Object System.Security.Cryptography.X509Certificates.X509Store("My", $storeLocation) + try { + $store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly) + $match = @($store.Certificates | Where-Object { $_.Thumbprint -eq $certThumbprint }) + if ($match.Count -gt 0) { + # Prefer a copy that carries the private key. + $withKey = @($match | Where-Object { $_.HasPrivateKey } | Select-Object -First 1) + $certificate = if ($withKey.Count -gt 0) { $withKey[0] } else { $match[0] } + $resolvedLoc = $loc + break + } + } + catch { + # This store could not be opened/read (e.g. access denied to + # LocalMachine\My); skip it and try the other store. + } + finally { + $store.Close() + } + } + if (-not $certificate) { + Write-LogHost "ERROR: Certificate with thumbprint '$certThumbprint' not found in CurrentUser\My or LocalMachine\My." -ForegroundColor Red + throw "Certificate not found" + } + if (-not $certificate.HasPrivateKey) { + Write-LogHost "ERROR: Certificate '$certThumbprint' was found in $resolvedLoc\My but has no accessible private key (only the public .cer appears to be installed, or the running account cannot read the key)." -ForegroundColor Red + throw "Certificate has no usable private key" + } + if ($resolvedLoc -ne $requestedLoc) { + Write-LogHost " -> Certificate located in $resolvedLoc\My (requested store was $requestedLoc\My)." -ForegroundColor Gray + } + # Pin the resolved cert object for the whole run + token refreshes. Passing + # -Certificate (not -CertificateThumbprint) makes the lookup deterministic and + # independent of which stores the SDK searches; pinning prevents the + # SafeCertContext handle from being GC-invalidated on later token requests + # (same rationale as the -ClientCertificatePath branch). + $script:AuthConfig.CertThumbprint = $certThumbprint + $script:AuthConfig.CertStoreLocation = $resolvedLoc + $script:AuthConfig.CertObject = $certificate + $script:AuthConfig.CanReauthenticate = $true + Connect-MgGraph -TenantId $appTenantId -ClientId $appClientId -Certificate $certificate -NoWelcome -ErrorAction Stop + } + elseif (-not [string]::IsNullOrWhiteSpace($certPath)) { + Write-LogHost " -> Authenticating with certificate file $certPath" -ForegroundColor Gray + # CRITICAL: EphemeralKeySet keeps the private key in process memory only (no temp + # .pfx-key file in %APPDATA%\Microsoft\Crypto\) and ties the SafeCertContext lifetime + # to the X509Certificate2 object itself. Without this flag, default DefaultKeySet/UserKeySet + # storage causes Azure.Identity's ClientCertificateCredential to throw + # 'm_safeCertContext is an invalid handle' on every token request after the first. + # Exportable is required so the SDK can re-export the key for token signing. + $flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet ` + -bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable + $cert = $null + try { + if ($certPasswordPlain) { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($certPath, $certPasswordPlain, $flags) + } + else { + # 3-arg ctor with empty password is required so the explicit flags apply. + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($certPath, [string]'', $flags) + } + # Store cert path and password for re-authentication + $script:AuthConfig.CertPath = $certPath + if ($certPasswordSecure) { $script:AuthConfig.CertPassword = $certPasswordSecure.Copy() } + # CRITICAL: Pin the cert object on $script: scope so it lives for the entire run. + # Azure.Identity's ClientCertificateCredential keeps a managed reference but does + # NOT clone the SafeCertContext - if the cert is GC'd or Disposed, every later + # token request fails with 'invalid handle'. The cert MUST NOT be disposed in + # the finally block; it is released at script exit when $script:AuthConfig is torn down. + $script:AuthConfig.CertObject = $cert + $script:AuthConfig.CanReauthenticate = $true + Connect-MgGraph -TenantId $appTenantId -ClientId $appClientId -Certificate $cert -NoWelcome -ErrorAction Stop + } + finally { + # Intentionally NOT disposing $cert here - it must outlive Connect-MgGraph for the + # duration of the script run. EphemeralKeySet ensures no on-disk artifacts to clean up; + # the cert is fully released at script exit when $script:AuthConfig is torn down. + if ($certPasswordPlain) { + Clear-Variable -Name certPasswordPlain -Force -ErrorAction SilentlyContinue + } + } + } + else { + Write-LogHost "ERROR: Provide either -ClientSecret, -ClientCertificateThumbprint, or -ClientCertificatePath for AppRegistration auth." -ForegroundColor Red + throw "No credential material supplied for AppRegistration" + } + + if ($certPasswordSecure) { + Clear-Variable -Name certPasswordSecure -Force -ErrorAction SilentlyContinue + } + } + } + + # Build a context-aware connection-success message + $connectMsg = if ($AuthMethod.ToLower() -eq 'appregistration') { + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + "Successfully connected to Microsoft Graph (Phase 1 of 2: app-only audit context). The Phase 2 (Agent 365) interactive sign-in will be requested NEXT, up-front, before audit work begins." + } else { + "Successfully connected to Microsoft Graph (app-only context)." + } + } else { + "Successfully connected to Microsoft Graph (delegated user context)." + } + Write-LogHost $connectMsg -ForegroundColor Green + + # Record token issue time for proactive refresh tracking + $script:AuthConfig.TokenIssueTime = Get-Date + + # Initialize shared auth state for thread jobs (enables proactive token refresh) + $tokenInfo = Get-GraphAccessTokenWithExpiry + if ($tokenInfo) { + $script:SharedAuthState.Token = $tokenInfo.Token + $script:SharedAuthState.ExpiresOn = $tokenInfo.ExpiresOn + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.AuthMethod = $AuthMethod.ToLower() + Write-LogHost " Token expires: $($tokenInfo.ExpiresOn.ToString('yyyy-MM-dd HH:mm:ss')) UTC (source: $($tokenInfo.Source))" -ForegroundColor Gray + } + + # Get and display current context + $context = Get-MgContext + # Detect dual-context run: AppRegistration audit phase + delegated Agent 365 phase. + # In that case, defer the per-line Tenant/Account/Scopes display until AFTER Phase 2 + # sign-in completes, so the log shows BOTH phases honestly side-by-side. Capture the + # Phase 1 context now into script-scope vars so the combined emitter can use them. + # + # (app-only modes) - or the same delegated context (delegated modes) - so there is no + # separate Phase 2 context to combine and nothing to defer. Always emit the auth-context + # display inline below for every auth mode. + $script:DeferAuthContextDisplay = $false + if ($script:DeferAuthContextDisplay) { + $script:Phase1Context = [pscustomobject]@{ + TenantId = $context.TenantId + Account = $context.Account + GrantedRequired = @($RequiredScopes | Where-Object { $context.Scopes -contains $_ }) + RequiredScopes = @($RequiredScopes) + } + Write-LogHost " Phase 1 (audit) connected." -ForegroundColor Gray + } else { + Write-LogHost " Tenant ID: $($context.TenantId)" -ForegroundColor Gray + $maskedAccount = Get-MaskedUsername -Username $context.Account + if ([string]::IsNullOrWhiteSpace($maskedAccount)) { + # App-only contexts have no signed-in user; surface the auth mode instead of a blank value. + $maskedAccount = if ($AuthMethod -eq 'AppRegistration') { '(app-only / AppRegistration - no interactive user)' } + elseif ($AuthMethod -eq 'ManagedIdentity') { + if ($env:AZURE_CLIENT_ID) { "(app-only / ManagedIdentity - clientId $env:AZURE_CLIENT_ID)" } + else { '(app-only / ManagedIdentity - system-assigned)' } + } + else { '(no signed-in user)' } + } + Write-LogHost " Account: $maskedAccount" -ForegroundColor Gray + # Filter displayed scopes to those required by this script (avoids confusion when the + # user/app holds additional consented scopes unrelated to PAX). Granted required scopes + # are shown; any required scopes missing from the token are surfaced separately below. + $grantedRequired = @($RequiredScopes | Where-Object { $context.Scopes -contains $_ }) + Write-LogHost " Scopes: $($grantedRequired -join ', ')" -ForegroundColor Gray + } + + # Trigger Graph API version detection early (before queries start) + $null = Get-GraphAuditApiUri -Path 'queries' + + # Validate required scopes are present + # NOTE: For app-only (AppRegistration) contexts, the access token carries permissions + # in the 'roles' claim (application permissions), not the 'scp' claim. The Graph SDK + # only populates $context.Scopes from 'scp', so $context.Scopes is empty/partial for + # app-only auth even when admin-consented application permissions are fully granted. + # A scope-membership check would falsely flag every required permission as missing. + # App permissions are validated server-side at API call time, so we skip the warning. + $isAppOnlyContext = $false + if ($context) { + $isAppOnlyContext = ([string]::IsNullOrWhiteSpace($context.Account)) -or ($context.AuthType -eq 'AppOnly') + } + $missingScopes = @() + if (-not $isAppOnlyContext) { + foreach ($scope in $RequiredScopes) { + if ($context.Scopes -notcontains $scope) { + $missingScopes += $scope + } + } + } + + if ($missingScopes.Count -gt 0) { + Write-LogHost "" + Write-LogHost "WARNING: Missing required scope(s):" -ForegroundColor Yellow + foreach ($scope in $missingScopes) { + Write-LogHost " • $scope" -ForegroundColor Yellow + } + Write-LogHost "" + Write-LogHost "Script may fail when accessing audit logs." -ForegroundColor Yellow + Write-LogHost "Consider re-authenticating with full permissions." -ForegroundColor Yellow + Write-LogHost "" + } + + $script:Connected = $true + } + catch { + Write-LogHost "ERROR: Graph API authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "" + Write-LogHost "Troubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure you have AuditLogsQuery.Read.All permission" -ForegroundColor White + Write-LogHost " 2. Verify Azure AD role (Compliance/Security Administrator)" -ForegroundColor White + Write-LogHost " 3. Check network connectivity to Microsoft Graph API" -ForegroundColor White + Write-LogHost " 4. Try a different authentication method (-Auth parameter)" -ForegroundColor White + Write-LogHost " 5. Use -UseEOM switch to fall back to EOM mode" -ForegroundColor White + Write-LogHost "" + throw + } + } +} + +# ============================================== +# ACCESS TOKEN EXTRACTION HELPER +# ============================================== +function Get-GraphAccessToken { + <# + .SYNOPSIS + Extracts the current access token from an active Microsoft Graph session. + + .DESCRIPTION + Microsoft Graph PowerShell SDK 2.x does NOT expose AccessToken via Get-MgContext + for security reasons. This function reliably extracts the token by making a + lightweight request and extracting the Authorization header. + + Primary method: HTTP request header extraction (reliable in SDK 2.x) + Fallback method: Get-MgContext.AccessToken (for older SDK versions) + + .OUTPUTS + [string] The access token, or $null if extraction fails + #> + [CmdletBinding()] + param() + + # Primary method: Extract token from HTTP response headers (reliable in SDK 2.x) + try { + $response = Invoke-MgGraphRequest -Method GET -Uri 'https://graph.microsoft.com/v1.0/$metadata' -OutputType HttpResponseMessage -ErrorAction Stop + $token = $response.RequestMessage.Headers.Authorization.Parameter + if ($token) { + return $token + } + } + catch { + # HTTP method failed, try fallback + } + + # Fallback method: Get-MgContext.AccessToken (works in older SDK versions) + try { + $context = Get-MgContext -ErrorAction SilentlyContinue + if ($context -and $context.AccessToken) { + return $context.AccessToken + } + } + catch { + # Fallback also failed + } + + return $null +} + +# ============================================== +# ACCESS TOKEN WITH EXPIRY EXTRACTION (for shared auth state) +# ============================================== +function Get-GraphAccessTokenWithExpiry { + <# + .SYNOPSIS + Extracts access token AND expiry time from the active Microsoft Graph session. + + .DESCRIPTION + Decodes the JWT access token to extract the actual 'exp' (expiry) claim. + This is more reliable than Azure.Identity reflection and doesn't cause + extra authentication popups. + + JWT tokens have three base64-encoded parts: header.payload.signature + The payload contains the 'exp' claim as a Unix timestamp. + + Falls back to 50-minute estimated expiry if JWT decode fails. + + .OUTPUTS + [hashtable] with Token (string) and ExpiresOn (DateTime) properties + Returns $null if no token can be extracted + #> + [CmdletBinding()] + param() + + $result = @{ + Token = $null + ExpiresOn = $null + Source = 'unknown' + } + + # First, get the token using existing reliable method + $result.Token = Get-GraphAccessToken + if (-not $result.Token) { + return $null + } + + # Try to decode JWT to get actual expiry from 'exp' claim + # JWT format: base64url(header).base64url(payload).signature + try { + $tokenParts = $result.Token.Split('.') + if ($tokenParts.Count -ge 2) { + # Decode the payload (second part) + $payloadBase64 = $tokenParts[1] + + # Add padding if needed (base64url uses no padding) + $paddingNeeded = 4 - ($payloadBase64.Length % 4) + if ($paddingNeeded -lt 4) { + $payloadBase64 += ('=' * $paddingNeeded) + } + + # Convert base64url to standard base64 (replace - with +, _ with /) + $payloadBase64 = $payloadBase64.Replace('-', '+').Replace('_', '/') + + # Decode and parse JSON + $payloadBytes = [Convert]::FromBase64String($payloadBase64) + $payloadJson = [System.Text.Encoding]::UTF8.GetString($payloadBytes) + $payload = $payloadJson | ConvertFrom-Json + + if ($payload.exp) { + # 'exp' is Unix timestamp (seconds since 1970-01-01 UTC) + $unixEpoch = [DateTime]::new(1970, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) + $result.ExpiresOn = $unixEpoch.AddSeconds($payload.exp) + $result.Source = 'JWT' + + # Calculate time remaining for logging + $timeRemaining = $result.ExpiresOn - (Get-Date).ToUniversalTime() + if ($timeRemaining.TotalMinutes -gt 0) { + Write-Verbose "Token expires in $([int]$timeRemaining.TotalMinutes) minutes (from JWT 'exp' claim)" + } + + return $result + } + } + } + catch { + # JWT decode failed, use fallback + Write-Verbose "JWT decode failed: $($_.Exception.Message)" + } + + # Fallback: estimate 50-minute expiry from now (observed token lifetime ~45-60 minutes) + # With 5-minute buffer, proactive refresh triggers at ~45 min mark + $result.ExpiresOn = (Get-Date).ToUniversalTime().AddMinutes(50) + $result.Source = 'estimated' + + return $result +} + +# ============================================== +# TOKEN REFRESH FUNCTION FOR LONG-RUNNING OPERATIONS +# ============================================== +function Invoke-TokenRefresh { + <# + .SYNOPSIS + Forces re-authentication for AppRegistration auth mode to get fresh access token. + + .DESCRIPTION + When using App Registration authentication (client secret or certificate), + this function reconnects to Microsoft Graph to obtain a fresh access token. + This is critical for long-running operations that exceed the default OAuth + token lifetime (~60-90 minutes). + + For interactive auth modes, this function returns $false as re-authentication + would require user interaction. + + .PARAMETER Force + Force re-authentication even if token doesn't appear expired. + + .OUTPUTS + [PSCustomObject] with Success ($true/$false) and NewToken properties + #> + [CmdletBinding()] + param( + [switch]$Force + ) + + $result = [PSCustomObject]@{ + Success = $false + NewToken = $null + Message = "" + AuthMethod = $script:AuthConfig.Method + } + + # Check if we can re-authenticate + if (-not $script:AuthConfig.CanReauthenticate) { + $result.Message = "Auth method '$($script:AuthConfig.Method)' does not support automatic re-authentication" + return $result + } + + # Validate we have stored config + if ($script:AuthConfig.Method -ne 'AppRegistration') { + $result.Message = "Only AppRegistration auth mode supports automatic token refresh" + return $result + } + + if ([string]::IsNullOrWhiteSpace($script:AuthConfig.TenantId) -or + [string]::IsNullOrWhiteSpace($script:AuthConfig.ClientId)) { + $result.Message = "Missing TenantId or ClientId in stored auth config" + return $result + } + + Write-LogHost " [TOKEN-REFRESH] Attempting re-authentication using AppRegistration..." -ForegroundColor Cyan + + try { + # Disconnect first to ensure clean state + try { + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + } catch { } + + # Re-authenticate based on stored credential type + $reconnected = $false + + # Try client secret first + if ($script:AuthConfig.ClientSecret) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with client secret..." -ForegroundColor Gray + $credential = New-Object System.Management.Automation.PSCredential( + $script:AuthConfig.ClientId, + $script:AuthConfig.ClientSecret + ) + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientSecretCredential $credential ` + -NoWelcome -ErrorAction Stop + $reconnected = $true + } + # Try certificate thumbprint + elseif ($script:AuthConfig.CertThumbprint) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with certificate thumbprint..." -ForegroundColor Gray + if ($script:AuthConfig.CertObject) { + # Reuse the cert object resolved + pinned at initial login (carries the + # CurrentUser/LocalMachine resolution; avoids re-searching the stores). + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -Certificate $script:AuthConfig.CertObject ` + -NoWelcome -ErrorAction Stop + } + else { + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -CertificateThumbprint $script:AuthConfig.CertThumbprint ` + -NoWelcome -ErrorAction Stop + } + $reconnected = $true + } + # Try certificate file + elseif ($script:AuthConfig.CertPath) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with certificate file..." -ForegroundColor Gray + # Prefer the cert object pinned at initial login - it already has the correct + # EphemeralKeySet+Exportable storage flags applied, and reusing it avoids any + # chance of leaking a second key file or invalidating the credential's handle. + $cert = $script:AuthConfig.CertObject + if (-not $cert) { + $flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet ` + -bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable + if ($script:AuthConfig.CertPassword) { + $plainPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto( + [Runtime.InteropServices.Marshal]::SecureStringToBSTR($script:AuthConfig.CertPassword) + ) + try { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2( + $script:AuthConfig.CertPath, $plainPassword, $flags + ) + } + finally { + Clear-Variable -Name plainPassword -Force -ErrorAction SilentlyContinue + } + } + else { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2( + $script:AuthConfig.CertPath, [string]'', $flags + ) + } + # Re-pin so subsequent refreshes can reuse this same instance. + $script:AuthConfig.CertObject = $cert + } + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -Certificate $cert ` + -NoWelcome -ErrorAction Stop + # Intentionally NOT disposing $cert - must outlive Connect-MgGraph for the rest + # of the run. Cleanup happens at script exit when $script:AuthConfig is released. + $reconnected = $true + } + + if ($reconnected) { + # Use reliable token extraction helper (HTTP method primary) + $result.NewToken = Get-GraphAccessToken + if ($result.NewToken) { + $result.Success = $true + $result.Message = "Successfully refreshed token" + + # Update token timing and reset auth failure flags + $script:AuthConfig.TokenIssueTime = Get-Date + $script:TokenAcquiredTime = Get-Date + $script:AuthFailureDetected = $false + $script:Auth401MessageShown = $false # Reset for next auth failure cycle + + Write-LogHost " [TOKEN-REFRESH] Successfully obtained fresh access token" -ForegroundColor Green + Write-LogHost " [TOKEN-REFRESH] Token acquired at $(Get-Date -Format 'HH:mm:ss') - proactive refresh at 30-minute age" -ForegroundColor DarkGray + } + else { + $result.Message = "Reconnected but could not extract access token" + Write-LogHost " [TOKEN-REFRESH] ✗ $($result.Message)" -ForegroundColor Red + } + } + else { + $result.Message = "No valid credential found in stored auth config" + } + } + catch { + $result.Message = "Re-authentication failed: $($_.Exception.Message)" + Write-LogHost " [TOKEN-REFRESH] ✗ $($result.Message)" -ForegroundColor Red + } + + return $result +} + +# ============================================== +# PROACTIVE TOKEN REFRESH FOR LONG-RUNNING OPERATIONS +# ============================================== +function Refresh-GraphTokenIfNeeded { + <# + .SYNOPSIS + Proactively refreshes the Graph access token if it's nearing expiry. + + .DESCRIPTION + Checks SharedAuthState.ExpiresOn and refreshes token if less than 10 minutes + remain before expiry. Uses Azure.Identity for interactive auth modes, or + Invoke-TokenRefresh for AppRegistration mode. + + This function is called from the main thread's job monitoring loop to ensure + thread jobs always have a valid token in SharedAuthState. + + IMPORTANT: Includes cooldown logic to prevent spam - only attempts refresh + once per 5 minutes. If silent refresh fails, sets AuthFailureDetected to + trigger interactive re-auth prompt. + + .PARAMETER BufferMinutes + Refresh if token expires within this many minutes. Default: 5. + + .OUTPUTS + $true - Token was refreshed successfully (silent or interactive) + $false - No refresh needed (token still valid) or within cooldown period + 'Quit' - User chose to quit at the re-auth prompt + + CRITICAL: Callers MUST check for 'Quit' return and handle gracefully! + #> + [CmdletBinding()] + param( + [int]$BufferMinutes = 5 + ) + + # Check if we have shared auth state + if (-not $script:SharedAuthState.ExpiresOn) { + return $false + } + + $now = (Get-Date).ToUniversalTime() + $expiresOn = $script:SharedAuthState.ExpiresOn + $minutesRemaining = ($expiresOn - $now).TotalMinutes + + # PROACTIVE REFRESH FOR APPREG: Refresh at 30-minute token age (not just near expiry) + # AppRegistration can refresh silently, so we do this proactively to avoid 401s + $needsProactiveRefresh = $false + if ($script:AuthConfig.Method -eq 'AppRegistration' -and $script:AuthConfig.CanReauthenticate) { + if ($script:AuthConfig.TokenIssueTime) { + $tokenAge = (Get-Date) - $script:AuthConfig.TokenIssueTime + if ($tokenAge.TotalMinutes -gt 30) { + $needsProactiveRefresh = $true + Write-LogHost " [TOKEN] Token age: $([Math]::Round($tokenAge.TotalMinutes, 1)) minutes - proactive refresh triggered" -ForegroundColor Yellow + } + } + } + + if ($minutesRemaining -gt $BufferMinutes -and -not $needsProactiveRefresh) { + return $false # Token still valid, no refresh needed + } + + # COOLDOWN CHECK: Auth-mode-aware cooldown between refresh attempts + # AppReg: 45 seconds (silent client_credentials grant - cheap and fast) + # Interactive: 5 minutes (avoids spamming browser/prompt windows) + $cooldownMinutes = if ($script:AuthConfig.Method -eq 'AppRegistration') { 0.75 } else { 5 } + if ($script:LastProactiveRefreshAttempt) { + $timeSinceLastAttempt = ((Get-Date) - $script:LastProactiveRefreshAttempt).TotalMinutes + if ($timeSinceLastAttempt -lt $cooldownMinutes) { + return $false + } + } + $script:LastProactiveRefreshAttempt = Get-Date + + # Log appropriate message based on trigger reason + if (-not $needsProactiveRefresh) { + Write-LogHost " [TOKEN] Token expires in $([Math]::Round($minutesRemaining, 1)) minutes - attempting proactive refresh..." -ForegroundColor Yellow + } + + # Try to refresh using Azure.Identity (uses cached MSAL tokens, may prompt if needed) + $tokenInfo = Get-GraphAccessTokenWithExpiry + if ($tokenInfo -and $tokenInfo.Token -ne $script:SharedAuthState.Token) { + # Validate the new token is actually valid (ExpiresOn must be > 2 min in the future) + # Protects against stale MSAL cache returning already-expired tokens (e.g., after process suspension) + $tokenExpiresOn = $tokenInfo.ExpiresOn + $nowUtc = (Get-Date).ToUniversalTime() + $minutesUntilExpiry = ($tokenExpiresOn - $nowUtc).TotalMinutes + if ($minutesUntilExpiry -le 2) { + Write-LogHost " [TOKEN] WARNING: Refreshed token is already expired or near-expiry (expires in $([Math]::Round($minutesUntilExpiry, 1)) min) - forcing full re-authentication" -ForegroundColor Red + # Fall through to Invoke-TokenRefresh -Force below + } else { + # Got a genuinely valid new token + $script:SharedAuthState.Token = $tokenInfo.Token + $script:SharedAuthState.ExpiresOn = $tokenInfo.ExpiresOn + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.RefreshCount++ + + Write-LogHost " [TOKEN] Token refreshed silently (expires: $($tokenInfo.ExpiresOn.ToString('HH:mm:ss')) UTC, refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green + Write-LogHost " [TOKEN] Note: In-flight queries may still require re-auth before this expiration" -ForegroundColor DarkGray + return $true + } + } + + # Azure.Identity didn't give us a new token (or it was stale), try AppRegistration refresh if available + if ($script:AuthConfig.CanReauthenticate) { + $refreshResult = Invoke-TokenRefresh -Force + if ($refreshResult.Success) { + $script:SharedAuthState.Token = $refreshResult.NewToken + $script:SharedAuthState.ExpiresOn = (Get-Date).ToUniversalTime().AddMinutes(50) + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.RefreshCount++ + $script:AuthConfig.TokenIssueTime = Get-Date # Reset age timer for proactive refresh + + Write-LogHost " [TOKEN] Token refreshed via AppRegistration (refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green + return $true + } + } + + # SILENT REFRESH FAILED + # For AppRegistration + Force: FATAL exit (true headless operation) + # For AppRegistration without Force: Fall back to interactive prompt + # For interactive modes: Prompt user for re-authentication + Write-LogHost " [TOKEN] [!] Silent token refresh failed - interactive re-authentication required" -ForegroundColor Red + + # AppRegistration mode with -Force: Silent refresh failure is fatal (no interactive fallback for headless runs) + if ($script:AuthConfig.Method -eq 'AppRegistration' -and $Force) { + Write-LogHost " [TOKEN] FATAL: AppRegistration token refresh failed. Cannot continue headless (-Force mode)." -ForegroundColor Red + Write-LogHost " [TOKEN] Check: client secret expiration, certificate validity, or API permissions." -ForegroundColor Yellow + return 'Quit' + } + + # Interactive modes OR AppRegistration without -Force: prompt user for re-authentication + $refreshResult = Invoke-TokenRefreshPrompt + if ($refreshResult -eq 'Quit') { + # User chose to quit - return special value for callers to handle + return 'Quit' + } + + # User pressed R and successfully re-authenticated + # Invoke-TokenRefreshPrompt already updated SharedAuthState and reset AuthFailureDetected + return $true +} + +# ============================================== +# CHECKPOINT/RESUME FUNCTIONS FOR LONG-RUNNING OPERATIONS +# ============================================== + +function Initialize-CheckpointForNewRun { + <# + .SYNOPSIS + Creates new checkpoint structure for a fresh run (not resume mode). + .DESCRIPTION + Initializes checkpoint data structure with all processing parameters, + creates _PARTIAL output filename, and saves initial checkpoint file to disk. + On resume, all parameters are restored from checkpoint to ensure consistency. + #> + param( + [Parameter(Mandatory)] + [string]$OutputPath, + + [Parameter(Mandatory)] + [string]$BaseOutputFileName, + + [Parameter(Mandatory)] + [string]$RunTimestamp, + + [Parameter(Mandatory)] + [datetime]$StartDate, + + [Parameter(Mandatory)] + [datetime]$EndDate, + + [Parameter()] + [hashtable]$AllParameters + ) + + # Create _PARTIAL filename + $fileNameWithoutExt = [System.IO.Path]::GetFileNameWithoutExtension($BaseOutputFileName) + $fileExt = [System.IO.Path]::GetExtension($BaseOutputFileName) + $partialFileName = "${fileNameWithoutExt}_PARTIAL${fileExt}" + $script:PartialOutputPath = Join-Path $OutputPath $partialFileName + + # Create checkpoint file path (hidden file with dot prefix) + $script:CheckpointPath = Join-Path $OutputPath ".pax_checkpoint_${RunTimestamp}.json" + + # Acquire exclusive lock on this checkpoint so a second replica running against + # the same OutputPath cannot silently corrupt our state. Fails loudly if another + # live PAX process is already using this checkpoint. + script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath + + # Initialize checkpoint data structure with comprehensive parameter snapshot + $script:CheckpointData = @{ + version = 2 # Bumped version for expanded parameter storage + # Structured schema/compatibility metadata. Read-Checkpoint + # uses these for forward/backward compatibility checks (Test-CheckpointCompatibility). + # `checkpointSchemaVersion` is a SemVer-style identifier for the file format. The + # legacy integer `version` field is preserved for older readers. + checkpointSchemaVersion = '2.1.0' + compatibilityMinimumVersion = '1.11.0' + createdByVersion = $ScriptVersion + createdUtc = (Get-Date).ToUniversalTime().ToString('o') + checkpointType = 'PurviewAudit' + runTimestamp = $RunTimestamp + created = (Get-Date).ToUniversalTime().ToString('o') + lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + parameters = @{ + # Date range + startDate = $StartDate.ToUniversalTime().ToString('o') + endDate = $EndDate.ToUniversalTime().ToString('o') + + # Activity/Record filtering + activityTypes = if ($AllParameters.ActivityTypes) { @($AllParameters.ActivityTypes) } else { @() } + recordTypes = if ($AllParameters.RecordTypes) { @($AllParameters.RecordTypes) } else { @() } + serviceTypes = if ($AllParameters.ServiceTypes) { @($AllParameters.ServiceTypes) } else { @() } + userIds = if ($AllParameters.UserIds) { @($AllParameters.UserIds) } else { @() } + groupNames = if ($AllParameters.GroupNames) { @($AllParameters.GroupNames) } else { @() } + + # Agent filtering + agentId = if ($AllParameters.AgentId) { @($AllParameters.AgentId) } else { @() } + agentsOnly = [bool]$AllParameters.AgentsOnly + excludeAgents = [bool]$AllParameters.ExcludeAgents + + # Prompt filtering + promptFilter = if ($AllParameters.PromptFilter) { $AllParameters.PromptFilter } else { $null } + + # Schema/Explosion settings + explodeArrays = [bool]$AllParameters.ExplodeArrays + explodeDeep = [bool]$AllParameters.ExplodeDeep + explosionThreads = if ($AllParameters.ExplosionThreads) { $AllParameters.ExplosionThreads } else { 0 } + flatDepth = if ($AllParameters.FlatDepth) { $AllParameters.FlatDepth } else { 120 } + streamingSchemaSample = if ($AllParameters.StreamingSchemaSample) { $AllParameters.StreamingSchemaSample } else { 5000 } + streamingChunkSize = if ($AllParameters.StreamingChunkSize) { $AllParameters.StreamingChunkSize } else { 5000 } + + # M365/User info bundles + includeM365Usage = [bool]$AllParameters.IncludeM365Usage + includeUserInfo = [bool]$AllParameters.IncludeUserInfo + includeCopilotInteraction = [bool]$AllParameters.IncludeCopilotInteraction + excludeCopilotInteraction = [bool]$AllParameters.ExcludeCopilotInteraction + includeAgent365Info = [bool]$AllParameters.IncludeAgent365Info + onlyAgent365Info = [bool]$AllParameters.OnlyAgent365Info + + # Partitioning + blockHours = if ($AllParameters.BlockHours) { $AllParameters.BlockHours } else { 0.5 } + partitionHours = if ($AllParameters.PartitionHours) { $AllParameters.PartitionHours } else { 0 } + maxPartitions = if ($AllParameters.MaxPartitions) { $AllParameters.MaxPartitions } else { 160 } + + # Output settings + # Persist the USER-SUPPLIED -OutputPath (SharePoint URL / Fabric OneLake URL / + # local path), NOT the script-rewritten local scratch dir. At checkpoint-save + # time $OutputPath has already been redirected to $script:RemoteScratchDir for + # remote modes (the parse-time redirect block earlier in the script), so saving + # $OutputPath here would lose the destination URL and a -Resume run would + # re-resolve the tier as 'Local' — no upload sweep would ever fire. + # + # $script:DestRaw['Purview'] is the canonical source of truth. It is populated + # at parse-time from the ORIGINAL CLI-bound value (the SharePoint / Fabric URL + # or local path) BEFORE the scratch redirect runs, and is REPOPULATED on each + # resume from the restored value BEFORE the resume-side scratch redirect + # rewrites $OutputPath — so it carries the user-supplied destination across + # both fresh runs and resumes. Falls back to $OutputPath only for runs where + # -OutputPath was never bound and $script:DestRaw['Purview'] is unset + # (default PSScriptRoot). Tier-agnostic: identical behavior for Local, + # SharePoint, and Fabric (OneLake) modes. + outputPath = if ($script:DestRaw -and $script:DestRaw['Purview']) { [string]$script:DestRaw['Purview'] } else { $OutputPath } + exportWorkbook = [bool]$AllParameters.ExportWorkbook + combineOutput = [bool]$AllParameters.CombineOutput + # Deidentify: persisted so a resumed run keeps the original run's anonymization. + # -Deidentify cannot be passed on the resume command line (resume allow-list rejects + # it), so the checkpoint is the sole source of truth for it on resume. + deidentify = [bool]$AllParameters.Deidentify + + # FillerLabel: persisted (resolved hierarchy-filler mode + literal) so a resumed + # AIO/AIBV rollup keeps the original run's level-filler choice. -FillerLabel / + # -FillerLabelText cannot be passed on resume (allow-list), so the checkpoint is the + # sole source of truth for them on resume. + fillerLabelMode = [string]$AllParameters.FillerLabel + fillerLabelText = [string]$AllParameters.FillerLabelText + + # Append-mode targets. Persisted so a resumed run continues to merge + # against the same target file/dim, even if the resume command line omits the + # -Append* parameter. Empty string means "not set on this run". + appendFile = if ($AllParameters.AppendFile) { [string]$AllParameters.AppendFile } else { '' } + appendUserInfo = if ($AllParameters.AppendUserInfo) { [string]$AllParameters.AppendUserInfo } else { '' } + appendAgent365Info = if ($AllParameters.AppendAgent365Info) { [string]$AllParameters.AppendAgent365Info } else { '' } + + # Per-data-type destinations (captured so resume re-targets correctly). + # Storage tier is inferred from each value's form at resume time; no separate + # tier field is persisted. + outputPathUserInfo = if ($AllParameters.OutputPathUserInfo) { [string]$AllParameters.OutputPathUserInfo } else { '' } + outputPathAgent365Info = if ($AllParameters.OutputPathAgent365Info) { [string]$AllParameters.OutputPathAgent365Info } else { '' } + outputPathLog = if ($AllParameters.OutputPathLog) { [string]$AllParameters.OutputPathLog } else { '' } + outputDestinationType = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + + # Rollup post-processor mode (None|Rollup|RollupPlusRaw) and the resolved + # embedded-processor mode (None|CopilotInteraction|M365Bundle). Persisted so resume + # runs can restore the original rollup intent when the user does not pass a switch + # on the resume command line. If a rollup switch IS passed on resume, the resume + # value wins (last-write-wins) and these fields are overwritten on the next save. + rollupMode = if ([bool]$AllParameters.Rollup) { 'Rollup' } elseif ([bool]$AllParameters.RollupPlusRaw) { 'RollupPlusRaw' } else { 'None' } + processorMode = if ($script:RollupProcessorMode) { $script:RollupProcessorMode } else { 'None' } + # Resolved rollup dashboard (None|AIO|AIBV|M365). Persisted so a resumed + # CopilotInteraction run restores the correct --profile (AIO vs AIBV cannot + # be re-derived from -IncludeM365Usage alone). Last-write-wins on resume. + rollupDashboard = if ($script:RollupDashboard) { $script:RollupDashboard } else { 'None' } + + # Auth (method only - no secrets) + auth = if ($AllParameters.Auth) { $AllParameters.Auth } else { 'WebLogin' } + tenantId = if ($AllParameters.TenantId) { $AllParameters.TenantId } else { $null } + clientId = if ($AllParameters.ClientId) { $AllParameters.ClientId } else { $null } + # Note: ClientSecret is NOT stored for security + + # Other settings + resultSize = if ($AllParameters.ResultSize) { $AllParameters.ResultSize } else { 10000 } + maxConcurrency = if ($AllParameters.MaxConcurrency) { $AllParameters.MaxConcurrency } else { 10 } + maxMemoryMB = if ($AllParameters.MaxMemoryMB) { $AllParameters.MaxMemoryMB } else { 0 } + useEOM = [bool]$AllParameters.UseEOM + autoCompleteness = [bool]$AllParameters.AutoCompleteness + includeTelemetry = [bool]$AllParameters.IncludeTelemetry + } + outputFiles = @{ + partialCsv = $partialFileName + finalCsv = $BaseOutputFileName + } + partitions = @{ + total = 0 + blockHours = if ($AllParameters.BlockHours) { $AllParameters.BlockHours } else { 0.5 } + completed = @() + queryCreated = @() + } + statistics = @{ + totalRecordsSaved = 0 + partitionsComplete = 0 + partitionsQueryCreated = 0 + partitionsRemaining = 0 + } + explosion = @{ + status = 'NotStarted' # NotStarted, InProgress, Completed + recordsProcessed = 0 + rowsGenerated = 0 + lastUpdateTime = $null + } + } + + # Save initial checkpoint + Save-CheckpointToDisk + + return $script:PartialOutputPath +} + +function Sync-FabricResumeMirror { + <# + .SYNOPSIS + Mirrors local resume artifacts to OneLake Files/.pax_resume//. + .DESCRIPTION + Uploads the checkpoint JSON, every changed .pax_incremental/*.jsonl shard, + and the *_PARTIAL.csv companion to a durable OneLake folder so a Fabric + container restart can rehydrate the run via Restore-FabricResumeMirror. + + No-op unless the Purview destination tier is Fabric. Failures throw so the + calling try/catch (in Save-CheckpointToDisk) can abort the run and keep the + local + mirror artifact set consistent. + + Per-file change tracking lives in $script:FabricResumeMirrorState keyed by + full local path with "{size}:{ticks}" stamps to avoid re-uploading + unchanged shards on every checkpoint write. + #> + [CmdletBinding()] + param() + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return } + if (-not $script:CheckpointPath -or -not (Test-Path -LiteralPath $script:CheckpointPath)) { return } + if (-not $global:ScriptRunTimestamp) { return } + + if (-not $script:FabricResumeMirrorState) { $script:FabricResumeMirrorState = @{} } + $mirrorBase = ".pax_resume/$global:ScriptRunTimestamp" + + # Checkpoint JSON — always upload (small, mutates every call). + $cpName = Split-Path -Leaf $script:CheckpointPath + Send-FileToOneLake -LocalPath $script:CheckpointPath -RemoteFileName "$mirrorBase/$cpName" + + # .pax_incremental shards — upload only those whose size or mtime changed since last sync. + $cpDir = Split-Path -Parent $script:CheckpointPath + $incDir = Join-Path $cpDir ".pax_incremental" + if (Test-Path -LiteralPath $incDir -PathType Container) { + foreach ($f in (Get-ChildItem -LiteralPath $incDir -Filter '*.jsonl' -File -ErrorAction SilentlyContinue)) { + $key = $f.FullName + $stamp = "{0}:{1}" -f $f.Length, $f.LastWriteTimeUtc.Ticks + if ($script:FabricResumeMirrorState[$key] -eq $stamp) { continue } + Send-FileToOneLake -LocalPath $f.FullName -RemoteFileName "$mirrorBase/.pax_incremental/$($f.Name)" + $script:FabricResumeMirrorState[$key] = $stamp + } + } + + # *_PARTIAL.csv — upload if present and changed. + if ($script:PartialOutputPath -and (Test-Path -LiteralPath $script:PartialOutputPath)) { + $pf = Get-Item -LiteralPath $script:PartialOutputPath + $pkey = $pf.FullName + $pstamp = "{0}:{1}" -f $pf.Length, $pf.LastWriteTimeUtc.Ticks + if ($script:FabricResumeMirrorState[$pkey] -ne $pstamp) { + Send-FileToOneLake -LocalPath $pf.FullName -RemoteFileName "$mirrorBase/$($pf.Name)" + $script:FabricResumeMirrorState[$pkey] = $pstamp + } + } +} + +function Restore-FabricResumeMirror { + <# + .SYNOPSIS + Hydrates local resume artifacts from OneLake Files/.pax_resume//. + .DESCRIPTION + Used on startup of a Fabric run when a resume is requested and the local + working copy is missing or partial (typical after a container restart). + Lists the mirror folder via the ADLS Gen2 List Paths API and downloads + every file into the supplied LocalDir, preserving the .pax_incremental + subfolder layout. Returns the number of files downloaded. + + Throws on partial-download failure so callers can abort rather than + resume from a torn artifact set. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [string] $LocalDir + ) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return 0 } + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return 0 } + throw + } + if (-not $resp -or -not $resp.Content) { return 0 } + $parsed = try { $resp.Content | ConvertFrom-Json -ErrorAction Stop } catch { $null } + if (-not $parsed -or -not $parsed.paths) { return 0 } + + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + } + + $downloaded = 0 + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + foreach ($entry in $parsed.paths) { + $isDir = $entry.isDirectory + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } + $name = "$($entry.name)" + $rest = if ($name.StartsWith($prefix)) { $name.Substring($prefix.Length) } else { Split-Path -Leaf $name } + if (-not $rest) { continue } + $dest = Join-Path $LocalDir $rest + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/$rest" -DestinationPath $dest + $downloaded++ + } + return $downloaded +} + +function Remove-FabricResumeMirror { + <# + .SYNOPSIS + Deletes the OneLake Files/.pax_resume// mirror after a successful run. + .DESCRIPTION + Best-effort cleanup. Failures are logged at Verbose only — they do not + abort the run because the customer-visible outputs are already durable + and a leftover mirror folder is purely cosmetic. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return } + if (-not $script:FabricResolved) { return } + $resolved = $script:FabricResolved + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem`?recursive=true" + try { + $null = Invoke-FabricWebRequest -Uri $dfsPath -Method DELETE + if ($script:FabricResumeMirrorState) { $script:FabricResumeMirrorState.Clear() } + } catch { + Write-Verbose ("Resume mirror cleanup failed: {0}" -f $_.Exception.Message) + } +} + +function Restore-AllFabricResumeMirrors { + <# + .SYNOPSIS + Discovers all in-progress run mirrors under OneLake Files/.pax_resume/ and + downloads their artifacts to a local directory for auto-discover resume. + .DESCRIPTION + Lists Files/.pax_resume/ recursively, then for each / folder + downloads every file (checkpoint JSON, *_PARTIAL.csv, .pax_incremental/*.jsonl) + into LocalDir, flattening the / prefix because every artifact + filename already embeds the run timestamp. Returns the number of files + downloaded. No-op on Local/SharePoint tiers. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $LocalDir) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return 0 } + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume" } else { ".pax_resume" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return 0 } + throw + } + if (-not $resp -or -not $resp.Content) { return 0 } + $parsed = try { $resp.Content | ConvertFrom-Json -ErrorAction Stop } catch { $null } + if (-not $parsed -or -not $parsed.paths) { return 0 } + + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + } + + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $count = 0 + foreach ($e in $parsed.paths) { + $isDir = $e.isDirectory + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } + $n = "$($e.name)" + if (-not $n.StartsWith($prefix)) { continue } + $rest = $n.Substring($prefix.Length) + if (-not $rest) { continue } + # rest = "/" or "/.pax_incremental/". + # Flatten the / prefix; filenames already embed it. + $slash = $rest.IndexOf('/') + if ($slash -lt 0) { continue } + $localRel = $rest.Substring($slash + 1) + $dest = Join-Path $LocalDir $localRel + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$rest" -DestinationPath $dest + $count++ + } + return $count +} + +function Save-CsvAtomic { + <# + .SYNOPSIS + Atomic CSV write helper using temp+rename, mirroring Save-CheckpointToDisk. + .DESCRIPTION + Replaces direct `Export-Csv -Path $target` calls at output sites that previously + risked a half-written file on Ctrl+C / power loss / container OOM-kill. + Writes to "$Path.tmp" first, then Move-Item -Force overwrites the target in a + single filesystem operation. On any failure the temp file is removed and the + original $Path (if any) is left untouched. + + For remote tiers (SharePoint / Fabric) the caller still passes a LOCAL scratch + path; remote upload happens later via the existing upload pipeline. So this + helper only needs to handle the local atomic-rename case. + + Encoding/quote handling mirrors what the existing call sites used (UTF-8, + NoTypeInformation) so output bytes are unchanged for successful writes. + .PARAMETER InputObject + Rows to export. Accepts arrays, lists, or single objects via pipeline. + .PARAMETER Path + Final destination path. Will be replaced atomically when the write succeeds. + .PARAMETER NoTypeInformation + Forwarded to Export-Csv. Default $true (matches all current call sites). + .PARAMETER Encoding + Forwarded to Export-Csv. Default 'UTF8' (matches all current call sites). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory, ValueFromPipeline)] + [AllowNull()] + $InputObject, + + [Parameter(Mandatory)] + [string]$Path, + + [switch]$NoTypeInformation = $true, + [string]$Encoding = 'UTF8' + ) + begin { + $rows = New-Object System.Collections.Generic.List[object] + } + process { + if ($null -ne $InputObject) { + # Accept arrays piped as a single object, single objects, or per-pipeline-element streams. + if ($InputObject -is [System.Collections.IEnumerable] -and -not ($InputObject -is [string])) { + foreach ($r in $InputObject) { if ($null -ne $r) { [void]$rows.Add($r) } } + } else { + [void]$rows.Add($InputObject) + } + } + } + end { + $tmp = "$Path.tmp" + try { + # Ensure destination directory exists (caller may have skipped this). + $destDir = Split-Path -Path $Path -Parent + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + + if ($rows.Count -eq 0) { + # Header-only fallback: write an empty file (best-effort header derived from $InputObject's first row would be ambiguous here). + # Match the behavior of `Export-Csv -InputObject @()`: produce an empty file with no header. + Set-Content -LiteralPath $tmp -Value '' -Encoding $Encoding -NoNewline + } else { + $rows | Export-Csv -Path $tmp -NoTypeInformation:$NoTypeInformation -Encoding $Encoding -Force -ErrorAction Stop + } + + # Atomic replace. On Windows, Move-Item -Force will overwrite an existing file. + Move-Item -LiteralPath $tmp -Destination $Path -Force -ErrorAction Stop + } + catch { + # Best-effort cleanup of temp; rethrow so callers see the failure. + try { if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } } catch {} + throw + } + } +} + +# ========================================================================== +# CHECKPOINT LOCKING (G-9) +# ========================================================================== +# Defends against two replicas of the same PAX run racing on the same checkpoint +# file. The race surface is real only when an operator deliberately sets ACA Job +# parallelism > 1 (or runs two PAX invocations against the same OutputPath in +# the same wall-clock second). The default ACA Job replicaCompletionCount=1 +# path is safe; this is defense-in-depth for the misconfigured case. +# +# Lock file: ".lock" +# Contents: JSON { pid, host, started, runTimestamp } +# Stale detection: +# - If the lock file's pid is dead on this machine -> treat as stale, take over. +# - If pid is alive on this machine -> reject (loud error, exit non-zero). +# - If the lock file is on a different host (different "host" field) AND was +# last touched more than $script:CheckpointLockStaleTtl ago -> treat as +# stale, take over (cannot verify a foreign PID). +# The exclusive FileStream we open on the lock file is held for the lifetime of +# the run, so a concurrent acquire on the same Windows host will fail at the +# filesystem level even if the JSON-based check races. +$script:CheckpointLockStream = $null +$script:CheckpointLockPath = $null +$script:CheckpointLockStaleTtl = [TimeSpan]::FromMinutes(10) + +function script:Acquire-CheckpointLock { + param( + [Parameter(Mandatory)][string]$CheckpointPath + ) + $lockPath = "$CheckpointPath.lock" + + # Inspect any existing lock for stale-takeover eligibility BEFORE attempting open. + if (Test-Path -LiteralPath $lockPath -PathType Leaf) { + $existing = $null + try { $existing = Get-Content -LiteralPath $lockPath -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop } catch {} + $stale = $false + $reason = '' + if (-not $existing) { + $stale = $true; $reason = 'unreadable lock file (corrupt or empty)' + } + elseif ($existing.host -eq [System.Net.Dns]::GetHostName()) { + # Same-host: definitive PID check. + $alive = $false + try { $alive = [bool](Get-Process -Id ([int]$existing.pid) -ErrorAction Stop) } catch { $alive = $false } + if (-not $alive) { $stale = $true; $reason = "owning PID $($existing.pid) is no longer running on this host" } + } + else { + # Foreign host: fall back to age-based TTL. + try { + $age = (Get-Date) - ([datetime]$existing.started) + if ($age -gt $script:CheckpointLockStaleTtl) { + $stale = $true + $reason = "foreign-host lock from $($existing.host) is older than $($script:CheckpointLockStaleTtl.TotalMinutes) min (age $([int]$age.TotalMinutes) min)" + } + } catch { $stale = $true; $reason = 'foreign-host lock has unparseable timestamp' } + } + + if ($stale) { + Write-LogHost " Checkpoint lock at $lockPath is stale ($reason); taking over." -ForegroundColor Yellow + try { Remove-Item -LiteralPath $lockPath -Force -ErrorAction Stop } catch { + throw "G-9: Could not remove stale checkpoint lock at $lockPath`: $($_.Exception.Message)" + } + } + else { + $ownerDesc = if ($existing) { "pid=$($existing.pid) host=$($existing.host) started=$($existing.started)" } else { '(unknown owner)' } + throw "G-9: Another PAX run is using this checkpoint ($ownerDesc). Refusing to start. If you know the other run is dead, delete the lock file manually: $lockPath" + } + } + + # Open the lock file exclusively (CreateNew + FileShare.None). Hold the stream for the lifetime of the run. + try { + $stream = [System.IO.File]::Open($lockPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + } + catch { + throw "G-9: Failed to acquire checkpoint lock at $lockPath`: $($_.Exception.Message). Another process may have raced in." + } + + # Stamp the lock with our identity so other replicas can do stale detection later. + $payload = @{ + pid = $PID + host = [System.Net.Dns]::GetHostName() + started = (Get-Date).ToUniversalTime().ToString('o') + runTimestamp = $global:ScriptRunTimestamp + } | ConvertTo-Json -Compress + $bytes = [System.Text.Encoding]::UTF8.GetBytes($payload) + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush() + + $script:CheckpointLockStream = $stream + $script:CheckpointLockPath = $lockPath +} + +function script:Release-CheckpointLock { + if ($script:CheckpointLockStream) { + try { $script:CheckpointLockStream.Dispose() } catch {} + $script:CheckpointLockStream = $null + } + if ($script:CheckpointLockPath -and (Test-Path -LiteralPath $script:CheckpointLockPath)) { + try { Remove-Item -LiteralPath $script:CheckpointLockPath -Force -ErrorAction SilentlyContinue } catch {} + } + $script:CheckpointLockPath = $null +} + +function Save-CheckpointToDisk { + <# + .SYNOPSIS + Writes current checkpoint data to disk atomically. + .DESCRIPTION + Uses temp file + rename pattern for atomic writes to prevent corruption. + + This function SELF-GATES on $script:CheckpointEnabled. Callers are still + expected to gate at the call site, but if a future caller forgets, this + guard turns "silently writes a checkpoint when disabled" into a safe no-op. + #> + + # Self-gate. Do NOT remove without revisiting the corresponding static tests. + if (-not $script:CheckpointEnabled) { + return + } + + if (-not $script:CheckpointPath -or -not $script:CheckpointData) { + return + } + + try { + # Update timestamp + $script:CheckpointData.lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + + # Update statistics + $script:CheckpointData.statistics.partitionsComplete = $script:CheckpointData.partitions.completed.Count + $script:CheckpointData.statistics.partitionsQueryCreated = $script:CheckpointData.partitions.queryCreated.Count + $script:CheckpointData.statistics.partitionsRemaining = $script:CheckpointData.partitions.total - + $script:CheckpointData.statistics.partitionsComplete - + $script:CheckpointData.statistics.partitionsQueryCreated + + # Write to temp file first (atomic write pattern) + $tempPath = "$($script:CheckpointPath).tmp" + $script:CheckpointData | ConvertTo-Json -Depth 10 | Set-Content -Path $tempPath -Encoding UTF8 -Force + + # Remove destination first if it exists (Move-Item -Force doesn't always work on Windows) + if (Test-Path $script:CheckpointPath) { + Remove-Item -Path $script:CheckpointPath -Force -ErrorAction SilentlyContinue + } + + # Rename to final (atomic on most filesystems) + Move-Item -Path $tempPath -Destination $script:CheckpointPath -Force + + # Fabric tier: mirror the full resume artifact set (checkpoint JSON, + # .pax_incremental/*.jsonl, *_PARTIAL.csv) to a durable OneLake path inside + # the same try block so either both the local write and the mirror upload + # succeed or the run aborts. Local and SharePoint tiers keep resume artifacts + # local-only (those hosts are not ephemeral). See Sync-FabricResumeMirror. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + # Fabric-tier mirror failure must abort: a torn artifact set on the durable + # mirror would silently corrupt a future resume. Local checkpoint write + # failures remain a soft warning. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + throw ("Checkpoint mirror to OneLake failed; aborting to preserve resume integrity: {0}" -f $_.Exception.Message) + } + Write-LogHost " Warning: Failed to save checkpoint: $($_.Exception.Message)" -ForegroundColor Yellow + } +} + +function Save-Checkpoint { + <# + .SYNOPSIS + Updates checkpoint with partition state change and saves to disk. + .PARAMETER PartitionIndex + The partition index (1-based). + .PARAMETER State + 'QueryCreated' or 'Completed' + .PARAMETER QueryId + The server-assigned query ID. + .PARAMETER PartitionStart + Partition start time (optional - looked up from partitionStatus if not provided). + .PARAMETER PartitionEnd + Partition end time (optional - looked up from partitionStatus if not provided). + .PARAMETER RecordCount + Number of records (only for Completed state). + .PARAMETER Force + Just save current checkpoint state to disk without updating partition info. + #> + param( + [Parameter()] + [int]$PartitionIndex, + + [Parameter()] + [ValidateSet('QueryCreated', 'Completed')] + [string]$State, + + [Parameter()] + [string]$QueryId, + + [Parameter()] + [datetime]$PartitionStart, + + [Parameter()] + [datetime]$PartitionEnd, + + [Parameter()] + [int]$RecordCount = 0, + + [Parameter()] + [switch]$Force + ) + + if (-not $script:CheckpointData) { + return + } + + # If -Force is specified, just save current state to disk without updating partition info + if ($Force) { + Save-CheckpointToDisk + return + } + + # For normal calls, require the mandatory parameters + if (-not $PartitionIndex -or -not $State -or -not $QueryId) { + Write-Verbose "Save-Checkpoint: Missing required parameters (PartitionIndex, State, QueryId) - skipping" + return + } + + # Look up partition times from partitionStatus if not provided + if (-not $PartitionStart -or -not $PartitionEnd) { + $partitionInfo = $script:partitionStatus[$PartitionIndex] + if ($partitionInfo -and $partitionInfo.Partition) { + # The Partition object has PStart and PEnd properties + if (-not $PartitionStart -and $partitionInfo.Partition.PStart) { $PartitionStart = $partitionInfo.Partition.PStart } + if (-not $PartitionEnd -and $partitionInfo.Partition.PEnd) { $PartitionEnd = $partitionInfo.Partition.PEnd } + } + + # If still missing, we can't proceed with checkpoint save + if (-not $PartitionStart -or -not $PartitionEnd) { + Write-Verbose "Save-Checkpoint: Could not determine partition times for index $PartitionIndex - skipping checkpoint update" + return + } + } + + $partitionEntry = @{ + index = $PartitionIndex + start = $PartitionStart.ToUniversalTime().ToString('o') + end = $PartitionEnd.ToUniversalTime().ToString('o') + queryId = $QueryId + } + + if ($State -eq 'QueryCreated') { + $partitionEntry.createdAt = (Get-Date).ToUniversalTime().ToString('o') + + # Add to queryCreated list (if not already there) + $existing = $script:CheckpointData.partitions.queryCreated | Where-Object { $_.index -eq $PartitionIndex } + if (-not $existing) { + $script:CheckpointData.partitions.queryCreated += $partitionEntry + } + } + elseif ($State -eq 'Completed') { + $partitionEntry.records = $RecordCount + + # Remove from queryCreated if present + $script:CheckpointData.partitions.queryCreated = @( + $script:CheckpointData.partitions.queryCreated | Where-Object { $_.index -ne $PartitionIndex } + ) + + # Add to completed list (if not already there) + $existing = $script:CheckpointData.partitions.completed | Where-Object { $_.index -eq $PartitionIndex } + if (-not $existing) { + $script:CheckpointData.partitions.completed += $partitionEntry + $script:CheckpointData.statistics.totalRecordsSaved += $RecordCount + } + } + + # Save to disk + Save-CheckpointToDisk +} + +function Test-CheckpointCompatibility { + <# + .SYNOPSIS + Validates that a parsed checkpoint can safely be resumed by this script version. + .DESCRIPTION + Defends against three failure modes: + 1. Legacy integer `version` outside the supported range (1..2). + 2. `checkpointSchemaVersion` from a FUTURE major release — warned but allowed + when within the same major (forward-compat best effort), rejected on major + mismatch. + 3. `compatibilityMinimumVersion` higher than the running script version — hard + reject (the checkpoint explicitly says this script is too old). + The check is intentionally conservative: it never silently up/down-converts the + structure on disk, and it always emits actionable recovery guidance. + .OUTPUTS + $true -> safe to resume. + $false -> incompatible; caller must abort the resume. + #> + param( + [Parameter(Mandatory)] $CheckpointData, + [Parameter(Mandatory)] [string]$RunningScriptVersion + ) + + # 1) Legacy integer version range (preserves prior behaviour). + if (-not $CheckpointData.version -or [int]$CheckpointData.version -lt 1 -or [int]$CheckpointData.version -gt 2) { + Write-LogHost "ERROR: Unsupported checkpoint legacy version: $($CheckpointData.version). Supported range is 1..2." -ForegroundColor Red + Write-LogHost " Recovery: start a fresh run without -Resume, or use a checkpoint produced by PAX 1.11.x." -ForegroundColor Yellow + return $false + } + + # 2/3) New structured fields are OPTIONAL (older checkpoints predate them). When + # present, they are authoritative. + $schemaVer = [string]$CheckpointData.checkpointSchemaVersion + $minVerStr = [string]$CheckpointData.compatibilityMinimumVersion + $createdBy = [string]$CheckpointData.createdByVersion + + try { $running = [version]$RunningScriptVersion } catch { $running = $null } + try { $minVer = if ($minVerStr) { [version]$minVerStr } else { $null } } catch { $minVer = $null } + try { $created = if ($createdBy) { [version]$createdBy } else { $null } } catch { $created = $null } + + if ($minVer -and $running -and $running -lt $minVer) { + Write-LogHost "ERROR: Checkpoint requires PAX >= $minVerStr but running version is $RunningScriptVersion." -ForegroundColor Red + Write-LogHost " Recovery: upgrade PAX to a version >= $minVerStr or start a fresh run without -Resume." -ForegroundColor Yellow + return $false + } + + if ($schemaVer) { + # Parse the schema version. Only the leading "major.minor" pair gates compatibility. + $parts = $schemaVer.Split('.') + $schemaMajor = 0; [void][int]::TryParse($parts[0], [ref]$schemaMajor) + if ($schemaMajor -gt 2) { + Write-LogHost "ERROR: Checkpoint schema major version $schemaMajor is newer than this script supports (max major 2)." -ForegroundColor Red + Write-LogHost " Recovery: upgrade PAX or start a fresh run without -Resume." -ForegroundColor Yellow + return $false + } + } + + # Created-by drift is informational only (not all script edits change the on-disk shape). + if ($created -and $running -and $created -gt $running) { + Write-LogHost " [WARN] Checkpoint was produced by PAX $createdBy; running PAX $RunningScriptVersion is older. Resume will proceed but newer fields may be ignored." -ForegroundColor Yellow + } + + return $true +} + +function Read-Checkpoint { + <# + .SYNOPSIS + Loads and validates a checkpoint file. + .PARAMETER CheckpointPath + Path to the checkpoint JSON file. + .OUTPUTS + $true if valid and loaded, $false if invalid. + #> + param( + [Parameter(Mandatory)] + [string]$CheckpointPath + ) + + if (-not (Test-Path $CheckpointPath)) { + Write-LogHost "ERROR: Checkpoint file not found: $CheckpointPath" -ForegroundColor Red + return $false + } + + try { + $data = Get-Content -Path $CheckpointPath -Raw | ConvertFrom-Json -AsHashtable + + # Validate version (supports version 1 and 2) + if (-not $data.version -or $data.version -gt 2) { + Write-LogHost "ERROR: Unsupported checkpoint version: $($data.version). This script supports versions 1-2." -ForegroundColor Red + return $false + } + + # Structured compatibility check (schema version, + # compatibilityMinimumVersion, createdByVersion). Hard-rejects checkpoints from + # the future or from releases that explicitly require a newer PAX. + if (-not (Test-CheckpointCompatibility -CheckpointData $data -RunningScriptVersion $ScriptVersion)) { + return $false + } + + # Validate required fields + if (-not $data.runTimestamp -or -not $data.outputFiles -or -not $data.partitions) { + Write-LogHost "ERROR: Checkpoint file is missing required fields" -ForegroundColor Red + return $false + } + + # Get output directory from checkpoint path + $outputDir = Split-Path $CheckpointPath -Parent + $partialCsvPath = Join-Path $outputDir $data.outputFiles.partialCsv + + # Note: We don't require _PARTIAL.csv to exist - the actual data is in .pax_incremental/*.jsonl files + # The _PARTIAL.csv is only created when merging at completion, or may not exist yet + + # Check for incremental save data if there are completed partitions + $completedPartitions = @($data.partitionStates.PSObject.Properties | Where-Object { $_.Value.state -eq 'Completed' }) + if ($completedPartitions.Count -gt 0) { + $incrementalDir = Join-Path $outputDir ".pax_incremental" + $hasIncrementalData = $false + $incrementalRecordCount = 0 + + if (Test-Path $incrementalDir) { + $jsonlFiles = Get-ChildItem -Path $incrementalDir -Filter "*.jsonl" -ErrorAction SilentlyContinue + if ($jsonlFiles -and $jsonlFiles.Count -gt 0) { + $hasIncrementalData = $true + # Count records in files + foreach ($file in $jsonlFiles) { + $incrementalRecordCount += (Get-Content $file.FullName | Measure-Object -Line).Lines + } + } + } + + $expectedRecords = ($completedPartitions | ForEach-Object { $_.Value.recordCount } | Measure-Object -Sum).Sum + + if (-not $hasIncrementalData) { + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " WARNING: INCREMENTAL DATA MISSING" -ForegroundColor Red + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + Write-LogHost " Checkpoint shows $($completedPartitions.Count) completed partition(s) with ~$expectedRecords records," -ForegroundColor Yellow + Write-LogHost " but the .pax_incremental folder is missing or empty." -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost " Expected location: $incrementalDir" -ForegroundColor White + Write-LogHost "" + Write-LogHost " If you continue, data from completed partitions will be LOST." -ForegroundColor Red + Write-LogHost " The remaining partitions will be re-queried, but previous data cannot be recovered." -ForegroundColor Red + Write-LogHost "" + Write-LogHost " OPTIONS:" -ForegroundColor Cyan + Write-LogHost " 1. Restore the .pax_incremental folder from backup (if available)" -ForegroundColor White + Write-LogHost " 2. Start a fresh run without -Resume (will re-query all partitions)" -ForegroundColor White + Write-LogHost " 3. Continue anyway and accept data loss from completed partitions" -ForegroundColor White + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + + # G-6 noninteractive guard: a noninteractive host (container, ACA Job, scheduled + # task, CI runner) cannot answer a Read-Host. The original code would hang + # forever on stdin. Fail-fast with a clear message that points at the only + # two correct recoveries (restore the incremental folder, or start fresh). + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected (no console available to confirm data loss)." -ForegroundColor Red + Write-LogHost " Resume aborted to prevent silent data loss." -ForegroundColor Red + Write-LogHost " Recovery: restore the .pax_incremental folder from backup OR re-run without -Resume." -ForegroundColor Yellow + return $false + } + + $response = Read-Host "Continue with potential data loss? (yes/no)" + if ($response -notmatch '^y(es)?$') { + Write-LogHost "Resume cancelled. Consider starting a fresh run." -ForegroundColor Yellow + return $false + } + Write-LogHost "Continuing with resume despite missing incremental data..." -ForegroundColor Yellow + } elseif ($incrementalRecordCount -lt ($expectedRecords * 0.9)) { + # Warn if incremental count is significantly less than expected (allow 10% variance for counting differences) + Write-LogHost "" + Write-LogHost " [WARN] Incremental data may be incomplete:" -ForegroundColor Yellow + Write-LogHost " Checkpoint expects ~$expectedRecords records from completed partitions" -ForegroundColor Yellow + Write-LogHost " Found $incrementalRecordCount records in .pax_incremental" -ForegroundColor Yellow + Write-LogHost "" + } + } + + # Load into script scope + $script:CheckpointPath = $CheckpointPath + $script:CheckpointData = $data + $script:PartialOutputPath = $partialCsvPath + $script:IsResumeMode = $true + + return $true + } + catch { + Write-LogHost "ERROR: Failed to parse checkpoint file: $($_.Exception.Message)" -ForegroundColor Red + return $false + } +} + +function Find-Checkpoints { + <# + .SYNOPSIS + Discovers checkpoint files in the specified output directory. + .PARAMETER OutputPath + Directory to search for checkpoint files. + .OUTPUTS + Array of checkpoint info objects sorted by LastUpdated (newest first). + #> + param( + [Parameter(Mandatory)] + [string]$OutputPath + ) + + if (-not (Test-Path $OutputPath)) { + return @() + } + + $checkpointFiles = Get-ChildItem -Path $OutputPath -Filter ".pax_checkpoint_*.json" -Force -ErrorAction SilentlyContinue + + if (-not $checkpointFiles -or $checkpointFiles.Count -eq 0) { + return @() + } + + $checkpoints = @() + + foreach ($file in $checkpointFiles) { + try { + $data = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json -AsHashtable + + $checkpoints += [PSCustomObject]@{ + Path = $file.FullName + FileName = $file.Name + RunTimestamp = $data.runTimestamp + LastUpdated = script:Parse-DateSafe $data.lastUpdated + StartDate = if ($data.parameters.startDate) { $d = script:Parse-DateSafe $data.parameters.startDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + EndDate = if ($data.parameters.endDate) { $d = script:Parse-DateSafe $data.parameters.endDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + PartitionsComplete = $data.statistics.partitionsComplete + PartitionsTotal = $data.partitions.total + RecordsSaved = $data.statistics.totalRecordsSaved + } + } + catch { + # Skip invalid checkpoint files + continue + } + } + + # Sort by LastUpdated descending (newest first) + return $checkpoints | Sort-Object -Property LastUpdated -Descending +} + +function Select-Checkpoint { + <# + .SYNOPSIS + Prompts user to select from multiple checkpoint files. + .PARAMETER Checkpoints + Array of checkpoint info objects from Find-Checkpoints. + .OUTPUTS + Selected checkpoint path, or $null if user quits. + #> + param( + [Parameter(Mandatory)] + [array]$Checkpoints + ) + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Multiple checkpoint files found. Select one to resume:" -ForegroundColor Cyan + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" + + for ($i = 0; $i -lt $Checkpoints.Count; $i++) { + $cp = $Checkpoints[$i] + $num = $i + 1 + Write-Host " [$num] $($cp.LastUpdated.ToString('yyyy-MM-dd HH:mm')) | $($cp.StartDate) to $($cp.EndDate) | $($cp.PartitionsComplete)/$($cp.PartitionsTotal) partitions | $($cp.RecordsSaved.ToString('N0')) records" -ForegroundColor White + Write-Host " $($cp.FileName)" -ForegroundColor DarkGray + Write-Host "" + } + + Write-Host " [Q] Quit (do not resume)" -ForegroundColor Yellow + Write-Host "" + + # G-6 noninteractive guard: cannot prompt for a checkpoint selection on a + # container/ACA Job/CI runner. Reject explicitly with a recovery hint instead + # of letting Read-Host hang on stdin. Operators who actually want resume in + # noninteractive contexts must pass -Resume . + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: cannot prompt to choose among $($Checkpoints.Count) checkpoint(s)." -ForegroundColor Red + Write-Host " Re-run with an explicit path: -Resume `"`"" -ForegroundColor Yellow + Write-Host " Or run on an interactive host to use the selection menu." -ForegroundColor Yellow + return $null + } + + Send-PromptNotification + while ($true) { + $choice = Read-Host " Enter selection (1-$($Checkpoints.Count)) or 'Q' to quit" + + if ($choice -eq 'Q' -or $choice -eq 'q') { + return $null + } + + $selection = 0 + if ([int]::TryParse($choice, [ref]$selection)) { + if ($selection -ge 1 -and $selection -le $Checkpoints.Count) { + return $Checkpoints[$selection - 1] + } + } + + Write-Host " Invalid selection. Please enter a number 1-$($Checkpoints.Count) or 'Q' to quit." -ForegroundColor Red + } +} + +function Remove-Checkpoint { + <# + .SYNOPSIS + Deletes checkpoint file after successful completion. + #> + + if ($script:CheckpointPath -and (Test-Path $script:CheckpointPath)) { + try { + Remove-Item -Path $script:CheckpointPath -Force + } + catch { + Write-LogHost " Warning: Could not delete checkpoint file: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Release the exclusive checkpoint lock acquired at run start. + script:Release-CheckpointLock + + $script:CheckpointPath = $null + $script:CheckpointData = $null +} + +function Get-PartitionsToProcess { + <# + .SYNOPSIS + Categorizes partitions based on checkpoint state for resume. + .PARAMETER AllPartitions + Array of all partition objects for the date range. + .OUTPUTS + Hashtable with ToSkip, ToFetchOnly, and ToCreateAndFetch arrays. + #> + param( + [Parameter(Mandatory)] + [array]$AllPartitions + ) + + $result = @{ + ToSkip = @() # Already completed - skip entirely + ToFetchOnly = @() # Query exists on server - just fetch records + ToCreateAndFetch = @() # Start fresh - create query then fetch + } + + if (-not $script:CheckpointData) { + # No checkpoint - all partitions need full processing + $result.ToCreateAndFetch = $AllPartitions + return $result + } + + $completedIndices = @{} + $queryCreatedIndices = @{} + + # Build lookup tables from checkpoint (use string keys for reliable comparison) + foreach ($cp in $script:CheckpointData.partitions.completed) { + $completedIndices["$($cp.index)"] = $cp + } + foreach ($qc in $script:CheckpointData.partitions.queryCreated) { + $queryCreatedIndices["$($qc.index)"] = $qc + } + + # Categorize each partition + foreach ($partition in $AllPartitions) { + $idx = "$($partition.Index)" # Convert to string for reliable comparison + + if ($completedIndices.ContainsKey($idx)) { + $result.ToSkip += $partition + } + elseif ($queryCreatedIndices.ContainsKey($idx)) { + # Add QueryId to partition for fetch-only processing + $partition | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue $queryCreatedIndices[$idx].queryId -Force + $result.ToFetchOnly += $partition + } + else { + $result.ToCreateAndFetch += $partition + } + } + + return $result +} + +function Test-ShouldPromptTokenRefresh { + <# + .SYNOPSIS + Checks if token refresh handling is needed. + .DESCRIPTION + Reactive detection: Returns true only when a 401 Unauthorized error + has been detected, indicating the token has actually expired. + No proactive time-based prompts - only triggers on real auth failures. + + For AppRegistration mode, this still returns true on 401 so the + auth handling block can perform automatic silent token refresh. + For interactive modes, this triggers a user prompt. + .OUTPUTS + $true if auth failure detected and refresh handling is needed, $false otherwise. + #> + + # Reactive detection: Return true when 401 error detected (for ALL auth methods) + # The auth handling block will determine whether to auto-refresh (AppRegistration) + # or prompt the user (interactive modes) + return $script:AuthFailureDetected +} + +function Invoke-TokenRefreshPrompt { + <# + .SYNOPSIS + Handles token refresh for interactive auth modes (WebLogin/DeviceCode). + .DESCRIPTION + Triggered reactively when a 401 Unauthorized error is detected. + First attempts silent token refresh (SDK may have valid refresh token cached). + If silent refresh fails, prompts user to re-authenticate or quit. + .OUTPUTS + 'Refreshed' - Token refreshed successfully (silent or interactive) + 'Quit' - User chose to exit + #> + + $tokenAge = if ($script:TokenAcquiredTime) { (Get-Date) - $script:TokenAcquiredTime } else { $null } + + # ═══════════════════════════════════════════════════════════════════════════ + # NOTE: We intentionally do NOT attempt automatic token refresh here. + # On Windows, Connect-MgGraph with InteractiveBrowserCredential ALWAYS opens + # a browser popup. If the user is away, this popup sits waiting, and when + # they return and press 'R' to re-auth, a SECOND popup appears. + # By going straight to the user prompt, we ensure only ONE popup when ready. + # ═══════════════════════════════════════════════════════════════════════════ + + # ═══════════════════════════════════════════════════════════════════════════ + # Prompt user for interactive re-authentication (single popup, user-initiated) + # ═══════════════════════════════════════════════════════════════════════════ + + # Get progress info for display + $completedCount = if ($script:CheckpointData) { $script:CheckpointData.statistics.partitionsComplete } else { 0 } + $totalCount = if ($script:CheckpointData) { $script:CheckpointData.partitions.total } else { 0 } + $recordsSaved = if ($script:CheckpointData) { $script:CheckpointData.statistics.totalRecordsSaved } else { 0 } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " [!] AUTHENTICATION EXPIRED - Re-authentication Required" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Authentication failure detected (401 Unauthorized)." -ForegroundColor White + Write-Host " Your session token has expired. Please re-authenticate to continue." -ForegroundColor White + Write-Host "" + if ($tokenAge) { + Write-Host " Session duration: $([Math]::Round($tokenAge.TotalMinutes, 0)) minutes" -ForegroundColor Gray + } + Write-Host " Progress: $completedCount/$totalCount partitions complete | $($recordsSaved.ToString('N0')) records saved to disk" -ForegroundColor White + Write-Host " Auth method: $($Auth) (interactive)" -ForegroundColor Gray + Write-Host "" + Write-Host " [R] Re-authenticate now (recommended)" -ForegroundColor Green + Write-Host " [Q] Quit and save progress (resume later with -Resume)" -ForegroundColor Cyan + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + + # G-6 noninteractive guard: on a container/ACA Job/CI runner there is nobody at + # the keyboard to choose R/Q. Auto-choose Q ('quit and save progress'); the + # operator can resume the checkpoint later from an interactive host. Logging + # the auto-choice prevents the silent stdin hang the original code would have + # produced. + if (script:Test-IsNonInteractive) { + Write-Host "" -ForegroundColor Yellow + Write-Host " Noninteractive host detected: cannot prompt for re-authentication." -ForegroundColor Yellow + Write-Host " Auto-selecting [Q] - progress will be saved to the checkpoint." -ForegroundColor Yellow + Write-Host " Resume from an interactive host with: -Resume `"$($script:CheckpointPath)`"" -ForegroundColor Yellow + return 'Quit' + } + + Send-PromptNotification + while ($true) { + $choice = Read-Host " Enter choice (R/Q)" + + switch ($choice.ToUpper()) { + 'R' { + Write-Host "" + Write-Host " Re-authenticating..." -ForegroundColor Cyan + + try { + # Disconnect and reconnect + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Connect-PurviewAudit -AuthMethod $Auth -UseEOMMode $false + + # Update token timing and reset auth failure flags + $script:TokenAcquiredTime = Get-Date + $script:AuthFailureDetected = $false + $script:Auth401MessageShown = $false # Reset for next auth failure cycle + + Write-Host " Re-authentication successful. Resuming execution..." -ForegroundColor Green + Write-Host " Failed partitions will be retried with fresh token." -ForegroundColor Green + Write-Host "" + + return 'Refreshed' + } + catch { + Write-Host " ✗ Re-authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-Host " Please try again or quit." -ForegroundColor Yellow + } + } + 'Q' { + Write-Host "" + Show-CheckpointExitMessage + return 'Quit' + } + default { + Write-Host " Invalid choice. Please enter R or Q." -ForegroundColor Red + } + } + } +} + +function Merge-IncrementalSaves { + <# + .SYNOPSIS + Merges incremental JSON save files into the main allLogs collection. + .DESCRIPTION + Called at the end of execution to consolidate partition data that was + saved incrementally during execution. In resume mode, only merges data + from partitions that were skipped (already completed at start of run), + since partitions completed during this run already have data in memory. + .PARAMETER AllLogs + Reference to the synchronized ArrayList to add records to. + .PARAMETER OutputDirectory + The output directory containing the .pax_incremental folder. + .PARAMETER CleanupAfterMerge + If true, deletes the incremental files after successful merge. + .PARAMETER OnlyPartitionIndices + If specified, only merge files for these partition indices. + Used in resume mode to avoid double-counting data from partitions + that completed during this run. + #> + param( + [Parameter(Mandatory = $true)] + [System.Collections.ArrayList]$AllLogs, + + [Parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [Parameter(Mandatory = $false)] + [bool]$CleanupAfterMerge = $true, + + [Parameter(Mandatory = $false)] + [int[]]$OnlyPartitionIndices = $null + ) + + $incrementalDir = Join-Path $OutputDirectory ".pax_incremental" + + if (-not (Test-Path $incrementalDir)) { + return 0 + } + + $incrementalFiles = Get-ChildItem -Path $incrementalDir -Filter "*.jsonl" -ErrorAction SilentlyContinue + + if (-not $incrementalFiles -or $incrementalFiles.Count -eq 0) { + return 0 + } + + $mergedCount = 0 + $filesProcessed = 0 + $filesSkipped = 0 + + $filterMsg = if ($OnlyPartitionIndices) { " (filtering for partitions: $($OnlyPartitionIndices -join ', '))" } else { "" } + Write-LogHost " [MERGE] Found $($incrementalFiles.Count) incremental save files$filterMsg..." -ForegroundColor Cyan + + foreach ($file in $incrementalFiles) { + try { + # If filtering by partition indices, check if this file matches + # Filename format: Part{N}_timestamp_qid-{QueryId}_Nrecords.jsonl (recovery files use qid-recovery) + if ($OnlyPartitionIndices) { + $partMatch = [regex]::Match($file.Name, '^Part(\d+)_') + if ($partMatch.Success) { + $filePartitionIndex = [int]$partMatch.Groups[1].Value + if ($filePartitionIndex -notin $OnlyPartitionIndices) { + $filesSkipped++ + # Don't delete - this file is for a partition completed in this run + continue + } + } + } + + # Read JSON Lines (NDJSON) format - one record per line + $lines = Get-Content -Path $file.FullName -Encoding utf8 + $fileRecordCount = 0 + + foreach ($line in $lines) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + + try { + $record = $line | ConvertFrom-Json + [void]$AllLogs.Add($record) + $fileRecordCount++ + } catch { + # Skip malformed lines but continue processing + Write-Verbose "Skipped malformed line in $($file.Name)" + } + } + + $mergedCount += $fileRecordCount + if ($fileRecordCount -gt 0) { $filesProcessed++ } + + if ($CleanupAfterMerge) { + Remove-Item -Path $file.FullName -Force -ErrorAction SilentlyContinue + } + } + catch { + Write-LogHost " [WARN] Failed to merge $($file.Name): $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Clean up directory if empty + if ($CleanupAfterMerge) { + $remainingFiles = Get-ChildItem -Path $incrementalDir -ErrorAction SilentlyContinue + if (-not $remainingFiles -or $remainingFiles.Count -eq 0) { + Remove-Item -Path $incrementalDir -Force -ErrorAction SilentlyContinue + } + } + + if ($mergedCount -gt 0) { + Write-LogHost " [MERGE] Merged $($mergedCount.ToString('N0')) records from $filesProcessed incremental files" -ForegroundColor Green + } + + return $mergedCount +} + +function Merge-IncrementalSaves-Streaming { + <# + .SYNOPSIS + Memory-efficient streaming merge of incremental JSONL files directly to CSV. + .DESCRIPTION + Instead of loading all records into memory, this function streams records + from incremental JSONL files directly to the final CSV output using batched + writes and explicit garbage collection between files. This prevents memory + exhaustion when merging millions of records. + .PARAMETER OutputFile + The final CSV file path to write merged data to. + .PARAMETER OutputDirectory + The output directory containing the .pax_incremental folder. + .PARAMETER OnlyPartitionIndices + If specified, only merge files for these partition indices. + .PARAMETER Columns + The column schema to use for CSV output. If not specified, uses default 7-column schema. + .PARAMETER RunTimestamp + The script run timestamp used to filter incremental files to only those from the current run. + Prevents stale files from prior runs being merged into the output. + .RETURNS + The total number of records merged. + #> + param( + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [Parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [Parameter(Mandatory = $false)] + [int[]]$OnlyPartitionIndices = $null, + + [Parameter(Mandatory = $false)] + [string[]]$Columns = $null, + + [Parameter(Mandatory = $false)] + [System.Collections.Generic.HashSet[string]]$ExcludeRecordIds = $null, + + [Parameter(Mandatory = $false)] + [ref]$ActivityCounts = $null, + + [Parameter(Mandatory = $false)] + [string]$RunTimestamp = $null + ) + + $incrementalDir = Join-Path $OutputDirectory ".pax_incremental" + + if (-not (Test-Path $incrementalDir)) { + Write-LogHost " [MERGE-STREAM] No incremental directory found" -ForegroundColor Yellow + return 0 + } + + # Filter by run timestamp to avoid merging stale files from prior runs + $jsonlFilter = if ($RunTimestamp) { "*_${RunTimestamp}_*.jsonl" } else { "*.jsonl" } + $allFiles = Get-ChildItem -Path $incrementalDir -Filter $jsonlFilter -ErrorAction SilentlyContinue + if (-not $allFiles -or $allFiles.Count -eq 0) { + Write-LogHost " [MERGE-STREAM] No incremental files found$(if ($RunTimestamp) { " for run $RunTimestamp" })" -ForegroundColor Yellow + return 0 + } + + # Sort files by partition number for consistent output ordering + $files = $allFiles | Sort-Object { + if ($_.Name -match 'Part(\d+)_') { [int]$Matches[1] } else { 999999 } + } + + # Filter by partition indices if specified + if ($OnlyPartitionIndices) { + $files = $files | Where-Object { + $partMatch = [regex]::Match($_.Name, '^Part(\d+)_') + if ($partMatch.Success) { + [int]$partMatch.Groups[1].Value -in $OnlyPartitionIndices + } else { + $false + } + } + } + + if (-not $files -or @($files).Count -eq 0) { + Write-LogHost " [MERGE-STREAM] No matching incremental files for specified partitions" -ForegroundColor Yellow + return 0 + } + + # When multiple JSONL files exist for the same partition (from retries with different QueryIds), + # keep only the largest file per partition. This prevents duplicate records from partial first attempts + # being merged alongside the full retry result. + $filesByPartition = @{} + foreach ($f in @($files)) { + if ($f.Name -match '^Part(\d+)_') { + $pIdx = [int]$Matches[1] + if (-not $filesByPartition.ContainsKey($pIdx) -or $f.Length -gt $filesByPartition[$pIdx].Length) { + $filesByPartition[$pIdx] = $f + } + } + } + $deduplicatedFiles = @($filesByPartition.Values | Sort-Object { if ($_.Name -match 'Part(\d+)_') { [int]$Matches[1] } else { 999999 } }) + $removedFileCount = @($files).Count - $deduplicatedFiles.Count + if ($removedFileCount -gt 0) { + Write-LogHost " [MERGE-STREAM] Removed $removedFileCount duplicate partition file(s) from prior retry attempts — keeping largest per partition" -ForegroundColor DarkYellow + } + $files = $deduplicatedFiles + + $fileCount = @($files).Count + Write-LogHost " [MERGE-STREAM] Streaming $fileCount incremental files to CSV..." -ForegroundColor Cyan + + # Use default 8-column schema if not specified (non-explosion mode) + # Column names match Purview UI audit export: RecordId, CreationDate, RecordType, Operation, UserId, AuditData, AssociatedAdminUnits, AssociatedAdminUnitsNames + if (-not $Columns) { + $Columns = @('RecordId', 'CreationDate', 'RecordType', 'Operation', 'UserId', 'AuditData', 'AssociatedAdminUnits', 'AssociatedAdminUnitsNames') + } + + $totalMerged = 0 + $filesProcessed = 0 + $headerWritten = $false + $batchSize = 5000 + $startTime = Get-Date + $lastProgressTime = Get-Date + + # Track seen RecordIds for deduplication (seed with any in-memory RecordIds already written to CSV) + $seenIds = if ($ExcludeRecordIds) { New-Object System.Collections.Generic.HashSet[string] ($ExcludeRecordIds) } else { New-Object System.Collections.Generic.HashSet[string] } + $duplicatesSkipped = 0 + + foreach ($file in $files) { + $filesProcessed++ + $partNum = if ($file.Name -match 'Part(\d+)_') { $Matches[1] } else { "?" } + + try { + # Open CSV writer on first file with data + if (-not $headerWritten) { + Open-CsvWriter -Path $OutputFile -Columns $Columns + $headerWritten = $true + } + + # Stream records from this file in batches + $batch = New-Object System.Collections.Generic.List[object] + $fileRecords = 0 + + # Use StreamReader instead of Get-Content pipeline for ~5-10x faster file reading + $reader = [System.IO.StreamReader]::new($file.FullName, [System.Text.Encoding]::UTF8) + while ($null -ne ($line = $reader.ReadLine())) { + if (-not [string]::IsNullOrWhiteSpace($line)) { + try { + $record = $line | ConvertFrom-Json + + # Deduplicate by RecordId + $recordId = $null + if ($record.Identity) { $recordId = $record.Identity } + elseif ($record.Id) { $recordId = $record.Id } + elseif ($record.RecordId) { $recordId = $record.RecordId } + + if ($recordId -and $seenIds.Contains($recordId)) { + $script:StreamingMergeDuplicatesSkipped++ + continue # Skip duplicate — 'continue' works in while loop (was 'return' in ForEach-Object) + } + if ($recordId) { [void]$seenIds.Add($recordId) } + + # Client-side date-range trimming for streaming path + if ($script:TrimStartDateUTC -or $script:TrimEndDateUTC) { + $recDate = script:Parse-DateSafe $record.CreationDate + if ($recDate) { + $recDateUtc = $recDate.ToUniversalTime() + if ($script:TrimStartDateUTC -and $recDateUtc -lt $script:TrimStartDateUTC) { $script:DateTrimCount++; continue } + if ($script:TrimEndDateUTC -and $recDateUtc -ge $script:TrimEndDateUTC) { $script:DateTrimCount++; continue } + } + } + + # Parse AuditData for Operation if needed + $auditData = $record.AuditData + $parsedAudit = if ($record.PSObject.Properties['_ParsedAuditData']) { + $record._ParsedAuditData + } else { + try { $auditData | ConvertFrom-Json -ErrorAction SilentlyContinue } catch { $null } + } + $opValue = if ($parsedAudit -and $parsedAudit.Operation) { $parsedAudit.Operation } else { $record.Operations } + + # Track per-activity counts for Activity Type Breakdown + if ($ActivityCounts -and $opValue) { + if (-not $ActivityCounts.Value.ContainsKey($opValue)) { $ActivityCounts.Value[$opValue] = 0 } + $ActivityCounts.Value[$opValue]++ + } + + # Create normalized record matching expected schema (column names match Purview UI export) + $normalizedRecord = [pscustomobject]@{ + RecordId = if ($record.RecordId) { $record.RecordId } elseif ($record.Identity) { $record.Identity } elseif ($record.Id) { $record.Id } elseif ($parsedAudit -and $parsedAudit.Id) { $parsedAudit.Id } else { $null } + CreationDate = if ($record.CreationDate) { + $dt = script:Parse-DateSafe $record.CreationDate; if ($dt) { $dt.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) } else { $record.CreationDate } + } else { '' } + RecordType = $record.RecordType + Operation = $opValue + UserId = if ($record.UserId) { $record.UserId } elseif ($record.UserIds) { $record.UserIds } else { '' } + AuditData = $auditData + AssociatedAdminUnits = $(try { if ($parsedAudit.AssociatedAdminUnits) { $parsedAudit.AssociatedAdminUnits } elseif ($record.AssociatedAdminUnits) { $record.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($parsedAudit.AssociatedAdminUnitsNames) { $parsedAudit.AssociatedAdminUnitsNames } elseif ($record.AssociatedAdminUnitsNames) { $record.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + } + + $batch.Add($normalizedRecord) + $fileRecords++ + + # Write batch when full + if ($batch.Count -ge $batchSize) { + Write-CsvRows -Rows $batch -Columns $Columns + $totalMerged += $batch.Count + $batch.Clear() + } + } catch { + # Skip malformed lines + Write-Verbose "Skipped malformed line in $($file.Name): $($_.Exception.Message)" + } + } + } + # Dispose StreamReader to release file handle + if ($reader) { $reader.Dispose(); $reader = $null } + + # Flush remaining batch for this file + if ($batch.Count -gt 0) { + Write-CsvRows -Rows $batch -Columns $Columns + $totalMerged += $batch.Count + $batch.Clear() + } + + # Progress reporting every 30 seconds + $now = Get-Date + if (($now - $lastProgressTime).TotalSeconds -ge 30) { + $elapsed = ($now - $startTime).TotalSeconds + $rate = if ($elapsed -gt 0) { [int]($totalMerged / $elapsed) } else { 0 } + Write-LogHost " [MERGE-STREAM] Progress: $filesProcessed/$fileCount files | $($totalMerged.ToString('N0')) records | ~$rate rec/sec" -ForegroundColor DarkCyan + $lastProgressTime = $now + } + + # Reduced GC frequency from every file to every 5th file for better throughput + $batch = $null + if ($filesProcessed % 5 -eq 0) { + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + } + + } catch { + # Ensure StreamReader is disposed on error to release file handle + if ($reader) { try { $reader.Dispose() } catch {} ; $reader = $null } + Write-LogHost " [WARN] Failed to stream merge $($file.Name): $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Close CSV writer + if ($headerWritten) { + Close-CsvWriter + } + + # Final stats + $totalElapsed = (Get-Date) - $startTime + $finalRate = if ($totalElapsed.TotalSeconds -gt 0) { [int]($totalMerged / $totalElapsed.TotalSeconds) } else { 0 } + + Write-LogHost " [MERGE-STREAM] Streaming merge complete: $($totalMerged.ToString('N0')) records from $filesProcessed files" -ForegroundColor Green + Write-LogHost " [MERGE-STREAM] Time: $([Math]::Round($totalElapsed.TotalSeconds, 1))s | Rate: $finalRate rec/sec" -ForegroundColor DarkGray + if ($duplicatesSkipped -gt 0 -or $script:StreamingMergeDuplicatesSkipped -gt 0) { + $totalDupes = $duplicatesSkipped + $script:StreamingMergeDuplicatesSkipped + Write-LogHost " [MERGE-STREAM] Duplicates skipped: $totalDupes" -ForegroundColor DarkGray + } + + # Clear the HashSet to free memory + $seenIds.Clear() + $seenIds = $null + [GC]::Collect() + + return $totalMerged +} + +function Show-CheckpointExitMessage { + <# + .SYNOPSIS + Displays checkpoint save confirmation and resume instructions. + #> + + if (-not $script:CheckpointData -or -not $script:CheckpointPath) { + return + } + + $completedCount = if ($script:CheckpointData.statistics.partitionsComplete) { $script:CheckpointData.statistics.partitionsComplete } else { 0 } + $queryCreatedCount = if ($script:CheckpointData.statistics.partitionsQueryCreated) { $script:CheckpointData.statistics.partitionsQueryCreated } else { 0 } + $totalCount = if ($script:CheckpointData.partitions.total) { $script:CheckpointData.partitions.total } else { 0 } + $remaining = $totalCount - $completedCount - $queryCreatedCount + $recordsSaved = if ($script:CheckpointData.statistics.totalRecordsSaved) { $script:CheckpointData.statistics.totalRecordsSaved } else { 0 } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host " PROGRESS SAVED" -ForegroundColor Green + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host "" + # The checkpoint file and the local _PARTIAL file always live on the local + # scratch path (they are required for -Resume; the checkpoint isn't uploaded + # until the run completes successfully). Display them as full local paths so + # the user can copy/paste them into -Resume verbatim. + # + # The run log is written to local scratch throughout the run. On successful + # completion the end-of-run upload sweep pushes it to the remote destination + # (SP/Fabric); on an interrupted run (Ctrl+C / failure) the upload sweep does + # NOT fire, so the log lives only at the local scratch path until the next + # -Resume completes successfully. Always display the literal local path here + # (consistent with Checkpoint: and Partial data:) so the user can inspect / + # attach the in-progress log immediately. When a remote destination is bound, + # print a follow-up "Uploads to:" note so the user knows where the log will + # land on successful completion. + Write-Host " Checkpoint: $script:CheckpointPath" -ForegroundColor White + # Live in-progress records stream into /.pax_incremental/*.jsonl + # shards as each partition completes. The companion _PARTIAL.csv ($script:PartialOutputPath) + # is only materialized at end-of-run merge or on successful completion, so it + # generally does NOT exist on an interrupted Ctrl+C exit. Point the user at + # the .pax_incremental folder (the actual durable source of partial data); if + # a _PARTIAL.csv already exists alongside it, list that on a follow-up line. + $_partialBaseDir = $null + # Defensive: Split-Path -LiteralPath throws "Parameter set cannot be resolved" + # when handed $null, an empty string, or a non-string (array / hashtable from a + # malformed checkpoint outputFiles entry). Coerce to a non-empty string before + # splitting and swallow any residual parsing failure so the exit banner never + # crashes after a real upstream error has already been reported. + $_pop = if ($script:PartialOutputPath) { [string]$script:PartialOutputPath } else { '' } + if ($_pop -and $_pop.Trim()) { + try { $_partialBaseDir = Split-Path -LiteralPath $_pop -Parent -ErrorAction Stop } catch { $_partialBaseDir = $null } + } + if (-not $_partialBaseDir -and $script:CheckpointPath) { + $_cpp = [string]$script:CheckpointPath + if ($_cpp -and $_cpp.Trim()) { + try { $_partialBaseDir = Split-Path -LiteralPath $_cpp -Parent -ErrorAction Stop } catch { $_partialBaseDir = $null } + } + } + $_incDir = if ($_partialBaseDir) { Join-Path $_partialBaseDir '.pax_incremental' } else { $null } + if ($_incDir) { + Write-Host " Partial data: $_incDir\*.jsonl" -ForegroundColor White + } else { + Write-Host " Partial data: (incremental .jsonl shards under scratch .pax_incremental/)" -ForegroundColor White + } + if ($script:PartialOutputPath -and (Test-Path -LiteralPath $script:PartialOutputPath)) { + Write-Host " Merged CSV (so far): $script:PartialOutputPath" -ForegroundColor DarkGray + } + if ($script:LogFile) { + Write-Host " Log file: $script:LogFile" -ForegroundColor White + $_logRemoteTarget = $null + try { + if ($script:DestTier -and $script:DestTier.ContainsKey('Log') -and $script:DestTier['Log'] -and $script:DestTier['Log'] -ne 'Local') { + $_logFolder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey('Log') -and $script:DestParentUrl['Log']) { + $script:DestParentUrl['Log'] + } elseif ($script:DestRaw -and $script:DestRaw.ContainsKey('Log') -and $script:DestRaw['Log']) { + $script:DestRaw['Log'] + } else { $null } + if ($_logFolder) { + $_logLeaf = [System.IO.Path]::GetFileName($script:LogFile) + $_logRemoteTarget = ($_logFolder.TrimEnd('/','\') + '/' + $_logLeaf) + } + } + } catch {} + if ($_logRemoteTarget) { + Write-Host " (uploads to $_logRemoteTarget on successful completion)" -ForegroundColor DarkGray + } + } + Write-Host " Records saved: $($recordsSaved.ToString('N0'))" -ForegroundColor White + # Build the Partitions status line as a single string before emission so the + # global Write-Host proxy (which mirrors every Write-Host call to the log + # file as ONE timestamped [INFO] entry regardless of -NoNewline) writes one + # log line instead of 2-3 fragmented entries. The console output is the same + # either way — both forms produce an identical + # "Partitions: X/Y complete[, N queries pending][, M not started]" line — + # but a multi-call -NoNewline approach produces orphan log lines like + # ", 1 queries pending" on their own timestamp because -NoNewline only + # concatenates on the host stream, not in the proxy's Add-Content path. + $partitionLine = " Partitions: $completedCount/$totalCount complete" + if ($queryCreatedCount -gt 0) { $partitionLine += ", $queryCreatedCount queries pending" } + if ($remaining -gt 0) { $partitionLine += ", $remaining not started" } + Write-Host $partitionLine -ForegroundColor White + Write-Host "" + Write-Host " To resume later:" -ForegroundColor Cyan + # Resume credential hint. -Auth / -TenantId / -ClientId are restored from the + # checkpoint on -Resume (see the resume restore block), so they are NOT + # re-supplied on the resume command line. Only AppRegistration needs credential + # material on every invocation, because the secret / certificate is never + # persisted to the checkpoint by design — and the hint must reflect the SAME + # credential kind the operator originally used (client secret vs. certificate + # thumbprint vs. PFX path), detected from the promoted $script:Client* values. + # Interactive modes (WebLogin / DeviceCode / Credential / Silent) and + # ManagedIdentity need nothing beyond -Resume. + if ($Auth -eq 'AppRegistration') { + $_resumeCred = + if (-not [string]::IsNullOrWhiteSpace([string]$script:ClientCertificateThumbprint)) { "-ClientCertificateThumbprint ''" } + elseif (-not [string]::IsNullOrWhiteSpace([string]$script:ClientCertificatePath)) { "-ClientCertificatePath '' [-ClientCertificatePassword '']" } + else { "-ClientSecret ''" } + Write-Host " -Resume `"$($script:CheckpointPath)`" $_resumeCred" -ForegroundColor White + } else { + Write-Host " -Resume `"$($script:CheckpointPath)`"" -ForegroundColor White + } + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green +} + +function Complete-CheckpointRun { + <# + .SYNOPSIS + Finalizes successful run: renames _PARTIAL file, deletes checkpoint. + .PARAMETER FinalOutputPath + The final output path (without _PARTIAL). + #> + param( + [Parameter(Mandatory)] + [string]$FinalOutputPath + ) + + # Rename _PARTIAL output (and log) to final names ONLY if the _PARTIAL file still exists. + # In CSV-split / ExportWorkbook modes the intermediate _PARTIAL.csv may already have + # been deleted by upstream code paths; in that case skip the rename but STILL proceed + # with checkpoint deletion below so the checkpoint file is not orphaned. + if ($script:PartialOutputPath -and (Test-Path $script:PartialOutputPath)) { + try { + # Rename _PARTIAL to final + if (Test-Path $FinalOutputPath) { + # Final file already exists - add timestamp to avoid overwrite + $dir = Split-Path $FinalOutputPath -Parent + $name = [System.IO.Path]::GetFileNameWithoutExtension($FinalOutputPath) + $ext = [System.IO.Path]::GetExtension($FinalOutputPath) + $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + $FinalOutputPath = Join-Path $dir "${name}_${timestamp}${ext}" + } + + Move-Item -Path $script:PartialOutputPath -Destination $FinalOutputPath -Force + + # Rename log file (remove _PARTIAL suffix) + $partialLogPath = $script:LogFile + if ($partialLogPath -and (Test-Path $partialLogPath) -and $partialLogPath -match '_PARTIAL\.log$') { + $finalLogPath = $partialLogPath -replace '_PARTIAL\.log$', '.log' + if (Test-Path $finalLogPath) { + $logDir = Split-Path $finalLogPath -Parent + $logName = [System.IO.Path]::GetFileNameWithoutExtension($finalLogPath) + $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + $finalLogPath = Join-Path $logDir "${logName}_${timestamp}.log" + } + Move-Item -Path $partialLogPath -Destination $finalLogPath -Force + $script:LogFile = $finalLogPath + } + + $script:PartialOutputPath = $null + } + catch { + Write-LogHost " Warning: Could not finalize output file: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Delete checkpoint (always — caller has determined the run succeeded). + Remove-Checkpoint +} + +function Get-UserLicenseData { + <# + .SYNOPSIS + Fetches user license information from Microsoft Graph API. + + .DESCRIPTION + Queries Graph API for subscribedSkus and builds lookup hashtables for: + - User license assignments (userId -> list of SKU names) + - Copilot license detection (userId -> has Copilot license) + + Dynamic Copilot license detection (no hardcoded SKU GUIDs): + 1. From /subscribedSkus, collect every servicePlanId whose servicePlanName matches '*COPILOT*'. + 2. For each user, scan assignedPlans for entries where capabilityStatus == 'Enabled' + AND servicePlanId is in the discovered Copilot plan set. + This honors capability state (admin-disabled service plans are correctly excluded) + and auto-adapts to any current or future Copilot SKU (M365, EDU, Sales, Finance, GCC, etc.). + Requires only User.Read.All + Organization.Read.All (no additional permissions). + + .OUTPUTS + Hashtable with two keys: + - UserLicenses: @{userId = @('SKU1', 'SKU2')} + - UserHasCopilot: @{userId = $true/$false} + #> + + Write-LogHost "" + Write-LogHost "Fetching license data from Microsoft Graph API..." -ForegroundColor Cyan + + try { + # Fetch all subscribed SKUs with assigned user data + $uri = "https://graph.microsoft.com/v1.0/subscribedSkus" + $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + $skus = $response.value + + Write-LogHost " Found $($skus.Count) SKU(s) in tenant" -ForegroundColor Gray + + # Build SKU lookup: skuId -> skuPartNumber + # Also dynamically discover Copilot service plan IDs by name pattern (no hardcoded GUIDs). + $skuLookup = @{} + $copilotPlanIds = New-Object System.Collections.Generic.HashSet[string] + + foreach ($sku in $skus) { + $skuId = $sku.skuId + $skuName = $sku.skuPartNumber + $skuLookup[$skuId] = $skuName + if ($sku.servicePlans) { + foreach ($plan in $sku.servicePlans) { + if ($plan.servicePlanName -and $plan.servicePlanName -like '*COPILOT*' -and $plan.servicePlanId) { + [void]$copilotPlanIds.Add([string]$plan.servicePlanId) + } + } + } + } + Write-LogHost " Discovered $($copilotPlanIds.Count) Copilot service plan(s) in tenant" -ForegroundColor Gray + + # Build user license hashtables + $userLicenses = @{} # userId -> @('SKU1', 'SKU2') + $userHasCopilot = @{} # userId -> $true/$false + + # Fetch all users with licenses in batches + # assignedPlans is required for capability-aware Copilot detection (still under User.Read.All). + Write-LogHost " Fetching user license assignments..." -ForegroundColor Gray + $userUri = "https://graph.microsoft.com/v1.0/users?`$select=id,userPrincipalName,assignedLicenses,assignedPlans&`$top=999" + $userCount = 0 + $copilotUserCount = 0 + + do { + $userResponse = Invoke-MgGraphRequest -Method GET -Uri $userUri -ErrorAction Stop + $users = $userResponse.value + + foreach ($user in $users) { + $userId = $user.id + $upn = $user.userPrincipalName + + if (-not $userId -or -not $upn) { continue } + + $userCount++ + $licenses = @() + $hasCopilot = $false + + # Friendly SKU name list (unchanged): map assignedLicenses[].skuId -> skuPartNumber via $skuLookup + foreach ($license in $user.assignedLicenses) { + $skuId = $license.skuId + $skuName = if ($skuLookup.ContainsKey($skuId)) { $skuLookup[$skuId] } else { $skuId } + $licenses += $skuName + } + + # Capability-aware Copilot detection: any assignedPlan that is Enabled AND a Copilot service plan. + if ($copilotPlanIds.Count -gt 0 -and $user.assignedPlans) { + foreach ($plan in $user.assignedPlans) { + if ($plan.capabilityStatus -eq 'Enabled' -and $plan.servicePlanId -and $copilotPlanIds.Contains([string]$plan.servicePlanId)) { + $hasCopilot = $true + break + } + } + } + + # Store license data keyed by both ID and UPN for flexible lookup + if ($licenses.Count -gt 0) { + $userLicenses[$userId] = $licenses + $userLicenses[$upn] = $licenses + } + + $userHasCopilot[$userId] = $hasCopilot + $userHasCopilot[$upn] = $hasCopilot + + if ($hasCopilot) { + $copilotUserCount++ + } + } + + $userUri = $userResponse.'@odata.nextLink' + } while ($userUri) + + Write-LogHost " Processed $userCount user(s) with license assignments" -ForegroundColor Gray + Write-LogHost " Detected $copilotUserCount user(s) with Copilot licenses" -ForegroundColor Green + Write-LogHost "" + + return @{ + UserLicenses = $userLicenses + UserHasCopilot = $userHasCopilot + } + } + catch { + Write-LogHost "WARNING: Failed to fetch license data: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " License columns will be empty in export" -ForegroundColor Yellow + Write-LogHost "" + + # Return empty hashtables on failure + return @{ + UserLicenses = @{} + UserHasCopilot = @{} + } + } +} + +function ConvertTo-FlatEntraUsers { + <# + .SYNOPSIS + Flattens Entra user objects into CSV-friendly format. + + .DESCRIPTION + Converts Entra ID user objects with nested properties into flat tabular format. + Filters out non-user accounts (rooms, resources) based on userType validation. + Explodes arrays (proxyAddresses, manager) into individual columns. + + NOTE: This version excludes the 9 granular Entra license columns from the Graph script. + License data is added separately via Get-UserLicenseData() in MAC format (assignedLicenses, hasLicense). + + .PARAMETER Users + Array of user objects from Microsoft Graph API (with 35 properties + manager expansion). + + .OUTPUTS + Array of PSCustomObjects with 37 flattened columns (35 user properties + 5 manager columns, no license columns yet). + #> + param( + [Parameter(Mandatory = $true)] + [array]$Users + ) + + $flattenedUsers = @() + + foreach ($user in $Users) { + # Filter: Only include real user accounts (exclude rooms, resources, shared mailboxes) + # Room and resource mailboxes have specific characteristics: + # - userType is often null or not "Member"/"Guest" + # - They typically lack givenName and surname + # - They often have mail but no userPrincipalName with typical user format + + $userTypeValue = $user.userType + + # Skip if userType is null/empty (likely a room or resource) + if ([string]::IsNullOrWhiteSpace($userTypeValue)) { + continue + } + + # Only include users with userType = "Member" or "Guest" + # Rooms/resources typically have different userType values or null + if ($userTypeValue -ne 'Member' -and $userTypeValue -ne 'Guest') { + continue + } + + # Additional heuristic: Real users typically have either givenName or surname + # Room mailboxes typically have neither (only displayName) + # This is not foolproof but combined with userType check, it's quite reliable + $hasGivenName = -not [string]::IsNullOrWhiteSpace($user.givenName) + $hasSurname = -not [string]::IsNullOrWhiteSpace($user.surname) + + # If user has Member/Guest type but no name components, might be a shared resource + # Allow through if they have at least givenName OR surname OR if account is enabled + # (most room mailboxes are enabled but lack name components) + if (-not $hasGivenName -and -not $hasSurname -and $user.accountEnabled) { + # Additional check: if they have licenses assigned, likely a real user + if (-not $user.assignedLicenses -or $user.assignedLicenses.Count -eq 0) { + # No licenses and no name components - likely a room/resource + continue + } + } + + $flatUser = [ordered]@{} + + # Core Identity Properties (simple strings) + $flatUser['userPrincipalName'] = $user.userPrincipalName + $flatUser['displayName'] = $user.displayName + $flatUser['id'] = $user.id + $flatUser['mail'] = $user.mail + $flatUser['givenName'] = $user.givenName + $flatUser['surname'] = $user.surname + + # Job Properties + $flatUser['jobTitle'] = $user.jobTitle + $flatUser['department'] = $user.department + $flatUser['employeeType'] = $user.employeeType + $flatUser['employeeId'] = $user.employeeId + $flatUser['employeeHireDate'] = $user.employeeHireDate + + # Location Properties + $flatUser['officeLocation'] = $user.officeLocation + $flatUser['city'] = $user.city + $flatUser['state'] = $user.state + $flatUser['country'] = $user.country + $flatUser['postalCode'] = $user.postalCode + $flatUser['companyName'] = $user.companyName + + # Organizational Properties + $flatUser['employeeOrgData_division'] = if ($user.employeeOrgData) { $user.employeeOrgData.division } else { $null } + $flatUser['employeeOrgData_costCenter'] = if ($user.employeeOrgData) { $user.employeeOrgData.costCenter } else { $null } + + # Status Properties + $flatUser['accountEnabled'] = $user.accountEnabled + $flatUser['userType'] = $user.userType + $flatUser['createdDateTime'] = $user.createdDateTime + + # Usage Properties + $flatUser['usageLocation'] = $user.usageLocation + $flatUser['preferredLanguage'] = $user.preferredLanguage + + # Sync Properties + $flatUser['onPremisesSyncEnabled'] = $user.onPremisesSyncEnabled + $flatUser['onPremisesImmutableId'] = $user.onPremisesImmutableId + $flatUser['externalUserState'] = $user.externalUserState + + # Explode proxyAddresses array (Email aliases) + if ($user.proxyAddresses -and $user.proxyAddresses.Count -gt 0) { + $primarySMTP = $user.proxyAddresses | Where-Object { $_ -like 'SMTP:*' } | Select-Object -First 1 + $flatUser['proxyAddresses_Primary'] = if ($primarySMTP) { $primarySMTP -replace '^SMTP:', '' } else { $null } + $flatUser['proxyAddresses_Count'] = $user.proxyAddresses.Count + $flatUser['proxyAddresses_All'] = ($user.proxyAddresses -join '; ') + } + else { + $flatUser['proxyAddresses_Primary'] = $null + $flatUser['proxyAddresses_Count'] = 0 + $flatUser['proxyAddresses_All'] = $null + } + + # Handle manager object separately (flatten to individual columns) + if ($user.manager) { + $flatUser['manager_id'] = $user.manager.id + $flatUser['manager_displayName'] = $user.manager.displayName + $flatUser['manager_userPrincipalName'] = $user.manager.userPrincipalName + $flatUser['manager_mail'] = $user.manager.mail + $flatUser['manager_jobTitle'] = $user.manager.jobTitle + } + else { + $flatUser['manager_id'] = $null + $flatUser['manager_displayName'] = $null + $flatUser['manager_userPrincipalName'] = $null + $flatUser['manager_mail'] = $null + $flatUser['manager_jobTitle'] = $null + } + + # License columns will be added separately by Get-EntraUsersData() + # using Get-UserLicenseData() to provide MAC-format columns: + # - assignedLicenses (semicolon-separated SKU names) + # - hasLicense (Copilot detection boolean) + + # ===================================================================== + # Power BI AI-in-One Dashboard 2701 Template Compatibility Columns + # These alias columns map existing Graph API data to 2701 template column names + # ===================================================================== + $flatUser['ManagerID'] = $flatUser['manager_id'] + $flatUser['BusinessAreaLabel'] = $flatUser['employeeOrgData_division'] + $flatUser['CountryofEmployment'] = $flatUser['country'] + $flatUser['CompanyCodeLabel'] = $flatUser['companyName'] + $flatUser['CostCentreLabel'] = $flatUser['employeeOrgData_costCenter'] + $flatUser['UserName'] = $flatUser['displayName'] + + # Viva Insights-specific columns (not available from Microsoft Graph API) + # These are placeholders for template compatibility - data must come from HR systems + $flatUser['EffectiveDate'] = $null + $flatUser['FunctionType'] = $null + $flatUser['BusinessAreaCode'] = $null + $flatUser['OrgLevel_3Label'] = $null + + # Convert ordered hashtable to PSCustomObject for proper CSV export + $flattenedUsers += [PSCustomObject]$flatUser + } + + return $flattenedUsers +} + +function Get-EntraUsersData { + <#! + .SYNOPSIS + Collects and flattens Entra ID (Azure AD) user directory data and enriches with MAC-format license info. + + .DESCRIPTION + # New naming: Purview_Audit_CombinedUsageActivity[_EntraUsers]_timestamp.xlsx + $baseName = "Purview_Audit_CombinedUsageActivity" + if ($IncludeUserInfo -and -not $UseEOM) { $baseName += "_EntraUsers" } + $excelDescriptor = if ($IncludeUserInfo -and -not $UseEOM) { 'multi-tab workbook (CombinedActivity + EntraUsers)' } else { 'single-tab workbook' } + Write-LogHost "Output File: ${outputDir}${baseName}_.xlsx ($excelDescriptor)" -ForegroundColor White + Filters out non-user principals (rooms/resources) using userType + name heuristics identical to ConvertTo-FlatEntraUsers. + Flattens users via ConvertTo-FlatEntraUsers, then appends two MAC-aligned columns: + • assignedLicenses (semicolon-separated SKU `skuPartNumber` names) + • hasLicense (Copilot license boolean; renamed from UserHasCopilotLicense) + + License enrichment uses existing Get-UserLicenseData() hashtables (UserLicenses, UserHasCopilot). + Only called when -IncludeUserInfo is specified (Graph API mode). + + .OUTPUTS + Array[psobject] of flattened users (30 core columns + 5 manager columns + 2 license columns = 37 total). + + .NOTES + If Graph API call fails, returns empty array with warning. License enrichment silently skips if lookup missing. + #> + param( + [switch]$Quiet + ) + + $entraUsers = @() + try { + if (-not $Quiet) { Write-LogHost "Fetching Entra user directory (35 properties + manager)..." -ForegroundColor Cyan } + + # Properties mirrored from Graph script (excluding license arrays we purposefully omit) + $entraUserSelect = @( + 'userPrincipalName','displayName','id','mail','givenName','surname','jobTitle','department','employeeType','employeeId','employeeHireDate', + 'officeLocation','city','state','country','postalCode','companyName','accountEnabled','userType','createdDateTime','usageLocation', + 'preferredLanguage','onPremisesSyncEnabled','onPremisesImmutableId','externalUserState','employeeOrgData','proxyAddresses' + ) -join ',' + + $baseUri = "https://graph.microsoft.com/v1.0/users?`$select=$entraUserSelect&`$expand=manager&`$top=999" + $nextLink = $baseUri + # Generic List accumulator (was $rawUsers = @() with +=). On a mega-tenant + # (~400K+ users) the array-append pattern is O(n^2) — every += reallocates and + # copies the whole backing array — which is both slow and memory-heavy. A + # List[object] with AddRange appends amortized O(1). + $rawUsers = [System.Collections.Generic.List[object]]::new() + $loops = 0 + # Scale-aware paging ceiling. With $expand=manager, Graph caps each page far + # below the requested $top=999 (often ~100 rows/page), so a mega-tenant needs + # thousands of pages: at the ~100/page floor, 8000 pages covers ~800K users. + # The old fixed >2000 guard tripped mid-fetch on a ~412K-user tenant. This + # ceiling still catches a true runaway (e.g. a non-advancing @odata.nextLink). + $maxPages = 8000 + # Inner try/catch around paging ONLY: on a throttling/network error or the + # safety-abort throw, keep whatever pages were already collected and fall + # through to flatten/enrich so a PARTIAL directory is still emitted. The + # shortfall is flagged so the end-of-run summary and exit code report it + # instead of masking it behind "Enterprise Export Complete". + try { + while ($nextLink) { + $loops++ + $resp = Invoke-GraphRequest -Uri $nextLink -Method GET -ErrorAction Stop + if ($resp.value) { $rawUsers.AddRange([object[]]$resp.value) } + $nextLink = $resp.'@odata.nextLink' + # Progress heartbeat: large tenants page for several minutes; emit a count + # every 25 pages so the run is not mistaken for a hang. + if (-not $Quiet -and ($loops % 25 -eq 0)) { + Write-LogHost " ... $($rawUsers.Count) users retrieved so far ($loops pages)" -ForegroundColor DarkGray + } + if ($loops -gt $maxPages) { throw "Safety abort: excessive paging (>$maxPages pages)" } + } + } + catch { + $script:EntraUsersFetchFailed = $true + $script:EntraUsersFetchError = $_.Exception.Message + Write-LogHost "WARNING: Entra user directory paging did not complete: $($_.Exception.Message)" -ForegroundColor Yellow + } + + # If paging collected nothing there is no directory to emit: flag (if not + # already) and return empty so the writers skip and the summary/exit report it. + if ($rawUsers.Count -eq 0) { + if (-not $script:EntraUsersFetchFailed) { + $script:EntraUsersFetchFailed = $true + $script:EntraUsersFetchError = 'Graph returned no users for the directory query.' + } + return $entraUsers + } + # Partial directory: some pages succeeded before a paging failure. Emit what we + # have (schema-complete via the flatten/enrich below) but mark it partial. + if ($script:EntraUsersFetchFailed) { + $script:EntraUsersPartial = $true + if (-not $Quiet) { Write-LogHost " Emitting PARTIAL directory: $($rawUsers.Count) users collected before the failure." -ForegroundColor Yellow } + } + + if (-not $Quiet) { Write-LogHost " Retrieved $($rawUsers.Count) raw user objects" -ForegroundColor Gray } + $flattened = ConvertTo-FlatEntraUsers -Users $rawUsers + if (-not $Quiet) { Write-LogHost " Flattened to $($flattened.Count) user rows (filtered)" -ForegroundColor Gray } + + # License enrichment (MAC-format columns) + $licenseData = $script:LicenseData + foreach ($u in $flattened) { + $upn = $u.userPrincipalName + $assignedNames = $null + $hasCopilot = $false + if ($licenseData) { + # lookup by UPN then id for flexibility + if ($licenseData.UserLicenses.ContainsKey($upn)) { + $assignedNames = ($licenseData.UserLicenses[$upn] -join ';') + } elseif ($licenseData.UserLicenses.ContainsKey($u.id)) { + $assignedNames = ($licenseData.UserLicenses[$u.id] -join ';') + } + if ($licenseData.UserHasCopilot.ContainsKey($upn)) { + $hasCopilot = [bool]$licenseData.UserHasCopilot[$upn] + } elseif ($licenseData.UserHasCopilot.ContainsKey($u.id)) { + $hasCopilot = [bool]$licenseData.UserHasCopilot[$u.id] + } + } + Add-Member -InputObject $u -NotePropertyName 'assignedLicenses' -NotePropertyValue $assignedNames -Force + Add-Member -InputObject $u -NotePropertyName 'hasLicense' -NotePropertyValue $hasCopilot -Force + } + $entraUsers = $flattened + # Validate schema (non-fatal) + try { Test-EntraUsersSchema -Users $entraUsers -Quiet:$Quiet } catch { } + } + catch { + # Flatten/enrich (or any non-paging) failure. Flag it so the summary + exit + # code report the shortfall; preserve an inner paging-error reason if one was + # already captured. + $script:EntraUsersFetchFailed = $true + if (-not $script:EntraUsersFetchError) { $script:EntraUsersFetchError = $_.Exception.Message } + Write-LogHost "WARNING: Failed to collect Entra user directory: $($_.Exception.Message)" -ForegroundColor Yellow + } + return $entraUsers +} + +# ============================================== +# GRAPH API QUERY FUNCTIONS +# ============================================== +# REST-based audit log query functions for Microsoft Graph Security API + +function Test-Is429 { + <# + .SYNOPSIS + Safely detects 429 (Too Many Requests) throttling errors. + + .DESCRIPTION + Provides null-safe detection of 429 throttling responses from Graph API. + Handles PowerShell 7+ variations where .Response property may be null. + + Three-layer fallback strategy: + 1. Check .Response.StatusCode (when Response object exists) + 2. Check .Exception.Response.StatusCode directly (PS7+ pattern) + 3. Parse error message for '429' string (final fallback) + + .PARAMETER Exception + The caught exception object from try/catch block + + .OUTPUTS + $true if 429 throttling detected, $false otherwise + + .EXAMPLE + try { + Invoke-RestMethod -Uri $uri -Headers $headers + } + catch { + if (Test-Is429 -Exception $_) { + Start-Sleep -Seconds 60 + } + } + #> + + param( + [Parameter(Mandatory = $true)] + [System.Management.Automation.ErrorRecord]$Exception + ) + + # Layer 1: Check .Response.StatusCode (traditional method) + if ($Exception.Exception.Response -and $Exception.Exception.Response.StatusCode) { + if ($Exception.Exception.Response.StatusCode -eq 429 -or $Exception.Exception.Response.StatusCode -eq 'TooManyRequests') { + return $true + } + } + + # Layer 2: Check .Exception.Response.StatusCode directly (PS7+ sometimes skips wrapper) + if ($Exception.Exception.Response.StatusCode) { + if ($Exception.Exception.Response.StatusCode.value__ -eq 429) { + return $true + } + } + + # Layer 3: Parse error message as final fallback + $errorMessage = $Exception.Exception.Message + if ($errorMessage -match '429' -or $errorMessage -match 'Too Many Requests' -or $errorMessage -match 'TooManyRequests') { + return $true + } + + return $false +} + +function Invoke-GraphAuditQuery { + <# + .SYNOPSIS + Creates a new audit log query in Microsoft Graph Security API. + + .DESCRIPTION + Submits an audit log query request to Microsoft Graph Security API. + Returns a query ID that can be used to poll for status and retrieve results. + + The Graph API uses an asynchronous query model: + 1. Submit query (this function) - returns queryId + 2. Poll query status - wait for "succeeded" state + 3. Retrieve records - paginated results + + .PARAMETER DisplayName + Friendly name for the query (for tracking purposes) + + .PARAMETER StartDate + Start date/time for audit log query (ISO 8601 format) + + .PARAMETER EndDate + End date/time for audit log query (ISO 8601 format) + + .PARAMETER Operations + Array of operation types to query (e.g., 'CopilotInteraction') + + .PARAMETER RecordTypes + Optional record type filters to include in the Graph query body (passthrough). + + .PARAMETER ServiceTypes + Optional service/workload filters to include in the Graph query body (passthrough). + + .OUTPUTS + Query ID string if successful, $null if failed + #> + + param( + [Parameter(Mandatory = $true)] + [string]$DisplayName, + + [Parameter(Mandatory = $true)] + [Alias('FilterStartDateTime')] + [datetime]$StartDate, + + [Parameter(Mandatory = $true)] + [Alias('FilterEndDateTime')] + [datetime]$EndDate, + + [Parameter(Mandatory = $false)] + [Alias('OperationFilters')] + [string[]]$Operations, + + [Parameter(Mandatory = $false)] + [Alias('RecordTypeFilters')] + [string[]]$RecordTypes, + + [Parameter(Mandatory = $false)] + [Alias('ServiceFilter')] + [string[]]$ServiceTypes + ) + + try { + # Format dates to ISO 8601 format required by Graph API + $startDateStr = $StartDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + $endDateStr = $EndDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + + # Build request body + $body = @{ + displayName = $DisplayName + filterStartDateTime = $startDateStr + filterEndDateTime = $endDateStr + } + + # Fail-safe sanitizer: If operations include M365 usage ops, drop record/service filters + try { + $usageOps = $script:m365UsageActivityBundle + if (-not $usageOps) { $usageOps = $m365UsageActivityBundle } + $hasUsageOps = $false + if ($Operations -and $usageOps) { + $opsLower = @($Operations | ForEach-Object { $_.ToLowerInvariant() }) + $usageLower = @($usageOps | ForEach-Object { $_.ToLowerInvariant() }) + $hasUsageOps = ($opsLower | Where-Object { $usageLower -contains $_ }) | Select-Object -First 1 + } + if ($hasUsageOps) { + $RecordTypes = $null + $ServiceTypes = $null + } + } catch { } + + # Add operation filters if specified + if ($Operations -and $Operations.Count -gt 0) { + $body.operationFilters = @($Operations) + } + + # Add optional record/service filters (passthrough from caller) + if ($RecordTypes -and $RecordTypes.Count -gt 0) { + $body.recordTypeFilters = @($RecordTypes) + } + + if ($ServiceTypes -and $ServiceTypes.Count -gt 0) { + $body.serviceFilter = $ServiceTypes[0] + } + + # Log query details for troubleshooting (persisted to run log) + Write-LogHost "[INFO] Graph API Query Body:" -ForegroundColor Magenta + if ($Operations -and $Operations.Count -gt 0) { + Write-LogHost " operationFilters: $($Operations -join ', ')" -ForegroundColor DarkGray + } + if ($RecordTypes -and $RecordTypes.Count -gt 0) { + Write-LogHost " recordTypeFilters: $($RecordTypes -join ', ')" -ForegroundColor DarkGray + } + if ($ServiceTypes -and $ServiceTypes.Count -gt 0) { + Write-LogHost " serviceFilter: $($ServiceTypes[0])" -ForegroundColor DarkGray + } + $bodyJson = $body | ConvertTo-Json -Depth 10 + Write-LogHost $bodyJson -ForegroundColor DarkGray + + # Submit query via Graph API (auto-detects v1.0 or beta) + $uri = Get-GraphAuditApiUri -Path 'queries' + $response = Invoke-MgGraphRequest -Method POST -Uri $uri -Body $body -ErrorAction Stop + + if ($response -and $response.id) { + return $response.id + } + else { + Write-LogHost "WARNING: Graph API query submitted but no ID returned" -ForegroundColor Yellow + return $null + } + } + catch { + Write-LogHost "ERROR: Failed to submit Graph audit query: $($_.Exception.Message)" -ForegroundColor Red + try { + if ($_.Exception.Response) { + $respStream = $_.Exception.Response.GetResponseStream() + if ($respStream) { + $reader = New-Object System.IO.StreamReader($respStream) + $body = $reader.ReadToEnd() + $reader.Dispose() + if ($body) { Write-LogHost "GRAPH response body: $body" -ForegroundColor DarkGray } + } + } + } catch {} + return $null + } +} + +function Get-GraphAuditQueryStatus { + <# + .SYNOPSIS + Checks the status of a Graph API audit log query. + + .DESCRIPTION + Polls the Microsoft Graph Security API to check query execution status. + + Possible status values: + - notStarted: Query submitted but not yet processing + - queued: Query waiting in backend queue for available execution slot + - running: Query is executing + - succeeded: Query completed successfully, records ready + - failed: Query failed + - cancelled: Query was cancelled + + .PARAMETER QueryId + The query ID returned by Invoke-GraphAuditQuery + + .OUTPUTS + Hashtable with status information: @{ Status='succeeded'; RecordCount=1234 } + Returns $null if query check fails + #> + + param( + [Parameter(Mandatory = $true)] + [string]$QueryId + ) + + try { + $uri = Get-GraphAuditApiUri -Path "queries/$QueryId" + $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + + $result = @{ + QueryId = $QueryId + Status = $response.status + RecordCount = 0 + } + + # Some status responses include record count + if ($response.PSObject.Properties.Name -contains 'recordCount') { + $result.RecordCount = $response.recordCount + } + + return $result + } + catch { + Write-LogHost "ERROR: Failed to get Graph query status: $($_.Exception.Message)" -ForegroundColor Red + return $null + } +} + +function Get-GraphAuditRecords { + <# + .SYNOPSIS + Retrieves audit log records from a completed Graph API query. + + .DESCRIPTION + Fetches audit log records from Microsoft Graph Security API for a completed query. + Handles pagination automatically using @odata.nextLink. + + Only call this function after confirming query status is "succeeded". + + .PARAMETER QueryId + The query ID returned by Invoke-GraphAuditQuery + + .PARAMETER MaxRecords + Maximum number of records to retrieve (default: unlimited) + + .OUTPUTS + Array of audit log record objects, or empty array if none found + #> + + param( + [Parameter(Mandatory = $true)] + [string]$QueryId, + + [Parameter(Mandatory = $false)] + [int]$MaxRecords = 0 + ) + + try { + $allRecords = @() + $uri = Get-GraphAuditApiUri -Path "queries/$QueryId/records" + + do { + $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + + if ($response -and $response.value) { + $allRecords += $response.value + + # Check if we've hit the max records limit + if ($MaxRecords -gt 0 -and $allRecords.Count -ge $MaxRecords) { + $allRecords = $allRecords | Select-Object -First $MaxRecords + break + } + } + + # Check for pagination + $uri = $response.'@odata.nextLink' + + } while ($uri) + + return $allRecords + } + catch { + Write-LogHost "ERROR: Failed to retrieve Graph audit records: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +# ============================================== +# DATA NORMALIZATION FUNCTION +# ============================================== +# Converts Graph API audit records to EOM-compatible schema + +function ConvertFrom-GraphAuditRecord { + <# + .SYNOPSIS + Normalizes Graph API audit records to match EOM cmdlet output schema. + + .DESCRIPTION + Transforms Microsoft Graph Security API audit log records into the same + structure as Search-UnifiedAuditLog cmdlet output. This ensures the + existing explosion logic works identically regardless of data source. + + Graph API Schema → EOM Schema Mapping: + - auditLogRecordType → RecordType + - operation → Operations + - createdDateTime → CreationDate + - auditData → AuditData (JSON string) + - userPrincipalName → UserIds + - id → Identity (unique record identifier) + + .PARAMETER GraphRecords + Array of audit log records from Graph API (Get-GraphAuditRecords output) + + .OUTPUTS + Array of normalized records matching EOM schema structure + #> + + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [array]$GraphRecords + ) + + if (-not $GraphRecords -or $GraphRecords.Count -eq 0) { + return @() + } + + $normalized = @() + + foreach ($record in $GraphRecords) { + try { + # Create EOM-compatible object structure + $eomRecord = [PSCustomObject]@{ + RecordType = $null + CreationDate = $null + UserIds = $null + Operations = $null + AuditData = '{}' + } + # Map: auditLogRecordType → RecordType + if ($record.PSObject.Properties.Name -contains 'auditLogRecordType') { + $eomRecord.RecordType = $record.auditLogRecordType + } + + # Map: createdDateTime → CreationDate + if ($record.PSObject.Properties.Name -contains 'createdDateTime') { + try { + $eomRecord.CreationDate = script:Parse-DateSafe $record.createdDateTime + } + catch { + $eomRecord.CreationDate = $record.createdDateTime + } + } + + # Map: userPrincipalName → UserIds + if ($record.PSObject.Properties.Name -contains 'userPrincipalName') { + $eomRecord.UserIds = $record.userPrincipalName + } + + # Map: operation → Operations + if ($record.PSObject.Properties.Name -contains 'operation') { + $eomRecord.Operations = $record.operation + } + + # Map: id → Identity (unique identifier) + if ($record.PSObject.Properties.Name -contains 'id') { + $eomRecord.Identity = $record.id + } + + # Map: auditData → AuditData (must be JSON string for explosion logic) + # PERF: Also store _ParsedAuditData to avoid re-parsing during explosion + if ($record.PSObject.Properties.Name -contains 'auditData') { + $auditDataObj = $record.auditData + + # Store the already-parsed object for explosion optimization + $eomRecord | Add-Member -NotePropertyName '_ParsedAuditData' -NotePropertyValue $auditDataObj -Force + + # If auditData is already an object, convert to JSON string + if ($auditDataObj -is [string]) { + $eomRecord.AuditData = $auditDataObj + # String means it wasn't pre-parsed, clear _ParsedAuditData + $eomRecord._ParsedAuditData = $null + } + else { + # Convert object to JSON string (explosion logic expects string) + try { + $eomRecord.AuditData = ($auditDataObj | ConvertTo-Json -Depth 100 -Compress) + } + catch { + Write-LogHost "WARNING: Failed to serialize auditData for record $($eomRecord.Identity)" -ForegroundColor Yellow + $eomRecord.AuditData = '{}' + $eomRecord._ParsedAuditData = $null + } + } + } + else { + # No auditData present - create minimal valid JSON + $eomRecord.AuditData = '{}' + } + + $normalized += $eomRecord + } + catch { + Write-LogHost "WARNING: Failed to normalize Graph record: $($_.Exception.Message)" -ForegroundColor Yellow + # Continue processing remaining records + } + } + + return $normalized +} + +# ============================================================================= +# MICROSOFT AGENT 365 ENRICHMENT FUNCTIONS +# ============================================================================= +# Self-contained section. All functions are no-ops unless -IncludeAgent365Info +# or -OnlyAgent365Info is set. Reuses existing Graph SDK plumbing +# (Invoke-MgGraphRequest, Connect-MgGraph, token-refresh helpers, retry helpers). +# Output schema must exactly match temp/Synthetic_Agent365_May2026.csv (28 cols). +# Empty cells are intentional when the API does not return a value (no fabrication). +# Reference: https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api +# ============================================================================= + +# Module-scope caches populated during the Agent 365 phase. +$script:Agent365FrontierAvailable = $null # $true / $false / $null = untested +$script:Agent365DeveloperCache = @{} # appId/displayName -> resolved name +$script:Agent365AuditEnrichment = @{} # titleId/appId -> @{ Created=...; CreatedBy=... } +$script:Agent365InteractiveCtx = $false # set $true after secondary interactive sign-in (AppReg path) +$script:Agent365PreAuthCompleted = $false # set $true after eager up-front Agent 365 sign-in (AppReg + Agent365 combo) + +function Get-Agent365PackagesUri { + param([Parameter(Mandatory = $false)][string]$PackageId = '') + # Microsoft Agent 365 / Copilot package management API is currently published at /beta only. + # When/if the endpoint is also published at /v1.0, switch this constant. + $base = 'https://graph.microsoft.com/beta/copilot/admin/catalog/packages' + if ([string]::IsNullOrWhiteSpace($PackageId)) { return $base } + return ($base + '/' + [System.Uri]::EscapeDataString($PackageId)) +} + +function Connect-Agent365InteractiveContext { + <# + .SYNOPSIS + Establishes the Microsoft Graph context for the Agent 365 phase. + + Delegated auth modes (WebLogin / DeviceCode / Credential / Silent) already carry the + Agent 365 scopes consented at their initial interactive sign-in, so this is a no-op + for them (early return $true). + + App-only auth modes (AppRegistration certificate, AppRegistration client secret, + ManagedIdentity) reuse the EXISTING application Graph context. The app-only token must + already carry the APPLICATION app-role CopilotPackages.Read.All (plus Application.Read.All + for developer-name resolution via /applications), granted and admin-consented on the app + registration / managed-identity service principal. NO interactive sign-in and NO + Connect-MgGraph -Scopes call is performed - the catalog GET runs directly on the active + app-only context. Idempotent: safe to call multiple times. + #> + # Delegated modes: their initial sign-in already consented the Agent 365 scopes. + if ($Auth -ne 'AppRegistration' -and $Auth -ne 'ManagedIdentity') { return $true } + + # App-only modes: surface an APP-ONLY Phase 2 banner (no delegated user, no prompt) so the + # log is honest about the auth surface, then reuse the existing application context as-is. + $appOnlyLabel = if ($Auth -eq 'ManagedIdentity') { 'managed identity service principal' } else { 'app registration service principal' } + Write-LogHost "" + Write-LogHost "=== Phase 2: Agent 365 - APP-ONLY context (no interactive sign-in) ===" -ForegroundColor Cyan + Write-LogHost (" Reusing the existing app-only Graph context ({0})." -f $appOnlyLabel) -ForegroundColor Gray + Write-LogHost " Required APPLICATION permissions (granted + admin-consented on the SP):" -ForegroundColor White + Write-LogHost " [App-only] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor Yellow + Write-LogHost " [App-only] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor Yellow + Write-LogHost " Note: the catalog API is published at /beta only; the tenant must hold a" -ForegroundColor Gray + Write-LogHost " Microsoft Agent 365 license / program enrollment." -ForegroundColor Gray + Write-LogHost "" + return $true +} + + +function Invoke-Agent365EarlyInteractiveSignIn { + <# + .SYNOPSIS + No-op retained for call-site stability. + + .DESCRIPTION + Historically this performed an eager up-front interactive DELEGATED sign-in for the + Agent 365 phase under -Auth AppRegistration, because the catalog endpoint was assumed to + have no app-only Graph scope. The Agent Package Management API is read with + the APPLICATION app-role CopilotPackages.Read.All (+ Application.Read.All) on the app / + managed-identity service principal, so NO interactive sign-in is required in ANY auth mode: + - app-only modes (AppRegistration cert/secret, ManagedIdentity) reuse the existing + application context (see Connect-Agent365InteractiveContext); + - delegated modes (WebLogin / DeviceCode / Credential / Silent) already consented the + Agent 365 scopes at their initial sign-in. + This function is intentionally a no-op so the startup wiring that calls it stays stable. + + .OUTPUTS + $true always (no early sign-in is ever needed). + #> + return $true +} + +function Test-Agent365FrontierAccess { + <# + .SYNOPSIS + Probes the Agent Package Management API with a minimal GET to confirm + tenant enrollment in the Frontier program and adequate caller role. + .OUTPUTS + $true - access confirmed; agent phase may proceed + $false - access denied (403/404 or unsupported); banner 7d printed; phase skipped + #> + if ($null -ne $script:Agent365FrontierAvailable) { return $script:Agent365FrontierAvailable } + + $uri = (Get-Agent365PackagesUri) + '?$top=1' + try { + $null = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + $script:Agent365FrontierAvailable = $true + return $true + } catch { + $status = $null + try { if ($_.Exception.Response) { $status = [int]$_.Exception.Response.StatusCode } } catch {} + if ($status -in @(401,403,404)) { + # Message branches by auth surface: an app-only 403 means the APP/MI service + # principal lacks the app-role (or the tenant is unlicensed/not-enrolled), NOT that + # a signed-in user needs a directory role. Delegated text is kept byte-identical. + $isAppOnlyMode = ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') + Write-LogHost "" + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost "| Microsoft Agent 365 - catalog unavailable for this tenant |" -ForegroundColor Yellow + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost ("| The Agent Package Management API returned HTTP {0,-3}, indicating |" -f $status) -ForegroundColor Yellow + if ($isAppOnlyMode) { + Write-LogHost "| this tenant is not enrolled in the Microsoft Agent 365 program (or |" -ForegroundColor Yellow + Write-LogHost "| lacks a Microsoft Agent 365 license), OR the app registration / |" -ForegroundColor Yellow + Write-LogHost "| managed-identity service principal has NOT been granted the |" -ForegroundColor Yellow + Write-LogHost "| CopilotPackages.Read.All APPLICATION permission (admin-consented). |" -ForegroundColor Yellow + Write-LogHost "| App-only auth needs the app-role, not a user directory role. The |" -ForegroundColor Yellow + Write-LogHost "| Agent 365 CSV will be skipped for this run. |" -ForegroundColor Yellow + } else { + Write-LogHost "| this tenant is not enrolled in the Microsoft Agent 365 program (or |" -ForegroundColor Yellow + Write-LogHost "| lacks a Microsoft Agent 365 license), or the signed-in user lacks |" -ForegroundColor Yellow + Write-LogHost "| the AI Administrator / Global Administrator role. The Agent 365 CSV |" -ForegroundColor Yellow + Write-LogHost "| will be skipped for this run. |" -ForegroundColor Yellow + } + Write-LogHost "| |" -ForegroundColor Yellow + Write-LogHost "| Reference: |" -ForegroundColor Yellow + Write-LogHost "| https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api|" -ForegroundColor Yellow + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost "" + } else { + Write-LogHost (" Agent 365 probe failed (HTTP {0}): {1}" -f ($status, $_.Exception.Message)) -ForegroundColor Red + } + $script:Agent365FrontierAvailable = $false + return $false + } +} + +function Get-Agent365Packages { + <# + .SYNOPSIS + Returns all Agent 365 catalog packages (list view) with paging. + #> + $results = New-Object System.Collections.Generic.List[object] + $uri = Get-Agent365PackagesUri + $pageNum = 0 + while ($uri) { + $pageNum++ + try { + Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue + } catch {} + try { + $resp = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + } catch { + Write-LogHost (" WARNING: Agent 365 list page {0} failed: {1}" -f $pageNum, $_.Exception.Message) -ForegroundColor Yellow + break + } + if ($resp -and $resp.value) { + foreach ($p in $resp.value) { [void]$results.Add($p) } + } + $uri = $resp.'@odata.nextLink' + if ($pageNum -gt 500) { + Write-LogHost " WARNING: Agent 365 paging safety abort (>500 pages)" -ForegroundColor Yellow + break + } + } + return $results.ToArray() +} + +function Invoke-Agent365GraphWithRetry { + <# + .SYNOPSIS + Throttle-aware Graph GET for the Agent 365 catalog read path. Retries on HTTP 429 + and 5xx with exponential backoff (up to 5 attempts, min(60, 2^attempt) seconds), + honoring a Retry-After header when present. Non-throttle / non-5xx errors rethrow + immediately so callers keep their existing skip-with-warning behavior. Mirrors the + backoff convention used by the chunked remote-upload path. + #> + param([Parameter(Mandatory = $true)][string]$Uri) + $attempt = 0 + while ($true) { + try { + return Invoke-MgGraphRequest -Method GET -Uri $Uri -ErrorAction Stop + } catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 0) { if ($_.Exception.Message -match '429|Too Many Requests|TooManyRequests') { $status = 429 } } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } +} + +function Get-Agent365PackageDetail { + <# + .SYNOPSIS + Returns the full detail object for a single agent package, including elementDetails. + #> + param([Parameter(Mandatory = $true)][string]$PackageId) + $uri = Get-Agent365PackagesUri -PackageId $PackageId + try { + Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue + } catch {} + try { + return Invoke-Agent365GraphWithRetry -Uri $uri + } catch { + Write-LogHost (" WARNING: Agent 365 detail fetch failed for '{0}': {1}" -f $PackageId, $_.Exception.Message) -ForegroundColor Yellow + return $null + } +} + +function Resolve-Agent365DeveloperName { + <# + .SYNOPSIS + Resolves a developer/publisher name. Tries package's developer.name first, + falls back to /applications + /owners lookup keyed on appId. Cached. + #> + param( + [Parameter(Mandatory = $false)][string]$DeveloperName, + [Parameter(Mandatory = $false)][string]$AppId + ) + if ($DeveloperName) { return $DeveloperName } + if (-not $AppId) { return '' } + if ($script:Agent365DeveloperCache.ContainsKey($AppId)) { + return $script:Agent365DeveloperCache[$AppId] + } + $resolved = '' + try { + $appUri = "https://graph.microsoft.com/v1.0/applications?`$filter=appId eq '$AppId'&`$select=id,displayName,publisherDomain" + $appResp = Invoke-Agent365GraphWithRetry -Uri $appUri + if ($appResp -and $appResp.value -and $appResp.value.Count -gt 0) { + $app = $appResp.value[0] + if ($app.publisherDomain) { $resolved = $app.publisherDomain } + elseif ($app.displayName) { $resolved = $app.displayName } + # Optional owner lookup (first owner UPN) only if still blank + if (-not $resolved -and $app.id) { + try { + $ownerUri = "https://graph.microsoft.com/v1.0/applications/$($app.id)/owners?`$select=userPrincipalName,displayName" + $ownerResp = Invoke-Agent365GraphWithRetry -Uri $ownerUri + if ($ownerResp -and $ownerResp.value -and $ownerResp.value.Count -gt 0) { + $resolved = $ownerResp.value[0].displayName + if (-not $resolved) { $resolved = $ownerResp.value[0].userPrincipalName } + } + } catch { } + } + } + } catch { + # Application.Read.All may not be granted; degrade gracefully (no fabrication). + } + $script:Agent365DeveloperCache[$AppId] = $resolved + return $resolved +} + +function Get-Agent365AuditEnrichment { + <# + .SYNOPSIS + Runs a single narrow Graph audit query to retrieve agent create/publish events + and returns a hashtable keyed on agent id (titleId/appId/lower-cased displayName) + with @{ Created = [datetime]; CreatedBy = [string] }. Skipped when -OnlyAgent365Info. + .NOTES + Best-effort enrichment. If the audit query fails or returns no records, both + fields stay blank in the output (no fabrication). + #> + $enrichment = @{} + if ($OnlyAgent365Info) { return $enrichment } + if (-not $script:GraphConnected -and -not (Get-MgContext -ErrorAction SilentlyContinue)) { return $enrichment } + + # Resolve time window: prefer main-run StartDate/EndDate; else last 30 days. + $rangeStart = $null; $rangeEnd = $null + try { + if ($StartDate) { $rangeStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } + if ($EndDate) { $rangeEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } + } catch {} + if (-not $rangeStart) { $rangeStart = (Get-Date).ToUniversalTime().AddDays(-30) } + if (-not $rangeEnd) { $rangeEnd = (Get-Date).ToUniversalTime() } + + # Operation set covering agent/app create/publish/install. Conservative; missing + # operations simply mean blank columns for affected agents. + $ops = @( + 'AppCatalogPublishedAppCreated', + 'AppCatalogPublishedAppUpdated', + 'AgentCreated', + 'AgentPublished', + 'CopilotAgentInstalled' + ) + + Write-LogHost " Running narrow audit query for Agent 365 enrichment (Date created / Created by)..." -ForegroundColor DarkGray + $queryId = $null + try { + $queryId = Invoke-GraphAuditQuery ` + -DisplayName ("PAX_Agents365_Enrichment_{0}" -f (Get-Date -Format 'yyyyMMddHHmmss')) ` + -StartDate $rangeStart ` + -EndDate $rangeEnd ` + -Operations $ops + } catch { + Write-LogHost (" WARNING: Agent 365 enrichment query submit failed: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + return $enrichment + } + if (-not $queryId) { + Write-LogHost " WARNING: Agent 365 enrichment query did not return a query id; columns will be blank." -ForegroundColor Yellow + return $enrichment + } + + # Poll for query completion. Fortune-500 tenants routinely take 30-90 minutes + # for audit queries even on narrow operation/time filters. Use the same + # tolerance window the main audit phase uses (-MaxNetworkOutageMinutes is for + # transport errors only; query-completion timeout is a separate, longer cap). + # 4 hours is the documented Graph audit query SLA upper bound. + $pollTimeoutMinutes = 240 + $pollDeadline = (Get-Date).AddMinutes($pollTimeoutMinutes) + $pollIntervalSeconds = 15 + $lastLoggedMinute = -1 + $status = $null + Write-LogHost (" Polling enrichment query (timeout {0} min, refresh-token aware)..." -f $pollTimeoutMinutes) -ForegroundColor DarkGray + while ((Get-Date) -lt $pollDeadline) { + Start-Sleep -Seconds $pollIntervalSeconds + try { Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue } catch {} + try { $status = Get-GraphAuditQueryStatus -QueryId $queryId } catch { $status = $null } + if ($status -and $status.Status -in @('succeeded','failed','cancelled')) { break } + # Heartbeat once per minute to keep long polls visible in logs. + $elapsedMin = [int]((Get-Date) - $pollDeadline.AddMinutes(-$pollTimeoutMinutes)).TotalMinutes + if ($elapsedMin -ne $lastLoggedMinute -and ($elapsedMin % 5) -eq 0) { + $lastLoggedMinute = $elapsedMin + $st = if ($status -and $status.Status) { $status.Status } else { 'pending' } + Write-LogHost (" ... {0} min elapsed, status={1}" -f $elapsedMin, $st) -ForegroundColor DarkGray + } + # Gentle backoff: 15s -> 30s -> 60s after 2 and 10 minutes respectively. + if ($elapsedMin -ge 10 -and $pollIntervalSeconds -lt 60) { $pollIntervalSeconds = 60 } + elseif ($elapsedMin -ge 2 -and $pollIntervalSeconds -lt 30) { $pollIntervalSeconds = 30 } + } + if (-not $status -or $status.Status -ne 'succeeded') { + Write-LogHost (" WARNING: Agent 365 enrichment query did not succeed (status={0}); columns will be blank." -f ($status.Status)) -ForegroundColor Yellow + return $enrichment + } + + $records = @() + try { $records = Get-GraphAuditRecords -QueryId $queryId } catch { $records = @() } + if (-not $records -or $records.Count -eq 0) { return $enrichment } + + foreach ($rec in $records) { + try { + $auditObj = $null + if ($rec.PSObject.Properties.Name -contains 'auditData') { $auditObj = $rec.auditData } + if ($auditObj -is [string]) { try { $auditObj = $auditObj | ConvertFrom-Json -ErrorAction Stop } catch { $auditObj = $null } } + $created = $null + if ($rec.PSObject.Properties.Name -contains 'createdDateTime' -and $rec.createdDateTime) { + try { $created = [datetime]$rec.createdDateTime } catch {} + } + $createdBy = $null + if ($rec.PSObject.Properties.Name -contains 'userPrincipalName' -and $rec.userPrincipalName) { + $createdBy = [string]$rec.userPrincipalName + } + # Pull keys from auditData defensively (preview schema may shift). + $keys = New-Object System.Collections.Generic.List[string] + if ($auditObj) { + foreach ($p in 'TitleId','titleId','AppId','appId','PackageId','packageId','TeamsAppId','teamsAppId','DisplayName','displayName','Name','name') { + try { if ($auditObj.$p) { [void]$keys.Add(([string]$auditObj.$p).ToLowerInvariant()) } } catch {} + } + } + foreach ($k in $keys) { + if (-not $enrichment.ContainsKey($k)) { + $enrichment[$k] = @{ Created = $created; CreatedBy = $createdBy } + } + } + } catch { continue } + } + Write-LogHost (" Audit enrichment matched {0} agent identifier key(s)." -f $enrichment.Keys.Count) -ForegroundColor DarkGray + return $enrichment +} + +function ConvertTo-Agent365Row { + <# + .SYNOPSIS + Maps one package detail object to the exact 28-column synthetic schema. + Empty cells stay empty when fields are absent (no fabrication). + #> + param( + [Parameter(Mandatory = $true)][object]$Package, + [Parameter(Mandatory = $false)][hashtable]$AuditEnrichment + ) + function _g { param($obj, [string[]]$names) foreach ($n in $names) { try { if ($null -ne $obj.$n -and "$($obj.$n)" -ne '') { return $obj.$n } } catch {} } return '' } + function _gb { param($obj, [string[]]$names) foreach ($n in $names) { try { if ($null -ne $obj.$n) { return [bool]$obj.$n } } catch {} } return '' } + function _join { param($v, [string]$sep) if ($null -eq $v) { return '' }; if ($v -is [System.Collections.IEnumerable] -and -not ($v -is [string])) { return (@($v) -join $sep) } else { return [string]$v } } + function _fmtDate { param($v) if (-not $v) { return '' }; try { return ([datetime]$v).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ssZ') } catch { return [string]$v } } + + $elem = $null + try { $elem = $Package.elementDetails } catch {} + + $titleIdRaw = (_g $Package @('id','titleId','packageId')) + $titleId = '' + if ($titleIdRaw) { $titleId = if ($titleIdRaw -match '^T_') { [string]$titleIdRaw } else { 'T_' + [string]$titleIdRaw } } + + # Audit enrichment lookup (lowercase keys). Try titleId raw, appId, displayName. + $dateCreated = '' + $createdBy = '' + if ($AuditEnrichment) { + $probeKeys = @() + if ($titleIdRaw) { $probeKeys += ([string]$titleIdRaw).ToLowerInvariant() } + $appIdProbe = (_g $Package @('appId','applicationId')) + if ($appIdProbe) { $probeKeys += ([string]$appIdProbe).ToLowerInvariant() } + $dispProbe = (_g $Package @('displayName','name')) + if ($dispProbe) { $probeKeys += ([string]$dispProbe).ToLowerInvariant() } + foreach ($k in $probeKeys) { + if ($AuditEnrichment.ContainsKey($k)) { + $hit = $AuditEnrichment[$k] + if ($hit.Created) { $dateCreated = (_fmtDate $hit.Created) } + if ($hit.CreatedBy) { $createdBy = [string]$hit.CreatedBy } + break + } + } + } + # Fallback to package's own createdDateTime if audit didn't supply one + if (-not $dateCreated) { + $pkgCreated = (_g $Package @('createdDateTime','createdDate')) + if ($pkgCreated) { $dateCreated = (_fmtDate $pkgCreated) } + } + + $developer = Resolve-Agent365DeveloperName -DeveloperName (_g $Package @('developer.name')) -AppId (_g $Package @('appId','applicationId')) + # fallback: try direct nested object access + if (-not $developer) { + try { if ($Package.developer -and $Package.developer.name) { $developer = [string]$Package.developer.name } } catch {} + } + + [PSCustomObject][ordered]@{ + 'Name' = (_g $Package @('displayName','name')) + 'Supported in' = (_join (_g $Package @('supportedHosts','supportedClients')) ';') + 'Date created' = $dateCreated + 'Developer Name' = $developer + 'Type' = (_g $Package @('agentType','type')) + 'Version' = (_g $Package @('version')) + 'Availability' = (_g $Package @('availability','allowedUsersAndGroups')) + 'Created by' = $createdBy + 'Description' = (_g $Package @('description')) + 'Created in' = (_g $Package @('source','origin','createdIn')) + 'Last updated' = (_fmtDate (_g $Package @('lastModifiedDateTime','lastUpdatedDateTime'))) + 'Custom actions' = (_g $elem @('customActions')) + 'Title ID' = $titleId + 'Sensitivity' = (_g $Package @('sensitivity')) + 'Can read OneDrive and Sharepoint items'= (_gb $elem @('canReadOneDriveAndSharepointItems')) + 'OneDrive and Sharepoint items' = (_g $elem @('oneDriveAndSharepointItems')) + 'Can read OneDrive files' = (_gb $elem @('canReadOneDriveFiles')) + 'OneDrive files' = (_g $elem @('oneDriveFiles')) + 'OneDrive sites' = (_g $elem @('oneDriveSites')) + 'Can read Sharepoint sites and files' = (_gb $elem @('canReadSharepointSitesAndFiles')) + 'Sharepoint files' = (_g $elem @('sharepointFiles')) + 'Sharepoint sites' = (_g $elem @('sharepointSites')) + 'Can extend to Graph connector' = (_gb $elem @('canExtendToGraphConnector')) + 'Graph connector details' = (_g $elem @('graphConnectorDetails')) + 'Can generate images using user prompt' = (_gb $elem @('canGenerateImagesUsingUserPrompt')) + 'Can use code interpreter' = (_gb $elem @('canUseCodeInterpreter')) + 'Contains uploaded files' = (_gb $elem @('containsUploadedFiles')) + 'Uploaded files' = (_g $elem @('uploadedFiles')) + } +} + +function Export-Agent365Csv { + <# + .SYNOPSIS + Writes the Agent 365 CSV (UTF-8 BOM) to OutputPath using the run timestamp. + .DESCRIPTION + When -AppendAgent365Info is in effect, this function pre-reads the + target CSV and union-merges its rows with the current API-derived rows: the + current run's row wins on AgentId conflicts (freshest metadata) and any + target-only AgentId is carried forward (departed agents are not dropped). + .OUTPUTS + Full path to written file, or $null if no rows. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $Rows -or $Rows.Count -eq 0) { + Write-LogHost " Agent 365: no rows to write." -ForegroundColor Yellow + return $null + } + + # Union-merge with -AppendAgent365Info target when set. Failures here are + # non-fatal — fall back to current-run rows only. + $rowsToWrite = [System.Collections.Generic.List[object]]::new() + foreach ($r in $Rows) { [void]$rowsToWrite.Add($r) } + if ($AppendAgent365Info -and (Test-Path -LiteralPath $AppendAgent365Info -PathType Leaf)) { + try { + $currentAgentIds = New-Object System.Collections.Generic.HashSet[string] + foreach ($r in $Rows) { + $aid = $null + try { $aid = [string]$r.AgentId } catch {} + if ($aid) { [void]$currentAgentIds.Add($aid) } + } + $targetRows = @(Import-Csv -LiteralPath $AppendAgent365Info -Encoding UTF8) + $carried = 0 + foreach ($tr in $targetRows) { + $aid = $null + try { $aid = [string]$tr.AgentId } catch {} + if (-not $aid) { continue } + if (-not $currentAgentIds.Contains($aid)) { + [void]$rowsToWrite.Add($tr) + $carried++ + } + } + Write-LogHost (" -AppendAgent365Info: carried {0:N0} departed agent(s) from '{1}'; current run contributed {2:N0}." -f $carried, $AppendAgent365Info, $Rows.Count) -ForegroundColor DarkCyan + } + catch { + Write-LogHost (" WARNING: -AppendAgent365Info merge failed ({0}); writing current-run rows only." -f $_.Exception.Message) -ForegroundColor Yellow + } + } + + $ts = $global:ScriptRunTimestamp + if (-not $ts) { $ts = (Get-Date).ToString('yyyyMMdd_HHmmss') } + # Per-data-type destination. If -OutputPathAgent365Info resolves to a Local + # folder or file form, route the catalog there; otherwise inherit $OutputPath. Remote + # tiers (SharePoint/Fabric) continue to write to the scratch dir and upload from there. + $agentOutDir = $OutputPath + # Effective basename (honors -AppendAgent365Info target leaf so the merged file + # produced this run can be uploaded back over the original remote URL \u2014 matched + # by the upload sweep's Append-leaf filter). + $agentBasename = (& $script:GetEffectiveAgent365Basename) + try { + $dt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename $agentBasename + if ($dt -and $dt.IsBound -and $dt.Tier -eq 'Local') { + $agentOutDir = $dt.EffectiveDir + if ($dt.Basename) { $agentBasename = $dt.Basename } + if (-not (Test-Path -LiteralPath $agentOutDir -PathType Container)) { + New-Item -Path $agentOutDir -ItemType Directory -Force | Out-Null + } + } + } catch {} + $outFile = Join-Path $agentOutDir $agentBasename + try { + # Atomic write — Save-CsvAtomic uses temp+rename so a Ctrl+C / OOM-kill mid-write + # cannot leave a half-formed Agent365 CSV in the operator's output folder. + Save-CsvAtomic -InputObject ($rowsToWrite.ToArray()) -Path $outFile -NoTypeInformation -Encoding UTF8 + Write-LogHost (" Agent 365 CSV written: {0} ({1} rows)" -f (Get-DisplayPath -LocalPath $outFile), $rowsToWrite.Count) -ForegroundColor Green + return $outFile + } catch { + Write-LogHost (" ERROR: Failed to write Agent 365 CSV: {0}" -f $_.Exception.Message) -ForegroundColor Red + return $null + } +} + +function Add-Agent365WorkbookTab { + <# + .SYNOPSIS + Appends an 'Agents365' worksheet to the existing workbook (after EntraUsers tab). + No-op when -ExportWorkbook is not set or ImportExcel is unavailable. + #> + param( + [Parameter(Mandatory = $true)][string]$WorkbookPath, + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $ExportWorkbook) { return } + if (-not $Rows -or $Rows.Count -eq 0) { return } + if (-not (Get-Module -ListAvailable -Name ImportExcel)) { + Write-LogHost " Agent 365: ImportExcel not available; skipping workbook tab." -ForegroundColor Yellow + return + } + try { + Import-Module ImportExcel -Force -ErrorAction Stop + $Rows | Export-Excel -Path $WorkbookPath -WorksheetName 'Agents365' -AutoSize -BoldTopRow -FreezeTopRow -ErrorAction Stop + Write-LogHost (" Agent 365 worksheet appended to: {0}" -f (Get-DisplayPath -LocalPath $WorkbookPath)) -ForegroundColor Green + } catch { + Write-LogHost (" WARNING: Could not append Agents365 worksheet: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + } +} + +function Invoke-Agent365Phase { + <# + .SYNOPSIS + Top-level orchestrator for the Agent 365 phase. Called once from the main + flow when -IncludeAgent365Info or -OnlyAgent365Info is set. No-op otherwise. + .OUTPUTS + Hashtable: @{ CsvPath = ; Rows = } + #> + if (-not ($IncludeAgent365Info -or $OnlyAgent365Info)) { return @{ CsvPath = $null; Rows = @() } } + + Write-LogHost "" + Write-LogHost "============================================================" -ForegroundColor Cyan + Write-LogHost " Microsoft Agent 365 enrichment phase" -ForegroundColor Cyan + Write-LogHost "============================================================" -ForegroundColor Cyan + + # App-only modes (AppRegistration certificate/secret, ManagedIdentity): surface the app-only + # Phase 2 banner and reuse the EXISTING application Graph context (no interactive sign-in). + # Delegated modes skip this block and go straight to the Frontier probe using the Agent 365 + # scopes already consented at their initial sign-in. + if ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') { + if (-not (Connect-Agent365InteractiveContext)) { + Write-LogHost " Agent 365 phase aborted (app-only Graph context unavailable)." -ForegroundColor Red + return @{ CsvPath = $null; Rows = @() } + } + } + + # Frontier probe (prints banner 7d on 401/403/404 and returns false). + if (-not (Test-Agent365FrontierAccess)) { + return @{ CsvPath = $null; Rows = @() } + } + + # Audit enrichment (skipped when -OnlyAgent365Info). + $script:Agent365AuditEnrichment = Get-Agent365AuditEnrichment + + # List + per-package detail + Write-LogHost " Listing Agent 365 packages..." -ForegroundColor DarkGray + $listed = Get-Agent365Packages + if (-not $listed -or $listed.Count -eq 0) { + Write-LogHost " No Agent 365 packages returned by the catalog." -ForegroundColor Yellow + return @{ CsvPath = $null; Rows = @() } + } + Write-LogHost (" {0} package(s) listed; fetching details..." -f $listed.Count) -ForegroundColor DarkGray + + $rows = New-Object System.Collections.Generic.List[object] + $idx = 0 + foreach ($p in $listed) { + $idx++ + $pkgId = $null + try { $pkgId = $p.id } catch {} + if (-not $pkgId) { try { $pkgId = $p.titleId } catch {} } + if (-not $pkgId) { continue } + $detail = Get-Agent365PackageDetail -PackageId $pkgId + if (-not $detail) { continue } + try { + $row = ConvertTo-Agent365Row -Package $detail -AuditEnrichment $script:Agent365AuditEnrichment + [void]$rows.Add($row) + } catch { + Write-LogHost (" WARNING: Row build failed for package '{0}': {1}" -f $pkgId, $_.Exception.Message) -ForegroundColor Yellow + } + if (($idx % 25) -eq 0) { + Write-LogHost (" ... {0}/{1} packages processed" -f $idx, $listed.Count) -ForegroundColor DarkGray + } + } + + $csvPath = Export-Agent365Csv -Rows $rows.ToArray() + return @{ CsvPath = $csvPath; Rows = $rows.ToArray() } +} + +# ============================================================================= +# END MICROSOFT AGENT 365 ENRICHMENT FUNCTIONS +# ============================================================================= + +# +# Core live-mode functions providing connectivity and paged audit retrieval. +# NOTE: This function is now wrapped by Connect-PurviewAudit for EOM mode compatibility + +function Connect-ToComplianceCenter { + param() + if ($script:Connected) { return } + Write-LogHost "Connecting to Microsoft 365 Security & Compliance Center..." -ForegroundColor Cyan + # Ensure ExchangeOnlineManagement module is available + try { + $existingEOM = Get-Module -ListAvailable -Name ExchangeOnlineManagement | Sort-Object Version -Descending | Select-Object -First 1 + if (-not $existingEOM) { + Write-LogHost "Installing ExchangeOnlineManagement module (CurrentUser scope)..." -ForegroundColor Yellow + try { Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop } catch { Write-LogHost "Failed to install module: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + Import-Module ExchangeOnlineManagement -Force -ErrorAction Stop + } catch { + Write-LogHost "Module load/install failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } + + # Authentication modes (subset retained for stability) + try { + switch ($Auth.ToLower()) { + 'weblogin' { + try { + $exoCmd = Get-Command Connect-ExchangeOnline -ErrorAction Stop + $hasUseWeb = $exoCmd.Parameters.ContainsKey('UseWebLogin') + if ($hasUseWeb) { + Write-LogHost 'Using Connect-ExchangeOnline -UseWebLogin (parameter present).' -ForegroundColor DarkGray + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + else { + Write-LogHost 'UseWebLogin parameter not available in this host/module; invoking standard interactive Connect-ExchangeOnline.' -ForegroundColor Yellow + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + } + catch { Write-LogHost "WebLogin flow failed: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + 'devicecode' { + Connect-ExchangeOnline -ShowBanner:$false -Device | Out-Null + } + 'credential' { + $cred = Get-Credential -Message 'Enter admin credentials for Exchange Online' + Connect-ExchangeOnline -ShowBanner:$false -Credential $cred | Out-Null + } + default { + # Silent first, fallback to WebLogin + $silentOk = $true + try { Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null } catch { $silentOk = $false } + if (-not $silentOk) { + try { Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null } catch { Write-LogHost "Silent + fallback auth failed: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + } + } + $script:Connected = $true + Write-LogHost "Connected successfully." -ForegroundColor Green + } + catch { + Write-LogHost "Connection failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +# ============================================== +# DUAL-MODE DIAGNOSTICS FUNCTION +# ============================================== +# Unified capability check for both EOM and Graph API modes + +function Test-PurviewAuditCapability { + <# + .SYNOPSIS + Tests audit log query capability for either EOM or Graph API mode. + + .DESCRIPTION + Performs connectivity and permission checks based on active mode. + + EOM Mode: + - Verifies Search-UnifiedAuditLog cmdlet availability + - Performs probe query to test permissions + - Checks for proper role assignments + + Graph API Mode: + - Tests Graph API connectivity + - Verifies AuditLogsQuery.Read.All permissions + - Performs lightweight endpoint check + + .PARAMETER UseEOMMode + If true, test EOM capabilities. If false, test Graph API. + + .PARAMETER SkipChecks + If true, skip all diagnostic checks (for advanced scenarios) + + .OUTPUTS + $true if capability check passes, $false otherwise + #> + + param( + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false, + + [Parameter(Mandatory = $false)] + [bool]$SkipChecks = $false + ) + + if ($SkipChecks) { + Write-LogHost "Diagnostics: Skipped (per user request)" -ForegroundColor Gray + return $true + } + + if ($UseEOMMode) { + # ======================================== + # EOM MODE DIAGNOSTICS + # ======================================== + + Write-LogHost "Running EOM capability diagnostics..." -ForegroundColor Cyan + + # Check if cmdlet is available + $cmd = Get-Command Search-UnifiedAuditLog -ErrorAction SilentlyContinue + if (-not $cmd) { + Write-LogHost " ✗ DIAGNOSTIC FAILED: 'Search-UnifiedAuditLog' cmdlet not found" -ForegroundColor Red + Write-LogHost "" + Write-LogHost "Troubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure ExchangeOnlineManagement module v3+ is installed" -ForegroundColor White + Write-LogHost " 2. Try: Install-Module ExchangeOnlineManagement -Scope CurrentUser" -ForegroundColor White + Write-LogHost " 3. Verify authentication completed successfully" -ForegroundColor White + Write-LogHost "" + Write-LogHost "Role Requirements:" -ForegroundColor Yellow + Write-LogHost " • View-Only Audit Logs role (minimum)" -ForegroundColor White + Write-LogHost " • Compliance Management role group" -ForegroundColor White + Write-LogHost " • Organization Management role group" -ForegroundColor White + return $false + } + + # Perform probe query to test permissions + try { + $now = (Get-Date).ToUniversalTime() + $probeStart = $now.AddMinutes(-7) + $probeEnd = $now.AddMinutes(-6) + + # Lightweight probe with unlikely operation + $null = Search-UnifiedAuditLog -StartDate $probeStart -EndDate $probeEnd -Operations 'UserLoggedIn' -ResultSize 1 -ErrorAction Stop + + Write-LogHost " EOM capability check passed" -ForegroundColor Green + return $true + } + catch { + $msg = $_.Exception.Message + Write-LogHost " ✗ DIAGNOSTIC FAILED: Probe query failed" -ForegroundColor Red + Write-LogHost " Error: $msg" -ForegroundColor Yellow + Write-LogHost "" + + if ($msg -match 'is not within the current user|Access denied|not authorized|insufficient') { + Write-LogHost "Likely Cause: Missing required roles" -ForegroundColor Yellow + Write-LogHost " Add account to 'Audit Logs' role group in Microsoft Purview" -ForegroundColor White + } + elseif ($msg -match 'The term .*Search-UnifiedAuditLog.* is not recognized') { + Write-LogHost "Likely Cause: Module not loaded properly" -ForegroundColor Yellow + Write-LogHost " Try: Import-Module ExchangeOnlineManagement -Force" -ForegroundColor White + } + else { + Write-LogHost "General Guidance:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure Unified Audit Log is enabled tenant-wide" -ForegroundColor White + Write-LogHost " 2. Verify role assignments are properly configured" -ForegroundColor White + Write-LogHost " 3. Check for conditional access policies blocking access" -ForegroundColor White + } + + return $false + } + } + else { + # ======================================== + # GRAPH API MODE DIAGNOSTICS + # ======================================== + + Write-LogHost "Running Graph API capability diagnostics..." -ForegroundColor Cyan + + # Verify connected to Graph + try { + $context = Get-MgContext -ErrorAction Stop + + if (-not $context) { + Write-LogHost " ✗ DIAGNOSTIC FAILED: Not connected to Microsoft Graph" -ForegroundColor Red + Write-LogHost " Run Connect-PurviewAudit first to establish connection" -ForegroundColor Yellow + return $false + } + + # Check for required scopes + $requiredScope = 'AuditLogsQuery.Read.All' + if ($context.Scopes -notcontains $requiredScope) { + Write-LogHost " [!] WARNING: Missing required scope: $requiredScope" -ForegroundColor Yellow + Write-LogHost " Queries may fail without this permission" -ForegroundColor Yellow + } + } + catch { + Write-LogHost " ✗ DIAGNOSTIC FAILED: Unable to get Graph context" -ForegroundColor Red + Write-LogHost " Error: $($_.Exception.Message)" -ForegroundColor Yellow + return $false + } + + # Test Graph API endpoint connectivity + try { + # Test actual query endpoint with minimal test query + $testQueryBody = @{ + displayName = "PAX-Diagnostic-Test-$(Get-Date -Format 'HHmmss')" + filterStartDateTime = (Get-Date).AddMinutes(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + filterEndDateTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + operationFilters = @('UserLoggedIn') # Common activity type for quick test + } + + $createUri = Get-GraphAuditApiUri -Path 'queries' + $createResponse = Invoke-MgGraphRequest -Method POST -Uri $createUri -Body $testQueryBody -ErrorAction Stop if ($createResponse.id) { + Write-LogHost " Graph API capability check passed" -ForegroundColor Green + Write-LogHost " Successfully created test query (ID: $($createResponse.id))" -ForegroundColor Green + return $true + } + else { + Write-LogHost " ✗ DIAGNOSTIC WARNING: Query created but no ID returned" -ForegroundColor Yellow + return $false + } + } + catch { + $msg = $_.Exception.Message + + # Check if this is a throttling error (429 TooManyRequests) + $isThrottling = $msg -match 'TooManyRequests|429|Too many requests|throttl' + + if ($isThrottling) { + # Set flag so we don't show scary warning message later + $script:ThrottlingDetected = $true + + # Throttling detected - friendly terminal message, full details to log only + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor DarkYellow + Write-Host " [!] Graph API Throttling Detected (429 - Too Many Requests)" -ForegroundColor DarkYellow + Write-Host "============================================================================================================" -ForegroundColor DarkYellow + Write-Host "" + Write-Host " Microsoft Graph is currently rate-limiting requests to your tenant." -ForegroundColor White + Write-Host "" + Write-Host " How PAX handles throttling:" -ForegroundColor Cyan + Write-Host " • Automatic exponential backoff with retries" -ForegroundColor Gray + Write-Host " • Circuit breaker protection (pauses after repeated failures)" -ForegroundColor Gray + Write-Host " • Adaptive concurrency (reduces parallel requests)" -ForegroundColor Gray + Write-Host " • Real-time notifications (you'll see throttle events as they occur)" -ForegroundColor Gray + Write-Host "" + Write-Host " Recommendation: Graph API throttling typically clears within 5-10 minutes." -ForegroundColor Yellow + Write-Host "" # Interactive prompt (unless -Force is used for headless runs) + if (-not $Force) { + Write-Host " Options:" -ForegroundColor Cyan + Write-Host " [C] CONTINUE - Proceed with automatic throttling handling (may be slow)" -ForegroundColor Green + Write-Host " [E] EXIT - Stop gracefully and retry later (recommended if heavily throttled)" -ForegroundColor Red + Write-Host "" + + # Strict noninteractive guard: a noninteractive + # host cannot choose C/E. Auto-select C (CONTINUE) so the run progresses + # with PAX's existing throttling controls (exponential backoff, circuit + # breaker, adaptive concurrency). Operators who want a hard exit on throttling + # should run interactively or pre-empt via run-orchestration retry policy. + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: auto-selecting [C] CONTINUE." -ForegroundColor DarkGray + Write-Host "" + $choice = 'C' + } else { + Send-PromptNotification + $choice = Read-Host " Enter your choice [C/E]" + } + if ($choice -match '^E$|^Exit$') { + Write-Host "" + Write-Host " Exiting gracefully..." -ForegroundColor Yellow + Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Gray + + try { + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Write-Host " Disconnected successfully" -ForegroundColor Green + } + catch { + Write-Host " (Graph connection cleanup completed)" -ForegroundColor Gray + } + + Write-Host "" + Write-Host " Please wait 5-10 minutes before retrying." -ForegroundColor Cyan + Write-Host "" + + # Log the graceful exit + Write-Output "[DIAGNOSTIC] User chose to exit due to throttling. Will retry later." | Out-File -FilePath $LogFile -Append -Encoding utf8 + + exit 0 + } + else { + Write-Host "" + Write-Host " Proceeding with automatic throttling handling..." -ForegroundColor Green + Write-Host " Expect slower execution times while Graph API recovers." -ForegroundColor Gray + Write-Host "" + } + } + else { + # -Force flag present (headless/automation mode) - proceed automatically + Write-Host " -Force flag detected: Proceeding automatically with throttling handling..." -ForegroundColor Green + Write-Host " Expect slower execution times while Graph API recovers." -ForegroundColor Gray + Write-Host "" + } + + # Log full error details to log file only (not terminal) + Write-Output "[DIAGNOSTIC] Graph API throttling detected during capability check" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[DIAGNOSTIC] Full error details: $msg" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[DIAGNOSTIC] Continuing with automatic throttling handling enabled" | Out-File -FilePath $LogFile -Append -Encoding utf8 + } + else { + # Non-throttling error - show full details + Write-LogHost " ✗ DIAGNOSTIC FAILED: Graph API endpoint test failed" -ForegroundColor Red + Write-LogHost " Error: $msg" -ForegroundColor Yellow + Write-LogHost "" + + if ($msg -match 'Forbidden|403|Access.*denied|Insufficient privileges') { + Write-LogHost "Likely Cause: Missing required permissions" -ForegroundColor Yellow + Write-LogHost " Required: AuditLogsQuery.Read.All Graph API scope" -ForegroundColor White + Write-LogHost " Required: Azure AD role (Compliance/Security Administrator)" -ForegroundColor White + } + elseif ($msg -match 'Unauthorized|401') { + Write-LogHost "Likely Cause: Authentication issue" -ForegroundColor Yellow + Write-LogHost " Try disconnecting and reconnecting: Disconnect-MgGraph; Connect-PurviewAudit" -ForegroundColor White + } + else { + Write-LogHost "General Guidance:" -ForegroundColor Yellow + Write-LogHost " 1. Verify admin has consented to AuditLogsQuery.Read.All scope" -ForegroundColor White + Write-LogHost " 2. Check Azure AD role assignments" -ForegroundColor White + Write-LogHost " 3. Ensure network connectivity to graph.microsoft.com" -ForegroundColor White + } + } + + return $false + } + } +} + +# ============================================== +# DUAL-MODE GROUP EXPANSION FUNCTION +# ============================================== +# Expand distribution/security groups to individual user principal names + +function Expand-GroupToUsers { + <# + .SYNOPSIS + Expands a distribution or security group to individual user principal names. + + .DESCRIPTION + Retrieves members of a group using either EOM cmdlets or Graph API. + + EOM Mode: + - Uses Get-DistributionGroupMember cmdlet + - Accepts group display name or email address + - Returns PrimarySmtpAddress of members + + Graph API Mode: + - Uses Get-MgGroupMember cmdlet + - Requires group ObjectId (auto-resolved from display name) + - Returns userPrincipalName of user members + + .PARAMETER GroupIdentity + The group identifier. Can be: + - Display name (e.g., "Executive Leadership") + - Email address (e.g., "exec-team@contoso.com") + - ObjectId/GUID (Graph mode only) + + .PARAMETER UseEOMMode + If true, use EOM cmdlets. If false, use Graph API. + + .OUTPUTS + Array of user principal names (email addresses) + #> + + param( + [Parameter(Mandatory = $true)] + [string]$GroupIdentity, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + $members = @() + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Get-DistributionGroupMember + # ======================================== + + try { + Write-LogHost " Processing group (EOM): '$GroupIdentity'" -ForegroundColor Gray + + # Get-DistributionGroupMember works with display name or email + $groupMembers = Get-DistributionGroupMember -Identity $GroupIdentity -ErrorAction Stop + + $members = $groupMembers | Select-Object -ExpandProperty PrimarySmtpAddress + + Write-LogHost " Expanded: $($members.Count) member(s)" -ForegroundColor DarkGray + } + catch { + Write-LogHost " Warning: Failed to expand group '$GroupIdentity': $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Possible causes:" -ForegroundColor Yellow + Write-LogHost " • Group does not exist or name is misspelled" -ForegroundColor Gray + Write-LogHost " • Insufficient permissions (need Organization Management or similar)" -ForegroundColor Gray + Write-LogHost " • Group is not a distribution/mail-enabled group" -ForegroundColor Gray + } + } + else { + # ======================================== + # GRAPH API MODE: Get-MgGroupMember + # ======================================== + + try { + Write-LogHost " Processing group (Graph API): '$GroupIdentity'" -ForegroundColor Gray + + # Determine if we have an ObjectId (GUID) or display name + $groupId = $null + if ($GroupIdentity -match '^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') { + # Looks like a GUID, use directly + $groupId = $GroupIdentity + } + else { + # Display name or email - need to resolve to ObjectId + Write-LogHost " Resolving group ID from display name..." -ForegroundColor DarkGray + + # Try searching by display name first + $groupSearch = Get-MgGroup -Filter "displayName eq '$GroupIdentity'" -ErrorAction SilentlyContinue + + if (-not $groupSearch) { + # Try by mail/mailNickname + $groupSearch = Get-MgGroup -Filter "mail eq '$GroupIdentity'" -ErrorAction SilentlyContinue + } + + if ($groupSearch) { + $groupId = $groupSearch.Id + Write-LogHost " Resolved to ObjectId: $groupId" -ForegroundColor DarkGray + } + else { + throw "Unable to find group with identifier: $GroupIdentity" + } + } + + # Get group members (users only) + $groupMembers = Get-MgGroupMember -GroupId $groupId -All -ErrorAction Stop + + # Filter to users only and extract UPN + foreach ($member in $groupMembers) { + # Check if member is a user (not a nested group or service principal) + if ($member.AdditionalProperties.'@odata.type' -eq '#microsoft.graph.user') { + # Get full user object to retrieve userPrincipalName + $user = Get-MgUser -UserId $member.Id -ErrorAction SilentlyContinue + if ($user -and $user.UserPrincipalName) { + $members += $user.UserPrincipalName + } + } + } + + Write-LogHost " Expanded: $($members.Count) user member(s)" -ForegroundColor DarkGray + } + catch { + Write-LogHost " Warning: Failed to expand group '$GroupIdentity': $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Possible causes:" -ForegroundColor Yellow + Write-LogHost " • Group does not exist or identifier is invalid" -ForegroundColor Gray + Write-LogHost " • Insufficient permissions (need GroupMember.Read.All - or higher: Group.Read.All / Directory.Read.All)" -ForegroundColor Gray + Write-LogHost " • Network connectivity issues with Graph API" -ForegroundColor Gray + } + } + + return $members +} + +# ============================================== +# DUAL-MODE QUERY EXECUTION WRAPPER +# ============================================== +# Unified query function that routes to either EOM or Graph API + +function Invoke-PurviewAuditQuery { + <# + .SYNOPSIS + Executes an audit log query using either EOM or Graph API. + + .DESCRIPTION + Routes audit log queries to the appropriate backend: + - EOM Mode: Uses Search-UnifiedAuditLog cmdlet + - Graph API Mode: Uses async query pattern (create → poll → retrieve) + + Returns audit records in a normalized format compatible with downstream processing. + + .PARAMETER StartDate + Query start date (inclusive) + + .PARAMETER EndDate + Query end date (exclusive) + + .PARAMETER Operations + Activity type(s) to query + + .PARAMETER UserIds + Optional array of user principal names to filter by + + .PARAMETER ResultSize + Maximum number of records to retrieve (EOM mode) + + .PARAMETER UseEOMMode + If true, use EOM cmdlets. If false, use Graph API. + + .OUTPUTS + Array of audit log records in normalized schema + #> + + param( + [Parameter(Mandatory = $true)] + [datetime]$StartDate, + + [Parameter(Mandatory = $true)] + [datetime]$EndDate, + + [Parameter(Mandatory = $true)] + [string]$Operations, + + [Parameter(Mandatory = $false)] + [string[]]$UserIds, + + [Parameter(Mandatory = $false)] + [int]$ResultSize = 5000, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Use existing Search-UnifiedAuditLog logic + # ======================================== + + # Call existing retry wrapper (preserves all the sophisticated logic) + $results = Invoke-SearchUnifiedAuditLogWithRetry ` + -Start $StartDate ` + -End $EndDate ` + -Operation $Operations ` + -ResultSize $ResultSize ` + -UserIds $UserIds ` + -AutoSubdivide $true + + return $results + } + else { + # ======================================== + # GRAPH API MODE: Async query pattern + # ======================================== + + try { + # Step 1: Create async query + Write-Host " [Graph API] Creating async query for $Operations..." -ForegroundColor DarkGray + + # Use last included minute (EndDate - 1 minute) since end date is exclusive + $endDisplay = $EndDate.AddMinutes(-1) + $displayName = "PAX_Query_$($StartDate.ToString('yyyyMMdd_HHmm'))-$($endDisplay.ToString('yyyyMMdd_HHmm'))" + + # M365 usage mode requires operationFilters only (no recordType/service filters) + $recordTypesArg = $null + $serviceFilterArg = $null + if (-not $script:IncludeM365Usage) { + # Only populate filters when NOT in M365 usage mode + $recordTypesArg = $RecordTypes + $serviceFilterArg = $script:CurrentServiceFilter + if (-not $serviceFilterArg -and $ServiceTypes -and $ServiceTypes.Count -gt 0) { + $serviceFilterArg = $ServiceTypes[0] + } + } + + $queryId = Invoke-GraphAuditQuery ` + -DisplayName $displayName ` + -FilterStartDateTime $StartDate ` + -FilterEndDateTime $EndDate ` + -OperationFilters @($Operations) ` + -RecordTypeFilters $recordTypesArg ` + -ServiceFilter $serviceFilterArg + + if (-not $queryId) { + Write-Host " [Graph API] Failed to create query" -ForegroundColor Red + return @() + } + + Write-Host " [Graph API] Query created: $queryId" -ForegroundColor DarkGray + + # Step 2: Poll for completion + # Replaced fixed-count polling with time-budget model supporting extended outages (up to 30 minutes) + $effectiveOutageMinutes = if ($MaxNetworkOutageMinutes -and $MaxNetworkOutageMinutes -gt 0) { $MaxNetworkOutageMinutes } else { 30 } + $maxPollDurationSeconds = $effectiveOutageMinutes * 60 # Absolute cap for network outage tolerance + $pollInterval = 5 # Base interval (seconds) when healthy + $maxHealthyInterval = 15 # Cap interval when status retrieval succeeds + $pollStart = Get-Date + $pollCount = 0 + $queryComplete = $false + $networkErrorStreak = 0 + $networkOutageStart = $null + $lastNetMessage = $null # Throttle repetitive network messages + + Write-Host " [Graph API] Polling for query completion..." -ForegroundColor DarkGray + + # Transient network resilience variables + $transientPatterns = @('timed out','unable to connect','connection','remote name could not be resolved','temporarily unavailable') + + while (-not $queryComplete) { + # Network outage guard: only abort if we've been in a CONTINUOUS network outage + # exceeding the user's MaxNetworkOutageMinutes threshold. Normal polling + # continues indefinitely until the query succeeds, fails, or the user cancels. + if ($networkOutageStart) { + $outageElapsed = (Get-Date) - $networkOutageStart + if ($outageElapsed.TotalSeconds -ge $maxPollDurationSeconds) { + Write-Host " [NET] Polling aborted after $effectiveOutageMinutes minutes of continuous network outage" -ForegroundColor Red + break + } + } + Start-Sleep -Seconds $pollInterval + $pollCount++ + + $status = $null + try { + $status = Get-GraphAuditQueryStatus -QueryId $queryId -ErrorAction Stop + # Successful status retrieval resets outage tracking + $networkErrorStreak = 0 + if ($networkOutageStart) { + $outageDuration = (Get-Date) - $networkOutageStart + # Only log recovery if outage lasted > 1 minute (ignore brief connection blips) + if ($outageDuration.TotalMinutes -ge 1) { + Write-Host " [NET] Network recovered after $([Math]::Round($outageDuration.TotalMinutes,1)) minutes" -ForegroundColor Green + } + $networkOutageStart = $null + $lastNetMessage = $null + } + # Query status retrieved successfully; interval adjusted + # Tighten interval gradually back to healthy baseline + $pollInterval = [Math]::Max(5, [Math]::Min($pollInterval - 2, $maxHealthyInterval)) + } + catch { + $errMsg = $_.Exception.Message + if ($transientPatterns | Where-Object { $errMsg.ToLower().Contains($_) }) { + $networkErrorStreak++ + if (-not $networkOutageStart) { $networkOutageStart = Get-Date } + $outageElapsed = (Get-Date) - $networkOutageStart + # Throttle messages: only show if outage > 1 min OR first error with no recent message + if ($outageElapsed.TotalMinutes -ge 1 -or ($networkErrorStreak -eq 1 -and (-not $lastNetMessage -or ((Get-Date) - $lastNetMessage).TotalSeconds -ge 60))) { + if (-not $lastNetMessage -or ((Get-Date) - $lastNetMessage).TotalSeconds -ge 60) { + Write-Host " [NET] Poll $pollCount`: transient network issue (streak $networkErrorStreak, outage $([Math]::Round($outageElapsed.TotalMinutes,1))m)" -ForegroundColor Yellow + $lastNetMessage = Get-Date + } + } + # Dynamic backoff growth with ceiling (to avoid hammering during outage) + $pollInterval = [Math]::Min(90, [Math]::Round($pollInterval * 1.6 + (Get-Random -Minimum 2 -Maximum 6))) + continue + } else { + Write-Host " [Graph API] Non-transient status error: $errMsg" -ForegroundColor Red + break + } + } + + if (-not $status) { continue } + + Write-Host " [Graph API] Poll $pollCount`: Status=$($status.Status), RecordCount=$($status.RecordCount)" -ForegroundColor DarkGray + + switch ($status.Status) { + 'succeeded' { + $queryComplete = $true + Write-Host " [Graph API] Query completed: $($status.RecordCount) records available" -ForegroundColor Green + break + } + 'failed' { + Write-Host " [Graph API] Query failed" -ForegroundColor Red + return @() + } + 'cancelled' { + Write-Host " [Graph API] Query was cancelled" -ForegroundColor Yellow + return @() + } + default { continue } + } + } + + if (-not $queryComplete) { + Write-Host " [Graph API] Query polling aborted (network outage or non-transient error after $pollCount polls)" -ForegroundColor Yellow + return @() + } + + # Step 3: Retrieve records + Write-Host " [Graph API] Retrieving records..." -ForegroundColor DarkGray + + # Retrieve records with transient retry resilience + $graphRecords = $null + $recordStart = Get-Date + $recordAttempt = 0 + $maxRecordDurationSeconds = $effectiveOutageMinutes * 60 + $retrieveInterval = 4 + while (-not $graphRecords) { + $recordAttempt++ + if (((Get-Date) - $recordStart).TotalSeconds -ge $maxRecordDurationSeconds) { + Write-Host " [NET] Record retrieval aborted after $effectiveOutageMinutes minutes of network instability" -ForegroundColor Red + break + } + try { + # Graph API: MaxRecords=0 (unlimited) - 10K limit only applies to EOM mode + $graphRecords = Get-GraphAuditRecords -QueryId $queryId -MaxRecords 0 -ErrorAction Stop + } + catch { + $err = $_.Exception.Message + if ($transientPatterns | Where-Object { $err.ToLower().Contains($_) }) { + Write-Host " [NET] Transient record fetch issue (attempt $recordAttempt, elapsed $([Math]::Round(((Get-Date)-$recordStart).TotalMinutes,2))m): $err" -ForegroundColor Yellow + $retrieveInterval = [Math]::Min(90, [Math]::Round($retrieveInterval * 1.5 + (Get-Random -Minimum 1 -Maximum 5))) + Start-Sleep -Seconds $retrieveInterval + continue + } else { + Write-Host " [Graph API] Non-transient record fetch error: $err" -ForegroundColor Red + break + } + } + } + if (-not $graphRecords) { Write-Host " [NET] Retrieval failed after extended retry window" -ForegroundColor Red } + + if (-not $graphRecords -or $graphRecords.Count -eq 0) { + Write-Host " [Graph API] No records returned" -ForegroundColor Gray + return @() + } + + Write-Host " [Graph API] Retrieved $($graphRecords.Count) records, normalizing..." -ForegroundColor DarkGray + + # Step 4: Normalize to EOM-compatible schema + $normalized = @() + foreach ($record in $graphRecords) { + $normalizedRecord = ConvertFrom-GraphAuditRecord -GraphRecord $record + if ($normalizedRecord) { + $normalized += $normalizedRecord + } + } + + # Filter by UserIds if specified (Graph API doesn't support UPN filtering in query) + if ($UserIds -and $UserIds.Count -gt 0 -and $normalized.Count -gt 0) { + Write-Host " [Graph API] Applying client-side UserIds filter..." -ForegroundColor DarkGray + $beforeFilter = $normalized.Count + $normalized = $normalized | Where-Object { $UserIds -contains $_.UserIds } + Write-Host " [Graph API] Filtered: $beforeFilter → $($normalized.Count) records" -ForegroundColor DarkGray + } + + Write-Host " [Graph API] Normalization complete: $($normalized.Count) records ready" -ForegroundColor Green + + return $normalized + } + catch { + Write-Host " [Graph API] Query error: $($_.Exception.Message)" -ForegroundColor Red + Write-Host " [Graph API] Falling back to empty result set" -ForegroundColor Yellow + return @() + } + } +} + +# ============================================== +# DUAL-MODE DISCONNECTION FUNCTION +# ============================================== +# Unified disconnection for both EOM and Graph API modes + +function Disconnect-PurviewAudit { + <# + .SYNOPSIS + Disconnects from either Exchange Online or Microsoft Graph. + + .DESCRIPTION + Cleanly disconnects active sessions based on mode. + + EOM Mode: + - Calls Disconnect-ExchangeOnline + - No confirmation prompt + + Graph API Mode: + - Calls Disconnect-MgGraph + - Clears Graph context + + .PARAMETER UseEOMMode + If true, disconnect from EOM. If false, disconnect from Graph. + + .OUTPUTS + None + #> + + param( + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Disconnect-ExchangeOnline + # ======================================== + + try { + Write-LogHost "Disconnecting from Exchange Online..." -ForegroundColor Gray + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction Stop | Out-Null + Write-LogHost " Disconnected from Exchange Online" -ForegroundColor Green + } + catch { + # Silently handle - may not be connected or already disconnected + Write-LogHost " (Exchange Online disconnection skipped or already disconnected)" -ForegroundColor DarkGray + } + } + else { + # ======================================== + # GRAPH API MODE: Disconnect-MgGraph + # ======================================== + + try { + # Check if connected first + $context = Get-MgContext -ErrorAction SilentlyContinue + + if ($context) { + Write-LogHost "Disconnecting from Microsoft Graph..." -ForegroundColor Gray + Disconnect-MgGraph -ErrorAction Stop | Out-Null + Write-LogHost " Disconnected from Microsoft Graph" -ForegroundColor Green + } + else { + Write-LogHost " (Not connected to Microsoft Graph)" -ForegroundColor DarkGray + } + } + catch { + # Silently handle - may not be connected or already disconnected + Write-LogHost " (Microsoft Graph disconnection skipped or already disconnected)" -ForegroundColor DarkGray + } + } +} + +# Pre-query diagnostic: verify Search-UnifiedAuditLog availability & likely permission coverage. +# NOTE: This function is now wrapped by Test-PurviewAuditCapability for EOM mode compatibility +function Invoke-AuditCapabilityDiagnostics { + param() + if ($SkipDiagnostics) { return $true } + $cmd = Get-Command Search-UnifiedAuditLog -ErrorAction SilentlyContinue + if (-not $cmd) { + Write-LogHost "DIAGNOSTIC: 'Search-UnifiedAuditLog' cmdlet not found in this session." -ForegroundColor Red + Write-LogHost "Guidance: Ensure ExchangeOnlineManagement module (v3+) is installed and imported. Try: Install-Module ExchangeOnlineManagement -Scope CurrentUser" -ForegroundColor Yellow + Write-LogHost "Role Requirements: Membership in 'Audit Logs' (preferred) or 'View-Only Audit Logs' / appropriate Compliance role group." -ForegroundColor Yellow + return $false + } + # Attempt a minimal, very narrow harmless probe query (empty expected results) + try { + $now = (Get-Date).ToUniversalTime() + $probeStart = $now.AddMinutes(-7) + $probeEnd = $now.AddMinutes(-6) + # Use an operation that is unlikely to appear but valid syntactically + $null = Search-UnifiedAuditLog -StartDate $probeStart -EndDate $probeEnd -Operations 'UserLoggedIn' -ResultSize 1 -ErrorAction Stop + Write-LogHost "Diagnostics: Audit search cmdlet available (probe succeeded/no error)." -ForegroundColor DarkGray + return $true + } + catch { + $msg = $_.Exception.Message + Write-LogHost "DIAGNOSTIC: Probe audit search failed: $msg" -ForegroundColor Yellow + if ($msg -match 'is not within the current user' -or $msg -match 'Access denied' -or $msg -match 'not authorized' -or $msg -match 'insufficient') { + Write-LogHost "Likely Missing Roles: Add the account to 'Audit Logs' (Microsoft Purview) or at minimum 'View-Only Audit Logs'." -ForegroundColor Red + } + elseif ($msg -match 'The term .*Search-UnifiedAuditLog.* is not recognized') { + Write-LogHost "Module Issue: Cmdlet not loaded. Import-Module ExchangeOnlineManagement or update module version." -ForegroundColor Red + } + else { + Write-LogHost "General Guidance: Ensure Unified Audit Log is enabled tenant-wide & correct role assignments are in place." -ForegroundColor Yellow + } + return $false + } +} + +function Invoke-SearchUnifiedAuditLogWithRetry { + <# + Provides pagination & early 10K detection. + Adjustments: + * Honors $PacingMs but leaves adaptive / circuit breaker to caller. + * Maintains metrics.PagesFetched & global limit flags used by higher layers. + #> + param( + [Parameter(Mandatory)][datetime]$Start, + [Parameter(Mandatory)][datetime]$End, + [Parameter(Mandatory)][string]$Operation, + [Parameter(Mandatory)][int]$ResultSize, + [string[]]$UserIds, + [int]$MaxRetries = 3, + [bool]$AutoSubdivide = $true + ) + + $script:Hit10KLimit = $false + $script:LimitTimeWindow = "" + $allResults = New-Object System.Collections.ArrayList + $totalFetched = 0 + $pageNumber = 1 + $maxPages = 50 + $pageSize = [Math]::Min($ResultSize, 5000) + $useSessionPagination = $ResultSize -gt 5000 + $sessionId = if ($useSessionPagination) { [guid]::NewGuid().ToString() } else { $null } + + Write-LogHost (" Using {0} pagination (page size {1})" -f ($(if ($useSessionPagination){'session'} else {'standard'}), $pageSize)) -ForegroundColor DarkCyan + + try { + while ($totalFetched -lt $ResultSize -and $pageNumber -le $maxPages) { + $remainingNeeded = $ResultSize - $totalFetched + $currentPageSize = [Math]::Min($pageSize, $remainingNeeded) + $attempt = 0; $pageResults = $null + while ($attempt -le $MaxRetries) { + try { + $params = @{ StartDate = $Start; EndDate = $End; Operations = $Operation; ResultSize = $currentPageSize; ErrorAction = 'Stop' } + if ($UserIds) { $params['UserIds'] = $UserIds } + if ($useSessionPagination) { + $params['SessionId'] = $sessionId + $params['SessionCommand'] = if ($pageNumber -eq 1) { 'ReturnLargeSet' } else { 'ReturnNextPreviewPage' } + } + if ($PacingMs -gt 0) { Start-Sleep -Milliseconds $PacingMs } + if ($attempt -gt 0) { Write-LogHost " Retrying page $pageNumber (attempt $($attempt+1))" -ForegroundColor Yellow } + $pageResults = Search-UnifiedAuditLog @params + break + } + catch { + $attempt++ + if ($attempt -le $MaxRetries) { + $delay = [Math]::Min(30, [Math]::Pow(2, $attempt)) + Write-LogHost " Page $pageNumber failed: $($_.Exception.Message). Backoff ${delay}s" -ForegroundColor DarkYellow + Start-Sleep -Seconds $delay + if ($useSessionPagination -and $attempt -gt 1) { $sessionId = [guid]::NewGuid().ToString(); Write-LogHost " New session id for retry: $sessionId" -ForegroundColor DarkGray } + } else { + Write-LogHost " Page $pageNumber permanently failed after $attempt attempts" -ForegroundColor Red + throw + } + } + } + + if ($pageResults -and $pageResults.Count -gt 0) { + # Early 10K detection (first page result count meta) + if ($pageNumber -eq 1 -and $AutoSubdivide) { + try { + $est = $pageResults[0].ResultCount + if ($null -ne $est -and $est -ge 10000) { + Write-LogHost " [!] Estimated >=10K records in window – consider subdivision" -ForegroundColor Yellow + } + } catch {} + } + # Safe add - handle both array and single object returns + if ($pageResults -is [Array]) { + foreach ($item in $pageResults) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($pageResults) + } + $totalFetched += $pageResults.Count + # Hard stop enforcement: never return more than requested -ResultSize + if ($totalFetched -ge $ResultSize) { + if ($totalFetched -gt $ResultSize) { + $excess = $totalFetched - $ResultSize + # Trim excess items from tail + for ($trim = 0; $trim -lt $excess; $trim++) { [void]$allResults.RemoveAt($allResults.Count - 1) } + $totalFetched = $ResultSize + } + Write-LogHost " Requested result size $ResultSize reached (cumulative: $totalFetched) – stopping" -ForegroundColor DarkCyan + try { $script:metrics.PagesFetched += 1 } catch {} + break + } + try { $script:metrics.PagesFetched += 1 } catch {} + Write-LogHost " Page $pageNumber returned $($pageResults.Count) (cumulative: $totalFetched)" -ForegroundColor DarkCyan + if ($pageResults.Count -lt $currentPageSize) { break } + if ($totalFetched -ge 10000) { + $script:Hit10KLimit = $true + $script:LimitTimeWindow = "$(($Start).ToString('yyyy-MM-dd HH:mm')) to $(($End).ToString('yyyy-MM-dd HH:mm'))" + # SMART SUBDIVISION for EOM: Analyze timestamp distribution + if ($AutoSubdivide -and $allResults.Count -ge 10000) { + try { + $timestamps = @() + foreach ($rec in $allResults) { + if ($rec.CreationDate) { + $ts = script:Parse-DateSafe $rec.CreationDate; if ($ts) { $timestamps += $ts } + } + } + if ($timestamps.Count -gt 100) { + $sorted = $timestamps | Sort-Object + $coveredHours = ($sorted[-1] - $sorted[0]).TotalHours + $totalHours = ($End - $Start).TotalHours + if ($coveredHours -gt 0 -and $coveredHours -lt $totalHours) { + $recordsPerHour = 10000 / $coveredHours + $targetHours = 8000 / $recordsPerHour + $subdivFactor = [Math]::Max(2, [Math]::Ceiling($totalHours / $targetHours)) + Write-LogHost " [SMART SUBDIVISION] EOM: $([Math]::Round($coveredHours,2))h of $([Math]::Round($totalHours,2))h → suggest dividing by $subdivFactor" -ForegroundColor Cyan + } + } + } catch {} + } + Write-LogHost " 10K server limit reached in this window" -ForegroundColor Yellow + break + } + } else { + Write-LogHost " Page $pageNumber empty – stopping" -ForegroundColor DarkCyan + break + } + $pageNumber++ + } + + if ($pageNumber -gt $maxPages) { + Write-LogHost " Reached max page limit ($maxPages)" -ForegroundColor Yellow + } + Write-LogHost " Pagination complete: $($allResults.Count) records" -ForegroundColor Green + return $allResults.ToArray() + } + catch { + Write-LogHost " Pagination failed: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +# Wrapper for main processing (kept minimal for clarity) +function Invoke-PAXProcessingCore { + param() + try { + # Existing core logic already executed above in previous top-level scope. + # This wrapper intentionally left minimal to avoid structural parse issues. + } + catch { + Write-LogHost "Core processing error: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +$script:adaptiveThroughputBaseline = $null +$script:adaptiveLowLatencyStreak = 0 +$script:consecutiveBlockFailures = 0 +$script:circuitBreakerOpen = $false +$script:circuitBreakerOpenUntil = $null + +function Get-BackoffDelaySeconds { + param( + [Parameter(Mandatory)][int]$Attempt, + [Parameter(Mandatory)][double]$BaseSeconds, + [Parameter(Mandatory)][int]$MaxSeconds + ) + if ($Attempt -lt 1) { return 0 } + $raw = $BaseSeconds * [math]::Pow(2, ($Attempt - 1)) + return [math]::Min($MaxSeconds, $raw) +} + +function Test-CircuitBreakerTrip { + param( + [Parameter(Mandatory)][int]$ConsecutiveFailures, + [Parameter(Mandatory)][int]$Threshold + ) + return ($ConsecutiveFailures -ge $Threshold) +} + +$JsonDepth = 60 +$FlatDepthStandard = 6 +$FlatDepthDeep = 120 +$ExplosionPerRecordRowCap = 1000 +$script:TenantPrimaryDomain = $null +if (-not $script:TenantId) { $script:TenantId = $null } +$script:TenantIndicators = @() +$ForcedRawInputCsvExplosion = $false + +# Auth config storage for token refresh (AppRegistration mode) +$script:AuthConfig = @{ + Method = $null + TenantId = $null + ClientId = $null + ClientSecret = $null # SecureString + CertThumbprint = $null + CertPath = $null + CertPassword = $null # SecureString + CertStoreLocation = 'CurrentUser' + TokenIssueTime = $null + CanReauthenticate = $false +} + +# Shared auth state for thread job token refresh (synchronized hashtable for cross-thread access) +# Thread jobs read Token from this hashtable; main thread updates it proactively before expiry +$script:SharedAuthState = [hashtable]::Synchronized(@{ + Token = $null + ExpiresOn = $null + LastRefresh = $null + RefreshCount = 0 + AuthMethod = $null +}) + +# Azure (Fabric/OneLake) storage-audience token state. Independent of Graph because +# OneLake DFS uses a separate audience (https://storage.azure.com/) and a separate SDK +# (Az.Accounts), with its own MSAL token cache. Mirrors $script:SharedAuthState shape so +# the refresh code paths read uniformly. Single-thread access (uploads run on main thread), +# so a regular hashtable is sufficient. +$script:AzAuthState = @{ + Token = $null + ExpiresOn = $null # UTC DateTime + AcquiredAt = $null # local DateTime, used for age-cap proactive refresh + LastRefresh = $null + LastRefreshAttempt = $null # cooldown anchor + RefreshCount = 0 + AuthMethod = $null # 'ManagedIdentity' | 'AppRegistration' | 'Interactive' +} + +# Checkpoint/Resume state for long-running operations +$script:CheckpointPath = $null # Path to checkpoint JSON file +$script:CheckpointData = $null # Loaded/active checkpoint object +$script:IsResumeMode = $false # Whether we're resuming from checkpoint +$script:PartialOutputPath = $null # Path to _PARTIAL.csv file during execution +$script:OriginallySkippedPartitionIndices = @() # Partition indices that were already completed before this run (for resume mode) +$script:StreamingMergeDuplicatesSkipped = 0 # Count of duplicate records removed during streaming merge +$script:StreamingMergeDataLoss = $false # Whether streaming merge detected missing partition data + +# Token expiration detection (reactive - triggers on 401 Unauthorized) +$script:TokenAcquiredTime = $null # When current token was obtained +$script:AuthFailureDetected = $false # Set to $true when 401 error detected - triggers reauth prompt +$script:Auth401MessageShown = $false # Suppresses duplicate 401 error messages (reset after successful reauth) +$script:AuthPromptInProgress = $false # Debounce flag - prevents multiple auth prompts from triggering simultaneously + +# PowerShell version detection for parallel processing features +$script:IsPS7 = ($PSVersionTable.PSVersion.Major -ge 7) + +if ($RAWInputCSV) { $ForcedRawInputCsvExplosion = $true } + +$script:RegexTrueFalse = [regex]::new('^(?i:true|false)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:RegexYes1 = [regex]::new('^(?i:yes|1)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:RegexNo0 = [regex]::new('^(?i:no|0)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:LocaleDateParsingNotified = $false + +function script:Parse-DateSafe { + <# + .SYNOPSIS + Culture-invariant date parsing that handles Purview API date formats. + .DESCRIPTION + Purview API returns dates in US format (M/d/yyyy HH:mm:ss) regardless of client locale. + This function safely parses such dates on systems with non-US regional settings (e.g., UK). + #> + param([Parameter(Mandatory=$false)][AllowNull()][AllowEmptyString()]$DateValue) + + # Log once when running under non-US locale + if (-not $script:LocaleDateParsingNotified) { + $script:LocaleDateParsingNotified = $true + $currentCulture = [System.Threading.Thread]::CurrentThread.CurrentCulture.Name + if ($currentCulture -and $currentCulture -ne 'en-US') { + Write-LogHost " [DATE] Locale-aware date parsing active (Culture: $currentCulture)" -ForegroundColor DarkCyan + } + } + + # Already a DateTime? Return as-is + if ($DateValue -is [datetime]) { return $DateValue } + + # Null or empty? Return null + if ([string]::IsNullOrWhiteSpace($DateValue)) { return $null } + + $dateStr = [string]$DateValue + + # Try ISO 8601 formats first (most common from properly-formatted API responses) + $isoFormats = @( + 'yyyy-MM-ddTHH:mm:ss.fffffffK', + 'yyyy-MM-ddTHH:mm:ss.fffK', + 'yyyy-MM-ddTHH:mm:ssK', + 'yyyy-MM-ddTHH:mm:ss.fffffff', + 'yyyy-MM-ddTHH:mm:ss.fffZ', + 'yyyy-MM-ddTHH:mm:ssZ', + 'yyyy-MM-ddTHH:mm:ss.fff', + 'yyyy-MM-ddTHH:mm:ss', + 'yyyy-MM-dd HH:mm:ss.fff', + 'yyyy-MM-dd HH:mm:ss', + 'yyyy-MM-dd' + ) + + foreach ($fmt in $isoFormats) { + try { + return [datetime]::ParseExact($dateStr, $fmt, [System.Globalization.CultureInfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::AdjustToUniversal) + } + catch { } + } + + # Try US formats explicitly (what Purview actually returns - causes UK locale issues) + $usFormats = @( + 'M/d/yyyy HH:mm:ss', + 'M/d/yyyy h:mm:ss tt', + 'M/d/yyyy H:mm:ss', + 'MM/dd/yyyy HH:mm:ss', + 'M/d/yyyy', + 'MM/dd/yyyy' + ) + + foreach ($fmt in $usFormats) { + try { + return [datetime]::ParseExact($dateStr, $fmt, [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { } + } + + # Last resort: use InvariantCulture with Parse + try { + return [datetime]::Parse($dateStr, [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { + return $null + } +} + +function script:Format-DatePurviewFast($dt) { + if (-not $dt) { return '' } + try { + if ($dt -is [datetime]) { + return $dt.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + } + else { + $p = script:Parse-DateSafe $dt + if ($null -eq $p) { return '' } + return $p.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + } + } + catch { return '' } +} + +function script:BoolTFFast($v) { + if ($null -eq $v) { return '' } + if ($v -is [bool]) { return $v.ToString().ToUpper() } + $vStr = [string]$v + if ($script:RegexTrueFalse.IsMatch($vStr)) { return $vStr.ToUpper() } + if ($script:RegexYes1.IsMatch($vStr)) { return 'TRUE' } + if ($script:RegexNo0.IsMatch($vStr)) { return 'FALSE' } + return $vStr +} + +# Apply FlatDepth override (if provided) +try { + if ($PSBoundParameters.ContainsKey('FlatDepth')) { + $FlatDepthDeep = $FlatDepth + $FlatDepthStandard = [int][Math]::Min($FlatDepth, $FlatDepthStandard) + } +} catch {} + +function script:ToJsonIfObjectFast($v) { + if ($null -eq $v) { return '' } + if (Test-ScalarValue $v) { return $v } + try { return ($v | ConvertTo-Json -Depth $JsonDepth -Compress) } + catch { return [string]$v } +} + +function script:GetArrayFast($parent, [string]$name) { + $val = Get-SafeProperty $parent $name + if ($null -eq $val) { return @() } + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + return @($val) + } + return @($val) +} + +$effectiveExplodeForProgress = ($ExplodeDeep -or $ExplodeArrays -or $ForcedRawInputCsvExplosion) + +# MEMORY MANAGEMENT: Resolve MaxMemoryMB (-1 = auto 75% of system RAM, 0 = disabled, >0 = explicit limit) +$script:ResolvedMaxMemoryMB = $MaxMemoryMB +if ($MaxMemoryMB -eq -1) { + # Auto-detect: use 75% of total physical memory + try { + $totalRAM = [math]::Round((Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue).TotalPhysicalMemory / 1MB, 0) + $script:ResolvedMaxMemoryMB = [math]::Round($totalRAM * 0.75, 0) + Write-LogHost "Memory management: Auto-detected ${totalRAM}MB total RAM -> limit set to $($script:ResolvedMaxMemoryMB)MB (75%)" -ForegroundColor Cyan + } catch { + # Fallback if CIM fails (e.g., Linux/macOS) + $script:ResolvedMaxMemoryMB = 4096 + Write-LogHost "Memory management: Could not detect system RAM, defaulting to 4096MB limit" -ForegroundColor Yellow + } +} elseif ($MaxMemoryMB -eq 0) { + $script:ResolvedMaxMemoryMB = 0 + Write-LogHost "Memory management: DISABLED (-MaxMemoryMB 0)" -ForegroundColor DarkGray +} + +# Memory flush mode: enabled when ResolvedMaxMemoryMB > 0 AND explosion is disabled (explosion needs full $allLogs in memory) +$script:memoryFlushEnabled = ($script:ResolvedMaxMemoryMB -gt 0) -and (-not $ExplodeDeep) -and (-not $ExplodeArrays) -and (-not $ForcedRawInputCsvExplosion) +$script:memoryFlushed = $false # Track if we've flushed $allLogs during this run (affects export path) +$enableParallelSwitchUsed = $EnableParallel.IsPresent +if ($enableParallelSwitchUsed) { $ParallelMode = 'On' } + +function Get-ParallelActivationDecision { + param( + [array]$QueryPlan, + [string]$ParallelMode, + [int]$MaxParallelGroups, + [int]$MaxConcurrency + ) + $ps7 = ($PSVersionTable.PSVersion.Major -ge 7) + $totalGroups = $QueryPlan.Count + $totalActivities = ($QueryPlan | ForEach-Object { $_.Activities.Count } | Measure-Object -Sum).Sum + # Auto parallel eligibility heuristic: allow auto parallel when there's at least one group + # AND either >1 group OR a single group whose planned concurrency would yield >1 partition. + # (Single-activity multi-partition scenarios — e.g., CopilotInteraction with 3 partitions — + # also benefit from parallelism.) + $singleGroupMultiPartition = ($totalGroups -eq 1) -and ($QueryPlan[0].Concurrency -gt 1) + $autoEligible = $ps7 -and ($MaxParallelGroups -gt 0) -and ($MaxConcurrency -gt 1) -and ($totalGroups -ge 1) -and (($totalGroups -gt 1) -or $singleGroupMultiPartition) + + switch ($ParallelMode) { + 'On' { + return @{ Enabled = ($ps7 -and $MaxParallelGroups -gt 0 -and $MaxConcurrency -gt 0); Reason = if ($ps7) { 'Forced On' } else { 'PS < 7 (cannot parallel)' }; AutoEligible = $autoEligible } + } + 'Auto' { + return @{ Enabled = $autoEligible; Reason = if ($autoEligible) { 'Auto criteria met' } else { 'Auto criteria not met' }; AutoEligible = $autoEligible } + } + default { + return @{ Enabled = $false; Reason = 'Mode Off'; AutoEligible = $autoEligible } + } + } +} + +$weights = if ($effectiveExplodeForProgress) { @{ Query = 0.30; Explosion = 0.60; Export = 0.10 } } else { @{ Query = 0.80; Explosion = 0.00; Export = 0.20 } } +if ($RAWInputCSV) { + try { + $weights = @{ Parsing = 0.10; Query = 0.0; Explosion = 0.80; Export = 0.10 } + } + catch {} +} +$script:originalWeights = $weights.Clone() +$script:progressState = @{ Weights = $weights; Phase = 'Query'; Parsing = @{Current = 0; Total = 0 }; Query = @{Current = 0; Total = 0 }; Explode = @{Current = 0; Total = 0 }; Export = @{Current = 0; Total = 1 } } +function Set-ProgressPhase { param([ValidateSet('Parsing', 'Query', 'Explosion', 'Export', 'Complete')] [string]$Phase, [string]$Status = ''); $script:progressState.Phase = $Phase; Update-Progress -Status $Status } +function Update-Progress { + param( + [string]$Status = '', + [int]$BatchCurrent = 0, + [int]$BatchTotal = 0, + [int]$BatchRangeStart = 0, + [int]$BatchRangeEnd = 0, + [int]$BatchStartPercent = 0, + [int]$BatchEndPercent = 0, + [bool]$BatchTotalIsEstimate = $false + ) + $w = $script:progressState.Weights; $ps = $script:progressState.Parsing; $qs = $script:progressState.Query; $es = $script:progressState.Explode; $xs = $script:progressState.Export + $pPct = if ($ps.Total -gt 0 -and $w.ContainsKey('Parsing') -and $w.Parsing -gt 0) { [double]$ps.Current / [double]$ps.Total } else { 0.0 } + $qPct = if ($qs.Total -gt 0) { [double]$qs.Current / [double]$qs.Total } else { 0.0 } + $ePct = if ($es.Total -gt 0 -and $w.Explosion -gt 0) { [double]$es.Current / [double]$es.Total } else { 0.0 } + $xPct = if ($xs.Total -gt 0) { [double]$xs.Current / [double]$xs.Total } else { 0.0 } + # Zero-record weighting: emphasize Query progression when no records retrieved yet + if ($script:progressState.Phase -eq 'Query' -and ($script:metrics.TotalRecordsFetched -eq 0)) { + $w.Query = 1.0; $w.Explosion = 0.0; $w.Export = 0.0; if ($w.ContainsKey('Parsing')) { $w.Parsing = 0.0 } + } + # Restoration: Once at least one record has been fetched, revert weights if they were temporarily overridden. + elseif ($script:progressState.Phase -eq 'Query' -and ($script:metrics.TotalRecordsFetched -gt 0)) { + if ($script:originalWeights -and $w.Query -eq 1.0 -and $w.Explosion -eq 0.0 -and $w.Export -eq 0.0) { + foreach ($key in $script:originalWeights.Keys) { $w[$key] = $script:originalWeights[$key] } + } + } + # Calculate phase-specific progress details + $phase = $script:progressState.Phase + $pDetail = if ($w.ContainsKey('Parsing') -and $w.Parsing -gt 0 -and $ps.Total -gt 0) { "{0}/{1}({2}%)" -f $ps.Current, $ps.Total, ([int]([Math]::Round($pPct * 100))) } else { '' } + $qDetail = if ($w.Query -gt 0 -and $qs.Total -gt 0) { "{0}/{1}({2}%)" -f $qs.Current, $qs.Total, ([int]([Math]::Round($qPct * 100))) } else { '' } + if ($BatchRangeStart -ge 1 -and $BatchRangeEnd -ge 1 -and $es.Total -gt 0) { + if ($BatchStartPercent -ge 0 -and $BatchEndPercent -gt 0) { + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = if ($BatchTotal -ge 1) { " Batch: {0}/{1}({2}%-{3}%)" -f $BatchCurrent, $batchTotalDisplay, $BatchStartPercent, $BatchEndPercent } else { '' } + } + else { + $batchPct = if ($BatchTotal -gt 0 -and $BatchCurrent -gt 0) { [int]([Math]::Round(([double]$BatchCurrent / [double]$BatchTotal) * 100)) } else { 0 } + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = if ($BatchTotal -ge 1) { " Batch: {0}/{1}({2}%)" -f $BatchCurrent, $batchTotalDisplay, $batchPct } else { '' } + } + $explosionCounts = "Records {0}-{1}/{2}{3}" -f $BatchRangeStart, $BatchRangeEnd, $es.Total, $batchInfo + } + elseif ($BatchTotal -ge 1) { + $batchPct = if ($BatchTotal -gt 0 -and $BatchCurrent -gt 0) { [int]([Math]::Round(([double]$BatchCurrent / [double]$BatchTotal) * 100)) } else { 0 } + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = " Batch: {0}/{1}({2}%)" -f $BatchCurrent, $batchTotalDisplay, $batchPct + $explosionCounts = if ($es.Total -gt 0) { "Records {0}/{1}{2}" -f $es.Current, $es.Total, $batchInfo } else { "0/0" } + } + else { + $explosionCounts = if ($es.Total -gt 0) { "{0}/{1}({2}%)" -f $es.Current, $es.Total, ([int]([Math]::Round($ePct * 100))) } else { '0/0' } + } + $eDetail = if ($w.Explosion -gt 0) { + if ($phase -eq 'Explosion') { + " | $explosionCounts" + } + else { + " | Explosion: $explosionCounts" + } + } + else { '' } + $batchDetail = '' + $xDetail = if ($xs.Total -gt 0) { " | Export: {0}/{1}({2}%)" -f $xs.Current, $xs.Total, ([int]([Math]::Round($xPct * 100))) } else { ' | Export: 0/0' } + $parsingLabel = 'Pre-parsing JSON' + if (($AgentId -or $AgentsOnly -or $ExcludeAgents -or $PromptFilter) -and $phase -eq 'Parsing') { + $parsingLabel = 'Pre-parsing + Filtering' + } + $phasePrefix = switch ($phase) { 'Parsing' { $parsingLabel } 'Query' { 'Query' } 'Explosion' { 'Explosion' } 'Export' { 'Export' } 'Complete' { 'Complete' } default { $phase } } + if ($phase -eq 'Parsing' -and $pDetail) { + $composite = "${phasePrefix}: $pDetail$eDetail$batchDetail$xDetail" + } + elseif ($phase -eq 'Explosion' -and -not $qDetail) { + $composite = "Explosion: $explosionCounts$batchDetail$xDetail" + } + else { + $composite = if ($qDetail) { "${phasePrefix}: $qDetail$eDetail$batchDetail$xDetail" } else { "${phasePrefix}:$eDetail$batchDetail$xDetail" } + } + $statusText = if ($Status) { "$Status :: $composite" } else { $composite } + if ($statusText.Length -gt 180) { $statusText = $statusText.Substring(0, 177) + '...' } + # Placeholder for progress display compatibility +} +function Complete-Progress { + # Placeholder for progress display compatibility +} + +# Lightweight explicit progress tick to ensure visual movement in long zero-record scenarios. +function Write-ProgressTick { + # Placeholder for progress display compatibility +} + +$script:learnedActivityBlockSize = @{} +$script:globalLearnedBlockSize = $BlockHours + +function Get-QueryPlan { + param([string[]]$RequestedActivities) + # Normalize and deduplicate (DSPM logic already handled fallback, so no default here) + $normalized = @(); foreach ($a in $RequestedActivities) { if ($a -and -not ($normalized -contains $a)) { $normalized += $a } } + # If still empty after DSPM logic, something went wrong - but DSPM validation should prevent this + if ($normalized.Count -eq 0) { + Write-Host "ERROR: No activity types provided to Get-QueryPlan. This should not happen after DSPM validation." -ForegroundColor Red + exit 1 + } + $plan = @(); $i = 0 + + # DUAL-MODE QUERY PLANNING: + # Graph API mode: Combine all activity types into single group (Graph API accepts multiple operationFilters) + # EOM mode: Separate groups per activity type (Search-UnifiedAuditLog performs better with single activity) + if (-not $UseEOM) { + # Graph API mode: Single group with all activities combined + $plan += @{ + Name = "Combined: $($normalized -join ', ')"; + Group = 'GraphCombined'; + Activities = $normalized; + Concurrency = $MaxConcurrency + } + } + else { + # EOM mode: One group per activity type (sequential processing) + foreach ($a in $normalized) { + $i++ + $plan += @{ + Name = "Activity: $a"; + Group = 'EOM_Sequential'; + Activities = @($a); + Concurrency = $MaxConcurrency + } + } + } + return $plan +} + +function Update-LearnedBlockSize { + param([string]$ActivityType, [double]$BlockHours, [int]$RecordCount, [bool]$Success) + if ($Success) { + if ($RecordCount -eq $ResultSize) { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + $script:globalLearnedBlockSize = [Math]::Min($script:globalLearnedBlockSize, $newSize) + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h due to limit hit" -ForegroundColor Magenta + } + elseif ($RecordCount -gt ($ResultSize * 0.8)) { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.7) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h (high volume: $RecordCount records)" -ForegroundColor Magenta + } + elseif ($RecordCount -lt ($ResultSize * 0.1)) { + $newSize = [Math]::Min(24.0, $BlockHours * 1.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Increasing block size to $([math]::Round($newSize,2))h (low volume: $RecordCount records)" -ForegroundColor Magenta + } + elseif ($RecordCount -lt ($ResultSize * 0.05)) { + $newSize = [Math]::Min(24.0, $BlockHours * 2.0) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Increasing block size to $([math]::Round($newSize,2))h (very low volume: $RecordCount records)" -ForegroundColor Magenta + } + } else { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + $script:globalLearnedBlockSize = [Math]::Min($script:globalLearnedBlockSize, $newSize) + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h due to failure" -ForegroundColor Magenta + } +} +function Get-NextSmallerBlockSize { param([double]$CurrentSize) return [Math]::Max(0.001389, $CurrentSize / 2) } # Min 2 minutes + +function Get-OptimalBlockSize { param([string]$ActivityType) if ($script:learnedActivityBlockSize.ContainsKey($ActivityType)) { return $script:learnedActivityBlockSize[$ActivityType] } elseif ($script:globalLearnedBlockSize -ne $BlockHours) { return $script:globalLearnedBlockSize } else { return $BlockHours } } + +function Invoke-ActivityTimeWindowProcessing { + param( + [Parameter(Mandatory = $true)][string]$ActivityType, + [Parameter(Mandatory = $true)][datetime]$StartDate, + [Parameter(Mandatory = $true)][datetime]$EndDate, + [int]$PartitionIndex = 1, + [int]$TotalPartitions = 1, + [bool]$UseEOMMode = $false + ) + + Write-Host "Processing $ActivityType (partition $PartitionIndex/$TotalPartitions) from $($StartDate.ToString('yyyy-MM-dd HH:mm')) to $($EndDate.ToString('yyyy-MM-dd HH:mm'))..." -ForegroundColor White + $blockHours = Get-OptimalBlockSize -ActivityType $ActivityType + Write-Host " Using initial block size: $blockHours hours" -ForegroundColor DarkCyan + + $allResults = New-Object System.Collections.ArrayList + $current = $StartDate + $blockNumber = 1 + + while ($current -lt $EndDate) { + # Show progress BEFORE block processing to ensure visibility (before log output clears it) + Write-ProgressTick + + if ($script:circuitBreakerOpen) { + if ($script:circuitBreakerOpenUntil -and (Get-Date) -lt $script:circuitBreakerOpenUntil) { + Write-LogHost " Circuit breaker OPEN until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')) – skipping remaining blocks for $ActivityType" -ForegroundColor Red + break + } else { + $script:circuitBreakerOpen = $false + $script:consecutiveBlockFailures = 0 + Write-LogHost " Circuit breaker cooldown elapsed – resuming block processing" -ForegroundColor DarkGreen + } + } + if ($script:learnedActivityBlockSize.ContainsKey($ActivityType)) { + $blockHours = $script:learnedActivityBlockSize[$ActivityType] + } + + $blockEnd = $current.AddHours($blockHours) + if ($blockEnd -gt $EndDate) { $blockEnd = $EndDate } + + $actualBlockHours = [math]::Round(($blockEnd - $current).TotalHours, 2) + Write-Host " Block $blockNumber`: $($current.ToString('yyyy-MM-dd HH:mm')) to $($blockEnd.ToString('yyyy-MM-dd HH:mm')) ($($actualBlockHours)h)" -ForegroundColor Yellow + + try { + $results = Invoke-PurviewAuditQuery -StartDate $current -EndDate $blockEnd -Operations $ActivityType -ResultSize $ResultSize -UserIds $script:targetUsers -UseEOMMode $UseEOMMode + + if ($results -and $results.Count -gt 0) { + # Safe add - handle both array and single object + if ($results -is [Array]) { + foreach ($item in $results) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($results) + } + Write-Host " Added $($results.Count) records (total: $($allResults.Count))" -ForegroundColor Green + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $actualBlockHours -RecordCount $results.Count -Success $true + $script:consecutiveBlockFailures = 0 + } + else { + Write-Host " No records found in this block" -ForegroundColor Gray + $script:consecutiveBlockFailures = 0 + } + } + catch { + Write-Host " Block failed: $($_.Exception.Message)" -ForegroundColor Red + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $actualBlockHours -RecordCount 0 -Success $false + $script:consecutiveBlockFailures++ + $attemptNum = $script:consecutiveBlockFailures + $expDelay = [math]::Min($BackoffMaxSeconds, $BackoffBaseSeconds * [math]::Pow(2, ($attemptNum - 1))) + $jitterMs = Get-Random -Minimum 150 -Maximum 750 + $totalDelaySec = [math]::Round($expDelay,2) + [math]::Round($jitterMs/1000,2) + try { $script:metrics.BackoffTotalDelaySeconds += $totalDelaySec } catch {} + Write-LogHost " Reliability: Backoff delay $([math]::Round($expDelay,2))s + jitter $([math]::Round($jitterMs/1000,2))s (attempt $attemptNum)" -ForegroundColor DarkYellow + Start-Sleep -Seconds ([int][math]::Ceiling($expDelay)) + Start-Sleep -Milliseconds $jitterMs + if ($script:consecutiveBlockFailures -ge $CircuitBreakerThreshold) { + $script:circuitBreakerOpen = $true + $script:circuitBreakerOpenUntil = (Get-Date).AddSeconds($CircuitBreakerCooldownSeconds) + try { $script:metrics.CircuitBreakerTrips++ } catch {} + Write-LogHost " CIRCUIT BREAKER TRIPPED after $script:consecutiveBlockFailures consecutive block failures – cooling down for $CircuitBreakerCooldownSeconds seconds (until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')))" -ForegroundColor Magenta + break + } + if ($blockHours -gt 0.5) { + $smallerBlockHours = Get-NextSmallerBlockSize -CurrentSize $blockHours + Write-Host " Retrying with smaller $smallerBlockHours hour block..." -ForegroundColor Yellow + + try { + $blockEnd = $current.AddHours($smallerBlockHours) + if ($blockEnd -gt $EndDate) { $blockEnd = $EndDate } + + $results = Invoke-PurviewAuditQuery -StartDate $current -EndDate $blockEnd -Operations $ActivityType -ResultSize $ResultSize -UserIds $script:targetUsers -UseEOMMode $UseEOMMode + + if ($results -and $results.Count -gt 0) { + # Safe add - handle both array and single object + if ($results -is [Array]) { + foreach ($item in $results) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($results) + } + Write-Host " Smaller block succeeded: $($results.Count) records" -ForegroundColor Green + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $smallerBlockHours -RecordCount $results.Count -Success $true + $blockHours = $smallerBlockHours + $script:consecutiveBlockFailures = 0 + } + } + catch { + Write-Host " Smaller block also failed: $($_.Exception.Message)" -ForegroundColor Red + $script:consecutiveBlockFailures++ + $attemptNum = $script:consecutiveBlockFailures + $expDelay = [math]::Min($BackoffMaxSeconds, $BackoffBaseSeconds * [math]::Pow(2, ($attemptNum - 1))) + $jitterMs = Get-Random -Minimum 150 -Maximum 750 + $totalDelaySec = [math]::Round($expDelay,2) + [math]::Round($jitterMs/1000,2) + try { $script:metrics.BackoffTotalDelaySeconds += $totalDelaySec } catch {} + Write-LogHost " Reliability: Backoff delay $([math]::Round($expDelay,2))s + jitter $([math]::Round($jitterMs/1000,2))s (attempt $attemptNum)" -ForegroundColor DarkYellow + Start-Sleep -Seconds ([int][math]::Ceiling($expDelay)) + Start-Sleep -Milliseconds $jitterMs + if ($script:consecutiveBlockFailures -ge $CircuitBreakerThreshold) { + $script:circuitBreakerOpen = $true + $script:circuitBreakerOpenUntil = (Get-Date).AddSeconds($CircuitBreakerCooldownSeconds) + try { $script:metrics.CircuitBreakerTrips++ } catch {} + Write-LogHost " CIRCUIT BREAKER TRIPPED after $script:consecutiveBlockFailures consecutive block failures – cooling down for $CircuitBreakerCooldownSeconds seconds (until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')))" -ForegroundColor Magenta + break + } + } + } + } + + try { + if ($script:progressState.Query.Current -ge $script:progressState.Query.Total) { + $script:progressState.Query.Total += 1 + } + $script:progressState.Query.Current += 1 + $script:progressBlocksCompleted = ($script:progressBlocksCompleted + 1) + $script:progressBlockHoursSum = ($script:progressBlockHoursSum + $actualBlockHours) + if ($script:progressBlocksCompleted -gt 0) { + # --- Progress Estimation Logic (multi-partition accurate) --- + # 1. Estimate remaining blocks in the CURRENT partition. + # 2. Add an estimate for yet-to-start partitions based on the average blocks/partition so far. + # 3. Enforce a monotonic (non-decreasing) Query.Total so percent cannot jump to 100% early. + # This prevents Query.Total from shrinking between partitions and reporting premature 100% + # completion when later partitions have not yet started. + $avgBlock = $script:progressBlockHoursSum / $script:progressBlocksCompleted + $elapsedHours = $script:progressBlockHoursSum + $currentPartitionRangeHours = ($EndDate - $StartDate).TotalHours + $remainingHoursCurrentPartition = [Math]::Max(0.0, $currentPartitionRangeHours - $elapsedHours) + $remainingBlocksEstCurrent = if ($avgBlock -gt 0) { [Math]::Ceiling($remainingHoursCurrentPartition / $avgBlock) } else { 0 } + $remainingPartitions = if ($TotalPartitions -gt $PartitionIndex) { $TotalPartitions - $PartitionIndex } else { 0 } + $avgBlocksPerCompletedPartition = if ($PartitionIndex -gt 0) { [double]$script:progressBlocksCompleted / [double]$PartitionIndex } else { [double]$script:progressBlocksCompleted } + $futurePartitionBlocksEst = if ($remainingPartitions -gt 0 -and $avgBlocksPerCompletedPartition -gt 0) { [int][Math]::Ceiling($avgBlocksPerCompletedPartition * $remainingPartitions) } else { 0 } + $newCalcGlobal = $script:progressBlocksCompleted + $remainingBlocksEstCurrent + $futurePartitionBlocksEst + # Apply optional smoothing but NEVER allow total to decrease (monotonic total). + if ($ProgressSmoothingAlpha -gt 0 -and $script:progressState.Query.Total -gt 0) { + $smoothed = [int]([Math]::Round(($ProgressSmoothingAlpha * $newCalcGlobal) + ((1 - $ProgressSmoothingAlpha) * $script:progressState.Query.Total))) + $newTotalCandidate = [Math]::Max($script:progressState.Query.Total, $smoothed, $newCalcGlobal) + } else { + $newTotalCandidate = [Math]::Max($script:progressState.Query.Total, $newCalcGlobal) + } + $script:progressState.Query.Total = [Math]::Max($script:progressState.Query.Total, $newTotalCandidate, $script:progressBlocksCompleted) + } + Update-Progress + # Explicit tick for visibility even if Update-Progress weighting collapses. + Write-ProgressTick + } + catch {} + + $current = $blockEnd + $blockNumber++ + } + + Write-Host " Completed $ActivityType (partition $PartitionIndex/$TotalPartitions)`: $($allResults.Count) total records" -ForegroundColor Green + return $allResults.ToArray() +} + +# Note: Write-Log and Write-LogHost are defined earlier in the script (near line 670) + +function Open-CsvWriter { + param([string]$Path, [string[]]$Columns) + $enc = New-Object System.Text.UTF8Encoding($false) + # OPTIMIZATION: Use 1MB StreamWriter buffer (default is 1KB) to reduce write syscalls + $script:PAX_CsvWriter = [System.IO.StreamWriter]::new($Path, $false, $enc, 1048576) + $escapedCols = New-Object System.Collections.Generic.List[string] + foreach ($col in $Columns) { + $c = [string]$col + $needsQuote = ($c -match '[",\r\n]') -or $c.StartsWith(' ') -or $c.EndsWith(' ') + $escaped = $c -replace '"', '""' + if ($needsQuote) { $escaped = '"' + $escaped + '"' } + $escapedCols.Add($escaped) | Out-Null + } + $script:PAX_CsvWriter.WriteLine(($escapedCols -join ',')) +} +function Close-CsvWriter { if ($script:PAX_CsvWriter) { try { $script:PAX_CsvWriter.Flush(); $script:PAX_CsvWriter.Dispose() } catch {}; Remove-Variable PAX_CsvWriter -Scope Script -ErrorAction SilentlyContinue } } +function Write-CsvRows { + param([System.Collections.IEnumerable]$Rows, [string[]]$Columns) + if (-not $Rows) { return } + if (-not $script:PAX_CsvWriter) { throw "CSV writer not initialized" } + + # Pre-compile regex once (not per-cell) for significant performance gain + $needsQuotePattern = [regex]::new('[",\r\n]', [System.Text.RegularExpressions.RegexOptions]::Compiled) + + # OPTIMIZATION: Pre-allocate larger buffer (4MB) to reduce write syscalls + $sb = New-Object System.Text.StringBuilder(4194304) + $colCount = $Columns.Count + $fieldValues = New-Object string[] $colCount # Reuse array instead of creating List per row + + # OPTIMIZATION: Build column index lookup table for O(1) access by name + # This eliminates per-cell string lookups which were the main bottleneck + $columnIndex = @{} + for ($i = 0; $i -lt $colCount; $i++) { + $columnIndex[$Columns[$i]] = $i + } + + foreach ($row in $Rows) { + if ($null -eq $row) { continue } + + # Reset field values (faster than creating new array) + for ($i = 0; $i -lt $colCount; $i++) { $fieldValues[$i] = "" } + + # For hashtables, iterate keys and map to column index (much faster than iterating columns) + if ($row -is [hashtable]) { + foreach ($key in $row.Keys) { + if ($columnIndex.ContainsKey($key)) { + $idx = $columnIndex[$key] + $val = $row[$key] + + if ($null -eq $val) { continue } + + # Handle arrays/collections + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + try { $val = ($val | ForEach-Object { if ($_ -ne $null) { [string]$_ } else { '' } }) -join ';' } catch { $val = [string]$val } + } + + $s = [string]$val + # Use pre-compiled regex and avoid method calls where possible + if ($s.Length -gt 0 -and ($needsQuotePattern.IsMatch($s) -or $s[0] -eq ' ' -or $s[$s.Length - 1] -eq ' ')) { + $s = '"' + ($s -replace '"', '""') + '"' + } + $fieldValues[$idx] = $s + } + } + } else { + # OPTIMIZED: For PSObjects, iterate only populated properties (not all columns) + # This reduces iterations from ~163 columns to ~50 actual properties per row + foreach ($prop in $row.PSObject.Properties) { + $key = $prop.Name + if (-not $columnIndex.ContainsKey($key)) { continue } + + $idx = $columnIndex[$key] + $val = $prop.Value + + if ($null -eq $val) { continue } + + # Handle arrays/collections + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + try { $val = ($val | ForEach-Object { if ($_ -ne $null) { [string]$_ } else { '' } }) -join ';' } catch { $val = [string]$val } + } + + $s = [string]$val + if ($s.Length -gt 0 -and ($needsQuotePattern.IsMatch($s) -or $s[0] -eq ' ' -or $s[$s.Length - 1] -eq ' ')) { + $s = '"' + ($s -replace '"', '""') + '"' + } + $fieldValues[$idx] = $s + } + } + + [void]$sb.AppendLine(($fieldValues -join ',')) + # Flush at 4MB (increased from 1MB to reduce write syscalls) + if ($sb.Length -gt 4194304) { + $script:PAX_CsvWriter.Write($sb.ToString()) + [void]$sb.Clear() + } + } + if ($sb.Length -gt 0) { $script:PAX_CsvWriter.Write($sb.ToString()) } +} + +function Test-AgentFilter { + param( + [Parameter(Mandatory = $true)] + $ParsedAuditData, + [string[]]$AgentIdFilter, + [bool]$AgentsOnlyFilter + ) + if (-not $AgentIdFilter -and -not $AgentsOnlyFilter) { + return $true + } + $recordAgentId = $null + try { + if ($ParsedAuditData.AgentId) { + $recordAgentId = [string]$ParsedAuditData.AgentId + } + } + catch { + return $false + } + if ($AgentsOnlyFilter) { + if ([string]::IsNullOrWhiteSpace($recordAgentId)) { + return $false + } + if (-not $AgentIdFilter) { + return $true + } + } + if ($AgentIdFilter) { + if ([string]::IsNullOrWhiteSpace($recordAgentId)) { + return $false + } + foreach ($filterId in $AgentIdFilter) { + if ($recordAgentId -eq $filterId) { + return $true + } + } + return $false + } + return $true +} + +# Ensure output directory exists +if (-not (Test-Path $OutputPath)) { New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null } + +# Generate output filename with proper extension +$fileExtension = if ($ExportWorkbook) { "xlsx" } else { "csv" } +$filePrefix = "Purview_Audit" + +# Determine initial combine mode (needed early for filename decision) +$isCsv = (-not $ExportWorkbook) +$initialCsvCombine = if ($RAWInputCSV -and $isCsv) { $true } elseif ($isCsv) { $CombineOutput.IsPresent } else { $false } + +# Predict the combined-audit CSV leaf name upfront. When the run is gated to a +# single activity type ($ActivityTypes count == 1 and -IncludeM365Usage is NOT +# expanding the set), name the file Purview_Audit_UsageActivity__.csv +# directly so the log file (derived from $OutputFile basename) matches. The +# post-export dynamic single-activity downgrade is a secondary safety net for +# runs where the resulting data happens to contain only one activity type. +$_combinedAuditLeaf = "Purview_Audit_UsageActivity_CombinedActivityTypes_$global:ScriptRunTimestamp.csv" +if ($ActivityTypes -and @($ActivityTypes).Count -eq 1 -and -not $IncludeM365Usage) { + $_onlyActivityType = [string]$ActivityTypes[0] + if (-not [string]::IsNullOrWhiteSpace($_onlyActivityType)) { + $_safeActivityType = $_onlyActivityType -replace '[\\/:*?"<>|]', '_' + $_combinedAuditLeaf = "Purview_Audit_UsageActivity_${_safeActivityType}_$global:ScriptRunTimestamp.csv" + } +} + +# Determine script mode for logging and validation +$scriptMode = if ($RAWInputCSV) { + "Replay (RAWInputCSV)" +} elseif ($ExplodeDeep) { + "Deep Column Explosion (-ExplodeDeep)" +} elseif ($ExplodeArrays -or $ForcedRawInputCsvExplosion) { + if ($ForcedRawInputCsvExplosion -and -not $ExplodeArrays.IsPresent -and -not $ExplodeDeep.IsPresent) { "Array Explosion (-ExplodeArrays, RAWInput implied)" } else { "Array Explosion (-ExplodeArrays)" } +} else { + "Standard (1:1, no explosion)" +} + +# Resolve OutputFile path +# OnlyUserInfo mode: Output file is EntraUsers_MAClicensing (log file will match) +if ($OnlyUserInfo) { + $fileExtension = if ($ExportWorkbook) { "xlsx" } else { "csv" } + $OutputFile = Join-Path $OutputPath "EntraUsers_MAClicensing_$global:ScriptRunTimestamp.$fileExtension" +} +elseif ($AppendFile) { + # User provided filename, full local path, or remote (SharePoint/Fabric) URL. + # Classify the form BEFORE doing any path join — IsPathRooted returns $false + # for "https://..." which would cause Join-Path to mangle the URL into the + # local scratch dir. The remote pre-flight downstream pulls the remote file + # into local scratch and then redirects $AppendFile to that local copy. + $appendInputTier = if (Get-Command -Name Get-PathTier -ErrorAction SilentlyContinue) { + # Suppress Get-PathTier's exit-on-invalid behaviour for relative filenames + # by short-circuiting based on form first. + if ($AppendFile -match '^https?://') { Get-PathTier $AppendFile -SwitchName 'AppendFile' } + elseif ([System.IO.Path]::IsPathRooted($AppendFile)) { 'Local' } + else { 'Local' } # relative filename → resolves under $OutputPath (Local) + } else { 'Local' } + + # Rollup-output detection. The embedded Python processors emit rollup outputs + # with one of the following suffixes on the original raw stem (the M365Bundle + # processor appends a current-run timestamp; the CopilotInteraction processor + # does not): + # CopilotInteraction processor : '_Interactions.csv' + # '_Users.csv' + # M365Bundle processor (rollup) : '_Rollup_.csv' + # '_UserStats_.csv' + # '_SessionCohort_.csv' + # '_SessionStats_.csv' + # M365Bundle processor (event) : '_Exploded.csv' + # When the user passes any such file as -AppendFile UNDER -Rollup / -RollupPlusRaw + # (or under -IncludeM365Usage event-level mode), they are pointing at the previous + # run's ROLLUP/EXPLOSION target — NOT at a raw Purview audit CSV. The raw audit + # writer must NOT be re-pointed to the rollup leaf, or it will overwrite the + # rollup target with raw audit rows and produce zero actual raw output (defeating + # the whole point of -RollupPlusRaw). Instead: keep $OutputFile on its natural + # timestamped raw-audit name, and let -AppendFile flow through downstream: + # • CopilotInteraction: seed-map source merged inside Python. + # • M365Bundle (rollup): union-merged PS-side via Merge-M365RollupCsv after + # the processor emits its timestamped Rollup; sidecars are then regenerated + # from the merged union via --rebuild-sidecars-from-rollup. + # • M365Bundle (event): --exclude-record-ids source for de-duplication. + $appendLeaf = [System.IO.Path]::GetFileName($AppendFile) + # Rollup-shape match accepts both the anchored '_.csv' shape + # (typical for customer-renamed long-lived targets) and the timestamped + # '__.csv' shape (when the customer points + # AppendFile at a previous run's raw output without renaming). + $appendIsRollupShaped = ($appendLeaf -match '_(Interactions|Rollup|UserStats|SessionCohort|Exploded)(?:_\d{8}_\d{6})?\.csv$') + # The embedded Python processor only runs under -Rollup / -RollupPlusRaw. + # Outside those switches a rollup-shaped leaf is just a coincidence (the user + # is presumably appending to a real raw-audit CSV that happens to share the + # suffix), so leave the legacy raw-append behaviour intact in that case. + $rollupActive = ($Rollup -or $RollupPlusRaw) + $treatAppendAsRollupTarget = ($appendIsRollupShaped -and $rollupActive) + + # M365 rollup-mode leaf validation. When -IncludeM365Usage is set with -Rollup + # or -RollupPlusRaw, the embedded M365Bundle processor produces 4 outputs + # ('_Rollup_.csv', '_UserStats_.csv', '_SessionCohort_.csv', + # '_SessionStats_.csv'). Only the Rollup file supports the union-merge + # semantic as the AppendFile anchor (PS-side via Merge-M365RollupCsv); + # UserStats, SessionCohort, and SessionStats are recomputed/merged sidecars + # anchored off the AppendFile leaf stem at the post-processor regen step. + # Pointing -AppendFile at a sidecar (or at a CopilotInteraction _Interactions + # leaf, or at an M365 event-level _Exploded leaf) would either silently feed + # the processor the wrong schema or anchor sidecars at the wrong stem, so we + # reject upfront with a leaf-specific error. + if ($IncludeM365Usage -and $rollupActive) { + $m365LeafErr = $null + if ($appendLeaf -match '_UserStats(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_UserStats.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_SessionCohort(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_SessionCohort.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_SessionStats(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_SessionStats.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_Interactions(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_Interactions.csv' is the CopilotInteraction rollup shape, not M365. Remove -IncludeM365Usage or supply a '_Rollup.csv' leaf." } + elseif ($appendLeaf -match '_Exploded(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_Exploded.csv' is the M365 event-level output. To append under -Rollup/-RollupPlusRaw you need a '_Rollup.csv' anchor (event-level append via --exclude-record-ids is not currently exposed through -AppendFile)." } + elseif ($appendLeaf -notmatch '_Rollup(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "Under -IncludeM365Usage with -Rollup/-RollupPlusRaw the -AppendFile leaf must end in '_Rollup.csv' or '_Rollup_.csv' (the M365 rollup anchor). The UserStats, SessionCohort, and SessionStats sidecars are derived from the anchor stem and overwritten in-place." } + if ($m365LeafErr) { + Write-Host "" -ForegroundColor Red + Write-Host ("ERROR: -AppendFile leaf '{0}' is not valid in this mode: {1}" -f $appendLeaf, $m365LeafErr) -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Expected: a '_Rollup.csv' or '_Rollup_.csv' leaf (filename, full local path, or remote URL)." -ForegroundColor Yellow + Write-Host "Example: -AppendFile 'Purview_Audit_UsageActivity_CombinedActivityTypes_20260101_120000_Rollup.csv'" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "If you do not yet have a Rollup target on the destination, run once WITHOUT -AppendFile to produce the initial timestamped Rollup (plus its UserStats / SessionCohort / SessionStats sidecars)," -ForegroundColor Gray + Write-Host "then point -AppendFile at that Rollup on subsequent runs (keep the timestamp or rename — both shapes are accepted)." -ForegroundColor Gray + exit 1 + } + } + + if ($treatAppendAsRollupTarget) { + # Raw audit CSV uses the precomputed combined-audit leaf — which is the + # single-activity name when $ActivityTypes is exactly one type, otherwise + # CombinedActivityTypes. The post-export dynamic downgrade is suppressed + # under -AppendFile, so the upfront prediction is what the log file (and + # the final raw artifact) inherit. + $OutputFile = Join-Path $OutputPath $_combinedAuditLeaf + } + elseif ($appendInputTier -eq 'SharePoint' -or $appendInputTier -eq 'Fabric') { + # Remote URL: the final $OutputFile is the local scratch path the remote + # pre-flight will populate (Join-Path $OutputPath (GetFileName URL)). + $remoteName = [System.IO.Path]::GetFileName($AppendFile) + $OutputFile = Join-Path $OutputPath $remoteName + } + elseif ([System.IO.Path]::IsPathRooted($AppendFile)) { + # Full local path provided + $OutputFile = $AppendFile + } else { + # Relative filename - combine with OutputPath + $OutputFile = Join-Path $OutputPath $AppendFile + } + + # Validate file exists. For Local-tier outputs this is a direct Test-Path. For + # SharePoint / Fabric URLs, defer the existence check to the remote helpers + # downstream (the remote pre-flight at Connect time will pull the file into + # scratch; if the source does not exist there, the pull will raise a clear + # remote error). + # Skip the existence check entirely when we've routed AppendFile to the rollup + # processor — $OutputFile is now a fresh raw-audit path the writer will create, + # and the AppendFile target's existence is verified by the rollup pre-seed + # block (Test-Path -LiteralPath $AppendFile in the rollup setup downstream). + if (-not $treatAppendAsRollupTarget -and $appendInputTier -eq 'Local' -and -not (Test-Path $OutputFile)) { + Write-Host "ERROR: Cannot append to file - file does not exist: $OutputFile" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "The file must exist before you can append to it." -ForegroundColor Yellow + Write-Host "Either:" -ForegroundColor Green + Write-Host " 1. Create the file first by running without -AppendFile" -ForegroundColor Green + Write-Host " 2. Verify the path and filename are correct" -ForegroundColor Green + exit 1 + } + + # Note: Column validation happens after banner display +} +elseif ($ExportWorkbook) { + # Excel workbook mode - determine final filename upfront so log file matches + if ($CombineOutput) { + # Single-tab workbook + if ($IncludeUserInfo -and -not $UseEOM) { + $OutputFile = Join-Path $OutputPath "Purview_Audit_CombinedUsageActivity_EntraUsers_MAClicensing_$global:ScriptRunTimestamp.xlsx" + } else { + $OutputFile = Join-Path $OutputPath "Purview_Audit_CombinedUsageActivity_$global:ScriptRunTimestamp.xlsx" + } + } else { + # Multi-tab workbook + $OutputFile = Join-Path $OutputPath "Purview_Audit_MultiTab_$global:ScriptRunTimestamp.xlsx" + } +} elseif ($isCsv -and $initialCsvCombine) { + # CSV combined-activity naming; EntraUsers exported separately. + # Use the precomputed combined-audit leaf: when $ActivityTypes contains exactly + # one type (and -IncludeM365Usage is not expanding the set), the leaf is + # Purview_Audit_UsageActivity__.csv so the log file (derived from + # $OutputFile basename downstream) matches. Otherwise it is CombinedActivityTypes, + # and the post-export dynamic single-activity downgrade renames the file (and + # updates $OutputFile) when the actual data ends up containing a single type. + $OutputFile = Join-Path $OutputPath $_combinedAuditLeaf +} else { + $OutputFile = Join-Path $OutputPath "${filePrefix}_$global:ScriptRunTimestamp.$fileExtension" +} + +# -Deidentify data-consistency guard. Refuse to mix deidentified and raw data in an append +# target (corrupts user joins / distinct counts), and refuse a NON-rollup append into an +# already-deidentified file (the whole-file post-pass would re-hash existing rows). Runs for +# any -AppendFile / -AppendUserInfo, in both deid and non-deid runs. Local targets only; +# remote (SharePoint/Fabric) append URLs are validated downstream after scratch download. +if ($AppendFile -or $AppendUserInfo) { + $_deidResolve = { + param([string]$p) + if ([string]::IsNullOrWhiteSpace($p) -or $p -match '^https?://') { return $null } + if ([System.IO.Path]::IsPathRooted($p)) { return $p } + return (Join-Path $OutputPath $p) + } + $_deidGuardTargets = @() + if ($AppendFile) { + # Rollup appends target the user's rollup file ($AppendFile); non-rollup targets $OutputFile. + $_afTarget = if ($Rollup -or $RollupPlusRaw) { & $_deidResolve $AppendFile } else { $OutputFile } + if ($_afTarget) { $_deidGuardTargets += @{ Path = $_afTarget; Label = '-AppendFile' } } + } + if ($AppendUserInfo) { + $_auiTarget = & $_deidResolve $AppendUserInfo + if ($_auiTarget) { $_deidGuardTargets += @{ Path = $_auiTarget; Label = '-AppendUserInfo' } } + } + if ($_deidGuardTargets.Count -gt 0) { + try { + Assert-PaxDeidAppendConsistency -Targets $_deidGuardTargets -IsRollup ([bool]($Rollup -or $RollupPlusRaw)) + } catch { + Write-Host "" -ForegroundColor Red + Write-Host ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + } +} + +# Ensure output directory exists after possible OutputPath override +if (-not (Test-Path $OutputPath)) { New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null } + +# When ExportWorkbook mode, set up intermediate CSV path (OutputFile stays as final .xlsx for log naming) +if ($ExportWorkbook -and -not $AppendFile) { + # CSV intermediate file uses same base name but .csv extension + $script:CsvOutputFile = Join-Path $OutputPath ([System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + ".csv") +} else { + $script:CsvOutputFile = $OutputFile +} + +# ============================================================ +# CHECKPOINT SYSTEM: Initialize for new runs or set paths for resume +# ============================================================ +# Checkpoint is enabled for ALL auth modes (AppRegistration, WebLogin, DeviceCode) +# Enables resume after Ctrl+C, network interruptions, system restarts, or any failure +$script:CheckpointEnabled = (-not $RAWInputCSV) -and (-not $OnlyUserInfo) + +# NOTE: Use $ResumeSpecified here (set early via RemainingArgs parsing) rather than $script:IsResumeMode +# which is only set later during resume detection. This skips new checkpoint creation when -Resume is specified. +if ($ResumeSpecified) { + # Resume mode: checkpoint paths will be set later during resume detection + # Skip all checkpoint initialization here - it will be handled in the resume detection block + $script:FinalOutputPath = $null # Will be set during resume + $script:PartialOutputPath = $null # Will be set during resume + # Also skip log file setup - will be set after checkpoint is loaded + $script:DeferLogFileSetup = $true +} +elseif ($script:CheckpointEnabled) { + # New run with checkpoint enabled: Add _PARTIAL suffix + $script:FinalOutputPath = $OutputFile + $dir = Split-Path $OutputFile -Parent + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + $ext = [System.IO.Path]::GetExtension($OutputFile) + $script:PartialOutputPath = Join-Path $dir "${baseName}_PARTIAL${ext}" + $OutputFile = $script:PartialOutputPath + # For ExportWorkbook mode, CsvOutputFile must use .csv extension (not .xlsx from PartialOutputPath) + if ($ExportWorkbook) { + $script:CsvOutputFile = Join-Path $dir "${baseName}_PARTIAL.csv" + } else { + $script:CsvOutputFile = $script:PartialOutputPath + } + + # Initialize checkpoint file for this run with ALL parameters for complete state restoration + $baseFileName = Split-Path $script:FinalOutputPath -Leaf + $allParams = @{ + # Date range + StartDate = $StartDate + EndDate = $EndDate + # Activity/Record filtering + ActivityTypes = $ActivityTypes + RecordTypes = $RecordTypes + ServiceTypes = $ServiceTypes + UserIds = $UserIds + GroupNames = $GroupNames + # Agent filtering + AgentId = $AgentId + AgentsOnly = $AgentsOnly.IsPresent + ExcludeAgents = $ExcludeAgents.IsPresent + # Prompt filtering + PromptFilter = $PromptFilter + # Schema/Explosion + ExplodeArrays = $ExplodeArrays.IsPresent + ExplodeDeep = $ExplodeDeep.IsPresent + FlatDepth = $FlatDepth + StreamingSchemaSample = $StreamingSchemaSample + StreamingChunkSize = $StreamingChunkSize + # M365/User info + IncludeM365Usage = $IncludeM365Usage.IsPresent + IncludeUserInfo = $IncludeUserInfo.IsPresent + IncludeCopilotInteraction = $IncludeCopilotInteraction.IsPresent + ExcludeCopilotInteraction = $ExcludeCopilotInteraction.IsPresent + IncludeAgent365Info = $IncludeAgent365Info.IsPresent + OnlyAgent365Info = $OnlyAgent365Info.IsPresent + # Partitioning + BlockHours = $BlockHours + PartitionHours = $PartitionHours + MaxPartitions = $MaxPartitions + # Output + ExportWorkbook = $ExportWorkbook.IsPresent + CombineOutput = $CombineOutput.IsPresent + Deidentify = $Deidentify.IsPresent + # FillerLabel: resolved hierarchy-filler mode (none|self|manager|fixed) + literal label. + FillerLabel = $script:HierarchyFillMode + FillerLabelText = $script:HierarchyFillLabel + # uploads to the original destinations — without these, Initialize-CheckpointForNewRun + # persists empty strings and the resume restore has nothing to rehydrate). + OutputPathUserInfo = $OutputPathUserInfo + OutputPathAgent365Info = $OutputPathAgent365Info + OutputPathLog = $OutputPathLog + AppendFile = $AppendFile + AppendUserInfo = $AppendUserInfo + AppendAgent365Info = $AppendAgent365Info + # Rollup mode (required so the resume restore re-arms -Rollup / -RollupPlusRaw + # correctly; without these the post-processor falls back to '-Rollup' semantics + # and deletes the raw EntraUsers CSV that -RollupPlusRaw is supposed to preserve). + Rollup = $Rollup.IsPresent + RollupPlusRaw = $RollupPlusRaw.IsPresent + Dashboard = $Dashboard + # Auth (no secrets) + Auth = $Auth + TenantId = $TenantId + ClientId = $ClientId + # Other + ResultSize = $ResultSize + MaxConcurrency = $MaxConcurrency + MaxMemoryMB = $MaxMemoryMB + ExplosionThreads = $ExplosionThreads + UseEOM = $UseEOM.IsPresent + AutoCompleteness = $AutoCompleteness.IsPresent + IncludeTelemetry = $IncludeTelemetry.IsPresent + } + Initialize-CheckpointForNewRun -OutputPath $OutputPath -BaseOutputFileName $baseFileName -RunTimestamp $global:ScriptRunTimestamp -StartDate (script:Parse-DateSafe $StartDate) -EndDate (script:Parse-DateSafe $EndDate) -AllParameters $allParams | Out-Null +} +else { + # No checkpoint needed (AppRegistration mode or RAWInputCSV or OnlyUserInfo) + $script:FinalOutputPath = $OutputFile + $script:PartialOutputPath = $null +} + +# Update LogFile to match OutputFile base name (extension swapped to .log). +# When -OutputPathLog is bound to a Local path or Fabric Files/ URL, route the log +# there instead. Remote tiers (SharePoint/Fabric) for the audit destination still +# write the log to scratch and the existing run-finalization path uploads from there. +# Skip for resume mode - log file will be set after checkpoint is loaded +if (-not $script:DeferLogFileSetup) { + $logBaseName = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + # Under non-rollup -AppendFile, $OutputFile is the customer's AppendFile target + # (or the _PARTIAL variant of it under checkpoint mode), so deriving the log + # basename from it would (a) inherit the target's ORIGINAL creation timestamp + # and (b) inherit the target's per-activity-type suffix (e.g. + # "_UsageActivity_CopilotInteraction") that the standard non-append path + # does NOT put in the log filename. Non-append runs name their log + # ${filePrefix}_.log (i.e. Purview_Audit_.log) — derived + # from the initial pre-splitter $OutputFile basename which itself starts as + # ${filePrefix}_.csv before the per-activity splitter renames its + # outputs to the long form. Match that canonical convention here so the + # append run's log file is named identically to the non-append run's log: + # the run log is a per-run artifact and is decoupled from the AppendFile + # target CSV's leaf identity. The _PARTIAL suffix convention (stripped at + # successful run-completion) is preserved. + if ($AppendFile -and -not ($Rollup -or $RollupPlusRaw)) { + $_hadPartialSuffix = ($logBaseName -match '_PARTIAL$') + $_currentRunTs = [string]$global:ScriptRunTimestamp + $_canonicalPfx = if ($filePrefix) { [string]$filePrefix } else { 'Purview_Audit' } + $_canonicalLogBase = if ($_currentRunTs) { "${_canonicalPfx}_${_currentRunTs}" } else { $_canonicalPfx } + $logBaseName = if ($_hadPartialSuffix) { "${_canonicalLogBase}_PARTIAL" } else { $_canonicalLogBase } + } + $logDir = Split-Path $OutputFile -Parent + try { + $dtL = script:Resolve-DataTypePaths -DataType 'Log' -DefaultBasename ("{0}.log" -f $logBaseName) + if ($dtL -and $dtL.IsBound -and $dtL.Tier -eq 'Local') { + $logDir = $dtL.EffectiveDir + if ($dtL.Basename) { $logBaseName = [System.IO.Path]::GetFileNameWithoutExtension($dtL.Basename) } + if (-not (Test-Path -LiteralPath $logDir -PathType Container)) { + New-Item -Path $logDir -ItemType Directory -Force | Out-Null + } + } + } catch {} + $finalLogFile = Join-Path $logDir ("{0}.log" -f $logBaseName) + + # Bootstrap-log migration: if the current + # $script:LogFile is a bootstrap temp-path file, move its contents to the + # resolved final location so the entire run (including pre-resolution + # entries) lands in one file. Falls back to copy+delete if Move-Item + # can't cross a volume boundary. + if ($script:LogFileIsBootstrap -and $script:LogFile -and (Test-Path -LiteralPath $script:LogFile)) { + try { + if (-not (Test-Path -LiteralPath $logDir -PathType Container)) { + New-Item -Path $logDir -ItemType Directory -Force | Out-Null + } + Move-Item -LiteralPath $script:LogFile -Destination $finalLogFile -Force -ErrorAction Stop + } catch { + try { + $bootstrapContent = Get-Content -LiteralPath $script:LogFile -Raw -ErrorAction Stop + Set-Content -LiteralPath $finalLogFile -Value $bootstrapContent -Encoding UTF8 -ErrorAction Stop + Remove-Item -LiteralPath $script:LogFile -Force -ErrorAction SilentlyContinue + } catch { + Microsoft.PowerShell.Utility\Write-Host ("WARNING: Could not migrate bootstrap log to final location ({0} -> {1}): {2}" -f $script:LogFile, $finalLogFile, $_.Exception.Message) -ForegroundColor Yellow + } + } + $script:LogFileIsBootstrap = $false + } + + $script:LogFile = $finalLogFile + $LogFile = $script:LogFile +} + +# Flush buffered logs now that log file is finalized (skip for resume - will flush after checkpoint load) +if ($script:LogBuffer -and $script:LogBuffer.Count -gt 0) { + foreach ($entry in $script:LogBuffer) { + try { Add-Content -Path $script:LogFile -Value $entry -Encoding UTF8 -ErrorAction SilentlyContinue } catch {} + } + $script:LogBuffer.Clear() +} + +# Note: $scriptMode already defined earlier for validation - reformat for display consistency +$scriptModeDisplay = if ($ExplodeDeep) { "Deep Column Explosion" } elseif ($ExplodeArrays -or $ForcedRawInputCsvExplosion) { if ($ForcedRawInputCsvExplosion -and -not $ExplodeArrays.IsPresent -and -not $ExplodeDeep.IsPresent) { "Array Explosion (RAWInput implied)" } else { "Array Explosion" } } else { "Standard (1:1)" } + +# Clean display names — strip _PARTIAL suffix so the startup banner always shows the expected final filename. +# Then route through Get-DisplayPath so remote -OutputPath* runs show the remote +# destination URL (where the artifact actually lives after upload), not the local scratch path. +$displayOutputFile = $OutputFile -replace '_PARTIAL(?=\.[^.]+$)', '' +$displayLogFile = $LogFile -replace '_PARTIAL(?=\.log$)', '' +$displayOutputFile = Get-DisplayPath -LocalPath $displayOutputFile +$displayLogFile = Get-DisplayPath -LocalPath $displayLogFile + +# Skip banner output to log file in resume mode (log file not set yet - will be set after checkpoint loads) +if (-not $script:DeferLogFileSetup) { +@" +=== Portable Audit eXporter (PAX) - Purview Audit Log Exporter === +Script Start Time (UTC): $((Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss')) UTC +Script Version: v$ScriptVersion +Mode: $scriptModeDisplay +Date Range: $(if ($RAWInputCSV) { if ([string]::IsNullOrWhiteSpace($StartDate) -and [string]::IsNullOrWhiteSpace($EndDate)) { 'Full CSV (no date filter)' } else { "$StartDate (inclusive) to $EndDate (exclusive) (filters)" } } else { "$StartDate (inclusive) to $EndDate (exclusive)" }) +Output File: $displayOutputFile +Log File: $displayLogFile +======================================================== + +"@ | Out-File -FilePath $LogFile -Encoding UTF8 -Append +} + + +# Fast-path: ensure M365 usage bundle is applied before output summary in raw/replay scenarios +if ($IncludeM365Usage -and -not ($PSBoundParameters.ContainsKey('ActivityTypes'))) { + $ActivityTypes = @($m365UsageActivityBundle + $copilotBaseActivityType) | Select-Object -Unique + # Activity types will be displayed in "Activity Types for This Run" section +} + +# Display active mode (Replay, EOM, or Graph API) +# Resume: suppress the parse-time mode/permissions/rollup/start/mode banner. +# On -Resume only the bootstrap log path, $Auth=WebLogin default, and a NEW +# $startTimeStamp are in scope at this point — the real values are restored +# AFTER this block by Read-Checkpoint. The resume restore block and the +# post-restore Parameter Snapshot print the authoritative auth context, file +# paths, and timestamps. Without this gate the early banner shows +# DELEGATED / WebLogin / bootstrap-log / new-timestamp on every resume run. +if (-not $ResumeSpecified) { +Write-LogHost "" +if ($RAWInputCSV) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " REPLAY MODE: Offline CSV (no service connections)" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Source: $RAWInputCSV" -ForegroundColor White + Write-LogHost " Explosion: $scriptModeDisplay" -ForegroundColor White + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan +} +elseif ($UseEOM) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " QUERY MODE: Exchange Online Management" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " API Method: Search-UnifiedAuditLog cmdlet" -ForegroundColor White + Write-LogHost " Module: ExchangeOnlineManagement" -ForegroundColor White + Write-LogHost " Authentication: $Auth" -ForegroundColor White + Write-LogHost " Parallel Support: DISABLED (serial-only processing)" -ForegroundColor Yellow + Write-LogHost " Permissions: Exchange Online RBAC roles required" -ForegroundColor White + Write-LogHost " (View-Only Audit Logs, Compliance Management)" -ForegroundColor Gray + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan +} +else { + # Determine effective auth context(s) for this run: + # - audit phase context (app-only or delegated) + # - agent 365 phase context (always delegated when -IncludeAgent365Info / -OnlyAgent365Info) + # When -Auth AppRegistration is combined with Agent 365, BOTH contexts apply (sequentially). + # ManagedIdentity is treated as app-only (Connect-MgGraph -Identity uses application permissions + # from the user-assigned MI's service principal); it cannot satisfy Agent 365 (no interactive user). + $isAppOnlyAuth = ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') + $isManagedId = ($Auth -eq 'ManagedIdentity') + $auditContextLbl = if ($isManagedId) { 'APP-ONLY (managed identity / application permissions)' } + elseif ($isAppOnlyAuth) { 'APP-ONLY (application permissions)' } + else { 'DELEGATED (interactive user sign-in)' } + + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green + Write-LogHost " QUERY MODE: Microsoft Graph Security API (Default)" -ForegroundColor Green + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green + Write-LogHost " API Method: REST-based audit log queries" -ForegroundColor White + Write-LogHost " Module: Microsoft.Graph.Security" -ForegroundColor White + Write-LogHost " Authentication: $Auth (OAuth 2.0)" -ForegroundColor White + $parallelStatus = if ($PSVersionTable.PSVersion.Major -ge 7) { "AVAILABLE (PowerShell 7+)" } else { "LIMITED (PowerShell 5.1 detected)" } + Write-LogHost " Parallel Support: $parallelStatus" -ForegroundColor Green + + # ---- Auth Context summary ---- + Write-LogHost "" -ForegroundColor White + Write-LogHost " Auth Context: $auditContextLbl" -ForegroundColor White + if ($isAppOnlyAuth -and ($IncludeAgent365Info -or $OnlyAgent365Info)) { + Write-LogHost " Agent 365 phase reuses this SAME app-only context" -ForegroundColor Gray + Write-LogHost " (application app-role CopilotPackages.Read.All; no sign-in)." -ForegroundColor Gray + } + + # ---- Permissions list ---- + Write-LogHost "" -ForegroundColor White + Write-LogHost " Permissions Required for THIS run:" -ForegroundColor White + Write-LogHost " (Yellow = required this run; DarkGray = not needed this run)" -ForegroundColor Gray + + $tagLegendLines = New-Object System.Collections.Generic.List[string] + if ($isAppOnlyAuth) { + if ($isManagedId) { + $tagLegendLines.Add('[App-only] = application permission on managed identity service principal') + } else { + $tagLegendLines.Add('[App-only] = grant on app registration as Application permission') + } + } + if (-not $isAppOnlyAuth) { + $tagLegendLines.Add('[Delegated] = consented at interactive sign-in') + } + if (($IncludeAgent365Info -or $OnlyAgent365Info) -and (-not $isAppOnlyAuth)) { + $tagLegendLines.Add('[Role] = Entra directory role on signed-in user') + } + if ($script:RemoteOutputMode -eq 'Fabric') { + $tagLegendLines.Add('[Azure RBAC] = Azure role assignment on Fabric workspace (not a Graph scope)') + } + if ($tagLegendLines.Count -gt 0) { + Write-LogHost (" Tag legend: {0}" -f $tagLegendLines[0]) -ForegroundColor Gray + for ($legendIdx = 1; $legendIdx -lt $tagLegendLines.Count; $legendIdx++) { + Write-LogHost (" {0}" -f $tagLegendLines[$legendIdx]) -ForegroundColor Gray + } + } + + # Audit phase scopes (use the audit context tag) + $auditTag = if ($isAppOnlyAuth) { '[App-only] ' } else { '[Delegated]' } + if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Audit query (REQUIRED - this run reads audit logs):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery.Read.All (umbrella - all audit record types)" -f $auditTag) -ForegroundColor Yellow + } else { + $auditSkipReason = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-OnlyUserInfo' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Audit query (NOT NEEDED - {0} skips audit queries):" -f $auditSkipReason) -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery.Read.All (umbrella - all audit record types)" -f $auditTag) -ForegroundColor DarkGray + } + + # M365 usage bundle (audit context) + if ($IncludeM365Usage) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " M365 usage bundle (REQUIRED - because -IncludeM365Usage is set):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery-Exchange.Read.All" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} AuditLogsQuery-SharePoint.Read.All" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} AuditLogsQuery-OneDrive.Read.All" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " M365 usage bundle (only required with -IncludeM365Usage):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery-Exchange.Read.All" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} AuditLogsQuery-SharePoint.Read.All" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} AuditLogsQuery-OneDrive.Read.All" -f $auditTag) -ForegroundColor DarkGray + } + + # Entra directory + license enrichment (audit context) + if ($IncludeUserInfo -or $OnlyUserInfo) { + $userInfoTrigger = if ($OnlyUserInfo) { '-OnlyUserInfo' } else { '-IncludeUserInfo' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Entra directory + license enrichment (REQUIRED - {0} is set):" -f $userInfoTrigger) -ForegroundColor White + Write-LogHost (" {0} User.Read.All (read /users)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} Organization.Read.All (read /subscribedSkus)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Entra directory + license enrichment (only required with -IncludeUserInfo or -OnlyUserInfo):" -ForegroundColor White + Write-LogHost (" {0} User.Read.All (read /users)" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} Organization.Read.All (read /subscribedSkus)" -f $auditTag) -ForegroundColor DarkGray + } + + # Group expansion (audit context) + if ($GroupNames -and $GroupNames.Count -gt 0) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Group expansion (REQUIRED - because -GroupNames is set):" -ForegroundColor White + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/members)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Group expansion (only required with -GroupNames):" -ForegroundColor White + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/members)" -f $auditTag) -ForegroundColor DarkGray + } + + # Remote output - SharePoint (audit context: same Graph token reused for SP drives API) + if ($script:RemoteOutputMode -eq 'SharePoint') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - SharePoint (REQUIRED - because -OutputPath resolves to a SharePoint URL):" -ForegroundColor White + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive via /sites + /drives)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} Files.ReadWrite.All (PUT/createUploadSession to /drives/{{id}}/items)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost " Destination user/identity ALSO needs at least Member access on the" -ForegroundColor Gray + Write-LogHost " target SharePoint site (granted via SharePoint, not Graph)." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - SharePoint (only required when an -OutputPath* value is a SharePoint URL):" -ForegroundColor White + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive via /sites + /drives)" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} Files.ReadWrite.All (PUT/createUploadSession to /drives/{{id}}/items)" -f $auditTag) -ForegroundColor DarkGray + } + + # Remote output - Fabric / OneLake (separate token: storage.azure.com audience via Az.Accounts; + # Azure RBAC, not Graph scopes — this is the only section that uses the [Azure RBAC] tag) + if ($script:RemoteOutputMode -eq 'Fabric') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - Fabric/OneLake (REQUIRED - because -OutputPath resolves to a OneLake URL):" -ForegroundColor White + Write-LogHost " [Azure RBAC] Storage Blob Data Contributor on the Fabric workspace" -ForegroundColor Yellow + Write-LogHost " (assign at workspace scope to the audit identity)" -ForegroundColor Yellow + Write-LogHost " [Azure RBAC] Contributor on the Fabric workspace (Fabric portal)" -ForegroundColor Yellow + Write-LogHost " (Workspace settings -> Manage access; required by some" -ForegroundColor Yellow + Write-LogHost " tenants for OneLake DFS write access)" -ForegroundColor Yellow + Write-LogHost " NOTE: OneLake uses an INDEPENDENT token (audience https://storage.azure.com/)" -ForegroundColor Gray + Write-LogHost " acquired via Az.Accounts. Microsoft Graph scopes do NOT apply here." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - Fabric/OneLake (only required when an -OutputPath* value is a OneLake URL):" -ForegroundColor White + Write-LogHost " [Azure RBAC] Storage Blob Data Contributor on Fabric workspace" -ForegroundColor DarkGray + Write-LogHost " [Azure RBAC] Contributor on Fabric workspace (Fabric portal)" -ForegroundColor DarkGray + } + + # Managed identity advisory - shown only under -Auth ManagedIdentity + if ($isManagedId) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Managed identity bootstrap (REQUIRED - because -Auth ManagedIdentity is set):" -ForegroundColor White + Write-LogHost " All scopes above must be ADMIN-CONSENTED on the managed identity's" -ForegroundColor Yellow + Write-LogHost " service principal as APPLICATION permissions (not delegated). One-time" -ForegroundColor Yellow + Write-LogHost " bootstrap via fabric_resources/.../Prereqs/Grant-PAXPermissions.ps1." -ForegroundColor Yellow + if ($env:AZURE_CLIENT_ID) { + Write-LogHost (" AZURE_CLIENT_ID is set: {0}" -f $env:AZURE_CLIENT_ID) -ForegroundColor Gray + } else { + Write-LogHost " AZURE_CLIENT_ID is NOT set; system-assigned MI will be used (if any)." -ForegroundColor Gray + } + } + + # Agent 365 enrichment - ALWAYS uses delegated scopes; ALSO requires Entra role. + # Suppressed from the permissions table while the Agent 365 switches are parked at + # the temporarily-disabled gate. Restore this block when the switches are re-enabled. + <# + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + $agent365Trigger = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-IncludeAgent365Info' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Agent 365 enrichment (REQUIRED - {0} is set):" -f $agent365Trigger) -ForegroundColor White + if ($dualContextRun) { + Write-LogHost " NOTE: Performed in a SEPARATE delegated context (Phase 2)," -ForegroundColor Gray + Write-LogHost " prompted UP-FRONT immediately after Phase 1 connects." -ForegroundColor Gray + Write-LogHost " Your app registration is NOT used for these scopes." -ForegroundColor Gray + } + Write-LogHost " [Delegated] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor Yellow + Write-LogHost " [Delegated] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor Yellow + Write-LogHost " [Role] AI Administrator (preferred - least privilege)" -ForegroundColor Yellow + Write-LogHost " -- OR --" -ForegroundColor Gray + Write-LogHost " [Role] Global Administrator (alternative)" -ForegroundColor Yellow + Write-LogHost " The signed-in user MUST hold the Entra directory role above;" -ForegroundColor Gray + Write-LogHost " delegated Graph consent ALONE is not sufficient (returns 403)." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Agent 365 enrichment (only required with -IncludeAgent365Info or -OnlyAgent365Info):" -ForegroundColor White + Write-LogHost " [Delegated] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor DarkGray + Write-LogHost " [Delegated] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor DarkGray + Write-LogHost " [Role] AI Administrator OR Global Administrator (signed-in user)" -ForegroundColor DarkGray + } + #> + + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green +} +Write-LogHost "" + +# Per-data-type destination banner. Emitted whenever any -OutputPath* switch was supplied, +# or whenever the inferred Purview destination is remote (SP/Fabric). Shows resolved tier, +# full destination, and append surface for each data type in scope. Sits between the +# permissions banner and the rollup banner so users can see at a glance where each +# customer-visible artifact (CSV/XLSX/log) will land. +$anyDestBound = $false +foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + if ($script:DestIsBound.ContainsKey($k) -and $script:DestIsBound[$k]) { $anyDestBound = $true; break } +} +# Bound -Append* values are destination bindings too — they tell the +# run where the canonical artifact lives. Treat them as such so the banner doesn't +# go silent on a pure -AppendFile/-AppendUserInfo/-AppendAgent365Info run with no +# -OutputPath* in scope. +if (-not $anyDestBound -and ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info)) { + $anyDestBound = $true +} +if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { + $scratchDirDisplay = if ($OutputPath.Length -le 70) { $OutputPath } else { $OutputPath.Substring(0, 67) + '...' } + $bannerHeader = if ($script:RemoteOutputMode -ne 'None') { "REMOTE OUTPUT MODE: $script:RemoteOutputMode" } else { 'OUTPUT DESTINATIONS' } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost (" {0}" -f $bannerHeader) -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + if ($script:RemoteOutputMode -ne 'None') { + # Top-line destination URL removed: per-data-type rows below carry the + # authoritative destination for each stream (which may differ across + # streams even within a single tier). Keep only auth + scratch dir here. + Write-LogHost (" Auth : {0}" -f $Auth) -ForegroundColor White + Write-LogHost " Scratch dir : $scratchDirDisplay" -ForegroundColor White + } + # Per-data-type rows (one line per data type in scope). + $dataTypeRows = @( + @{ Key = 'Purview' ; Label = 'Purview audit '; ScopeCondition = $true ; AppendVar = $AppendFile ; AppendName = 'AppendFile' }, + @{ Key = 'UserInfo' ; Label = 'EntraUsers CSV '; ScopeCondition = ($IncludeUserInfo -or $OnlyUserInfo) ; AppendVar = $AppendUserInfo ; AppendName = 'AppendUserInfo' }, + @{ Key = 'Agent365Info' ; Label = 'Agent 365 CSV '; ScopeCondition = ($IncludeAgent365Info -or $OnlyAgent365Info) ; AppendVar = $AppendAgent365Info ; AppendName = 'AppendAgent365Info' }, + @{ Key = 'Log' ; Label = 'Run log '; ScopeCondition = $true ; AppendVar = $null ; AppendName = $null } + ) + # Always emit the per-data-type rows when the banner shows at all + # (a run with only -OutputPath — no per-stream override and no Append* — must + # still show what each stream resolves to). + Write-LogHost "" -ForegroundColor White + Write-LogHost " Per-data-type destinations:" -ForegroundColor White + foreach ($row in $dataTypeRows) { + if (-not $row.ScopeCondition) { continue } + $tier = if ($script:DestTier.ContainsKey($row.Key)) { $script:DestTier[$row.Key] } + elseif ($row.Key -eq 'Purview' -and $script:DestTier.ContainsKey('Purview')) { $script:DestTier['Purview'] } + elseif ($row.Key -eq 'Log' -and $script:DestTier.ContainsKey('Purview')) { + # Log inherits the Purview tier when -OutputPathLog wasn't bound — the + # log lives in the Purview scratch dir (Local) or is uploaded alongside + # the audit CSV on remote runs. + $script:DestTier['Purview'] + } + else { '(default)' } + # Path resolution: prefer the bound Append target (it IS the destination + # when -Append* is set), then a resolved FILE URL per row, then sensible + # defaults. + # + # Every row renders as / via Get-DisplayPath (or a + # pattern leaf in per-activity-split mode where the leaf varies by + # ActivityType), so adjacent rows in the same banner stay internally + # consistent (file URL on every row, not a mix of folder + file URLs) + # and give customers a verifiable destination per stream. The Run log + # row uses the same Get-DisplayPath transform via $displayLogFile so the + # banner and the 'Log File:' preamble agree on the final destination. + $path = if ($row.AppendName -and $row.AppendVar) { $row.AppendVar } + elseif ($row.Key -eq 'Purview') { + if ($OutputFile) { + # Single-CSV cases: -CombineOutput on, OR a single-activity-type + # run (where the file leaf is fully determined). In both cases + # $OutputFile is the authoritative source for the final leaf. + $_isSingleActivity = ($ActivityTypes -and @($ActivityTypes).Count -eq 1) + if ($CombineOutput -or $_isSingleActivity) { + $_pvFinal = $OutputFile -replace '_PARTIAL(?=\.[^.]+$)', '' + Get-DisplayPath -LocalPath $_pvFinal + } + else { + # Per-activity split: leaf varies by ActivityType — show a + # pattern URL so customers see the naming shape and the + # folder, just like the existing 'Output Files: ...' line. + $_outDir = Split-Path $OutputFile -Parent + $_dispDir = (Get-DisplayPath -LocalPath $_outDir -Directory).TrimEnd('/','\') + $_ts = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) -replace '.*_(\d{8}_\d{6}).*', '$1' + "${_dispDir}/Purview_Audit_UsageActivity__${_ts}.csv" + } + } + elseif ($script:DestRaw.ContainsKey('Purview')) { $script:DestRaw['Purview'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'UserInfo') { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound; + # otherwise inherit -OutputPath. Without consulting Resolve-DataTypePaths the + # banner unconditionally joined the EntraUsers leaf onto $OutputFile's parent + # (the Purview -OutputPath dir), masking any -OutputPathUserInfo override that + # the runtime writer correctly routed to. + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Get-DisplayPath -LocalPath (Join-Path $_uiDt.EffectiveDir $_uiDt.Basename) + } + elseif ($_uiDt -and $_uiDt.IsBound) { + # Remote per-stream override (Fabric / SharePoint URL) — surface the resolved file URL. + ($_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename) + } + elseif ($OutputFile -and $script:GetEffectiveEntraBasename) { + $_entraLeaf = & $script:GetEffectiveEntraBasename + $_entraLocal = Join-Path (Split-Path $OutputFile -Parent) $_entraLeaf + Get-DisplayPath -LocalPath $_entraLocal + } + elseif ($script:DestRaw.ContainsKey('UserInfo')) { $script:DestRaw['UserInfo'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'Agent365Info') { + # Symmetric fix to the UserInfo branch above: honor -OutputPathAgent365Info + # (or an Append-side promotion) before falling back to the Purview -OutputPath dir. + $_a365Dt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + if ($_a365Dt -and $_a365Dt.IsBound -and $_a365Dt.Tier -eq 'Local') { + Get-DisplayPath -LocalPath (Join-Path $_a365Dt.EffectiveDir $_a365Dt.Basename) + } + elseif ($_a365Dt -and $_a365Dt.IsBound) { + ($_a365Dt.EffectiveDir.TrimEnd('/') + '/' + $_a365Dt.Basename) + } + elseif ($OutputFile -and $script:GetEffectiveAgent365Basename) { + $_a365Leaf = & $script:GetEffectiveAgent365Basename + $_a365Local = Join-Path (Split-Path $OutputFile -Parent) $_a365Leaf + Get-DisplayPath -LocalPath $_a365Local + } + elseif ($script:DestRaw.ContainsKey('Agent365Info')) { $script:DestRaw['Agent365Info'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'Log') { + # Surface the resolved log destination URL (or final local path), + # NOT the local _PARTIAL scratch path. $script:LogFile is the + # _PARTIAL working copy throughout the run; strip the suffix and + # pass through Get-DisplayPath so remote-mode banners show the + # SharePoint / OneLake URL the log will land at — matching the + # 'Log File:' preamble line which uses the same transform via + # $displayLogFile. ($script:LogFile is not yet resolved to its + # final path at this point; only the local _PARTIAL copy exists + # until end-of-run upload + rename.) + if ($script:LogFile) { + $_logFinal = $script:LogFile -replace '_PARTIAL(?=\.log$)', '' + Get-DisplayPath -LocalPath $_logFinal + } + elseif ($script:RemoteOutputMode -ne 'None') { "Files/pax_run_logs/$($global:ScriptRunTimestamp)/" } + else { $OutputPath } + } + else { '(inherits -OutputPath)' } + # Never truncate. Customers need the full URL for verification and copy-paste. + $pathDisplay = $path + $appendMarker = '' + if ($row.AppendName -and $row.AppendVar) { + # Single consistent marker across all tiers / both first-time and + # subsequent appends. Init-vs-not distinction was inconsistent across + # tiers (Local could Test-Path, Remote could not) and produced mixed + # '[append-init:...]' / '[append:...]' tags within a single banner. + $appendMarker = " [append:$($row.AppendName)]" + } + Write-LogHost (" {0} [{1,-10}] {2}{3}" -f $row.Label, $tier, $pathDisplay, $appendMarker) -ForegroundColor White + } + # Per-tier truth in the Behavior summary. The Log stream in particular may + # stay local even on a remote-mode run (only the Purview/UserInfo/Agent365 + # streams have remote bindings here). + if ($script:RemoteOutputMode -ne 'None') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Behavior : Each stream is written to the tier shown above. Remote streams" -ForegroundColor White + Write-LogHost " are staged in scratch and uploaded at run completion." -ForegroundColor White + } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +# One-time legend explaining the Append/Merge tally vocabulary used by the +# various 'EntraUsers append-merge complete:' / 'Rollup: -AppendUserInfo +# merge:' / 'Rollup: -AppendFile merge:' lines emitted later in the run. +# The label 'Departed' in particular can mislead a casual reader into thinking +# rows were REMOVED from the file — they are not; 'Departed' rows are carried +# forward into the union with In_Latest_Append=FALSE. Gated on any -Append* +# switch being bound so the legend only appears when at least one merge tally +# will actually be emitted. +if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " APPEND/MERGE TALLY LEGEND" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Each '-Append*' run emits a merge tally with four counts:" -ForegroundColor White + Write-LogHost " Retained = rows present in BOTH the target file and this run's output (overlap)" -ForegroundColor White + Write-LogHost " New = rows added by THIS run only (current-only; not in target before)" -ForegroundColor White + Write-LogHost " Departed = rows present in TARGET ONLY (not seen this run); KEPT in the union" -ForegroundColor White + Write-LogHost " with In_Latest_Append=FALSE — NOT removed from the file" -ForegroundColor White + Write-LogHost " Union = total rows in the merged output (= Retained + New + Departed)" -ForegroundColor White + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +# Rollup post-processor banner. Emitted only when -Rollup or -RollupPlusRaw is in effect +# (either passed on the CLI or restored from checkpoint on resume). Sits immediately after +# the permissions banner so users see at a glance that an embedded Python post-processor +# will run at end-of-run, what mode, and what raw-CSV retention behavior to expect. +if ($Rollup -or $RollupPlusRaw) { + $rollupSwitchBannerName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $rollupRetention = if ($Rollup) { + switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Raw Purview CSV and Entra users CSV will be DELETED on processor success (only rolled-up output kept).' } + 'M365Bundle' { + if ($IncludeUserInfo) { + 'Raw Purview CSV will be DELETED on processor success; Entra users CSV will be KEPT (not consumed by this processor).' + } else { + 'Raw Purview CSV will be DELETED on processor success (only rolled-up output kept).' + } + } + default { 'Raw CSV(s) will be DELETED on processor success (only rolled-up output kept).' } + } + } + else { + switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Raw Purview CSV and Entra users CSV will be KEPT alongside the rolled-up output.' } + 'M365Bundle' { + if ($IncludeUserInfo) { + 'Raw Purview CSV and Entra users CSV will be KEPT alongside the rolled-up output.' + } else { + 'Raw Purview CSV will be KEPT alongside the rolled-up output.' + } + } + default { 'Raw CSV(s) will be KEPT alongside the rolled-up output.' } + } + } + $processorLabel = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { "Purview_CopilotInteraction_Processor v$($Script:EMBEDDED_PROCESSOR_COPILOT_VERSION) (--profile $($script:RollupDashboardProfile); inputs: Purview CSV + Entra users CSV)" } + 'M365Bundle' { "Purview_M365_Usage_Bundle_Explosion_Processor v$($Script:EMBEDDED_PROCESSOR_M365_VERSION) (input: Purview CSV)" } + default { 'unresolved' } + } + $rollupTargetDashboard = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { + if ($script:RollupDashboard -eq 'AIBV') { 'AI Business Value (Analytics-Hub)' } + else { 'AI-in-One (Analytics-Hub)' } + } + 'M365Bundle' { 'M365 Usage Analytics (Analytics-Hub)' } + default { 'unresolved' } + } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " ROLLUP POST-PROCESSOR ENABLED ($rollupSwitchBannerName)" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Purpose : Produces input files for the Microsoft Copilot Growth ROI Advisory Team's" -ForegroundColor White + Write-LogHost " Power BI templates at https://github.com/microsoft/Analytics-Hub." -ForegroundColor White + Write-LogHost " Output is NOT intended for any other downstream use." -ForegroundColor White + Write-LogHost " Target : $rollupTargetDashboard" -ForegroundColor White + Write-LogHost " Processor : $processorLabel" -ForegroundColor White + Write-LogHost " Retention : $rollupRetention" -ForegroundColor White + if ($IncludeAgent365Info) { + Write-LogHost " Companion : Agent365_.csv (point-in-time snapshot; ALWAYS retained — consumed by the same Analytics-Hub dashboards alongside the rollup output)." -ForegroundColor White + } + Write-LogHost " Runtime : Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ required (auto-installed if missing); 'orjson' auto-installed for speed (optional)." -ForegroundColor Gray + # Destination-aware Output line. The per-data-type destination banner above + # already shows the resolved tier/path for every stream, so this line just + # summarizes where the rollup output lands relative to the audit CSV. + # In M365Bundle mode the processor emits four files (Rollup + UserStats + + # SessionCohort + SessionStats sidecars) sharing a common stem, all written + # to the same destination; CopilotInteraction emits a single Interactions Fact CSV. + $rollupOutputLabel = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Rollup Fact CSV' } + 'M365Bundle' { 'Rollup CSV bundle (Rollup + UserStats + SessionCohort + SessionStats sidecars sharing the same stem)' } + default { 'Rollup CSV' } + } + $rollupOutputSummary = + if ($AppendFile) { + "$rollupOutputLabel destination follows the Append target shown in the Per-data-type destinations banner above." + } elseif ($script:RemoteOutputMode -ne 'None') { + "$rollupOutputLabel is uploaded to the $script:RemoteOutputMode destination resolved for the Purview audit stream (see banner above)." + } else { + "$rollupOutputLabel is written to the local Purview audit destination resolved above." + } + Write-LogHost (" Output : {0}" -f $rollupOutputSummary) -ForegroundColor Gray + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +$startTimeStamp = try { $script:metrics.StartTime.ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss') } catch { (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss') } +Write-LogHost ("Script execution started at $startTimeStamp UTC") -ForegroundColor White + +# For OnlyUserInfo mode, show simplified header (no Mode/DateRange/Purview output info) +if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + Write-LogHost "Mode: $scriptMode" -ForegroundColor White + ${rangeText} = if ($RAWInputCSV) { if ([string]::IsNullOrWhiteSpace($StartDate) -and [string]::IsNullOrWhiteSpace($EndDate)) { 'Full CSV (no date filter)' } else { "$StartDate (inclusive) to $EndDate (exclusive) (filters)" } } else { "$StartDate (inclusive) to $EndDate (exclusive)" } + Write-LogHost "Date Range: $rangeText" -ForegroundColor White +} elseif ($OnlyAgent365Info) { + Write-LogHost "Mode: OnlyAgent365Info (Microsoft Agent 365 catalog export only - no audit logs)" -ForegroundColor Cyan +} else { + Write-LogHost "Mode: OnlyUserInfo (Entra user and MAC licensing export only)" -ForegroundColor Cyan +} +} # end: if (-not $ResumeSpecified) — parse-time mode/permissions/rollup/start banner + +# --- Early build of $finalActivityTypes for user warning checks (before auth) --- +# This preview build is used ONLY for multi-output warning and PAYG billing warning +# The full/authoritative build happens later in the pipeline after authentication +$finalActivityTypes = @() +if ($PSBoundParameters.ContainsKey('ActivityTypes') -and $ActivityTypes) { + $finalActivityTypes += $ActivityTypes +} +if ($IncludeM365Usage) { + $finalActivityTypes += $m365UsageActivityBundle +} +# Add CopilotInteraction as default if no custom types and not excluded +$userProvidedCustomTypes = $PSBoundParameters.ContainsKey('ActivityTypes') +if (-not $ExcludeCopilotInteraction -and -not $userProvidedCustomTypes) { + if (-not ($finalActivityTypes -contains 'CopilotInteraction')) { + $finalActivityTypes += 'CopilotInteraction' + } +} +# Remove CopilotInteraction if explicitly excluded +if ($ExcludeCopilotInteraction) { + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne 'CopilotInteraction' } +} +$finalActivityTypes = $finalActivityTypes | Select-Object -Unique + +# --- Multi-Output Warning: Prompt when many files/tabs expected without -CombineOutput --- +$activityTypeCount = $finalActivityTypes.Count +$isMultiOutputScenario = ($activityTypeCount -gt 10) -and (-not $CombineOutput) +$outputType = if ($ExportWorkbook) { "tabs" } else { "CSV files" } + +if ($isMultiOutputScenario -and -not $Force) { + Write-LogHost "" + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-Host "WARNING: Multiple Output $outputType Detected" -ForegroundColor Yellow + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "You have $activityTypeCount activity types selected." -ForegroundColor Cyan + Write-LogHost "Without -CombineOutput, this will create $activityTypeCount separate $outputType." -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost "Recommendation:" -ForegroundColor Green + Write-LogHost " • Add -CombineOutput to merge all activity types into a single $(if ($ExportWorkbook) { 'tab' } else { 'CSV file' })" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "Do you want to continue with $activityTypeCount separate $outputType?" -ForegroundColor White + Write-LogHost "" + Write-LogHost " [Y] YES - Continue with separate $outputType (I understand there will be many $outputType)" -ForegroundColor Green + Write-LogHost " [C] COMBINE - Enable -CombineOutput and continue (single merged $(if ($ExportWorkbook) { 'tab' } else { 'CSV' }))" -ForegroundColor Cyan + Write-LogHost " [E] EXIT - Cancel script execution" -ForegroundColor Red + Write-LogHost "" + + # Strict noninteractive guard: on a container/ACA Job/CI + # runner there is nobody to choose Y/C/E. Default to [Y] CONTINUE so scheduled runs + # do not hang on stdin. Operators who want the [C] COMBINE behavior should pass + # -CombineOutput explicitly; the script's -Force path treats this prompt as Y anyway. + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected: auto-selecting [Y] - continuing with $activityTypeCount separate $outputType." -ForegroundColor DarkGray + Write-LogHost " Hint: pass -CombineOutput to merge into a single output, or -Force to suppress this banner." -ForegroundColor DarkGray + $multiOutput_choice = 'Y' + } else { + Send-PromptNotification + $multiOutput_choice = Read-Host "Enter your choice (Y/C/E)" + } + + if ($multiOutput_choice -eq 'Y' -or $multiOutput_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with $activityTypeCount separate output files..." -ForegroundColor Green + Write-LogHost "" + } + elseif ($multiOutput_choice -eq 'C' -or $multiOutput_choice -eq 'c') { + Write-LogHost "" + Write-LogHost "ENABLED: -CombineOutput mode" -ForegroundColor Green + Write-LogHost " All $activityTypeCount activity types will be merged into a single CSV file." -ForegroundColor Cyan + Write-LogHost "" + $CombineOutput = $true + } + else { + Write-LogHost "" + Write-LogHost "User choice: EXIT - Script execution cancelled" -ForegroundColor Red + Write-LogHost "" + exit 0 + } +} +elseif ($isMultiOutputScenario -and $Force) { + Write-LogHost "Force mode: Skipping multi-output warning ($activityTypeCount activity types, separate $outputType)" -ForegroundColor DarkGray + Write-LogHost " → Continuing with separate $outputType (use -CombineOutput to merge if desired)" -ForegroundColor DarkGray +} +# --- End Multi-Output Warning --- + +# --- DSPM for AI: Billing Information Warning --- +if (($finalActivityTypes -contains 'AIAppInteraction') -or ($finalActivityTypes -contains 'ConnectedAIAppInteraction') -or ($finalActivityTypes -contains 'AIInteraction')) { + if ($IncludeM365Usage -and -not ($finalActivityTypes -contains 'AIAppInteraction')) { + # Bundle-attributed DSPM signal: -IncludeM365Usage carries ConnectedAIAppInteraction + # as part of the curated activity bundle. The DSPM informational prompt is redundant + # because AIAppInteraction (the PAYG path) is not requested; silently bypass. + } + elseif (-not $Force) { + Write-LogHost "" + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-Host "INFORMATION: DSPM for AI Audit Logging - Billing Details" -ForegroundColor Cyan + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "DSPM Activity Types:" -ForegroundColor Cyan + Write-LogHost " • AIInteraction - FREE (Microsoft platforms: Copilot Studio, Azure AI Studio)" -ForegroundColor Green + Write-LogHost " • ConnectedAIAppInteraction - MIXED (FREE for Microsoft apps, PAYG for third-party)" -ForegroundColor Yellow + if ($finalActivityTypes -contains 'AIAppInteraction') { + Write-LogHost " • AIAppInteraction - PAYG BILLING REQUIRED (third-party AI like ChatGPT)" -ForegroundColor DarkYellow + } + Write-LogHost "" + # Check if AIAppInteraction is included - offer options + if ($finalActivityTypes -contains 'AIAppInteraction') { + Write-LogHost "[!] IMPORTANT: AIAppInteraction REQUIRES Microsoft Purview PAYG billing" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "PAYG Requirements:" -ForegroundColor Cyan + Write-LogHost " • Azure subscription linked to M365 tenant" -ForegroundColor Cyan + Write-LogHost " • Microsoft Purview PAYG billing enabled in Compliance portal" -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost "Do you have PAYG billing configured in your tenant?" -ForegroundColor White + Write-LogHost "" + Write-LogHost " [Y] YES - I have PAYG billing, continue with all DSPM types" -ForegroundColor Green + Write-LogHost " [N] NO - I don't have PAYG billing, remove AIAppInteraction and continue" -ForegroundColor Yellow + Write-LogHost " (Third-party AI records will NOT be included)" -ForegroundColor DarkGray + Write-LogHost " [E] EXIT - Cancel script execution" -ForegroundColor Red + Write-LogHost "" + + # Strict noninteractive guard: cannot prompt for + # PAYG confirmation on a container/CI runner. Fail-fast with explicit recovery + # guidance rather than silently including or removing AIAppInteraction. + if (script:Test-IsNonInteractive) { + Write-LogHost "ERROR: Cannot prompt for PAYG billing confirmation on a noninteractive host." -ForegroundColor Red + Write-LogHost " Recovery: re-run WITHOUT -ActivityTypes AIAppInteraction if PAYG is not configured," -ForegroundColor Yellow + Write-LogHost " OR pass -Force to acknowledge PAYG is configured and continue with AIAppInteraction." -ForegroundColor Yellow + exit 1 + } + + Send-PromptNotification + $payg_choice = Read-Host "Enter your choice (Y/N/E)" + + if ($payg_choice -eq 'Y' -or $payg_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with all DSPM types (AIInteraction, ConnectedAIAppInteraction, AIAppInteraction)..." -ForegroundColor Green + Write-LogHost "" + } + elseif ($payg_choice -eq 'N' -or $payg_choice -eq 'n') { + Write-LogHost "" + Write-LogHost "REMOVED: AIAppInteraction (third-party AI records will NOT be captured)" -ForegroundColor Yellow + Write-LogHost "Continuing with: AIInteraction, ConnectedAIAppInteraction (Microsoft platforms only)" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "Note: Without PAYG billing, only Microsoft-hosted AI activity will be captured." -ForegroundColor Yellow + Write-LogHost " Third-party AI apps (ChatGPT, etc.) require PAYG billing." -ForegroundColor Yellow + Write-LogHost "" + + # Set flag to remove AIAppInteraction during later rebuild + $script:RemoveAIAppInteraction = $true + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne 'AIAppInteraction' } + } + else { + Write-LogHost "" + Write-LogHost "User choice: EXIT - Script execution cancelled" -ForegroundColor Red + Write-LogHost "" + exit 0 + } + } + else { + # No AIAppInteraction - simple Y/N prompt + Write-LogHost "Note: AIAppInteraction (PAYG-only third-party AI) is NOT included." -ForegroundColor DarkGray + Write-LogHost " Only Microsoft-hosted AI activity will be captured (AIInteraction, ConnectedAIAppInteraction)." -ForegroundColor DarkGray + Write-LogHost "" + + # Strict noninteractive guard: default to [Y] + # CONTINUE because the surrounding banner is informational-only (no PAYG + # entitlements are required for the activity types that remain in scope). + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected: auto-selecting [Y] - continuing with DSPM for AI export." -ForegroundColor DarkGray + Write-LogHost "" + $payg_choice = 'Y' + } else { + Send-PromptNotification + $payg_choice = Read-Host "Continue with DSPM for AI export? (Y/N)" + } + + if ($payg_choice -eq 'Y' -or $payg_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with DSPM for AI export..." -ForegroundColor Green + Write-LogHost "" + } + else { + Write-LogHost "" + Write-LogHost "User choice: ABORT - DSPM for AI export declined" -ForegroundColor Red + Write-LogHost "Script execution cancelled by user." -ForegroundColor Yellow + Write-LogHost "" + exit 0 + } + } + } + else { + Write-LogHost "Force mode enabled: Skipping DSPM for AI billing information prompt" -ForegroundColor DarkGray + Write-LogHost "User choice: CONTINUE (Force mode - automatic acceptance)" -ForegroundColor Gray + } +} +# --- End PAYG Billing Warning --- + +# Resume: suppress the parse-time Output Directory / Output Files / Log File / +# Authentication / Append/Destination banner. On -Resume the destination state +# ($OutputFile, $script:RemoteOutputMode, $displayLogFile, $Auth, $AppendFile…) +# is restored AFTER this point by Read-Checkpoint and re-resolved by the +# post-restore destination block. The post-restore Parameter Snapshot prints +# the authoritative values. Without this gate the early banner shows the +# bootstrap log path, a fresh non-restored timestamp in the Output Files line, +# and stale Auth=WebLogin on every resume run. +if (-not $ResumeSpecified) { +# Output file/directory display based on export mode +# Note: If activity type switches are used, detailed filenames will be shown after activity types are finalized +# For OnlyUserInfo / OnlyAgent365Info modes, show only the relevant standalone files +if ($OnlyUserInfo) { + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + if ($ExportWorkbook) { + $entraOutputFile = Join-Path $outputDir "EntraUsers_MAClicensing_${global:ScriptRunTimestamp}.xlsx" + Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $entraOutputFile) (Entra users workbook)" -ForegroundColor White + } else { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_uiDt6 = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraOutputFile = if ($_uiDt6 -and $_uiDt6.IsBound -and $_uiDt6.Tier -eq 'Local') { Join-Path $_uiDt6.EffectiveDir $_uiDt6.Basename } + elseif ($_uiDt6 -and $_uiDt6.IsBound) { $_uiDt6.EffectiveDir.TrimEnd('/') + '/' + $_uiDt6.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveEntraBasename) } + if ($_uiDt6 -and $_uiDt6.IsBound -and $_uiDt6.Tier -ne 'Local') { Write-LogHost "Output File: $entraOutputFile" -ForegroundColor White } + else { Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $entraOutputFile)" -ForegroundColor White } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_aiDt6 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt6 -and $_aiDt6.IsBound -and $_aiDt6.Tier -eq 'Local') { Join-Path $_aiDt6.EffectiveDir $_aiDt6.Basename } + elseif ($_aiDt6 -and $_aiDt6.IsBound) { $_aiDt6.EffectiveDir.TrimEnd('/') + '/' + $_aiDt6.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt6 -and $_aiDt6.IsBound -and $_aiDt6.Tier -ne 'Local') { Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray } + else { Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray } + } +} elseif ($OnlyAgent365Info) { + # OnlyAgent365Info mode: only the Agents365 catalog CSV is produced + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_aiDt7 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt7 -and $_aiDt7.IsBound -and $_aiDt7.Tier -eq 'Local') { Join-Path $_aiDt7.EffectiveDir $_aiDt7.Basename } + elseif ($_aiDt7 -and $_aiDt7.IsBound) { $_aiDt7.EffectiveDir.TrimEnd('/') + '/' + $_aiDt7.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt7 -and $_aiDt7.IsBound -and $_aiDt7.Tier -ne 'Local') { Write-LogHost "Output File: $agent365PreviewFile (Microsoft Agent 365 catalog)" -ForegroundColor White } + else { Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $agent365PreviewFile) (Microsoft Agent 365 catalog)" -ForegroundColor White } +} elseif ($AppendFile) { + # AppendFile mode: the destination URL is already surfaced by the parameter + # snapshot's AppendFile row. Re-emitting an 'Output File: ...' + 'Mode: + # Appending to existing file' pair here duplicates that row (and in remote + # tiers it also leaks the scratch-shadow URL via $displayOutputFile). Only + # emit Agent 365 sibling info, which the snapshot does not cover. + if ($IncludeAgent365Info) { + # Agent 365 always writes a separate CSV alongside the appended file (or its own tab if Excel). + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit the LOCAL OutputFile parent. + $agent365TargetDir = Split-Path $OutputFile -Parent + if ($ExportWorkbook) { + Write-LogHost " Agent 365 Tab: Agents365 (appended to workbook)" -ForegroundColor Gray + } else { + $_aiDt8 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt8 -and $_aiDt8.IsBound -and $_aiDt8.Tier -eq 'Local') { Join-Path $_aiDt8.EffectiveDir $_aiDt8.Basename } + elseif ($_aiDt8 -and $_aiDt8.IsBound) { $_aiDt8.EffectiveDir.TrimEnd('/') + '/' + $_aiDt8.Basename } + else { Join-Path $agent365TargetDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt8 -and $_aiDt8.IsBound -and $_aiDt8.Tier -ne 'Local') { Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray } + else { Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray } + } + } +} elseif ($ExcludeCopilotInteraction) { + # Activity type switches present - defer detailed filename listing until after activity types are finalized + $outputDir = if ($ExportWorkbook) { + if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + } else { + Split-Path $OutputFile -Parent + } + Write-LogHost "Output Directory: $(Get-DisplayPath -LocalPath $outputDir -Directory)\" -ForegroundColor White + Write-LogHost " (Detailed filenames will be shown after activity types are finalized)" -ForegroundColor Gray +} elseif ($ExportWorkbook) { + # Excel mode: always one .xlsx file (combined tab or multiple tabs) + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + $displayDir = (Get-DisplayPath -LocalPath $outputDir -Directory).TrimEnd('/','\') + '/' + if ($CombineOutput) { + # New naming: Purview_Audit_CombinedUsageActivity[_EntraUsers]_timestamp.xlsx + $baseName = "Purview_Audit_CombinedUsageActivity" + if ($IncludeUserInfo -and -not $UseEOM) { $baseName += "_EntraUsers" } + Write-LogHost "Output File: ${displayDir}${baseName}_.xlsx (single-tab workbook)" -ForegroundColor White + } else { + Write-LogHost "Output File: ${displayDir}Purview_Audit_MultiTab_.xlsx (multi-tab workbook)" -ForegroundColor White + } + if ($IncludeAgent365Info) { Write-LogHost " Agent 365 Tab: Agents365 (appended to workbook after audit + EntraUsers)" -ForegroundColor Gray } +} else { + # CSV mode: combined file or separate files per activity type + if ($CombineOutput) { + # Single combined CSV file. Suppress the "(combined - all activity types)" + # qualifier when the run is gated to exactly one activity type — the file + # name itself already carries the single activity-type name in that case. + $_isSingleActivityRun = ($ActivityTypes -and @($ActivityTypes).Count -eq 1 -and -not $IncludeM365Usage) + $_combinedQualifier = if ($_isSingleActivityRun) { '' } else { ' (combined - all activity types)' } + Write-LogHost "Output File: ${displayOutputFile}${_combinedQualifier}" -ForegroundColor White + if ($IncludeUserInfo -and -not $UseEOM) { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound, so the + # preview line matches the runtime writer's actual destination instead of unconditionally + # joining the EntraUsers leaf onto $OutputFile's parent (the Purview -OutputPath dir). + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraFile = if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Join-Path $_uiDt.EffectiveDir $_uiDt.Basename + } elseif ($_uiDt -and $_uiDt.IsBound) { + $_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename + } else { + Join-Path (Split-Path $OutputFile -Parent) (& $script:GetEffectiveEntraBasename) + } + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -ne 'Local') { + Write-LogHost " Entra Users File: $entraFile" -ForegroundColor Gray + } else { + Write-LogHost " Entra Users File: $(Get-DisplayPath -LocalPath $entraFile)" -ForegroundColor Gray + } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; same + # rationale as the EntraUsers branch above. + $_aiDt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -eq 'Local') { + Join-Path $_aiDt.EffectiveDir $_aiDt.Basename + } elseif ($_aiDt -and $_aiDt.IsBound) { + $_aiDt.EffectiveDir.TrimEnd('/') + '/' + $_aiDt.Basename + } else { + Join-Path (Split-Path $OutputFile -Parent) (& $script:GetEffectiveAgent365Basename) + } + if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -ne 'Local') { + Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray + } else { + Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray + } + } + } else { + # Separate CSV files per activity type + $outputDir = Split-Path $OutputFile -Parent + $displayDir = (Get-DisplayPath -LocalPath $outputDir -Directory).TrimEnd('/','\') + $timestamp = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) -replace '.*_(\d{8}_\d{6}).*', '$1' + Write-LogHost "Output Directory: $displayDir\" -ForegroundColor White + Write-LogHost "Output Files: ${displayDir}\Purview_Audit_UsageActivity__${timestamp}.csv" -ForegroundColor Gray + if ($IncludeUserInfo -and -not $UseEOM) { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound, so the + # separate-files preview matches the runtime writer's actual destination instead of + # unconditionally joining the EntraUsers leaf onto $OutputFile's parent. + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraFile = if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Join-Path $_uiDt.EffectiveDir $_uiDt.Basename + } elseif ($_uiDt -and $_uiDt.IsBound) { + $_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename + } else { + Join-Path $outputDir (& $script:GetEffectiveEntraBasename) + } + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -ne 'Local') { + Write-LogHost " Entra Users: $entraFile" -ForegroundColor Gray + } else { + Write-LogHost " Entra Users: $(Get-DisplayPath -LocalPath $entraFile)" -ForegroundColor Gray + } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; same + # rationale as the EntraUsers branch above. + $_aiDt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -eq 'Local') { + Join-Path $_aiDt.EffectiveDir $_aiDt.Basename + } elseif ($_aiDt -and $_aiDt.IsBound) { + $_aiDt.EffectiveDir.TrimEnd('/') + '/' + $_aiDt.Basename + } else { + Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) + } + if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -ne 'Local') { + Write-LogHost " Agent 365: $agent365PreviewFile" -ForegroundColor Gray + } else { + Write-LogHost " Agent 365: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray + } + } + } +} + +Write-LogHost "Log File: $displayLogFile" -ForegroundColor White +if (-not $RAWInputCSV) { + Write-LogHost "Authentication: $Auth" -ForegroundColor White +} + +# Append-target / Fabric Delta destination summary. Only emitted when at least one +# is in effect, so non-append local runs see no extra banner noise. +if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or + ($script:RemoteOutputMode -eq 'Fabric' -and $script:RemoteOutputUrl -match '/Tables/?$')) { + Write-LogHost "Append / Destination:" -ForegroundColor Yellow + if ($AppendFile) { Write-LogHost (" -AppendFile : {0}" -f $AppendFile) -ForegroundColor Gray } + if ($AppendUserInfo) { Write-LogHost (" -AppendUserInfo : {0}" -f $AppendUserInfo) -ForegroundColor Gray } + if ($AppendAgent365Info) { Write-LogHost (" -AppendAgent365Info : {0}" -f $AppendAgent365Info) -ForegroundColor Gray } + if ($script:RemoteOutputMode -eq 'Fabric' -and $script:RemoteOutputUrl -match '/Tables/?$') { + Write-LogHost (" Fabric destination : {0} (CSV outputs land as Delta tables)" -f $script:RemoteOutputUrl) -ForegroundColor Gray + } +} +} # end: if (-not $ResumeSpecified) — parse-time output/log/auth/append banner + +if ($AgentId -or $AgentsOnly -or $ExcludeAgents -or $PromptFilter -or $UserIds -or $GroupNames) { + Write-LogHost "Filters:" -ForegroundColor Yellow + if ($AgentsOnly) { Write-LogHost " AgentsOnly: Only records with AgentId present" -ForegroundColor Gray } + if ($AgentId) { + $agentDisplay = if ($AgentId.Count -eq 1) { + "Specific AgentId: $($AgentId[0])" + } + elseif ($AgentId.Count -le 3) { + "Specific AgentIds ($($AgentId.Count)): " + ($AgentId -join '; ') + } + else { + "Specific AgentIds ($($AgentId.Count) total):" + } + Write-LogHost " $agentDisplay" -ForegroundColor Gray + if ($AgentId.Count -gt 3) { + for ($i = 0; $i -lt [Math]::Min(3, $AgentId.Count); $i++) { + $displayId = if ($AgentId[$i].Length -gt 80) { $AgentId[$i].Substring(0, 77) + '...' } else { $AgentId[$i] } + Write-LogHost " [$($i+1)] $displayId" -ForegroundColor DarkGray + } + if ($AgentId.Count -gt 3) { + Write-LogHost " ... and $($AgentId.Count - 3) more" -ForegroundColor DarkGray + } + } + } + if ($ExcludeAgents) { Write-LogHost " ExcludeAgents: Only records without AgentId" -ForegroundColor Gray } + if ($PromptFilter) { + $promptLabel = switch ($PromptFilter) { + 'Prompt' { 'Only prompts (Message_isPrompt = True)' } + 'Response' { 'Only responses (Message_isPrompt = False)' } + 'Both' { 'Both prompts and responses (Message_isPrompt = True or False)' } + 'Null' { 'Only records with no Message_isPrompt values (Null/Empty)' } + } + Write-LogHost " PromptFilter: $promptLabel" -ForegroundColor Gray + } + if ($UserIds -or $GroupNames) { + if ($UserIds) { + if ($UserIds.Count -eq 1) { Write-LogHost " UserIds: 1 user" -ForegroundColor Gray } else { Write-LogHost " UserIds: $($UserIds.Count) users" -ForegroundColor Gray } + } + if ($GroupNames) { + if ($GroupNames.Count -eq 1) { Write-LogHost " GroupNames: 1 group" -ForegroundColor Gray } else { Write-LogHost " GroupNames: $($GroupNames.Count) groups" -ForegroundColor Gray } + } + } +} + +Write-LogHost "=============================================" -ForegroundColor Cyan +Write-LogHost "" + +# Now perform AppendFile validation if needed (after banner display). +# +# Test-AppendFileCompatibility's sole purpose is to detect a mismatch between +# the existing file's explosion schema (column shape produced by -ExplodeArrays +# or -ExplodeDeep) and the current run's explosion params. In a Standard → +# Standard append both modes are 'Standard (no explosion)' by construction +# and a mismatch is impossible — the only outcomes are 'compatible' (no-op) +# or a spurious failure when the existing-file probe (Get-Content -First 1) +# can't read the file (e.g. remote AppendFile URL not yet downloaded to +# scratch, file briefly locked by another process). Gate the validator on +# explosion actually being in play OR a replay (-RAWInputCSV) run so the +# only people who see this check are the ones it was designed for. +if ($AppendFile -and ($ExplodeArrays -or $ExplodeDeep -or $RAWInputCSV)) { + + $validation = Test-AppendFileCompatibility ` + -FilePath $OutputFile ` + -IsExcel $ExportWorkbook ` + -ExplodeArrays:$ExplodeArrays ` + -ExplodeDeep:$ExplodeDeep + + if (-not $validation.Compatible) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: Explosion Parameter Mismatch - Cannot Append" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host "The existing file was created with different explosion parameters than" -ForegroundColor Yellow + Write-Host "the current command. Appending would create incompatible data structures." -ForegroundColor Yellow + Write-Host "" + Write-Host "Existing file: $OutputFile" -ForegroundColor White + Write-Host " Columns: $($validation.ExistingCount)" -ForegroundColor Gray + Write-Host " Mode: $($validation.ExistingMode.DisplayName)" -ForegroundColor Gray + Write-Host "" + Write-Host "Current command:" -ForegroundColor White + Write-Host " Mode: $($validation.CurrentMode.DisplayName)" -ForegroundColor Gray + Write-Host "" + Write-Host "Root Cause:" -ForegroundColor Cyan + Write-Host " Explosion parameters must match between original file and append operation." -ForegroundColor Yellow + Write-Host "" + Write-Host "Resolution Options:" -ForegroundColor Cyan + Write-Host " 1. Match the original file's parameters:" -ForegroundColor White + Write-Host " Use: $($validation.ExistingMode.DisplayName)" -ForegroundColor Gray + Write-Host " 2. Create new output file instead:" -ForegroundColor White + Write-Host " Remove -AppendFile parameter" -ForegroundColor Gray + + if ($ExportWorkbook) { + Write-Host "" + Write-Host "Note for Excel mode:" -ForegroundColor DarkGray + Write-Host " If parameters matched, mismatched columns would create timestamped" -ForegroundColor DarkGray + Write-Host " duplicate tabs instead of appending (no data loss)." -ForegroundColor DarkGray + } + else { + Write-Host "" + Write-Host "CRITICAL for CSV mode:" -ForegroundColor Yellow + Write-Host " CSV append with mismatched explosion parameters creates CORRUPTED files!" -ForegroundColor Yellow + Write-Host " This validation prevents data corruption by failing early." -ForegroundColor Yellow + } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + exit 1 + } + + Write-LogHost " Explosion parameters compatible ($($validation.CurrentMode.DisplayName)) - safe to append" -ForegroundColor Green +} + +if ($ExplodeDeep -and $ExplodeArrays) { Write-LogHost "Note: -ExplodeDeep takes precedence over -ExplodeArrays (arrays will still explode, plus deep flatten)." -ForegroundColor DarkYellow } +if ($ForcedRawInputCsvExplosion -and -not $ExplodeDeep -and -not $ExplodeArrays.IsPresent) { Write-LogHost "RAWInputCSV provided -> forcing Purview array explosion (non-exploded mode disabled)." -ForegroundColor Yellow } +if ($script:memoryFlushEnabled) { + $memSource = if ($MaxMemoryMB -eq -1) { "auto-detected" } else { "user-specified" } + Write-LogHost "Memory management: $($script:ResolvedMaxMemoryMB)MB limit ($memSource) - will flush to disk when exceeded" -ForegroundColor Cyan +} elseif ($script:ResolvedMaxMemoryMB -gt 0 -and ($ExplodeDeep -or $ExplodeArrays -or $ForcedRawInputCsvExplosion)) { + Write-LogHost "Note: Memory limit ($($script:ResolvedMaxMemoryMB)MB) ignored because explosion mode is active" -ForegroundColor DarkYellow +} + +if ($RAWInputCSV) { + # Replay mode (offline): no live EOM/Graph query; show only relevant parameters. + # The output-destinations cluster at the top mirrors the Graph-mode snapshot so + # operators see all file paths in one block. Replay forbids remote destination + # tiers in parameter validation, so the resolveAuxDisplayPath helper effectively + # always falls through to local Get-DisplayPath here — but the structure is kept + # identical for visual consistency between modes. + $resolveAuxDisplayPath = { + param([string]$DataTypeKey, [string]$FileName) + if ($script:DestTier -and $script:DestTier.ContainsKey($DataTypeKey) -and $script:DestTier[$DataTypeKey] -and $script:DestTier[$DataTypeKey] -ne 'Local') { + $folder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey($DataTypeKey) -and $script:DestParentUrl[$DataTypeKey]) { $script:DestParentUrl[$DataTypeKey] } else { $script:DestRaw[$DataTypeKey] } + return ($folder.TrimEnd('/','\') + '/' + $FileName) + } + # Local tier: honor a per-stream -OutputPath override (or an Append-side promotion of + # DestRaw) so the snapshot reflects the actual on-disk destination, not the inherited -OutputPath + # dir. Without this, EntraUsersOutput / Agents365Output displayed the -OutputPath parent even + # when the runtime writer was routing the stream to its own -OutputPath directory. + $auxDt = script:Resolve-DataTypePaths -DataType $DataTypeKey -DefaultBasename $FileName + if ($auxDt -and $auxDt.IsBound -and $auxDt.Tier -eq 'Local') { + return (Get-DisplayPath -LocalPath (Join-Path $auxDt.EffectiveDir $FileName)) + } + $localSibling = Join-Path (Split-Path $OutputFile -Parent) $FileName + return (Get-DisplayPath -LocalPath $localSibling) + } + + $paramSnapshot = [ordered]@{ + Mode = $scriptMode + RAWInputCSV = $RAWInputCSV + 'StartDate (inclusive)' = $StartDate + 'EndDate (exclusive)' = $EndDate + } + + # ── Output destinations cluster ────────────────────────────────────── + # Append-vs-Output suppression: when an -Append* flag is bound the + # resolved destination of that stream IS the Append target — emitting both + # OutputFile/EntraUsersOutput/Agents365Output AND the matching Append* line + # produces two identical URLs that read like a bug. Suppress the resolved- + # destination line in append mode; the Append* line below is the single + # source of truth for that stream. + # Purview stream + if ($PSBoundParameters.ContainsKey('AppendFile') -and $AppendFile) { + $paramSnapshot['AppendFile'] = $AppendFile + } else { + $paramSnapshot['OutputFile'] = $displayOutputFile + } + + # UserInfo stream (Entra users licensing export). Replay mode currently + # does not regenerate EntraUsers, but if -IncludeUserInfo is in scope the + # path is surfaced for parity with the standard snapshot. + if ($PSBoundParameters.ContainsKey('AppendUserInfo') -and $AppendUserInfo) { + $paramSnapshot['AppendUserInfo'] = $AppendUserInfo + } elseif (($IncludeUserInfo -or $OnlyUserInfo) -and -not $UseEOM) { + if ($ExportWorkbook) { + $paramSnapshot['EntraUsersOutput'] = 'Workbook Tab: EntraUsers' + } else { + $paramSnapshot['EntraUsersOutput'] = & $resolveAuxDisplayPath 'UserInfo' (& $script:GetEffectiveEntraBasename) + } + } + + # Agent365Info stream + if ($PSBoundParameters.ContainsKey('AppendAgent365Info') -and $AppendAgent365Info) { + $paramSnapshot['AppendAgent365Info'] = $AppendAgent365Info + } elseif ($IncludeAgent365Info -or $OnlyAgent365Info) { + if ($ExportWorkbook) { + $paramSnapshot['Agents365Output'] = 'Workbook Tab: Agents365' + } else { + $paramSnapshot['Agents365Output'] = & $resolveAuxDisplayPath 'Agent365Info' (& $script:GetEffectiveAgent365Basename) + } + } + + # Log stream + $paramSnapshot['LogFile'] = $displayLogFile + # ── End output destinations cluster ────────────────────────────────── + + $paramSnapshot['ActivityTypes'] = ($ActivityTypes -join ';') + $paramSnapshot['ExcludeCopilotInteraction'] = $ExcludeCopilotInteraction.IsPresent + $paramSnapshot['ExplodeArrays'] = $ForcedRawInputCsvExplosion + $paramSnapshot['ExplodeDeep'] = $ExplodeDeep.IsPresent + if ($ExplodeDeep -or $PSBoundParameters.ContainsKey('FlatDepth')) { $paramSnapshot['FlatDepth'] = $FlatDepth } + $paramSnapshot['UseEOM'] = $UseEOM.IsPresent + $paramSnapshot['MaxMemoryMB'] = $(if ($script:ResolvedMaxMemoryMB -eq 0) { 'Off' } else { "$($script:ResolvedMaxMemoryMB)MB" + $(if ($MaxMemoryMB -eq -1) { ' (auto)' } else { '' }) }) + $paramSnapshot['StatusIntervalSeconds'] = $StatusIntervalSeconds + $paramSnapshot['ExportWorkbook'] = $ExportWorkbook.IsPresent + $paramSnapshot['CombineOutput'] = $CombineOutput.IsPresent + $paramSnapshot['Deidentify'] = $Deidentify.IsPresent + $paramSnapshot['FillerLabel'] = $script:HierarchyFillMode + $paramSnapshot['FillerLabelText'] = $script:HierarchyFillLabel + $paramSnapshot['Force'] = $Force.IsPresent + $paramSnapshot['SkipDiagnostics'] = $SkipDiagnostics.IsPresent + $paramSnapshot['SkipVersionCheck'] = $SkipVersionCheck.IsPresent + $paramSnapshot['EmitMetricsJson'] = $EmitMetricsJson.IsPresent + $paramSnapshot['MetricsPath'] = $(if ($MetricsPath) { $MetricsPath } else { '' }) + $paramSnapshot['StreamingSchemaSample'] = $StreamingSchemaSample + $paramSnapshot['StreamingChunkSize'] = $StreamingChunkSize + $paramSnapshot['PSVersion'] = $PSVersionTable.PSVersion.ToString() + $paramSnapshot['PSEdition'] = $PSVersionTable.PSEdition + $paramSnapshot['HostName'] = $Host.Name + $paramSnapshot['HostVersion'] = $(try { $Host.Version.ToString() } catch { '' }) + + $copilotIncluded = $IncludeCopilotInteraction.IsPresent -or ($ActivityTypes -contains $copilotBaseActivityType) + $paramSnapshot['IncludeCopilotInteraction'] = $copilotIncluded +} +else { + # Smart parameter snapshot: Show only applicable parameters for the chosen query mode. + # + # Top of the snapshot is a single, contiguous "Output destinations" cluster so + # operators can audit every file the run will produce at a glance. Each stream + # (Purview / UserInfo / Agent365Info / Log) gets its lines kept together: + # 1. The user-supplied -OutputPath* (if bound) — shown verbatim (DestRaw) + # 2. The resolved artifact path — shown via Get-DisplayPath / DestParentUrl + # so SharePoint/Fabric tiers display as remote URLs, not local scratch + # 3. The companion -Append* (if bound) — shown verbatim + # Auxiliary helper resolves per-data-type display path for sibling artifacts + # (EntraUsers, Agent365): if the data type's tier is non-Local, use that + # tier's folder URL; otherwise fall through Get-DisplayPath (which maps onto + # the Purview remote URL when the Purview output is remote, or returns the + # local path verbatim when fully local). + $resolveAuxDisplayPath = { + param([string]$DataTypeKey, [string]$FileName) + if ($script:DestTier -and $script:DestTier.ContainsKey($DataTypeKey) -and $script:DestTier[$DataTypeKey] -and $script:DestTier[$DataTypeKey] -ne 'Local') { + $folder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey($DataTypeKey) -and $script:DestParentUrl[$DataTypeKey]) { $script:DestParentUrl[$DataTypeKey] } else { $script:DestRaw[$DataTypeKey] } + return ($folder.TrimEnd('/','\') + '/' + $FileName) + } + # Local tier: honor a per-stream -OutputPath override (or an Append-side promotion of + # DestRaw) so the snapshot reflects the actual on-disk destination, not the inherited -OutputPath + # dir. Without this, EntraUsersOutput / Agents365Output displayed the -OutputPath parent even + # when the runtime writer was routing the stream to its own -OutputPath directory. + $auxDt = script:Resolve-DataTypePaths -DataType $DataTypeKey -DefaultBasename $FileName + if ($auxDt -and $auxDt.IsBound -and $auxDt.Tier -eq 'Local') { + return (Get-DisplayPath -LocalPath (Join-Path $auxDt.EffectiveDir $FileName)) + } + $localSibling = Join-Path (Split-Path $OutputFile -Parent) $FileName + return (Get-DisplayPath -LocalPath $localSibling) + } + + $paramSnapshot = [ordered]@{ + 'StartDate (inclusive)' = $StartDate + 'EndDate (exclusive)' = $EndDate + } + + # ── Output destinations cluster ────────────────────────────────────── + # Append-vs-Output suppression: see RAWInputCSV branch for rationale. + # Purview stream + if ($PSBoundParameters.ContainsKey('AppendFile') -and $AppendFile) { + $paramSnapshot['AppendFile'] = $AppendFile + } else { + $paramSnapshot['OutputFile'] = $displayOutputFile + } + + # UserInfo stream (Entra users licensing export) + if ($PSBoundParameters.ContainsKey('AppendUserInfo') -and $AppendUserInfo) { + $paramSnapshot['AppendUserInfo'] = $AppendUserInfo + } elseif (($IncludeUserInfo -or $OnlyUserInfo) -and -not $UseEOM) { + if ($ExportWorkbook) { + $paramSnapshot['EntraUsersOutput'] = 'Workbook Tab: EntraUsers' + } else { + $paramSnapshot['EntraUsersOutput'] = & $resolveAuxDisplayPath 'UserInfo' (& $script:GetEffectiveEntraBasename) + } + } + + # Agent365Info stream + if ($PSBoundParameters.ContainsKey('AppendAgent365Info') -and $AppendAgent365Info) { + $paramSnapshot['AppendAgent365Info'] = $AppendAgent365Info + } elseif ($IncludeAgent365Info -or $OnlyAgent365Info) { + if ($ExportWorkbook) { + $paramSnapshot['Agents365Output'] = 'Workbook Tab: Agents365' + } else { + $paramSnapshot['Agents365Output'] = & $resolveAuxDisplayPath 'Agent365Info' (& $script:GetEffectiveAgent365Basename) + } + } + + # Log stream + $paramSnapshot['LogFile'] = $displayLogFile + # ── End output destinations cluster ────────────────────────────────── + + # Authentication (both modes, but different usage) + if ($UseEOM) { + # EOM mode: Auth parameter controls connection + $paramSnapshot['Auth'] = $Auth + } else { + # Graph API mode: Auth parameter controls connection + $paramSnapshot['Auth'] = $Auth + # Capture AppRegistration context (without exposing secrets) + $paramSnapshot['TenantId'] = $(if ($TenantId) { $TenantId } elseif ($env:GRAPH_TENANT_ID) { '[GRAPH_TENANT_ID]' } else { '' }) + $paramSnapshot['ClientId'] = $(if ($ClientId) { $ClientId } elseif ($env:GRAPH_CLIENT_ID) { '[GRAPH_CLIENT_ID]' } else { '' }) + if ($PSBoundParameters.ContainsKey('ClientSecret') -or $env:GRAPH_CLIENT_SECRET) { + $paramSnapshot['ClientSecret'] = '[securestring provided]' + } + if ($PSBoundParameters.ContainsKey('ClientCertificateThumbprint')) { + $paramSnapshot['ClientCertificateThumbprint'] = $ClientCertificateThumbprint + } + if ($PSBoundParameters.ContainsKey('ClientCertificateStoreLocation')) { + $paramSnapshot['ClientCertificateStoreLocation'] = $ClientCertificateStoreLocation + } + if ($PSBoundParameters.ContainsKey('ClientCertificatePath')) { + $paramSnapshot['ClientCertificatePath'] = $ClientCertificatePath + } + if ($PSBoundParameters.ContainsKey('ClientCertificatePassword')) { + $paramSnapshot['ClientCertificatePassword'] = '[securestring provided]' + } + } + + # Query parameters specific to each mode + if ($UseEOM) { + # EOM-only: Search-UnifiedAuditLog parameters + $paramSnapshot['BlockHours'] = $BlockHours + $paramSnapshot['ResultSize'] = $ResultSize + $paramSnapshot['PacingMs'] = $PacingMs + $paramSnapshot['DisableAdaptive'] = $DisableAdaptive.IsPresent + $paramSnapshot['MemoryPressureMB'] = $MemoryPressureMB + $paramSnapshot['BackoffBaseSeconds'] = $BackoffBaseSeconds + $paramSnapshot['BackoffMaxSeconds'] = $BackoffMaxSeconds + $paramSnapshot['CircuitBreakerThreshold'] = $CircuitBreakerThreshold + $paramSnapshot['CircuitBreakerCooldownSeconds'] = $CircuitBreakerCooldownSeconds + $paramSnapshot['ProgressSmoothingAlpha'] = $ProgressSmoothingAlpha + # Adaptive-pacing controls — surfaced whenever -UseEOM is in play so the + # effective tuning that drove the run is auditable. These do nothing under + # -DisableAdaptive but are still informative for post-run analysis. + $paramSnapshot['HighLatencyMs'] = $HighLatencyMs + $paramSnapshot['LowLatencyMs'] = $LowLatencyMs + $paramSnapshot['LowLatencyConsecutive'] = $LowLatencyConsecutive + $paramSnapshot['ThroughputDropPct'] = $ThroughputDropPct + $paramSnapshot['ThroughputSmoothingAlpha'] = $ThroughputSmoothingAlpha + $paramSnapshot['AdaptiveConcurrencyCeiling'] = $AdaptiveConcurrencyCeiling + $paramSnapshot['AutoCompleteness'] = $AutoCompleteness.IsPresent + } else { + # Graph-only: parallel/partition orchestration + $paramSnapshot['ParallelMode'] = $ParallelMode + $paramSnapshot['MaxParallelGroups'] = $MaxParallelGroups + $paramSnapshot['EnableParallel'] = $EnableParallel.IsPresent + $paramSnapshot['PartitionHours'] = if ($PartitionHours -gt 0) { $PartitionHours } else { 'auto' } + $paramSnapshot['MaxPartitions'] = $MaxPartitions + $paramSnapshot['MaxNetworkOutageMinutes'] = $MaxNetworkOutageMinutes + $paramSnapshot['IncludeUserInfo'] = $IncludeUserInfo.IsPresent + $paramSnapshot['OnlyUserInfo'] = $OnlyUserInfo.IsPresent + $paramSnapshot['IncludeTelemetry'] = $IncludeTelemetry.IsPresent + } + + # Crossover (both modes) + $paramSnapshot['MaxConcurrency'] = $MaxConcurrency + $paramSnapshot['MaxMemoryMB'] = $(if ($script:ResolvedMaxMemoryMB -eq 0) { 'Off' } else { "$($script:ResolvedMaxMemoryMB)MB" + $(if ($MaxMemoryMB -eq -1) { ' (auto)' } else { '' }) }) + $paramSnapshot['StatusIntervalSeconds'] = $StatusIntervalSeconds + + # Common toggles and output options + $paramSnapshot['UseEOM'] = $UseEOM.IsPresent + $paramSnapshot['ExportWorkbook'] = $ExportWorkbook.IsPresent + $paramSnapshot['CombineOutput'] = $CombineOutput.IsPresent + $paramSnapshot['Deidentify'] = $Deidentify.IsPresent + $paramSnapshot['FillerLabel'] = $script:HierarchyFillMode + $paramSnapshot['FillerLabelText'] = $script:HierarchyFillLabel + # AppendFile / AppendUserInfo / AppendAgent365Info live in the output-destinations + # cluster at the top of the snapshot, kept next to their corresponding -OutputPath* + # and resolved artifact paths so operators see the full file-routing picture in one + # block. They are intentionally not repeated here. + $paramSnapshot['Force'] = $Force.IsPresent + $paramSnapshot['SkipDiagnostics'] = $SkipDiagnostics.IsPresent + $paramSnapshot['SkipVersionCheck'] = $SkipVersionCheck.IsPresent + $paramSnapshot['Rollup'] = $Rollup.IsPresent + $paramSnapshot['RollupPlusRaw'] = $RollupPlusRaw.IsPresent + $paramSnapshot['Dashboard'] = $Dashboard + $paramSnapshot['EmitMetricsJson'] = $EmitMetricsJson.IsPresent + $paramSnapshot['MetricsPath'] = $(if ($MetricsPath) { $MetricsPath } else { '' }) + $paramSnapshot['StreamingSchemaSample'] = $StreamingSchemaSample + $paramSnapshot['StreamingChunkSize'] = $StreamingChunkSize + + # Common parameters (work in both modes) + $paramSnapshot['ActivityTypes'] = ($ActivityTypes -join ';') + $paramSnapshot['RecordTypes'] = $(if ($RecordTypes) { ($RecordTypes -join ';') } else { '' }) + $paramSnapshot['ServiceTypes'] = $(if ($ServiceTypes) { ($ServiceTypes -join ';') } else { '' }) + $copilotIncluded = $IncludeCopilotInteraction.IsPresent -or ($ActivityTypes -contains $copilotBaseActivityType) + $paramSnapshot['IncludeCopilotInteraction'] = $copilotIncluded + $paramSnapshot['IncludeM365Usage'] = $IncludeM365Usage.IsPresent + $paramSnapshot['IncludeAgent365Info'] = $IncludeAgent365Info.IsPresent + $paramSnapshot['OnlyAgent365Info'] = $OnlyAgent365Info.IsPresent + $paramSnapshot['ExcludeCopilotInteraction'] = $ExcludeCopilotInteraction.IsPresent + + # Post-processing filters (work in both modes - applied during/after explosion) + $paramSnapshot['AgentsOnly'] = $AgentsOnly.IsPresent + $paramSnapshot['AgentId'] = $(if ($AgentId) { ($AgentId -join ';') } else { '' }) + $paramSnapshot['ExcludeAgents'] = $ExcludeAgents.IsPresent + $paramSnapshot['UserId'] = $(if ($UserIds) { ($UserIds -join ';') } else { '' }) + + # GroupNames only works in live mode (requires auth for expansion) + if (-not $UseEOM -and $GroupNames) { + $paramSnapshot['GroupName'] = ($GroupNames -join ';') + } + + $paramSnapshot['PromptFilter'] = $(if ($PromptFilter) { $PromptFilter } else { '' }) + + # Output format parameters (work in both modes) + $paramSnapshot['ExplodeArrays'] = ($ExplodeArrays.IsPresent -or $ForcedRawInputCsvExplosion -or $ExplodeDeep.IsPresent) + $paramSnapshot['ExplodeDeep'] = $ExplodeDeep.IsPresent + # FlatDepth governs how many nested levels -ExplodeDeep walks. Surface it whenever + # ExplodeDeep is in play, OR whenever the user explicitly set it on the CLI, so + # operators can verify depth was honored. + if ($ExplodeDeep -or $PSBoundParameters.ContainsKey('FlatDepth')) { $paramSnapshot['FlatDepth'] = $FlatDepth } + $paramSnapshot['ExplosionThreads'] = $(if ($ExplosionThreads -eq 0) { 'auto' } else { $ExplosionThreads }) + + # Environment info + $paramSnapshot['PSVersion'] = $PSVersionTable.PSVersion.ToString() + $paramSnapshot['PSEdition'] = $PSVersionTable.PSEdition + $paramSnapshot['HostName'] = $Host.Name + $paramSnapshot['HostVersion'] = $(try { $Host.Version.ToString() } catch { '' }) + + # EntraUsersOutput / Agents365Output are emitted up in the output-destinations + # cluster at the top of the snapshot so all per-stream file paths stay together. + # The earlier injection-via-rebuild logic was removed when that cluster was added. +} +# Parameter Snapshot will be displayed after DSPM processing (see line ~4150) + +# Predeclare script-scope collections to satisfy StrictMode before first access +if (-not (Get-Variable -Name DeepExtraColumns -Scope Script -ErrorAction SilentlyContinue)) { $script:DeepExtraColumns = $null } + +<# +===================================================================== + Operational Logic +===================================================================== +#> + +function Find-AllArrays { + param( + $Data, + [string]$Path = '', + [int]$Depth = 0, + [hashtable]$Arrays + ) + if ($null -eq $Data) { return @{} } + if (-not $Arrays) { $Arrays = @{} } + if ($Depth -gt 6) { return $Arrays } + if ($null -eq $Data) { return $Arrays } + + $isArray = ($Data -is [System.Collections.IEnumerable] -and -not ($Data -is [string]) -and (($Data -is [System.Collections.IList]) -or $Data.GetType().IsArray)) + if ($isArray) { + $key = if ($Path) { $Path } else { 'root' } + if (-not $Arrays.ContainsKey($key)) { + $Arrays[$key] = [pscustomobject]@{ Path = $Path; Data = $Data; Count = ($Data | Measure-Object).Count } + } + } + + $props = $null + if ($Data -is [System.Management.Automation.PSObject]) { $props = $Data.PSObject.Properties } + elseif ($Data -is [System.Collections.IDictionary]) { $props = $Data.GetEnumerator() } + + if ($props) { + foreach ($p in $props) { + $name = if ($p -is [System.Collections.DictionaryEntry]) { $p.Key } else { $p.Name } + $val = if ($p -is [System.Collections.DictionaryEntry]) { $p.Value } else { $p.Value } + $childPath = if ($Path) { "$Path.$name" } else { $name } + Find-AllArrays -Data $val -Path $childPath -Depth ($Depth + 1) -Arrays $Arrays | Out-Null + } + } + # Note: Do NOT recurse into array elements - arrays are treated as terminal values + # that will be converted to JSON strings for predictable column names + return $Arrays +} + +function Test-ScalarValue { param($v) ($null -eq $v -or $v -is [string] -or $v -is [char] -or $v -is [bool] -or $v -is [int] -or $v -is [long] -or $v -is [double] -or $v -is [decimal] -or $v -is [float] -or $v -is [datetime] -or $v -is [guid]) } + +function Import-CsvToDataTable { + <# + .SYNOPSIS + Imports a CSV file directly into a System.Data.DataTable using fast .NET StreamReader. + + .DESCRIPTION + This is 10-50x faster than Import-Csv | ConvertTo-DataTable for large files because it: + 1. Uses .NET StreamReader instead of PowerShell's Import-Csv + 2. Avoids creating intermediate PSObjects + 3. Parses CSV directly into DataTable rows + + .PARAMETER Path + The path to the CSV file to import. + + .OUTPUTS + System.Data.DataTable + #> + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + $dataTable = New-Object System.Data.DataTable + $reader = $null + + try { + $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8) + $lineNum = 0 + $columns = @() + + while ($null -ne ($line = $reader.ReadLine())) { + # Parse CSV line (handles quoted fields with commas) + $fields = [System.Collections.Generic.List[string]]::new() + $field = [System.Text.StringBuilder]::new() + $inQuotes = $false + + for ($i = 0; $i -lt $line.Length; $i++) { + $c = $line[$i] + if ($c -eq '"') { + if ($inQuotes -and $i + 1 -lt $line.Length -and $line[$i + 1] -eq '"') { + [void]$field.Append('"') + $i++ + } else { + $inQuotes = -not $inQuotes + } + } elseif ($c -eq ',' -and -not $inQuotes) { + [void]$fields.Add($field.ToString()) + [void]$field.Clear() + } else { + [void]$field.Append($c) + } + } + [void]$fields.Add($field.ToString()) + + if ($lineNum -eq 0) { + # Header row - create columns + $columns = $fields.ToArray() + foreach ($col in $columns) { + [void]$dataTable.Columns.Add($col, [string]) + } + } else { + # Data row + $row = $dataTable.NewRow() + for ($j = 0; $j -lt [Math]::Min($columns.Count, $fields.Count); $j++) { + $val = $fields[$j] + $row[$j] = if ([string]::IsNullOrEmpty($val)) { [DBNull]::Value } else { $val } + } + [void]$dataTable.Rows.Add($row) + } + $lineNum++ + } + } + finally { + if ($reader) { $reader.Dispose() } + } + + return ,$dataTable +} + +function ConvertTo-DataTable { + <# + .SYNOPSIS + Converts an array of PSObjects to a System.Data.DataTable for high-performance Excel export. + + .DESCRIPTION + Export-Excel with piped PSObjects processes cells one-by-one (~400 cells/sec), which is extremely + slow for large datasets. Send-SQLDataToExcel with DataTable uses bulk insert and is 100-1000x faster. + This function converts PSObject arrays to DataTable format for use with Send-SQLDataToExcel. + + .PARAMETER InputObject + The array of PSObjects to convert to a DataTable. + + .OUTPUTS + System.Data.DataTable + #> + param( + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] + [object[]]$InputObject + ) + + begin { + $dataTable = New-Object System.Data.DataTable + $isFirstRow = $true + $columns = @() + } + + process { + foreach ($obj in $InputObject) { + if ($isFirstRow) { + $columns = @($obj.PSObject.Properties.Name) + foreach ($colName in $columns) { + [void]$dataTable.Columns.Add($colName, [string]) + } + $isFirstRow = $false + } + + $row = $dataTable.NewRow() + foreach ($colName in $columns) { + $val = $obj.$colName + $row[$colName] = if ($null -eq $val) { [DBNull]::Value } else { [string]$val } + } + [void]$dataTable.Rows.Add($row) + } + } + + end { + return ,$dataTable + } +} + +function Export-DataTableToExcel { + <# + .SYNOPSIS + High-performance Excel export using DataTable bulk insert method. + + .DESCRIPTION + Wrapper function that converts PSObjects to DataTable and exports using Send-SQLDataToExcel. + This is 100-1000x faster than piping to Export-Excel for large datasets. + + .PARAMETER Data + The array of PSObjects to export. + + .PARAMETER Path + The path to the Excel file. + + .PARAMETER WorksheetName + The name of the worksheet/tab. + #> + param( + [Parameter(Mandatory = $true)] + [object[]]$Data, + + [Parameter(Mandatory = $true)] + [string]$Path, + + [Parameter(Mandatory = $true)] + [string]$WorksheetName + ) + + $dataTable = $Data | ConvertTo-DataTable + Send-SQLDataToExcel -DataTable $dataTable -Path $Path -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' +} + +function ConvertTo-UniqueString { + param([object]$items, [char]$Sep = ';') + if ($null -eq $items) { return $null } + $set = New-Object System.Collections.Generic.HashSet[string] + foreach ($v in $items) { if ($null -ne $v -and $v -ne '') { [void]$set.Add([string]$v) } } + ([string]::Join($Sep, $set)) +} + +function ConvertTo-FlatColumns { + param([object]$Node, [string]$Prefix = '', [int]$MaxDepth = 60) + $cols = @{} + function Recurse([object]$n, [string]$p, [int]$d) { + if ($d -gt $MaxDepth) { return } + if ($null -eq $n) { if ($p) { $cols[$p.TrimEnd('.')] = $null }; return } + if (Test-ScalarValue $n) { if ($p) { $cols[$p.TrimEnd('.')] = $n }; return } + if ($n -is [System.Collections.IEnumerable] -and -not ($n -is [string]) -and -not ($n -is [System.Collections.IDictionary])) { + # Smart array handling: single-element arrays recurse without index, multi-element become JSON + $arr = @($n) + if ($arr.Count -eq 1) { + # Single element: recurse into it without adding index to path (clean column names) + Recurse -n $arr[0] -p $p -d ($d + 1) + } elseif ($arr.Count -gt 1) { + # Multiple elements: serialize to JSON (row explosion handles important arrays separately) + if ($p) { + try { $cols[$p.TrimEnd('.')] = ($n | ConvertTo-Json -Depth 10 -Compress -ErrorAction SilentlyContinue) } + catch { $cols[$p.TrimEnd('.')] = '' } + } + } else { + # Empty array + if ($p) { $cols[$p.TrimEnd('.')] = '' } + } + return + } + $props = $null; try { $props = $n.PSObject.Properties } catch {} + if ($props) { + foreach ($prop in $props) { $name = [string]$prop.Name; $child = $prop.Value; $cp = if ($p) { $p + $name + '.' } else { $name + '.' }; Recurse -n $child -p $cp -d ($d + 1) } + } + } + Recurse -n $Node -p $Prefix -d 0 + return $cols +} + +function To-RecordArray { + param($records) + $result = @() + if ($null -eq $records) { return $result } + $isEnumerable = ($records -is [System.Collections.IEnumerable]) + $isScalarish = ($records -is [string] -or $records -is [System.Management.Automation.PSObject] -or $records -is [System.Management.Automation.PSCustomObject]) + if ($isEnumerable -and -not $isScalarish) { + foreach ($r in $records) { $result += ,$r } + } + else { + $result += ,$records + } + return $result +} + +try { + $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path + $schemaHelper = Join-Path $scriptDir "..\..\scripts\lib\M365UsageSchema.ps1" + if (-not (Test-Path $schemaHelper)) { $schemaHelper = Join-Path $scriptDir "M365UsageSchema.ps1" } + if (Test-Path $schemaHelper) { . $schemaHelper } +} catch {} + +function Invoke-ReplayInlineExport { + param( + [Parameter(Mandatory)] [System.Collections.IEnumerable]$Logs + ) + Write-LogHost "Replay inline export starting..." -ForegroundColor Magenta + $exportTemp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_export_" + [guid]::NewGuid().ToString() + ".tmp") + # Fixed 153-column M code schema (matches live explosion output exactly) + $columnOrder = $PurviewExplodedHeader + Open-CsvWriter -Path $exportTemp -Columns $columnOrder + $total = 0 + $idx = 0 + $errCount = 0 + $errLimit = 25 + foreach ($log in $Logs) { + $idx++ + if ($idx % 5000 -eq 0) { Write-LogHost ("Replay inline progress: {0} records" -f $idx) -ForegroundColor DarkGray } + try { + $records = Convert-ToPurviewExplodedRecords -Record $log -Deep:$ExplodeDeep -PromptFilterValue $PromptFilter + $recordsArr = To-RecordArray $records + if ($recordsArr.Count -gt 0) { + $total += $recordsArr.Count + $emitSet = $recordsArr | ForEach-Object { $_ | Select-Object -Property $columnOrder } + $rowsOut = @($emitSet) + if ($rowsOut.Count -gt 0) { Write-CsvRows -Rows $rowsOut -Columns $columnOrder } + } + } catch { + $errCount++ + } + } + try { Close-CsvWriter } catch {} + try { Move-Item -Force -Path $exportTemp -Destination $OutputFile } catch {} + try { $script:metrics.TotalStructuredRows = $total } catch {} + Write-LogHost ("Replay inline export complete: {0} rows" -f $total) -ForegroundColor Green + + # Explosion summary for replay mode + Write-LogHost "" + Write-LogHost "=== REPLAY EXPLOSION SUMMARY ===" -ForegroundColor Cyan + Write-LogHost (" Input records: {0:N0}" -f $idx) -ForegroundColor White + Write-LogHost (" Output rows: {0:N0}" -f $total) -ForegroundColor White + if ($total -gt $idx) { + $explosionRatio = [Math]::Round($total / $idx, 2) + Write-LogHost (" Expansion: {0}x ({1:N0} additional rows from array explosion)" -f $explosionRatio, ($total - $idx)) -ForegroundColor Green + } elseif ($total -eq $idx) { + Write-LogHost " Expansion: 1:1 (no arrays exploded)" -ForegroundColor Yellow + } else { + Write-LogHost (" Reduction: {0:N0} records filtered out" -f ($idx - $total)) -ForegroundColor DarkYellow + } + if ($errCount -gt 0) { + Write-LogHost (" Errors: {0:N0} record(s) failed to process" -f $errCount) -ForegroundColor Red + } + # Replay summary label tracks append mode. + if ($AppendFile) { + Write-LogHost (" Appended to: {0}" -f $AppendFile) -ForegroundColor Gray + } else { + Write-LogHost (" Output file: {0}" -f (Get-DisplayPath -LocalPath $OutputFile)) -ForegroundColor Gray + } + Write-LogHost "" +} + +function Get-SafeProperty { param($obj, [string]$name) try { if ($null -ne $obj -and $obj.PSObject.Properties[$name]) { return $obj.($name) } } catch {}; return $null } + +# --- Purview Exploded Schema (153 columns — matches M code #"Changed Type" step exactly) --- +$PurviewExplodedHeader = @( + 'RecordId', 'CreationDate', 'RecordType', 'Operation', 'UserId', + 'AssociatedAdminUnits', 'AssociatedAdminUnitsNames', + '@odata.type', 'CreationTime', 'Id', 'OrganizationId', + 'ResultStatus', 'UserKey', 'UserType', 'Version', 'Workload', + 'ClientIP', 'ObjectId', 'AzureActiveDirectoryEventType', + 'ActorContextId', 'ActorIpAddress', 'InterSystemsId', 'IntraSystemId', + 'SupportTicketId', 'TargetContextId', 'ApplicationId', + 'DeviceProperties.OS', 'DeviceProperties.BrowserType', + 'ErrorNumber', + 'SiteUrl', 'SourceRelativeUrl', 'SourceFileName', 'SourceFileExtension', + 'ListId', 'ListItemUniqueId', 'WebId', 'ApplicationDisplayName', 'EventSource', + 'ItemType', 'SiteSensitivityLabelId', 'GeoLocation', 'IsManagedDevice', + 'DeviceDisplayName', 'ListBaseType', 'ListServerTemplate', + 'AuthenticationType', 'Site', 'DoNotDistributeEvent', 'HighPriorityMediaProcessing', + 'BrowserName', 'BrowserVersion', 'CorrelationId', 'Platform', 'UserAgent', + 'ActorInfoString', 'AppId', 'AuthType', 'ClientAppId', 'ClientIPAddress', + 'ClientInfoString', 'ExternalAccess', 'InternalLogonType', 'LogonType', + 'LogonUserSid', 'MailboxGuid', 'MailboxOwnerSid', 'MailboxOwnerUPN', + 'OrganizationName', 'OriginatingServer', 'SessionId', + 'TokenObjectId', 'TokenTenantId', 'TokenType', 'SaveToSentItems', + 'OperationCount', 'FileSizeBytes', + 'MeetingId', 'MeetingType', 'EventSignature', 'EventData', + 'Permission', 'SensitivityLabelId', 'SharingLinkScope', + 'TargetUserOrGroupType', 'TargetUserOrGroupName', + 'MeetingURL', 'ChatId', 'MessageId', 'MessageSizeInBytes', 'MessageType', + 'FormId', 'FormName', 'VideoId', 'VideoName', 'ChannelId', 'ViewDuration', + 'ClientRegion', 'CopilotLogVersion', 'TargetId', + 'TeamName', 'TeamGuid', 'ResponseId', 'IsAnonymous', 'DeviceType', + 'ChannelName', 'ChannelGuid', 'ChannelType', 'AppName', 'EnvironmentName', + 'PlanId', 'PlanName', 'TaskId', 'TaskName', 'PercentComplete', + 'CrossMailboxOperation', + 'RecordTypeNum', 'ResultStatus_Audit', + 'ModelId', 'ModelProvider', 'ModelFamily', + 'TokensTotal', 'TokensInput', 'TokensOutput', 'DurationMs', 'OutcomeStatus', + 'ConversationId', 'TurnNumber', 'RetryCount', 'ClientVersion', 'ClientPlatform', + 'AgentId', 'AgentName', 'AgentVersion', 'AgentCategory', 'ApplicationName', + 'AppHost', 'ThreadId', + 'Context_Id', 'Context_Type', + 'Message_Id', 'Message_isPrompt', + 'AccessedResource_Action', 'AccessedResource_PolicyDetails', 'AccessedResource_SiteUrl', + 'AISystemPlugin_Id', 'AISystemPlugin_Name', + 'ModelTransparencyDetails_ModelName', 'MessageIds', + 'AccessedResource_Name', 'AccessedResource_SensitivityLabel', + 'AccessedResource_ResourceType', 'SensitivityLabel', 'Context_Item' +) + +# --- M365 Usage Base Header --- +$M365UsageBaseHeader = @( + 'RecordId','CreationDate','RecordType','Operation','UserId','AuditData','AssociatedAdminUnits','AssociatedAdminUnitsNames','CreationTime','Id','OrganizationId','ResultStatus','UserKey','UserType','Version','Workload','ClientIP','ObjectId','AzureActiveDirectoryEventType','ExtendedProperties','ExtendedProperties.ResultStatusDetail','ExtendedProperties.Name','ExtendedProperties.Value','ExtendedProperties.UserAgent','ExtendedProperties.RequestType','ModifiedProperties','Actor','Actor.ID','Actor.Type','ActorContextId','ActorIpAddress','InterSystemsId','IntraSystemId','SupportTicketId','Target','Target.ID','Target.Type','TargetContextId','ApplicationId','DeviceProperties','DeviceProperties.OS','DeviceProperties.Name','DeviceProperties.Value','DeviceProperties.BrowserType','DeviceProperties.SessionId','ErrorNumber','ExtendedProperties.KeepMeSignedIn','DeviceProperties.Id','DeviceProperties.DisplayName','DeviceProperties.TrustType','ExtendedProperties.UserAuthenticationMethod','DeviceProperties.IsCompliant','DeviceProperties.IsCompliantAndManaged', + # SharePoint / OneDrive + 'SiteUrl','SourceRelativeUrl','SourceFileName','SourceFileExtension','ListId','ListItemUniqueId','WebId','ApplicationDisplayName','EventSource','ItemType','SiteSensitivityLabelId','GeoLocation','IsManagedDevice','DeviceDisplayName','ListBaseType','ListServerTemplate','AuthenticationType','Site','DoNotDistributeEvent','HighPriorityMediaProcessing', + # App Access Context + 'AppAccessContext.ClientAppId','AppAccessContext.ClientAppName','AppAccessContext.CorrelationId','AppAccessContext.AADSessionId','AppAccessContext.UniqueTokenId','AppAccessContext.AuthTime','AppAccessContext.TokenIssuedAtTime','AppAccessContext.UserObjectId','AppAccessContext.DeviceId' +) + +# --- Unified Replay Header (auto-detects all activity types) --- +# Scans input CSV to detect columns from any record type, merges with PurviewExplodedHeader for Copilot +# Skips CopilotEventData.* paths since explosion produces flat column names +function Get-UnifiedReplayHeader { + param( + [Parameter(Mandatory)][string]$RawCsvPath, + [int]$Sample = 500 + ) + # Base columns common to all activity types + $base = @('RecordId','CreationDate','RecordType','Operation','UserId','AuditData','AssociatedAdminUnits','AssociatedAdminUnitsNames','CreationTime','Id','OrganizationId','ResultStatus','UserKey','UserType','Version','Workload','ClientIP','ObjectId','AzureActiveDirectoryEventType','ExtendedProperties','ExtendedProperties.ResultStatusDetail','ExtendedProperties.Name','ExtendedProperties.Value','ExtendedProperties.UserAgent','ExtendedProperties.RequestType','ModifiedProperties','Actor','Actor.ID','Actor.Type','ActorContextId','ActorIpAddress','InterSystemsId','IntraSystemId','SupportTicketId','Target','Target.ID','Target.Type','TargetContextId','ApplicationId','DeviceProperties','DeviceProperties.OS','DeviceProperties.Name','DeviceProperties.Value','DeviceProperties.BrowserType','DeviceProperties.SessionId','ErrorNumber','ExtendedProperties.KeepMeSignedIn','DeviceProperties.Id','DeviceProperties.DisplayName','DeviceProperties.TrustType','ExtendedProperties.UserAuthenticationMethod','DeviceProperties.IsCompliant','DeviceProperties.IsCompliantAndManaged') + $aug = @( + 'SiteUrl','SourceRelativeUrl','SourceFileName','SourceFileExtension','ListId','ListItemUniqueId','WebId','ApplicationDisplayName','EventSource','ItemType','SiteSensitivityLabelId','GeoLocation','IsManagedDevice','DeviceDisplayName','ListBaseType','ListServerTemplate','AuthenticationType','Site','DoNotDistributeEvent','HighPriorityMediaProcessing', + 'AppAccessContext.ClientAppId','AppAccessContext.ClientAppName','AppAccessContext.CorrelationId','AppAccessContext.AADSessionId','AppAccessContext.UniqueTokenId','AppAccessContext.AuthTime','AppAccessContext.TokenIssuedAtTime','AppAccessContext.UserObjectId','AppAccessContext.DeviceId','AppAccessContext.@odata.type','AppAccessContext.APIId','AppAccessContext.IssuedAtTime' + ) + $detected = New-Object System.Collections.Generic.List[string] + $hasCopilot = $false + + # Recursively detect column paths from JSON, skipping CopilotEventData (handled by explosion with flat names) + function Add-Paths([object]$node, [string]$prefix, [System.Collections.Generic.List[string]]$collector) { + if ($null -eq $node) { return } + if (Test-ScalarValue $node) { if ($prefix) { $collector.Add($prefix) | Out-Null }; return } + if ($node -is [System.Collections.IEnumerable] -and $node -isnot [string]) { + foreach ($item in $node) { Add-Paths $item $prefix $collector } + return + } + if ($node.PSObject -and $node.PSObject.Properties) { + foreach ($prop in $node.PSObject.Properties) { + $pn = $prop.Name; $pv = $prop.Value + $path = if ($prefix) { "$prefix.$pn" } else { $pn } + # SKIP CopilotEventData - explosion handles these with flat column names + if ($pn -eq 'CopilotEventData') { continue } + # Special handling for Name/Value arrays (pivot into columns) + if ($pn -eq 'ExtendedProperties' -and $pv -is [System.Collections.IEnumerable]) { + foreach ($item in $pv) { try { if ($item.Name) { $collector.Add("ExtendedProperties.$($item.Name)") | Out-Null } } catch {} } + continue + } + if ($pn -eq 'DeviceProperties' -and $pv -is [System.Collections.IEnumerable]) { + foreach ($item in $pv) { try { if ($item.Name) { $collector.Add("DeviceProperties.$($item.Name)") | Out-Null } } catch {} } + continue + } + Add-Paths $pv $path $collector + } + } + } + if ($RawCsvPath -and (Test-Path $RawCsvPath)) { + try { + $rows = Import-Csv $RawCsvPath | Select-Object -First $Sample + foreach ($r in $rows) { + try { + $audit = $r.AuditData | ConvertFrom-Json -ErrorAction Stop + if ($audit) { + # Detect if any Copilot records exist + if ($audit.CopilotEventData) { $hasCopilot = $true } + Add-Paths $audit '' $detected + } + } catch {} + } + } catch {} + } + # Build unified header: base + augmented + detected (non-Copilot) + PurviewExplodedHeader (flat Copilot columns) + $header = New-Object System.Collections.Generic.List[string] + foreach ($c in $base) { if (-not $header.Contains($c)) { $header.Add($c) } } + foreach ($c in $aug) { if (-not $header.Contains($c)) { $header.Add($c) } } + foreach ($c in $detected) { if (-not $header.Contains($c)) { $header.Add($c) } } + # Always include flat Copilot columns from PurviewExplodedHeader (supports all activity types) + foreach ($c in $PurviewExplodedHeader) { if (-not $header.Contains($c)) { $header.Add($c) } } + try { + if ($RawCsvPath) { + $hdrPath = Join-Path (Split-Path $RawCsvPath -Parent) 'UnifiedReplayHeader.txt' + $header | Set-Content -Path $hdrPath -Encoding utf8 + } + } catch {} + return $header +} + +# --- M365 Usage Wide Header (used for replay of previously exported raw CSVs) --- +function Get-M365UsageWideHeader { + param( + [string]$RawCsvPath, + [int]$Sample = 500 + ) + # Delegate to unified header function + return Get-UnifiedReplayHeader -RawCsvPath $RawCsvPath -Sample $Sample +} + +# --- Entra Users Schema (47 columns) --- +# 30 core + 5 manager + 2 license columns + 10 Power BI template compatibility columns +$EntraUsersHeader = @( + 'userPrincipalName','displayName','id','mail','givenName','surname','jobTitle','department','employeeType','employeeId','employeeHireDate', + 'officeLocation','city','state','country','postalCode','companyName','employeeOrgData_division','employeeOrgData_costCenter', + 'accountEnabled','userType','createdDateTime','usageLocation','preferredLanguage','onPremisesSyncEnabled','onPremisesImmutableId','externalUserState', + 'proxyAddresses_Primary','proxyAddresses_Count','proxyAddresses_All', + 'manager_id','manager_displayName','manager_userPrincipalName','manager_mail','manager_jobTitle', + 'assignedLicenses','hasLicense', + # Power BI template compatibility columns (alias mappings) + 'ManagerID','BusinessAreaLabel','CountryofEmployment','CompanyCodeLabel','CostCentreLabel','UserName', + # Power BI template compatibility columns (null placeholders for Viva Insights fields) + 'EffectiveDate','FunctionType','BusinessAreaCode','OrgLevel_3Label' +) + +function Test-EntraUsersSchema { + param( + [Parameter(Mandatory=$true)][array]$Users, + [switch]$Quiet + ) + if (-not $Users -or $Users.Count -eq 0) { return } + $expected = $EntraUsersHeader + $actual = $Users[0].PSObject.Properties.Name + $missing = @(); foreach ($c in $expected) { if ($c -notin $actual) { $missing += $c } } + $extra = @(); foreach ($c in $actual) { if ($c -notin $expected) { $extra += $c } } + if ($missing.Count -gt 0 -or $extra.Count -gt 0) { + Write-LogHost ("WARNING: EntraUsers schema mismatch. Missing: {0}; Extra: {1}" -f ($missing -join ', '), ($extra -join ', ')) -ForegroundColor Yellow + } elseif (-not $Quiet) { + Write-LogHost "Validated EntraUsers schema ($($expected.Count) columns)." -ForegroundColor DarkGray + } +} + +# ═══════════════════════════════════════════════════════════════════════════════ +# FAST ROW CREATION HELPER +# Converts a hashtable to PSCustomObject in a single operation (avoids Add-Member overhead) +# Used by explosion logic to build rows efficiently via hashtable accumulation +# ═══════════════════════════════════════════════════════════════════════════════ +function New-FastRow { + <# + .SYNOPSIS + Creates a PSCustomObject from a hashtable in a single operation. + .DESCRIPTION + Builds a row by accumulating properties in a hashtable first, then converting + to PSCustomObject once. This is significantly faster than repeated Add-Member calls. + .PARAMETER Properties + A hashtable containing property names and values for the new object. + .EXAMPLE + $props = @{ Name = 'Test'; Value = 123 } + $row = New-FastRow -Properties $props + #> + [CmdletBinding()] + param([Parameter(Mandatory)][hashtable]$Properties) + return [PSCustomObject]$Properties +} + +$existingDeep = Get-Variable -Name DeepExtraColumns -Scope Script -ErrorAction SilentlyContinue +if (-not $existingDeep -or -not $script:DeepExtraColumns) { $script:DeepExtraColumns = New-Object System.Collections.Generic.List[string] } + +function Convert-ToPurviewExplodedRecords { + [CmdletBinding()] + param( + [Parameter(Mandatory)] $Record, + [switch]$Deep, + [switch]$PartialExplode, # NEW: Prompt-specific explosion only (preserves AuditData) + [string]$PromptFilterValue, + [switch]$SkipMetrics # Used by parallel replay to defer metrics aggregation to parent thread + ) + try { + $auditData = if ($Record.PSObject.Properties['_ParsedAuditData']) { $Record._ParsedAuditData } else { try { $Record.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $null } } + if (-not $auditData) { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringMissingAuditData++ + } + return @() + } + try { Profile-AuditData $auditData } catch {} + # Ensure helper is available when re-entering session (PS alias scoping) + if (-not (Get-Command Find-AllArrays -ErrorAction SilentlyContinue)) { Set-Alias -Name Find-AllArrays -Value Find-AllArrays -ErrorAction SilentlyContinue | Out-Null } + + $ced = Get-SafeProperty $auditData 'CopilotEventData' + if (-not $ced) { + # ── M code-aligned non-Copilot path: fixed 153-column extraction (no dynamic discovery) ── + # Produces exactly 1 row per record with all 153 M code columns populated from AuditData. + # No array explosion for non-Copilot records (matches M code behaviour). + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp). + $recordId = if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id } + $creationDate = script:Format-DatePurviewFast $Record.CreationDate + $creationTime = try { script:Format-DatePurviewFast $auditData.CreationTime } catch { '' } + $opValue = try { $auditData.Operation } catch { if ($Record.Operation) { $Record.Operation } else { $Record.Operations } } + $uidValue = try { $auditData.UserId } catch { if ($Record.UserId) { $Record.UserId } elseif ($Record.UserIds) { $Record.UserIds } else { '' } } + $recordType = $Record.RecordType + $resultStatus = Get-SafeProperty $auditData 'ResultStatus' + $recordTypeNum = try { [int]$recordType } catch { $recordType } + $applicationId = Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationId'), (Get-SafeProperty $auditData 'AppId'), (Get-SafeProperty $auditData 'ClientAppId')) + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp) + $devProps = Get-SafeProperty $auditData 'DeviceProperties' + $dpOS = ''; $dpBrowser = '' + if ($devProps -and ($devProps -is [System.Collections.IEnumerable])) { + foreach ($dp in $devProps) { + try { + if ($dp.Name -eq 'OS') { $dpOS = $dp.Value } + elseif ($dp.Name -eq 'BrowserType') { $dpBrowser = $dp.Value } + } catch {} + } + } + # AgentCategory + $agentIdVal = Get-SafeProperty $auditData 'AgentId' + $agentCat = '' + if ($agentIdVal) { + if ($agentIdVal -like "CopilotStudio.Declarative.*") { $agentCat = "Declarative Agent" } + elseif ($agentIdVal -like "CopilotStudio.CustomEngine.*") { $agentCat = "Custom Engine Agent" } + elseif ($agentIdVal -like "P_*") { $agentCat = "Declarative Agent (Purview)" } + else { $agentCat = "Other Agent" } + } + $rowObj = [PSCustomObject][ordered]@{ + RecordId = $recordId + CreationDate = $creationDate + RecordType = $recordType + Operation = $opValue + UserId = $uidValue + AssociatedAdminUnits = $(try { if ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } elseif ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } elseif ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + '@odata.type' = (Get-SafeProperty $auditData '@odata.type') + CreationTime = $creationTime + Id = (Get-SafeProperty $auditData 'Id') + OrganizationId = (Get-SafeProperty $auditData 'OrganizationId') + ResultStatus = $resultStatus + UserKey = (Get-SafeProperty $auditData 'UserKey') + UserType = (Get-SafeProperty $auditData 'UserType') + Version = (Get-SafeProperty $auditData 'Version') + Workload = (Get-SafeProperty $auditData 'Workload') + ClientIP = (Get-SafeProperty $auditData 'ClientIP') + ObjectId = (Get-SafeProperty $auditData 'ObjectId') + AzureActiveDirectoryEventType = (Get-SafeProperty $auditData 'AzureActiveDirectoryEventType') + ActorContextId = (Get-SafeProperty $auditData 'ActorContextId') + ActorIpAddress = (Get-SafeProperty $auditData 'ActorIpAddress') + InterSystemsId = (Get-SafeProperty $auditData 'InterSystemsId') + IntraSystemId = (Get-SafeProperty $auditData 'IntraSystemId') + SupportTicketId = (Get-SafeProperty $auditData 'SupportTicketId') + TargetContextId = (Get-SafeProperty $auditData 'TargetContextId') + ApplicationId = $applicationId + 'DeviceProperties.OS' = $dpOS + 'DeviceProperties.BrowserType' = $dpBrowser + ErrorNumber = (Get-SafeProperty $auditData 'ErrorNumber') + SiteUrl = (Get-SafeProperty $auditData 'SiteUrl') + SourceRelativeUrl = (Get-SafeProperty $auditData 'SourceRelativeUrl') + SourceFileName = (Get-SafeProperty $auditData 'SourceFileName') + SourceFileExtension = (Get-SafeProperty $auditData 'SourceFileExtension') + ListId = (Get-SafeProperty $auditData 'ListId') + ListItemUniqueId = (Get-SafeProperty $auditData 'ListItemUniqueId') + WebId = (Get-SafeProperty $auditData 'WebId') + ApplicationDisplayName = (Get-SafeProperty $auditData 'ApplicationDisplayName') + EventSource = (Get-SafeProperty $auditData 'EventSource') + ItemType = (Get-SafeProperty $auditData 'ItemType') + SiteSensitivityLabelId = (Get-SafeProperty $auditData 'SiteSensitivityLabelId') + GeoLocation = (Get-SafeProperty $auditData 'GeoLocation') + IsManagedDevice = (Get-SafeProperty $auditData 'IsManagedDevice') + DeviceDisplayName = (Get-SafeProperty $auditData 'DeviceDisplayName') + ListBaseType = (Get-SafeProperty $auditData 'ListBaseType') + ListServerTemplate = (Get-SafeProperty $auditData 'ListServerTemplate') + AuthenticationType = (Get-SafeProperty $auditData 'AuthenticationType') + Site = (Get-SafeProperty $auditData 'Site') + DoNotDistributeEvent = (Get-SafeProperty $auditData 'DoNotDistributeEvent') + HighPriorityMediaProcessing = (Get-SafeProperty $auditData 'HighPriorityMediaProcessing') + BrowserName = (Get-SafeProperty $auditData 'BrowserName') + BrowserVersion = (Get-SafeProperty $auditData 'BrowserVersion') + CorrelationId = (Get-SafeProperty $auditData 'CorrelationId') + Platform = (Get-SafeProperty $auditData 'Platform') + UserAgent = (Get-SafeProperty $auditData 'UserAgent') + ActorInfoString = (Get-SafeProperty $auditData 'ActorInfoString') + AppId = (Get-SafeProperty $auditData 'AppId') + AuthType = (Get-SafeProperty $auditData 'AuthType') + ClientAppId = (Get-SafeProperty $auditData 'ClientAppId') + ClientIPAddress = (Get-SafeProperty $auditData 'ClientIPAddress') + ClientInfoString = (Get-SafeProperty $auditData 'ClientInfoString') + ExternalAccess = (Get-SafeProperty $auditData 'ExternalAccess') + InternalLogonType = (Get-SafeProperty $auditData 'InternalLogonType') + LogonType = (Get-SafeProperty $auditData 'LogonType') + LogonUserSid = (Get-SafeProperty $auditData 'LogonUserSid') + MailboxGuid = (Get-SafeProperty $auditData 'MailboxGuid') + MailboxOwnerSid = (Get-SafeProperty $auditData 'MailboxOwnerSid') + MailboxOwnerUPN = (Get-SafeProperty $auditData 'MailboxOwnerUPN') + OrganizationName = (Get-SafeProperty $auditData 'OrganizationName') + OriginatingServer = (Get-SafeProperty $auditData 'OriginatingServer') + SessionId = (Get-SafeProperty $auditData 'SessionId') + TokenObjectId = (Get-SafeProperty $auditData 'TokenObjectId') + TokenTenantId = (Get-SafeProperty $auditData 'TokenTenantId') + TokenType = (Get-SafeProperty $auditData 'TokenType') + SaveToSentItems = (Get-SafeProperty $auditData 'SaveToSentItems') + OperationCount = (Get-SafeProperty $auditData 'OperationCount') + FileSizeBytes = (Get-SafeProperty $auditData 'FileSizeBytes') + MeetingId = (Get-SafeProperty $auditData 'MeetingId') + MeetingType = (Get-SafeProperty $auditData 'MeetingType') + EventSignature = (Get-SafeProperty $auditData 'EventSignature') + EventData = (Get-SafeProperty $auditData 'EventData') + Permission = (Get-SafeProperty $auditData 'Permission') + SensitivityLabelId = (Get-SafeProperty $auditData 'SensitivityLabelId') + SharingLinkScope = (Get-SafeProperty $auditData 'SharingLinkScope') + TargetUserOrGroupType = (Get-SafeProperty $auditData 'TargetUserOrGroupType') + TargetUserOrGroupName = (Get-SafeProperty $auditData 'TargetUserOrGroupName') + MeetingURL = (Get-SafeProperty $auditData 'MeetingURL') + ChatId = (Get-SafeProperty $auditData 'ChatId') + MessageId = (Get-SafeProperty $auditData 'MessageId') + MessageSizeInBytes = (Get-SafeProperty $auditData 'MessageSizeInBytes') + MessageType = (Get-SafeProperty $auditData 'MessageType') + FormId = (Get-SafeProperty $auditData 'FormId') + FormName = (Get-SafeProperty $auditData 'FormName') + VideoId = (Get-SafeProperty $auditData 'VideoId') + VideoName = (Get-SafeProperty $auditData 'VideoName') + ChannelId = (Get-SafeProperty $auditData 'ChannelId') + ViewDuration = (Get-SafeProperty $auditData 'ViewDuration') + ClientRegion = (Get-SafeProperty $auditData 'ClientRegion') + CopilotLogVersion = (Get-SafeProperty $auditData 'CopilotLogVersion') + TargetId = (Get-SafeProperty $auditData 'TargetId') + TeamName = (Get-SafeProperty $auditData 'TeamName') + TeamGuid = (Get-SafeProperty $auditData 'TeamGuid') + ResponseId = (Get-SafeProperty $auditData 'ResponseId') + IsAnonymous = (Get-SafeProperty $auditData 'IsAnonymous') + DeviceType = (Get-SafeProperty $auditData 'DeviceType') + ChannelName = (Get-SafeProperty $auditData 'ChannelName') + ChannelGuid = (Get-SafeProperty $auditData 'ChannelGuid') + ChannelType = (Get-SafeProperty $auditData 'ChannelType') + AppName = (Get-SafeProperty $auditData 'AppName') + EnvironmentName = (Get-SafeProperty $auditData 'EnvironmentName') + PlanId = (Get-SafeProperty $auditData 'PlanId') + PlanName = (Get-SafeProperty $auditData 'PlanName') + TaskId = (Get-SafeProperty $auditData 'TaskId') + TaskName = (Get-SafeProperty $auditData 'TaskName') + PercentComplete = (Get-SafeProperty $auditData 'PercentComplete') + CrossMailboxOperation = (Get-SafeProperty $auditData 'CrossMailboxOperation') + RecordTypeNum = $recordTypeNum + ResultStatus_Audit = $resultStatus + ModelId = (Get-SafeProperty $auditData 'ModelId') + ModelProvider = (Get-SafeProperty $auditData 'ModelProvider') + ModelFamily = (Get-SafeProperty $auditData 'ModelFamily') + TokensTotal = (Get-SafeProperty $auditData 'TokensTotal') + TokensInput = (Get-SafeProperty $auditData 'TokensInput') + TokensOutput = (Get-SafeProperty $auditData 'TokensOutput') + DurationMs = (Get-SafeProperty $auditData 'DurationMs') + OutcomeStatus = (Get-SafeProperty $auditData 'OutcomeStatus') + ConversationId = (Get-SafeProperty $auditData 'ConversationId') + TurnNumber = (Get-SafeProperty $auditData 'TurnNumber') + RetryCount = (Get-SafeProperty $auditData 'RetryCount') + ClientVersion = (Get-SafeProperty $auditData 'ClientVersion') + ClientPlatform = (Get-SafeProperty $auditData 'ClientPlatform') + AgentId = $agentIdVal + AgentName = (Get-SafeProperty $auditData 'AgentName') + AgentVersion = (Get-SafeProperty $auditData 'AgentVersion') + AgentCategory = $agentCat + ApplicationName = (Get-SafeProperty $auditData 'ApplicationName') + SensitivityLabel = (Get-SafeProperty $auditData 'SensitivityLabel') + # CED sub-fields — empty for non-Copilot records + AppHost = '' + ThreadId = '' + Context_Id = '' + Context_Type = '' + Message_Id = '' + Message_isPrompt = '' + AccessedResource_Action = '' + AccessedResource_PolicyDetails = '' + AccessedResource_SiteUrl = '' + AISystemPlugin_Id = '' + AISystemPlugin_Name = '' + ModelTransparencyDetails_ModelName = '' + MessageIds = '' + AccessedResource_Name = '' + AccessedResource_SensitivityLabel = '' + AccessedResource_ResourceType = '' + Context_Item = '' + } + if ($Deep) { + # Deep flatten entire AuditData for each row (no raw JSON) + $flatAudit = ConvertTo-FlatColumns -Node $auditData -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flatAudit.Keys) { if (-not $rowObj.PSObject.Properties[$k]) { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flatAudit[$k] -Force } } + } + return @($rowObj) + } + $messages = script:GetArrayFast $ced 'Messages' + if ($PromptFilterValue) { + $filteredMessages = New-Object System.Collections.Generic.List[object] + if ($PromptFilterValue -eq 'Null') { + foreach ($msg in $messages) { if ($null -eq $msg.isPrompt) { $filteredMessages.Add($msg) } } + } + elseif ($PromptFilterValue -eq 'Both') { + foreach ($msg in $messages) { if ($null -ne $msg.isPrompt) { $filteredMessages.Add($msg) } } + } + else { + $targetValue = ($PromptFilterValue -eq 'Prompt') + foreach ($msg in $messages) { try { if ($msg.isPrompt -eq $targetValue) { $filteredMessages.Add($msg) } } catch {} } + } + $messages = $filteredMessages + if ($messages.Count -eq 0) { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringPromptFiltered++ + } + return @() + } + } + $contexts = script:GetArrayFast $ced 'Contexts' + $resources = script:GetArrayFast $ced 'AccessedResources' + $pluginsRaw = script:GetArrayFast $ced 'AISystemPlugin' + $modelDetRaw = script:GetArrayFast $ced 'ModelTransparencyDetails' + $messageIds = script:GetArrayFast $ced 'MessageIds' + + # DSPM for AI: Extract SensitivityLabels array + $sensitivityLabels = script:GetArrayFast $ced 'SensitivityLabels' + + # DSPM for AI: Determine activity type for conditional 2-level explosion + $activityType = try { $auditData.Operation } catch { $null } + + # DSPM for AI: Extract 2nd-level arrays (for full explosion mode) + $plugins = $null + $recordingSessions = $null + $contextItems = $null + + if (-not $PartialExplode) { + # Full explosion mode: Extract 2nd-level arrays for row count calculation + if ($activityType -eq 'ConnectedAIAppInteraction' -and $appIdentityRaw) { + $plugins = script:GetArrayFast $appIdentityRaw 'Plugins' + } + if ($activityType -eq 'CopilotInteraction' -and $contexts.Count -gt 0) { + # Find max Items[] count across all Contexts + $maxItemsCount = 0 + foreach ($ctx in $contexts) { + if ($ctx) { + $items = script:GetArrayFast $ctx 'Items' + if ($items -and $items.Count -gt $maxItemsCount) { + $maxItemsCount = $items.Count + } + } + } + if ($maxItemsCount -gt 0) { + $contextItems = $maxItemsCount # Store count for row calculation + } + } + } + + if ($PromptFilterValue) { $rowCount = [Math]::Max(1, $messages.Count) } else { + # DSPM for AI: Include all arrays in row count calculation (including AISystemPlugin and ModelTransparencyDetails) + $arrayCounts = @(1, $messages.Count, $contexts.Count, $resources.Count, $sensitivityLabels.Count, $pluginsRaw.Count, $modelDetRaw.Count) + + # Full explosion: include 2nd-level arrays in row count + if (-not $PartialExplode) { + if ($plugins) { $arrayCounts += $plugins.Count } + if ($recordingSessions) { $arrayCounts += $recordingSessions.Count } + if ($contextItems) { $arrayCounts += $contextItems } + } + + $rowCount = ($arrayCounts | Measure-Object -Maximum).Maximum + } + # Removed $plugin0 and $model0 - now using indexed access in row loop for full explosion + $creationDate = script:Format-DatePurviewFast $Record.CreationDate + $creationTime = try { script:Format-DatePurviewFast $auditData.CreationTime } catch { '' } + $appIdentityRaw = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'AppIdentity'), (Get-SafeProperty $ced 'AppIdentity'))) + $applicationId = Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationId'), (Get-SafeProperty $auditData 'AppId'), (Get-SafeProperty $auditData 'ClientAppId')) + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp) + $devProps = Get-SafeProperty $auditData 'DeviceProperties' + $dpOS = ''; $dpBrowser = '' + if ($devProps -and ($devProps -is [System.Collections.IEnumerable])) { + foreach ($dp in $devProps) { + try { + if ($dp.Name -eq 'OS') { $dpOS = $dp.Value } + elseif ($dp.Name -eq 'BrowserType') { $dpBrowser = $dp.Value } + } catch {} + } + } + $appHost = (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AppHost'), (Get-SafeProperty $auditData 'AppHost'), (Get-SafeProperty $auditData 'Workload'))) + $clientRegion = (Get-SafeProperty $auditData 'ClientRegion') + $agentId = (Get-SafeProperty $auditData 'AgentId') + $agentName = (Get-SafeProperty $auditData 'AgentName') + $agentVersion = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'AgentVersion'), (Get-SafeProperty $ced 'AgentVersion'), (Get-SafeProperty $ced 'Version'))) + + # Agent categorization based on AgentId pattern + $agentCategory = "" + if ($agentId) { + if ($agentId -like "CopilotStudio.Declarative.*") { + $agentCategory = "Declarative Agent" + } elseif ($agentId -like "CopilotStudio.CustomEngine.*") { + $agentCategory = "Custom Engine Agent" + } elseif ($agentId -like "P_*") { + $agentCategory = "Declarative Agent (Purview)" + } elseif ($agentId) { + $agentCategory = "Other Agent" + } + } + + # With -IncludeUserInfo, license data now appears only in EntraUsers output + # (or in combined mode via left join) + + $appName = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationName'), (Get-SafeProperty $ced 'HostAppName'), (Get-SafeProperty $ced 'ClientAppName'))) + $threadId = (Get-SafeProperty $ced 'ThreadId') + $auditUserKey = try { $auditData.UserKey } catch { $null } + # $modelName moved to row loop for indexed access + $clientIP = (Get-SafeProperty $auditData 'ClientIP') + $organizationId = (Get-SafeProperty $auditData 'OrganizationId') + $version = (Get-SafeProperty $auditData 'Version') + $userType = (Get-SafeProperty $auditData 'UserType') + $copilotLogVersion = (Get-SafeProperty $auditData 'CopilotLogVersion') + $workload = (Get-SafeProperty $auditData 'Workload') + + # Extract fields to match ExplodeArrays output for Power BI compatibility + $auditDataId = try { $auditData.Id } catch { $null } + $recordTypeNum = try { $auditData.RecordType } catch { $null } + $resultStatusAudit = try { $auditData.ResultStatus } catch { $null } + $appId = try { $auditData.AppId } catch { $null } + $clientAppId = try { $auditData.ClientAppId } catch { $null } + $correlationId = try { $auditData.CorrelationId } catch { $null } + + # Model and token fields (same as ExplodeArrays) + $modelId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelId'), (Get-SafeProperty $ced 'ModelID'), (Get-SafeProperty $auditData 'ModelId')) + $modelProvider = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelProvider'), (Get-SafeProperty $ced 'Provider'), (Get-SafeProperty $ced 'ModelVendor')) + $modelFamily = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelFamily'), (Get-SafeProperty $ced 'ModelType')) + $usageNode = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'Usage'), (Get-SafeProperty $ced 'TokenUsage'), (Get-SafeProperty $ced 'Tokens'), (Get-SafeProperty $auditData 'Usage')) + $tokensTotal = $null; $tokensInput = $null; $tokensOutput = $null + if ($usageNode) { + function Local:Get-Num([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + $tokensTotal = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Total'), (Get-SafeProperty $usageNode 'TotalTokens'), (Get-SafeProperty $usageNode 'TokensTotal'))) + $tokensInput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Input'), (Get-SafeProperty $usageNode 'Prompt'), (Get-SafeProperty $usageNode 'InputTokens'), (Get-SafeProperty $usageNode 'TokensInput'))) + $tokensOutput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Output'), (Get-SafeProperty $usageNode 'Completion'), (Get-SafeProperty $usageNode 'OutputTokens'), (Get-SafeProperty $usageNode 'TokensOutput'))) + } + if (-not $tokensTotal -and ($tokensInput -or $tokensOutput)) { try { $tokensTotal = ($tokensInput + $tokensOutput) } catch {} } + + # Duration, outcome, conversation fields (same as ExplodeArrays) + function Local:Get-NumSafe([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + $durationMs = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'DurationMs'), (Get-SafeProperty $ced 'ElapsedMs'), (Get-SafeProperty $ced 'ProcessingTimeMs'), (Get-SafeProperty $ced 'LatencyMs'))) + $outcomeStatus = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'OutcomeStatus'), (Get-SafeProperty $ced 'Outcome'), (Get-SafeProperty $ced 'Result'), (Get-SafeProperty $ced 'Status')) + if ($outcomeStatus -is [bool]) { $outcomeStatus = if ($outcomeStatus) { 'Success' } else { 'Failure' } } + $conversationId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ConversationId'), (Get-SafeProperty $ced 'ConversationID'), (Get-SafeProperty $ced 'SessionId')) + $turnNumber = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'TurnNumber'), (Get-SafeProperty $ced 'TurnIndex'), (Get-SafeProperty $ced 'MessageIndex'))) + $retryCount = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'RetryCount'), (Get-SafeProperty $ced 'Retries'))) + $clientVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientVersion'), (Get-SafeProperty $ced 'Version'), (Get-SafeProperty $ced 'Build')) + $clientPlatform = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientPlatform'), (Get-SafeProperty $ced 'Platform'), (Get-SafeProperty $ced 'OS')) + + $baseSet = New-Object System.Collections.Generic.HashSet[string]; foreach ($c in $PurviewExplodedHeader) { $null = $baseSet.Add($c) } + $rows = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $rowCount; $i++) { + $rowObj = [PSCustomObject][ordered]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + CreationDate = $creationDate + RecordType = $Record.RecordType + Operation = $auditData.Operation + UserId = $auditData.UserId + AssociatedAdminUnits = $(try { if ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } elseif ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } elseif ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + '@odata.type' = (Get-SafeProperty $auditData '@odata.type') + CreationTime = $creationTime + Id = $auditDataId + OrganizationId = $organizationId + ResultStatus = $resultStatusAudit + UserKey = $auditUserKey + UserType = $userType + Version = $version + Workload = $workload + ClientIP = $clientIP + ObjectId = (Get-SafeProperty $auditData 'ObjectId') + AzureActiveDirectoryEventType = (Get-SafeProperty $auditData 'AzureActiveDirectoryEventType') + ActorContextId = (Get-SafeProperty $auditData 'ActorContextId') + ActorIpAddress = (Get-SafeProperty $auditData 'ActorIpAddress') + InterSystemsId = (Get-SafeProperty $auditData 'InterSystemsId') + IntraSystemId = (Get-SafeProperty $auditData 'IntraSystemId') + SupportTicketId = (Get-SafeProperty $auditData 'SupportTicketId') + TargetContextId = (Get-SafeProperty $auditData 'TargetContextId') + ApplicationId = $applicationId + 'DeviceProperties.OS' = $dpOS + 'DeviceProperties.BrowserType' = $dpBrowser + ErrorNumber = (Get-SafeProperty $auditData 'ErrorNumber') + SiteUrl = (Get-SafeProperty $auditData 'SiteUrl') + SourceRelativeUrl = (Get-SafeProperty $auditData 'SourceRelativeUrl') + SourceFileName = (Get-SafeProperty $auditData 'SourceFileName') + SourceFileExtension = (Get-SafeProperty $auditData 'SourceFileExtension') + ListId = (Get-SafeProperty $auditData 'ListId') + ListItemUniqueId = (Get-SafeProperty $auditData 'ListItemUniqueId') + WebId = (Get-SafeProperty $auditData 'WebId') + ApplicationDisplayName = (Get-SafeProperty $auditData 'ApplicationDisplayName') + EventSource = (Get-SafeProperty $auditData 'EventSource') + ItemType = (Get-SafeProperty $auditData 'ItemType') + SiteSensitivityLabelId = (Get-SafeProperty $auditData 'SiteSensitivityLabelId') + GeoLocation = (Get-SafeProperty $auditData 'GeoLocation') + IsManagedDevice = (Get-SafeProperty $auditData 'IsManagedDevice') + DeviceDisplayName = (Get-SafeProperty $auditData 'DeviceDisplayName') + ListBaseType = (Get-SafeProperty $auditData 'ListBaseType') + ListServerTemplate = (Get-SafeProperty $auditData 'ListServerTemplate') + AuthenticationType = (Get-SafeProperty $auditData 'AuthenticationType') + Site = (Get-SafeProperty $auditData 'Site') + DoNotDistributeEvent = (Get-SafeProperty $auditData 'DoNotDistributeEvent') + HighPriorityMediaProcessing = (Get-SafeProperty $auditData 'HighPriorityMediaProcessing') + BrowserName = (Get-SafeProperty $auditData 'BrowserName') + BrowserVersion = (Get-SafeProperty $auditData 'BrowserVersion') + CorrelationId = $correlationId + Platform = (Get-SafeProperty $auditData 'Platform') + UserAgent = (Get-SafeProperty $auditData 'UserAgent') + ActorInfoString = (Get-SafeProperty $auditData 'ActorInfoString') + AppId = $appId + AuthType = (Get-SafeProperty $auditData 'AuthType') + ClientAppId = $clientAppId + ClientIPAddress = (Get-SafeProperty $auditData 'ClientIPAddress') + ClientInfoString = (Get-SafeProperty $auditData 'ClientInfoString') + ExternalAccess = (Get-SafeProperty $auditData 'ExternalAccess') + InternalLogonType = (Get-SafeProperty $auditData 'InternalLogonType') + LogonType = (Get-SafeProperty $auditData 'LogonType') + LogonUserSid = (Get-SafeProperty $auditData 'LogonUserSid') + MailboxGuid = (Get-SafeProperty $auditData 'MailboxGuid') + MailboxOwnerSid = (Get-SafeProperty $auditData 'MailboxOwnerSid') + MailboxOwnerUPN = (Get-SafeProperty $auditData 'MailboxOwnerUPN') + OrganizationName = (Get-SafeProperty $auditData 'OrganizationName') + OriginatingServer = (Get-SafeProperty $auditData 'OriginatingServer') + SessionId = (Get-SafeProperty $auditData 'SessionId') + TokenObjectId = (Get-SafeProperty $auditData 'TokenObjectId') + TokenTenantId = (Get-SafeProperty $auditData 'TokenTenantId') + TokenType = (Get-SafeProperty $auditData 'TokenType') + SaveToSentItems = (Get-SafeProperty $auditData 'SaveToSentItems') + OperationCount = (Get-SafeProperty $auditData 'OperationCount') + FileSizeBytes = (Get-SafeProperty $auditData 'FileSizeBytes') + MeetingId = (Get-SafeProperty $auditData 'MeetingId') + MeetingType = (Get-SafeProperty $auditData 'MeetingType') + EventSignature = (Get-SafeProperty $auditData 'EventSignature') + EventData = (Get-SafeProperty $auditData 'EventData') + Permission = (Get-SafeProperty $auditData 'Permission') + SensitivityLabelId = (Get-SafeProperty $auditData 'SensitivityLabelId') + SharingLinkScope = (Get-SafeProperty $auditData 'SharingLinkScope') + TargetUserOrGroupType = (Get-SafeProperty $auditData 'TargetUserOrGroupType') + TargetUserOrGroupName = (Get-SafeProperty $auditData 'TargetUserOrGroupName') + MeetingURL = (Get-SafeProperty $auditData 'MeetingURL') + ChatId = (Get-SafeProperty $auditData 'ChatId') + MessageId = (Get-SafeProperty $auditData 'MessageId') + MessageSizeInBytes = (Get-SafeProperty $auditData 'MessageSizeInBytes') + MessageType = (Get-SafeProperty $auditData 'MessageType') + FormId = (Get-SafeProperty $auditData 'FormId') + FormName = (Get-SafeProperty $auditData 'FormName') + VideoId = (Get-SafeProperty $auditData 'VideoId') + VideoName = (Get-SafeProperty $auditData 'VideoName') + ChannelId = (Get-SafeProperty $auditData 'ChannelId') + ViewDuration = (Get-SafeProperty $auditData 'ViewDuration') + ClientRegion = $clientRegion + CopilotLogVersion = $copilotLogVersion + TargetId = (Get-SafeProperty $auditData 'TargetId') + TeamName = (Get-SafeProperty $auditData 'TeamName') + TeamGuid = (Get-SafeProperty $auditData 'TeamGuid') + ResponseId = (Get-SafeProperty $auditData 'ResponseId') + IsAnonymous = (Get-SafeProperty $auditData 'IsAnonymous') + DeviceType = (Get-SafeProperty $auditData 'DeviceType') + ChannelName = (Get-SafeProperty $auditData 'ChannelName') + ChannelGuid = (Get-SafeProperty $auditData 'ChannelGuid') + ChannelType = (Get-SafeProperty $auditData 'ChannelType') + AppName = (Get-SafeProperty $auditData 'AppName') + EnvironmentName = (Get-SafeProperty $auditData 'EnvironmentName') + PlanId = (Get-SafeProperty $auditData 'PlanId') + PlanName = (Get-SafeProperty $auditData 'PlanName') + TaskId = (Get-SafeProperty $auditData 'TaskId') + TaskName = (Get-SafeProperty $auditData 'TaskName') + PercentComplete = (Get-SafeProperty $auditData 'PercentComplete') + CrossMailboxOperation = (Get-SafeProperty $auditData 'CrossMailboxOperation') + RecordTypeNum = $(try { [int]$Record.RecordType } catch { $Record.RecordType }) + ResultStatus_Audit = $resultStatusAudit + ModelId = $modelId + ModelProvider = $modelProvider + ModelFamily = $modelFamily + TokensTotal = $tokensTotal + TokensInput = $tokensInput + TokensOutput = $tokensOutput + DurationMs = $durationMs + OutcomeStatus = $outcomeStatus + ConversationId = $conversationId + TurnNumber = $turnNumber + RetryCount = $retryCount + ClientVersion = $clientVersion + ClientPlatform = $clientPlatform + AgentId = $agentId + AgentName = $agentName + AgentVersion = $agentVersion + AgentCategory = $agentCategory + ApplicationName = (Get-SafeProperty $auditData 'ApplicationName') + SensitivityLabel = $(if ($i -lt $sensitivityLabels.Count) { try { [string]$sensitivityLabels[$i] } catch { '' } } else { '' }) + AppHost = $appHost + ThreadId = $threadId + Context_Id = $(if ($i -lt $contexts.Count -and $contexts[$i]) { try { Get-SafeProperty $contexts[$i] 'Id' } catch { '' } } else { '' }) + Context_Type = $(if ($i -lt $contexts.Count -and $contexts[$i]) { try { Get-SafeProperty $contexts[$i] 'Type' } catch { '' } } else { '' }) + Message_Id = $(if ($i -lt $messages.Count) { $msg = $messages[$i]; if ($msg -is [psobject]) { try { Get-SafeProperty $msg 'Id' } catch { '' } } else { $msg } } else { '' }) + Message_isPrompt = $(if ($i -lt $messages.Count) { $msg = $messages[$i]; if ($msg -is [psobject]) { try { script:BoolTFFast (Get-SafeProperty $msg 'isPrompt') } catch { '' } } else { '' } } else { '' }) + AccessedResource_Action = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'Action' } catch { '' } } else { '' }) + AccessedResource_PolicyDetails = $(if ($i -lt $resources.Count -and $resources[$i]) { try { script:ToJsonIfObjectFast (Get-SafeProperty $resources[$i] 'PolicyDetails') } catch { '' } } else { '' }) + AccessedResource_SiteUrl = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'SiteUrl' } catch { '' } } else { '' }) + AISystemPlugin_Id = $(if ($i -lt $pluginsRaw.Count -and $pluginsRaw[$i]) { try { Get-SafeProperty $pluginsRaw[$i] 'Id' } catch { '' } } else { '' }) + AISystemPlugin_Name = $(if ($i -lt $pluginsRaw.Count -and $pluginsRaw[$i]) { try { Get-SafeProperty $pluginsRaw[$i] 'Name' } catch { '' } } else { '' }) + ModelTransparencyDetails_ModelName = $(if ($i -lt $modelDetRaw.Count -and $modelDetRaw[$i]) { try { Get-SafeProperty $modelDetRaw[$i] 'ModelName' } catch { '' } } else { '' }) + MessageIds = $(if ($messageIds.Count -gt 0) { $messageIds -join ';' } else { '' }) + AccessedResource_Name = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'Name' } catch { '' } } else { '' }) + AccessedResource_SensitivityLabel = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'SensitivityLabel' } catch { '' } } else { '' }) + AccessedResource_ResourceType = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'ResourceType' } catch { '' } } else { '' }) + Context_Item = $( + if ($activityType -eq 'CopilotInteraction') { + if ($PartialExplode) { + if ($i -lt $contexts.Count -and $contexts[$i]) { + try { + $items = script:GetArrayFast $contexts[$i] 'Items' + if ($items -and $items.Count -gt 0) { + ($items | ForEach-Object { try { script:ToJsonIfObjectFast $_ } catch { '' } }) -join ';' + } else { '' } + } catch { '' } + } else { '' } + } else { + try { + $foundItem = $null + foreach ($ctx in $contexts) { + if ($ctx) { + $items = script:GetArrayFast $ctx 'Items' + if ($items -and $i -lt $items.Count) { + $foundItem = $items[$i] + break + } + } + } + if ($foundItem) { script:ToJsonIfObjectFast $foundItem } else { '' } + } catch { '' } + } + } else { '' } + ) + } + + # Partial explosion mode: Preserve AuditData column (full JSON) for downstream processing + if ($PartialExplode) { + try { + Add-Member -InputObject $rowObj -NotePropertyName 'AuditData' -NotePropertyValue $Record.AuditData -Force + } catch {} + } + + # DSPM for AI: 2-level explosion for ConnectedAIAppInteraction (AppIdentity.Plugins[]) + if ($activityType -eq 'ConnectedAIAppInteraction' -and $plugins) { + try { + if ($PartialExplode) { + # Partial mode: Semi-colon-joined JSON for all plugins + $pluginsList = ($plugins | ForEach-Object { try { script:ToJsonIfObjectFast $_ } catch { '' } }) -join ';' + if (-not $rowObj.PSObject.Properties['AppIdentity_Plugins']) { + Add-Member -InputObject $rowObj -NotePropertyName 'AppIdentity_Plugins' -NotePropertyValue $pluginsList -Force + if (-not $script:DeepExtraColumns.Contains('AppIdentity_Plugins')) { [void]$script:DeepExtraColumns.Add('AppIdentity_Plugins') } + } + } else { + # Full mode: One plugin per row + if ($i -lt $plugins.Count) { + $plugin = $plugins[$i] + $pluginJson = try { script:ToJsonIfObjectFast $plugin } catch { '' } + if (-not $rowObj.PSObject.Properties['AppIdentity_Plugin']) { + Add-Member -InputObject $rowObj -NotePropertyName 'AppIdentity_Plugin' -NotePropertyValue $pluginJson -Force + if (-not $script:DeepExtraColumns.Contains('AppIdentity_Plugin')) { [void]$script:DeepExtraColumns.Add('AppIdentity_Plugin') } + } + } + } + } catch {} + } + + if ($Deep) { + if ($ced) { + $flat = ConvertTo-FlatColumns -Node $ced -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flat.Keys) { if ($baseSet.Contains($k)) { continue }; if (-not $rowObj.PSObject.Properties[$k]) { if (-not $script:DeepExtraColumns.Contains($k)) { [void]$script:DeepExtraColumns.Add($k) }; try { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flat[$k] -Force } catch {} } } + } + if ($auditData) { + $auditDataClone = [PSCustomObject]@{} + foreach ($prop in $auditData.PSObject.Properties) { if ($prop.Name -ne 'CopilotEventData') { Add-Member -InputObject $auditDataClone -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force } } + $flatAudit = ConvertTo-FlatColumns -Node $auditDataClone -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flatAudit.Keys) { if ($baseSet.Contains($k)) { continue }; if (-not $rowObj.PSObject.Properties[$k]) { if (-not $script:DeepExtraColumns.Contains($k)) { [void]$script:DeepExtraColumns.Add($k) }; try { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flatAudit[$k] -Force } catch {} } } + } + } + $rows.Add($rowObj) | Out-Null + } + if (-not $SkipMetrics -and $rows.Count -gt 1) { try { $script:metrics.ExplosionEvents += 1; $script:metrics.ExplosionRowsFromEvents += ($rows.Count - 1); if ($rows.Count -gt $script:metrics.ExplosionMaxPerRecord) { $script:metrics.ExplosionMaxPerRecord = $rows.Count } } catch {} } + return $rows + } + catch { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringParseFailures++ + } + Write-LogHost "Failed Purview explosion: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +function Select-FirstNonNull { param([object[]]$Values) foreach ($v in $Values) { if ($null -ne $v -and ('' -ne [string]$v)) { return $v } } return $null } + +function Convert-ToStructuredRecord { + # Uses proven stable implementation for record conversion + param( + [Parameter(Mandatory = $true)] $Record, + [bool]$EnableExplosion = $false + ) + try { + function Local:Get-Num([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + function Local:Add-OrUpdate([pscustomobject]$obj, [string]$name, $value) { try { if ($obj.PSObject.Properties[$name]) { $obj.PSObject.Properties[$name].Value = $value } else { Add-Member -InputObject $obj -NotePropertyName $name -NotePropertyValue $value -Force } } catch {} } + # Use pre-parsed AuditData if available + $auditData = if ($Record.PSObject.Properties['_ParsedAuditData']) { $Record._ParsedAuditData } else { try { $Record.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $null } } + if (-not $auditData) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringMissingAuditData++ + return @() + } + + # NON-EXPLOSION MODE: Return 8-column compact record matching Purview UI export schema + if (-not $EnableExplosion -and -not $ExplodeDeep) { + $compactRecord = [pscustomobject]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + CreationDate = $Record.CreationDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + RecordType = $Record.RecordType + Operation = $(try { $auditData.Operation } catch { if ($Record.Operation) { $Record.Operation } else { $Record.Operations } }) + UserId = if ($Record.UserId) { $Record.UserId } elseif ($Record.UserIds) { $Record.UserIds } else { '' } + AuditData = $Record.AuditData + AssociatedAdminUnits = $(try { if ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } elseif ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } elseif ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + } + return @($compactRecord) + } + + # EXPLOSION MODE: Extract and flatten all fields from AuditData (no raw JSON columns) + $ced = Get-SafeProperty $auditData 'CopilotEventData' + $modelId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelId'), (Get-SafeProperty $ced 'ModelID'), (Get-SafeProperty $auditData 'ModelId')) + $modelProvider = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelProvider'), (Get-SafeProperty $ced 'Provider'), (Get-SafeProperty $ced 'ModelVendor')) + $modelFamily = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelFamily'), (Get-SafeProperty $ced 'ModelType')) + $usageNode = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'Usage'), (Get-SafeProperty $ced 'TokenUsage'), (Get-SafeProperty $ced 'Tokens'), (Get-SafeProperty $auditData 'Usage')) + $tokensTotal = $null; $tokensInput = $null; $tokensOutput = $null + if ($usageNode) { + $tokensTotal = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Total'), (Get-SafeProperty $usageNode 'TotalTokens'), (Get-SafeProperty $usageNode 'TokensTotal'))) + $tokensInput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Input'), (Get-SafeProperty $usageNode 'Prompt'), (Get-SafeProperty $usageNode 'InputTokens'), (Get-SafeProperty $usageNode 'TokensInput'))) + $tokensOutput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Output'), (Get-SafeProperty $usageNode 'Completion'), (Get-SafeProperty $usageNode 'OutputTokens'), (Get-SafeProperty $usageNode 'TokensOutput'))) + } + if (-not $tokensTotal -and ($tokensInput -or $tokensOutput)) { try { $tokensTotal = ($tokensInput + $tokensOutput) } catch {} } + $durationMs = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'DurationMs'), (Get-SafeProperty $ced 'ElapsedMs'), (Get-SafeProperty $ced 'ProcessingTimeMs'), (Get-SafeProperty $ced 'LatencyMs'))) + $outcomeStatus = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'OutcomeStatus'), (Get-SafeProperty $ced 'Outcome'), (Get-SafeProperty $ced 'Result'), (Get-SafeProperty $ced 'Status')) + if ($outcomeStatus -is [bool]) { $outcomeStatus = if ($outcomeStatus) { 'Success' } else { 'Failure' } } + $conversationId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ConversationId'), (Get-SafeProperty $ced 'ConversationID'), (Get-SafeProperty $ced 'SessionId')) + $turnNumber = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'TurnNumber'), (Get-SafeProperty $ced 'TurnIndex'), (Get-SafeProperty $ced 'MessageIndex'))) + $retryCount = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'RetryCount'), (Get-SafeProperty $ced 'Retries'))) + $clientVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientVersion'), (Get-SafeProperty $ced 'Version'), (Get-SafeProperty $ced 'Build')) + $clientPlatform = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientPlatform'), (Get-SafeProperty $ced 'Platform'), (Get-SafeProperty $ced 'OS')) + $agentId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentId'), (Get-SafeProperty $ced 'AgentID'), (Get-SafeProperty $ced 'AssistantId')) + $agentName = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentName'), (Get-SafeProperty $ced 'AssistantName')) + $agentVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentVersion'), (Get-SafeProperty $ced 'Version')) + + # Agent categorization based on AgentId pattern + $agentCategory = "" + if ($agentId) { + if ($agentId -like "CopilotStudio.Declarative.*") { + $agentCategory = "Declarative Agent" + } elseif ($agentId -like "CopilotStudio.CustomEngine.*") { + $agentCategory = "Custom Engine Agent" + } elseif ($agentId -like "P_*") { + $agentCategory = "Declarative Agent (Purview)" + } elseif ($agentId) { + $agentCategory = "Other Agent" + } + } + + # With -IncludeUserInfo, license data now appears only in EntraUsers output + # (or in combined mode via left join) + + $appIdentity = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AppIdentity'), (Get-SafeProperty $ced 'ApplicationId'), (Get-SafeProperty $ced 'HostAppId')) + $applicationName = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ApplicationName'), (Get-SafeProperty $ced 'HostAppName'), (Get-SafeProperty $ced 'ClientAppName')) + $suggestions = (Get-SafeProperty $ced 'Suggestions'); if (-not $suggestions) { $suggestions = Get-SafeProperty $ced 'SuggestionList' } + $actions = Get-SafeProperty $ced 'Actions' + $references = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'References'), (Get-SafeProperty $ced 'Sources'), (Get-SafeProperty $ced 'Citations')) + $participants = Get-SafeProperty $ced 'Participants' + function Local:Measure-Collection($items, [string]$prefix) { + $result = @{}; if (-not $items) { return $result }; $arr = @($items); if ($arr.Count -eq 0) { return $result } + $result["${prefix}Count"] = $arr.Count; $types = New-Object System.Collections.Generic.HashSet[string]; $latencies = @(); $edits = @(); $accepted = 0; $success = 0; $failure = 0 + foreach ($s in $arr) { + foreach ($cand in @('Type', 'SuggestionType', 'Name', 'Kind', 'ActionType')) { try { if ($s.PSObject.Properties[$cand]) { [void]$types.Add([string]$s.$cand); break } } catch {} } + foreach ($lat in @('LatencyMs', 'DurationMs', 'ElapsedMs')) { try { if ($s.PSObject.Properties[$lat]) { $v = Local:Get-Num $s.$lat; if ($null -ne $v) { $latencies += $v; break } } } catch {} } + foreach ($ed in @('EditCount', 'Edits', 'EditsCount')) { try { if ($s.PSObject.Properties[$ed]) { $v = Local:Get-Num $s.$ed; if ($null -ne $v) { $edits += $v; break } } } catch {} } + foreach ($acc in @('Accepted', 'IsAccepted', 'Success', 'Succeeded')) { try { if ($s.PSObject.Properties[$acc]) { $val = $s.$acc; if ($val -is [bool]) { if ($val) { $accepted++ } } elseif ($val -match '^(?i:true|yes|1|success)') { $accepted++ } } } catch {} } + foreach ($succ in @('Success', 'Succeeded')) { try { if ($s.PSObject.Properties[$succ]) { $val = $s.$succ; if ($val -is [bool]) { if ($val) { $success++ } else { $failure++ } } elseif ($val -match '^(?i:true|yes|1|success)') { $success++ } else { $failure++ } } } catch {} } + } + if ($types.Count -gt 0) { $result["${prefix}Types"] = [string]::Join(';', [array]$types) } + if ($latencies.Count -gt 0) { $result["${prefix}AvgLatencyMs"] = [math]::Round(($latencies | Measure-Object -Average).Average, 2) } + if ($edits.Count -gt 0) { $result["${prefix}AvgEdits"] = [math]::Round(($edits | Measure-Object -Average).Average, 2); $result["${prefix}TotalEdits"] = ($edits | Measure-Object -Sum).Sum } + if ($accepted -gt 0) { $result["${prefix}Accepted"] = $accepted; $result["${prefix}AcceptanceRate"] = [math]::Round(($accepted / $arr.Count) * 100, 2) } + if ($success -gt 0 -or $failure -gt 0) { $result["${prefix}Success"] = $success; $result["${prefix}Failure"] = $failure } + return $result + } + $suggestAgg = Local:Measure-Collection $suggestions 'Suggestions' + $actionAgg = Local:Measure-Collection $actions 'Actions' + $refAgg = Local:Measure-Collection $references 'References' + $partAgg = Local:Measure-Collection $participants 'Participants' + $baseRecord = [pscustomobject]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + RecordType = $Record.RecordType + CreationDate = $Record.CreationDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + ResultStatus = $Record.ResultStatus + ResultCount = $Record.ResultCount + Identity = $Record.Identity + IsValid = $Record.IsValid + ObjectState = $Record.ObjectState + Id = $auditData.Id + CreationTime = & { $ct = script:Parse-DateSafe $auditData.CreationTime; if ($ct) { $ct.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) } else { $auditData.CreationTime } } + Operation = $auditData.Operation + OrganizationId = $auditData.OrganizationId + RecordTypeNum = $auditData.RecordType + ResultStatus_Audit = $auditData.ResultStatus + UserKey = $auditData.UserKey + UserType = $auditData.UserType + Version = $auditData.Version + Workload = $auditData.Workload + UserId = $auditData.UserId + AppId = $auditData.AppId + ClientAppId = $auditData.ClientAppId + CorrelationId = $auditData.CorrelationId + ModelId = $modelId + ModelProvider = $modelProvider + ModelFamily = $modelFamily + TokensTotal = $tokensTotal + TokensInput = $tokensInput + TokensOutput = $tokensOutput + DurationMs = $durationMs + OutcomeStatus = $outcomeStatus + ConversationId = $conversationId + TurnNumber = $turnNumber + RetryCount = $retryCount + ClientVersion = $clientVersion + ClientPlatform = $clientPlatform + AgentId = $agentId + AgentName = $agentName + AgentVersion = $agentVersion + AgentCategory = $agentCategory + AppIdentity = $appIdentity + ApplicationName = $applicationName + } + # Flatten AppAccessContext for Copilot/AI records as well + try { + $aac = Get-SafeProperty $auditData 'AppAccessContext' + if ($aac -and -not (Test-ScalarValue $aac)) { + $flatAac = ConvertTo-FlatColumns -Node $aac -Prefix 'AppAccessContext.' -MaxDepth $FlatDepthStandard + foreach ($k in $flatAac.Keys) { if (-not $baseRecord.PSObject.Properties[$k]) { Add-Member -InputObject $baseRecord -NotePropertyName $k -NotePropertyValue $flatAac[$k] -Force } } + if ($baseRecord.PSObject.Properties['AppAccessContext']) { $baseRecord.PSObject.Members.Remove('AppAccessContext') } + } + elseif ($aac -and (Test-ScalarValue $aac)) { + if (-not $baseRecord.PSObject.Properties['AppAccessContext']) { Add-Member -InputObject $baseRecord -NotePropertyName 'AppAccessContext' -NotePropertyValue $aac -Force } + } + } catch {} + foreach ($k in $suggestAgg.Keys) { Add-OrUpdate $baseRecord $k $suggestAgg[$k] } + foreach ($k in $actionAgg.Keys) { Add-OrUpdate $baseRecord $k $actionAgg[$k] } + foreach ($k in $refAgg.Keys) { Add-OrUpdate $baseRecord $k $refAgg[$k] } + foreach ($k in $partAgg.Keys) { Add-OrUpdate $baseRecord $k $partAgg[$k] } + + # If not doing array explosion, return base record now + if (-not $EnableExplosion) { return @($baseRecord) } + $rows = @($baseRecord) + $arraysToExplode = @( + @{ Name = 'Suggestions'; Data = $suggestions; Prefix = 'Suggestion'; Enabled = $suggestions }, + @{ Name = 'Actions'; Data = $actions; Prefix = 'Action'; Enabled = $actions }, + @{ Name = 'References'; Data = $references; Prefix = 'Reference'; Enabled = $references }, + @{ Name = 'Participants'; Data = $participants; Prefix = 'Participant'; Enabled = $participants } + ) + $maxRows = $ExplosionPerRecordRowCap + foreach ($entry in $arraysToExplode) { + if (-not $entry.Enabled) { continue } + $dataArr = @($entry.Data); if ($dataArr.Count -eq 0) { continue } + $newRows = New-Object System.Collections.ArrayList + foreach ($r in $rows) { + $idx = 0 + foreach ($el in $dataArr) { + $nr = [pscustomobject]@{} + foreach ($p in $r.PSObject.Properties) { Add-Member -InputObject $nr -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force } + Add-OrUpdate $nr ("ArrayIndex_{0}" -f $entry.Name) $idx + if ($el) { + foreach ($prop in $el.PSObject.Properties) { + $pname = ("{0}_{1}" -f $entry.Prefix, $prop.Name) + if ($nr.PSObject.Properties[$pname]) { continue } + $val = $prop.Value + if (Test-ScalarValue $val) { Add-OrUpdate $nr $pname $val } else { try { Add-OrUpdate $nr $pname ($val | ConvertTo-Json -Depth $JsonDepth -Compress) } catch {} } + } + } + [void]$newRows.Add($nr); $idx++ + if ($newRows.Count -gt $maxRows) { break } + } + if ($newRows.Count -gt $maxRows) { break } + } + $rows = @($newRows) + if ($rows.Count -gt $maxRows) { break } + } + if ($rows.Count -gt $maxRows) { foreach ($r in $rows) { Add-OrUpdate $r 'ExplosionTruncated' $true }; $rows = $rows[0..($maxRows - 1)]; try { $script:metrics.ExplosionTruncated = $true } catch {} } + if ($ExplodeDeep -and $ced) { + for ($i = 0; $i -lt $rows.Count; $i++) { + $r = $rows[$i] + $flat = ConvertTo-FlatColumns -Node $ced -Prefix '' -MaxDepth $FlatDepthStandard + foreach ($ck in $flat.Keys) { if (-not $r.PSObject.Properties[$ck]) { Add-OrUpdate $r $ck $flat[$ck] } } + } + } + return $rows + } + catch { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringParseFailures++ + Write-LogHost "Failed to process record: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +try { + # Unregister the early exit handler since catch/finally will handle Ctrl+C from this point + # This prevents duplicate "Script Interrupted" messages + Unregister-Event -SourceIdentifier PowerShell.Exiting -ErrorAction SilentlyContinue + + # ============================================================ + # RESUME MODE VALIDATION - Ensure no conflicting parameters + # ============================================================ + if ($PSBoundParameters.ContainsKey('Resume')) { + # Resume mode is standalone - only auth-related parameters allowed + $allowedWithResume = @( + 'Resume', + 'Force', + 'Auth', + 'TenantId', + 'ClientId', + 'ClientSecret', + 'ClientCertificateThumbprint', + 'ClientCertificateStoreLocation', + 'ClientCertificatePath', + 'ClientCertificatePassword', + # Standard PowerShell common parameters + 'Verbose', + 'Debug', + 'ErrorAction', + 'WarningAction', + 'InformationAction', + 'ErrorVariable', + 'WarningVariable', + 'InformationVariable', + 'OutVariable', + 'OutBuffer', + 'PipelineVariable' + ) + + $invalidParams = @($PSBoundParameters.Keys | Where-Object { $_ -notin $allowedWithResume }) + + if ($invalidParams.Count -gt 0) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: Invalid parameters used with -Resume" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Resume mode restores ALL settings from the checkpoint file." -ForegroundColor Yellow + Write-Host " You cannot specify other parameters (they would be ignored or cause inconsistency)." -ForegroundColor Yellow + Write-Host "" + Write-Host " Invalid parameters:" -ForegroundColor White + foreach ($p in $invalidParams) { + Write-Host " - $p" -ForegroundColor Red + } + Write-Host "" + Write-Host " ALLOWED with -Resume:" -ForegroundColor Green + Write-Host " -Resume [path] Checkpoint file (or auto-discover)" -ForegroundColor Gray + Write-Host " -Force Use most recent checkpoint without prompting" -ForegroundColor Gray + Write-Host " -Auth Override authentication method" -ForegroundColor Gray + Write-Host " -TenantId Tenant ID (for AppRegistration)" -ForegroundColor Gray + Write-Host " -ClientId Client ID (for AppRegistration)" -ForegroundColor Gray + Write-Host " -ClientSecret Client secret (for AppRegistration)" -ForegroundColor Gray + Write-Host "" + Write-Host " Example usage:" -ForegroundColor Cyan + Write-Host ' .\Script.ps1 -Resume' -ForegroundColor White + Write-Host ' .\Script.ps1 -Resume -Auth DeviceCode' -ForegroundColor White + Write-Host ' .\Script.ps1 -Resume "C:\path\.pax_checkpoint_xxx.json" -Force' -ForegroundColor White + Write-Host "" + exit 1 + } + } + + # ============================================================ + # RESUME MODE DETECTION - Check for checkpoint to resume + # ============================================================ + if ($ResumeSpecified) { + Write-LogHost "" + Write-LogHost "========================================" -ForegroundColor Cyan + Write-LogHost " RESUME MODE DETECTED" -ForegroundColor Cyan + Write-LogHost "========================================" -ForegroundColor Cyan + Write-LogHost "" + + if ($Resume -ne '') { + # Explicit checkpoint path provided + Write-LogHost "Loading checkpoint from explicit path: $Resume" -ForegroundColor Yellow + $checkpointLoadSuccess = Read-Checkpoint -CheckpointPath $Resume + if (-not $checkpointLoadSuccess) { + Write-LogHost "ERROR: Failed to load checkpoint file. Cannot resume." -ForegroundColor Red + exit 1 + } + $script:CheckpointPath = $Resume + # Lock the resumed checkpoint so a second replica cannot race on it. + try { script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath } catch { + Write-LogHost ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + # Read-Checkpoint sets $script:CheckpointData on success + $checkpointData = $script:CheckpointData + } + else { + # Auto-discover checkpoints in OutputPath + $searchPath = if ($OutputPath) { $OutputPath } else { (Get-Location).Path } + + # Fabric tier: resume artifacts are mirrored to OneLake Files/.pax_resume/ + # inside each Save-CheckpointToDisk call so a container restart can resume. + # Hydrate the local search path from the mirror before Find-Checkpoints runs. + # Local and SharePoint tiers keep artifacts local-only — those hosts are not + # ephemeral and a remote prefetch would be needless I/O. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + try { + $restored = Restore-AllFabricResumeMirrors -LocalDir $searchPath + if ($restored -gt 0) { + Write-LogHost "Hydrated $restored resume artifact(s) from OneLake mirror" -ForegroundColor DarkGray + } + } catch { + Write-LogHost ("WARNING: Resume mirror hydration failed: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + } + } + + Write-LogHost "Searching for checkpoints in: $searchPath" -ForegroundColor Yellow + + $checkpoints = Find-Checkpoints -OutputPath $searchPath + + if ($checkpoints.Count -eq 0) { + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " NO CHECKPOINT FILES FOUND" -ForegroundColor Red + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + Write-LogHost " Searched in: $searchPath" -ForegroundColor White + Write-LogHost "" + Write-LogHost " Checkpoint files are named: .pax_checkpoint_YYYYMMDD_HHMMSS.json" -ForegroundColor Gray + Write-LogHost " They are saved in the same folder as the _PARTIAL.csv output file." -ForegroundColor Gray + Write-LogHost "" + Write-LogHost " COMMON LOCATIONS TO CHECK:" -ForegroundColor Yellow + Write-LogHost " • The 'output' folder where you typically save exports" -ForegroundColor White + Write-LogHost " • The folder shown in the Ctrl+C message when the run was interrupted" -ForegroundColor White + Write-LogHost " • Look for _PARTIAL.csv files - the checkpoint is in the same folder" -ForegroundColor White + Write-LogHost "" + Write-LogHost " HOW TO RESUME:" -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost " Option 1: Specify the folder containing the checkpoint:" -ForegroundColor White + Write-LogHost " -Resume -OutputPath `"C:\path\to\output\folder`"" -ForegroundColor Green + Write-LogHost "" + Write-LogHost " Option 2: Specify the checkpoint file directly:" -ForegroundColor White + Write-LogHost " -Resume `"C:\path\to\.pax_checkpoint_20260120_123456.json`"" -ForegroundColor Green + Write-LogHost "" + Write-LogHost " Option 3: Run from the folder containing the checkpoint:" -ForegroundColor White + Write-LogHost " cd `"C:\path\to\output\folder`"" -ForegroundColor Green + Write-LogHost " pwsh -File `"...\PAX_Purview_Audit_Log_Processor.ps1`" -Resume" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + exit 1 + } + elseif ($checkpoints.Count -eq 1) { + $selectedCheckpoint = $checkpoints[0] + Write-LogHost "Found checkpoint: $($selectedCheckpoint.FileName)" -ForegroundColor Green + } + else { + # Multiple checkpoints found + if ($Force) { + # Use most recent without prompting + $selectedCheckpoint = $checkpoints | Sort-Object { $_.LastUpdated } -Descending | Select-Object -First 1 + Write-LogHost "Multiple checkpoints found. -Force specified, using most recent:" -ForegroundColor Yellow + Write-LogHost " $($selectedCheckpoint.FileName)" -ForegroundColor White + } + else { + # Prompt user to select + $selectedCheckpoint = Select-Checkpoint -Checkpoints $checkpoints + if (-not $selectedCheckpoint) { + Write-LogHost "No checkpoint selected. Exiting." -ForegroundColor Yellow + exit 0 + } + } + } + + $script:CheckpointPath = $selectedCheckpoint.Path + # Lock the resumed checkpoint so a second replica cannot race on it. + try { script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath } catch { + Write-LogHost ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + $checkpointLoadSuccess = Read-Checkpoint -CheckpointPath $script:CheckpointPath + if (-not $checkpointLoadSuccess) { + Write-LogHost "ERROR: Failed to load checkpoint file. Cannot resume." -ForegroundColor Red + exit 1 + } + # Read-Checkpoint sets $script:CheckpointData on success + $checkpointData = $script:CheckpointData + } + + # Note: $script:CheckpointData and $script:IsResumeMode already set by Read-Checkpoint + + # Display resume summary + $completedCount = if ($checkpointData.partitions.completed) { $checkpointData.partitions.completed.Count } else { 0 } + $queryCreatedCount = if ($checkpointData.partitions.queryCreated) { $checkpointData.partitions.queryCreated.Count } else { 0 } + $totalPartitions = if ($checkpointData.partitions.total) { $checkpointData.partitions.total } else { 0 } + + Write-LogHost "" + Write-LogHost "Checkpoint loaded successfully:" -ForegroundColor Green + Write-LogHost " Original Run: $($checkpointData.runTimestamp)" -ForegroundColor White + $cpStartDate = if ($checkpointData.parameters.startDate) { $d = script:Parse-DateSafe $checkpointData.parameters.startDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + $cpEndDate = if ($checkpointData.parameters.endDate) { $d = script:Parse-DateSafe $checkpointData.parameters.endDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + Write-LogHost " Date Range: $cpStartDate to $cpEndDate" -ForegroundColor White + Write-LogHost " Total Partitions: $totalPartitions" -ForegroundColor White + Write-LogHost " Completed: $completedCount" -ForegroundColor Green + Write-LogHost " Query Created: $queryCreatedCount (will attempt data fetch)" -ForegroundColor Yellow + Write-LogHost " Remaining: $($totalPartitions - $completedCount)" -ForegroundColor Cyan + Write-LogHost "" + + # ============================================================ + # RESTORE ALL PARAMETERS FROM CHECKPOINT + # ============================================================ + Write-LogHost "Restoring parameters from checkpoint..." -ForegroundColor DarkGray + $cp = $checkpointData.parameters + + # Restore original run timestamp so incremental files use consistent naming + # This ensures all partition files (original run + resumes) share the same timestamp + if ($checkpointData.runTimestamp) { + $global:ScriptRunTimestamp = $checkpointData.runTimestamp + Write-LogHost " Restored original run timestamp: $($global:ScriptRunTimestamp)" -ForegroundColor DarkGray + } + + # Date range (required) - using locale-safe parsing + $parsedStart = script:Parse-DateSafe $cp.startDate + if (-not $parsedStart) { throw "Failed to parse checkpoint startDate: $($cp.startDate)" } + # Checkpoint stores the date as a UTC instant (local midnight -> ToUniversalTime at save). + # Parse-DateSafe returns a UTC-kind value, so convert back to local before formatting the + # calendar date; otherwise positive-UTC-offset hosts (e.g. IST) render the prior day, + # shifting the trim window by one day on resume. + $StartDate = $parsedStart.ToLocalTime().ToString('yyyy-MM-dd') + + $parsedEnd = script:Parse-DateSafe $cp.endDate + if (-not $parsedEnd) { throw "Failed to parse checkpoint endDate: $($cp.endDate)" } + # See StartDate note above: convert the UTC-kind parse result back to local before + # formatting so the restored end date matches the originally requested calendar date. + $EndDate = $parsedEnd.ToLocalTime().ToString('yyyy-MM-dd') + + # Activity/Record filtering + if ($cp.activityTypes -and $cp.activityTypes.Count -gt 0) { $ActivityTypes = $cp.activityTypes } + if ($cp.recordTypes -and $cp.recordTypes.Count -gt 0) { $RecordTypes = $cp.recordTypes } + if ($cp.serviceTypes -and $cp.serviceTypes.Count -gt 0) { $ServiceTypes = $cp.serviceTypes } + if ($cp.userIds -and $cp.userIds.Count -gt 0) { $UserIds = $cp.userIds } + if ($cp.groupNames -and $cp.groupNames.Count -gt 0) { $GroupNames = $cp.groupNames } + + # Agent filtering + if ($cp.agentId -and $cp.agentId.Count -gt 0) { $AgentId = $cp.agentId } + if ($cp.agentsOnly) { $AgentsOnly = [switch]$true } + if ($cp.excludeAgents) { $ExcludeAgents = [switch]$true } + + # Prompt filtering + if ($cp.promptFilter) { $PromptFilter = $cp.promptFilter } + + # Schema/Explosion settings + if ($cp.explodeArrays) { $ExplodeArrays = [switch]$true } + if ($cp.explodeDeep) { $ExplodeDeep = [switch]$true } + if ($cp.flatDepth) { $FlatDepth = $cp.flatDepth } + if ($cp.streamingSchemaSample) { $StreamingSchemaSample = $cp.streamingSchemaSample } + if ($cp.streamingChunkSize) { $StreamingChunkSize = $cp.streamingChunkSize } + # Allow user to override explosion threads on resume (different machine/load) + if (-not $PSBoundParameters.ContainsKey('ExplosionThreads') -and $cp.explosionThreads) { $ExplosionThreads = $cp.explosionThreads } + + # M365/User info bundles + if ($cp.includeM365Usage) { $IncludeM365Usage = [switch]$true } + if ($cp.includeUserInfo) { $IncludeUserInfo = [switch]$true } + if ($cp.Contains('includeDSPMForAI') -and $cp.includeDSPMForAI) { + Write-LogHost "Resume checkpoint references legacy -IncludeDSPMForAI switch; ignoring (no longer supported)." -ForegroundColor Yellow + } + if ($cp.includeCopilotInteraction) { $IncludeCopilotInteraction = [switch]$true } + if ($cp.excludeCopilotInteraction) { $ExcludeCopilotInteraction = [switch]$true } + if ($cp.includeAgent365Info) { $IncludeAgent365Info = [switch]$true } + # Note: onlyAgent365Info is intentionally not restored on resume - it would imply + # no audit phase to resume from. Original validation blocks -OnlyAgent365Info + -Resume. + + # Partitioning + if ($cp.blockHours) { $BlockHours = $cp.blockHours } + if ($cp.partitionHours) { $PartitionHours = $cp.partitionHours } + if ($cp.maxPartitions) { $MaxPartitions = $cp.maxPartitions } + + # Output settings + if ($cp.outputPath) { $OutputPath = $cp.outputPath } + if ($cp.exportWorkbook) { $ExportWorkbook = [switch]$true } + if ($cp.combineOutput) { $CombineOutput = [switch]$true } + # Deidentify is checkpoint-driven on resume (the resume allow-list blocks it on the + # command line), so restore it here as the sole source of truth for a resumed run. + if ($cp.Contains('deidentify') -and $cp.deidentify) { $Deidentify = [switch]$true; $script:PaxDeidEnabled = $true } + + # FillerLabel is checkpoint-driven on resume (the allow-list blocks it on the command + # line); restore the resolved hierarchy-filler mode + literal as the sole source of truth. + if ($cp.Contains('fillerLabelMode') -and $cp.fillerLabelMode) { $script:HierarchyFillMode = [string]$cp.fillerLabelMode } + if ($cp.Contains('fillerLabelText')) { $script:HierarchyFillLabel = [string]$cp.fillerLabelText } + + # Per-stream destinations and append targets. Persisted by New-Checkpoint; + # restored here so a -Resume run does not require the user to re-supply any + # destination switch. Resume-allowed-list already blocks the user from passing + # these on the resume command line, so checkpoint is the sole source of truth. + if ($cp.Contains('outputPathUserInfo') -and $cp.outputPathUserInfo) { $OutputPathUserInfo = [string]$cp.outputPathUserInfo } + if ($cp.Contains('outputPathAgent365Info') -and $cp.outputPathAgent365Info) { $OutputPathAgent365Info = [string]$cp.outputPathAgent365Info } + if ($cp.Contains('outputPathLog') -and $cp.outputPathLog) { $OutputPathLog = [string]$cp.outputPathLog } + if ($cp.Contains('appendFile') -and $cp.appendFile) { $AppendFile = [string]$cp.appendFile } + if ($cp.Contains('appendUserInfo') -and $cp.appendUserInfo) { $AppendUserInfo = [string]$cp.appendUserInfo } + if ($cp.Contains('appendAgent365Info') -and $cp.appendAgent365Info) { $AppendAgent365Info = [string]$cp.appendAgent365Info } + + # ============================================================ + # RESUME: Re-resolve destination state from RESTORED variable values. + # The parse-time destination-resolution block (search for "destSwitches") + # keys off $PSBoundParameters, but on a -Resume run the user MUST NOT pass + # -OutputPath* / -Append* (resume-allowed-list rejects them) — so at parse + # time every $script:DestIsBound[*] was false and $script:RemoteOutputMode + # stayed 'None'. Now that the checkpoint restore above repopulated $OutputPath, + # $OutputPathUserInfo, $OutputPathAgent365Info, $OutputPathLog, $AppendFile, + # $AppendUserInfo, $AppendAgent365Info, re-derive the destination hashtables + # so the end-of-run upload sweep + display helpers route correctly. Without + # this, $script:RemoteOutputMode stays 'None' and no files ever upload to SP/Fabric. + # "Bound-ness" is inferred from a non-empty value (the new-run path uses + # $PSBoundParameters because string parameter defaults are '' which is falsy here too). + # ============================================================ + $script:DestTier = @{} + $script:DestRaw = @{} + $script:DestIsBound = @{} + $resumeDestSwitches = @( + @{ Key = 'Purview' ; Name = 'OutputPath' ; Value = $OutputPath ; AllowFabricFilesOnly = $false }, + @{ Key = 'UserInfo' ; Name = 'OutputPathUserInfo' ; Value = $OutputPathUserInfo ; AllowFabricFilesOnly = $false }, + @{ Key = 'Agent365Info' ; Name = 'OutputPathAgent365Info' ; Value = $OutputPathAgent365Info ; AllowFabricFilesOnly = $false }, + @{ Key = 'Log' ; Name = 'OutputPathLog' ; Value = $OutputPathLog ; AllowFabricFilesOnly = $true } + ) + foreach ($ds in $resumeDestSwitches) { + $hasValue = -not [string]::IsNullOrWhiteSpace($ds.Value) + $script:DestIsBound[$ds.Key] = $hasValue + if ($hasValue) { + $tier = script:Get-PathTier -Value $ds.Value -SwitchName $ds.Name -AllowFabricFilesOnly:$ds.AllowFabricFilesOnly + $script:DestTier[$ds.Key] = $tier + $script:DestRaw[$ds.Key] = $ds.Value.Trim() + } + } + $script:AppendIsBound = @{} + $script:AppendRaw = @{} + $script:AppendIsRemote = @{} + $resumeAppendSwitches = @( + @{ Key = 'Purview' ; Name = 'AppendFile' ; Value = $AppendFile }, + @{ Key = 'UserInfo' ; Name = 'AppendUserInfo' ; Value = $AppendUserInfo }, + @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info } + ) + foreach ($as in $resumeAppendSwitches) { + $hasValue = -not [string]::IsNullOrWhiteSpace($as.Value) + $script:AppendIsBound[$as.Key] = $hasValue + if (-not $hasValue) { continue } + $v = $as.Value.Trim() + $script:AppendRaw[$as.Key] = $v + $looksUrl = $v -match '^https?://' + $looksRooted = $v -match '^[A-Za-z]:[\\/]' -or $v -match '^/' + if ($looksUrl -or $looksRooted) { + $aTier = script:Get-PathTier -Value $v -SwitchName $as.Name + $script:AppendIsRemote[$as.Key] = ($aTier -ne 'Local') + if (-not $script:DestTier.ContainsKey($as.Key)) { + $script:DestTier[$as.Key] = $aTier + $script:DestRaw[$as.Key] = $v + } + } else { + $script:AppendIsRemote[$as.Key] = $false + } + } + # Promote legacy back-compat variables. + $script:RemoteOutputMode = 'None' + $script:RemoteOutputUrl = $null + if ($script:DestTier.ContainsKey('Purview')) { + switch ($script:DestTier['Purview']) { + 'SharePoint' { $script:RemoteOutputMode = 'SharePoint'; $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') } + 'Fabric' { $script:RemoteOutputMode = 'Fabric'; $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') } + } + } + if ($script:RemoteOutputMode -eq 'None') { + foreach ($k in @('UserInfo','Agent365Info')) { + if (-not $script:DestTier.ContainsKey($k)) { continue } + $t = $script:DestTier[$k] + if ($t -eq 'SharePoint' -or $t -eq 'Fabric') { + $script:RemoteOutputMode = $t + $script:RemoteOutputUrl = $script:DestRaw[$k].TrimEnd('/') + break + } + } + } + # Folder-URL normalization for DestParentUrl and RemoteOutputUrl (mirrors the + # parse-time block at $script:NormalizeFolderUrl). + $script:DestParentUrl = @{} + $resumeNormalizeFolderUrl = { + param([string]$u) + if ([string]::IsNullOrWhiteSpace($u)) { return $u } + if ($u -match '/[^/]+\.(csv|log|xlsx|json|parquet|txt)$') { + return ($u -replace '/[^/]+\.[A-Za-z0-9]+$','') + } + return $u + } + if ($script:RemoteOutputUrl) { $script:RemoteOutputUrl = & $resumeNormalizeFolderUrl $script:RemoteOutputUrl } + foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + if (-not $script:DestTier.ContainsKey($k)) { continue } + if ($script:DestTier[$k] -eq 'Local') { continue } + $script:DestParentUrl[$k] = & $resumeNormalizeFolderUrl $script:DestRaw[$k] + } + Write-LogHost " Resume: re-resolved destination state (RemoteOutputMode=$script:RemoteOutputMode, tiers=$(($script:DestTier.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ', '))" -ForegroundColor DarkGray + + # Resume-side scratch-dir rewrite. Mirrors the parse-time scratch-dir rewrite + # (search for "Redirect OutputPath to a per-run scratch dir") for the -Resume + # code path. Once we know the destination is SharePoint / Fabric, downstream + # code must continue to treat $OutputPath as the LOCAL scratch directory + # (where _PARTIAL.csv and .pax_incremental live) — exactly as it did during + # the interrupted run. The scratch dir on a -Resume is the parent of the + # checkpoint file (every PAX_/ scratch is self-contained). Without this + # rewrite, downstream code would re-use the remote URL string as a local + # Join-Path target and crash / write nowhere. Tier-agnostic: applies + # identically to SharePoint and Fabric (OneLake) modes. + if ($script:RemoteOutputMode -ne 'None') { + $script:RemoteScratchDir = Split-Path -Parent $script:CheckpointPath + $sep = [System.IO.Path]::DirectorySeparatorChar + if (-not $script:RemoteScratchDir.EndsWith($sep)) { $script:RemoteScratchDir = $script:RemoteScratchDir + $sep } + $OutputPath = $script:RemoteScratchDir + Write-LogHost " Resume: remote mode active - scratch dir = $script:RemoteScratchDir (remote URL = $script:RemoteOutputUrl)" -ForegroundColor DarkGray + } + + # Restore rollup post-processor intent from checkpoint, with last-write-wins semantics: + # if the user passed -Rollup or -RollupPlusRaw on the resume command line, the resume + # switch wins and the checkpoint value is ignored. If the user passed neither, restore + # the original run's rollup mode from the checkpoint so a resumed run produces the same + # outputs as the original would have. + $resumeUserPassedRollup = $PSBoundParameters.ContainsKey('Rollup') -or $PSBoundParameters.ContainsKey('RollupPlusRaw') + if (-not $resumeUserPassedRollup -and $cp.rollupMode) { + switch ([string]$cp.rollupMode) { + 'Rollup' { $Rollup = [System.Management.Automation.SwitchParameter]::new($true); $RollupPlusRaw = [System.Management.Automation.SwitchParameter]::new($false) } + 'RollupPlusRaw' { $RollupPlusRaw = [System.Management.Automation.SwitchParameter]::new($true); $Rollup = [System.Management.Automation.SwitchParameter]::new($false) } + default { } + } + if ($cp.processorMode -and ($cp.processorMode -in @('CopilotInteraction','M365Bundle','None'))) { + $script:RollupProcessorMode = [string]$cp.processorMode + } + if ($Rollup -or $RollupPlusRaw) { + $restoredSwitch = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + Write-LogHost " Restored rollup mode from checkpoint: $restoredSwitch (processorMode=$($script:RollupProcessorMode))" -ForegroundColor DarkGray + } + } + elseif ($resumeUserPassedRollup -and $cp.rollupMode -and $cp.rollupMode -ne 'None') { + # Resume's switch wins; just log the override for visibility. + $nowSwitch = if ($Rollup) { '-Rollup' } elseif ($RollupPlusRaw) { '-RollupPlusRaw' } else { '' } + Write-LogHost " Rollup mode overridden on resume: checkpoint='$($cp.rollupMode)' -> resume='$nowSwitch' (last-write-wins)." -ForegroundColor Yellow + } + + # Dashboard restore (independent last-write-wins): an explicit -Dashboard on the + # resume CLI wins; otherwise restore the original run's dashboard so a resumed + # CopilotInteraction run keeps its AIO/AIBV profile instead of defaulting to AIO. + # Guarded to valid values so the [ValidateSet] on $Dashboard never throws on assign. + $resumeUserPassedDashboard = $PSBoundParameters.ContainsKey('Dashboard') + if (-not $resumeUserPassedDashboard -and $cp.rollupDashboard -and ([string]$cp.rollupDashboard -in @('AIO', 'AIBV', 'M365'))) { + $Dashboard = [string]$cp.rollupDashboard + Write-LogHost " Restored rollup dashboard from checkpoint: $Dashboard" -ForegroundColor DarkGray + } + + # Re-derive $script:RollupProcessorMode AFTER all resume-restore lines above have + # merged checkpoint state (especially IncludeM365Usage). This guarantees the wire-up + # at end-of-run picks the right embedded processor even when the resume CLI omitted + # the modifier switches. The gating BLOCKERS are not re-run here — they were validated + # on the original run and the resume restore preserves all gating-relevant switches + # (ExportWorkbook, OnlyUserInfo, OnlyAgent365Info, AppendFile, + # RAWInputCSV, UseEOM); if any of those were set, the original run would have already + # hard-failed before producing a checkpoint to resume from. + if ($Rollup -or $RollupPlusRaw) { + if ($IncludeM365Usage) { + $script:RollupProcessorMode = 'M365Bundle' + $script:RollupDashboard = 'M365' + $script:RollupDashboardProfile = $null + } + else { + $script:RollupProcessorMode = 'CopilotInteraction' + # Honor the restored / resume-CLI dashboard so a resumed AIBV run is not + # silently downgraded to the AIO default. Only AIO/AIBV reach here + # (M365 implies -IncludeM365Usage -> the M365Bundle branch above). + $script:RollupDashboard = if ($Dashboard) { $Dashboard.ToUpperInvariant() } else { 'AIO' } + if ($script:RollupDashboard -notin @('AIO', 'AIBV')) { $script:RollupDashboard = 'AIO' } + $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() + if (-not $IncludeUserInfo) { + $IncludeUserInfo = [System.Management.Automation.SwitchParameter]::new($true) + Write-LogHost " Resume: auto-enabled -IncludeUserInfo for CopilotInteraction-mode rollup." -ForegroundColor Cyan + } + } + } + + # Auth - only restore if user didn't override + if (-not $PSBoundParameters.ContainsKey('Auth') -and $cp.auth) { $Auth = $cp.auth } + if (-not $PSBoundParameters.ContainsKey('TenantId') -and $cp.tenantId) { $TenantId = $cp.tenantId } + if (-not $PSBoundParameters.ContainsKey('ClientId') -and $cp.clientId) { $ClientId = $cp.clientId } + + # Re-promote restored auth identifiers to script scope. The original L1808-L1812 + # promotion ran at parse time, BEFORE the checkpoint restored TenantId/ClientId + # from the snapshot. Without this re-promotion Connect-PurviewAudit would see + # the empty default values and fail to authenticate even when the user supplied + # -ClientSecret on the resume command line. + $script:TenantId = $TenantId + $script:ClientId = $ClientId + + # Resume preflight for AppRegistration: ClientSecret / cert material is never + # persisted in the checkpoint (by design). If the restored Auth is AppRegistration + # and no credential material was supplied on the resume command line, fail fast + # with an actionable error rather than crashing inside Connect-PurviewAudit on + # the first audit query attempt. + if (([string]$Auth) -ieq 'AppRegistration' ` + -and [string]::IsNullOrEmpty([string]$script:ClientSecret) ` + -and [string]::IsNullOrEmpty([string]$script:ClientCertificateThumbprint) ` + -and [string]::IsNullOrEmpty([string]$script:ClientCertificatePath)) { + Write-LogHost "" -ForegroundColor Red + Write-LogHost "═════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " ERROR: -Resume against an AppRegistration checkpoint requires credential material" -ForegroundColor Red + Write-LogHost "═════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" -ForegroundColor Red + Write-LogHost (" The original run used -Auth AppRegistration (TenantId={0}, ClientId={1})." -f $script:TenantId, $script:ClientId) -ForegroundColor Yellow + Write-LogHost " Secrets / certificate material are NOT persisted in the checkpoint by design." -ForegroundColor Yellow + Write-LogHost " Re-supply one of the credential switches on the resume command line:" -ForegroundColor Yellow + Write-LogHost "" -ForegroundColor Yellow + Write-LogHost " -ClientSecret " -ForegroundColor Green + Write-LogHost " -ClientCertificateThumbprint " -ForegroundColor Green + Write-LogHost " -ClientCertificatePath [-ClientCertificatePassword ]" -ForegroundColor Green + Write-LogHost "" -ForegroundColor Yellow + Write-LogHost " Example:" -ForegroundColor Cyan + Write-LogHost (" pwsh -NoLogo -NoProfile -File `"