diff --git a/Config/ConversionTable.csv b/Config/ConversionTable.csv index 9c354ea9db054..2d0f3cadd3984 100644 --- a/Config/ConversionTable.csv +++ b/Config/ConversionTable.csv @@ -1864,6 +1864,7 @@ Microsoft 365 E3,SPE_E3,05e9a617-0261-4cee-bb44-138d3ef5d965,KAIZALA_O365_P3,aeb Microsoft 365 E3,SPE_E3,05e9a617-0261-4cee-bb44-138d3ef5d965,FORMS_PLAN_E3,2789c901-c14e-48ab-a76a-be334d9d793a,Microsoft Forms (Plan E3) Microsoft 365 E3,SPE_E3,05e9a617-0261-4cee-bb44-138d3ef5d965,MDE_LITE,292cc034-7b7c-4950-aaf5-943befd3f1d4,Microsoft Defender for Endpoint Plan 1 Microsoft 365 E3,SPE_E3,05e9a617-0261-4cee-bb44-138d3ef5d965,MICROSOFT_SEARCH,94065c59-bc8e-4e8b-89e5-5138d471eaff,Microsoft Search +Microsoft 365 E3,SPE_E3,05e9a617-0261-4cee-bb44-138d3ef5d965,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Microsoft 365 E3 - Unattended License,SPE_E3_RPA1,c2ac2ee4-9bb1-47e4-8541-d689c7e83371,SHAREPOINTENTERPRISE,5dbe027f-2339-4123-9542-606e4d348a72,SharePoint (Plan 2) Microsoft 365 E3 - Unattended License,SPE_E3_RPA1,c2ac2ee4-9bb1-47e4-8541-d689c7e83371,PROJECT_O365_P2,31b4e2fc-4cd6-4e7d-9c1b-41407303bd66,Project for Office (Plan E3) Microsoft 365 E3 - Unattended License,SPE_E3_RPA1,c2ac2ee4-9bb1-47e4-8541-d689c7e83371,MCOSTANDARD,0feaeb32-d00e-4d66-bd5a-43b5b83db82c,Skype for Business Online (Plan 2) @@ -2147,6 +2148,7 @@ Microsoft 365 E3_USGOV_DOD,SPE_E3_USGOV_DOD,d61d61cc-f992-433f-a577-5bd016037eeb Microsoft 365 E3_USGOV_DOD,SPE_E3_USGOV_DOD,d61d61cc-f992-433f-a577-5bd016037eeb,AAD_PREMIUM,41781fb2-bc02-4b7c-bd55-b576c07bb09d,Microsoft Entra ID P1 Microsoft 365 E3_USGOV_DOD,SPE_E3_USGOV_DOD,d61d61cc-f992-433f-a577-5bd016037eeb,SHAREPOINTENTERPRISE,5dbe027f-2339-4123-9542-606e4d348a72,SharePoint Online (Plan 2) Microsoft 365 E3_USGOV_DOD,SPE_E3_USGOV_DOD,d61d61cc-f992-433f-a577-5bd016037eeb,RMS_S_ENTERPRISE,bea4c11e-220a-4e6d-8eb8-8ea15d019f90,Microsoft Microsoft Entra Rights +Microsoft 365 E3_USGOV_DOD,SPE_E3_USGOV_DOD,d61d61cc-f992-433f-a577-5bd016037eeb,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,OFFICESUBSCRIPTION,43de0ff5-c92c-492b-9116-175376d08c38,Office 365 ProPlus Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,STREAM_O365_E3,9e700747-8b1d-45e5-ab8d-ef187ceec156,Microsoft Stream for O365 E3 SKU Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,TEAMS_AR_GCCHIGH,9953b155-8aef-4c56-92f3-72b0487fce41,Microsoft Teams for GCCHigh (AR) @@ -2161,6 +2163,7 @@ Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1 Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,RMS_S_PREMIUM,6c57d4b6-3b23-47a5-9bc9-69f17b4947b3,Azure Information Protection Premium P Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,RMS_S_ENTERPRISE,bea4c11e-220a-4e6d-8eb8-8ea15d019f90,Microsoft Microsoft Entra Rights Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,ADALLOM_S_DISCOVERY,932ad362-64a8-4783-9106-97849a1a30b9,Cloud App Security Discovery +Microsoft 365 E3_USGOV_GCCHIGH,SPE_E3_USGOV_GCCHIGH,ca9d1dd9-dfe9-4fef-b97c-9bc1ea3c3658,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Microsoft 365 E5,SPE_E5,06ebc4ee-1bb5-47dd-8120-11324bc54e06,Deskless,8c7d2df8-86f0-4902-b2ed-a0458298f3b3,Microsoft StaffHub Microsoft 365 E5,SPE_E5,06ebc4ee-1bb5-47dd-8120-11324bc54e06,MCOSTANDARD,0feaeb32-d00e-4d66-bd5a-43b5b83db82c,Skype for Business Online (Plan 2) Microsoft 365 E5,SPE_E5,06ebc4ee-1bb5-47dd-8120-11324bc54e06,SHAREPOINTENTERPRISE,5dbe027f-2339-4123-9542-606e4d348a72,SharePoint (Plan 2) @@ -2925,6 +2928,8 @@ Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7 Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7-c6a9a1999067,WINDOWSUPDATEFORBUSINESS_DEPLOYMENTSERVICE,7bf960f6-2cd9-443a-8046-5dbff9558365,Windows Update for Business Deployment Service Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7-c6a9a1999067,Defender_for_Iot_Enterprise,99cd49a9-0e54-4e07-aea1-d8d9f5f704f5,Defender for IoT - Enterprise IoT Security Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7-c6a9a1999067,MESH_AVATARS_ADDITIONAL_FOR_TEAMS,3efbd4ed-8958-4824-8389-1321f8730af8,Avatars for Teams (additional) +Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7-c6a9a1999067,CLOUD_PKI,795aec3a-93a2-45be-92c4-47b9a76340ca,Microsoft Cloud PKI +Microsoft 365 E5 Suite features,M365_E5_SUITE_COMPONENTS,99cc8282-2f74-4954-83b7-c6a9a1999067,3_PARTY_APP_PATCH,3afa0b92-83ef-41c1-8d64-586ab882a951,Intune Enterprise Application Management Microsoft 365 E5 with Calling Minutes,SPE_E5_CALLINGMINUTES,a91fc4e0-65e5-4266-aa76-4037509c1626,PREMIUM_ENCRYPTION,617b097b-4b93-4ede-83de-5f075bb5fb2f,Premium Encryption in Office 365 Microsoft 365 E5 with Calling Minutes,SPE_E5_CALLINGMINUTES,a91fc4e0-65e5-4266-aa76-4037509c1626,BI_AZURE_P2,70d33638-9c74-4d01-bfd3-562de28bd4ba,Power BI Pro Microsoft 365 E5 with Calling Minutes,SPE_E5_CALLINGMINUTES,a91fc4e0-65e5-4266-aa76-4037509c1626,POWERAPPS_O365_P3,9c0dab89-a30c-4117-86e7-97bda240acd2,Power Apps for Office 365 (Plan 3) @@ -3535,6 +3540,7 @@ Microsoft 365 G3 GCC,M365_G3_GOV,e823ca47-49c4-46b3-b38d-ca11d5abe3d2,POWERAPPS_ Microsoft 365 G3 GCC,M365_G3_GOV,e823ca47-49c4-46b3-b38d-ca11d5abe3d2,INTUNE_A,c1ec4a95-1f05-45b3-a911-aa3fa01094f5,Microsoft Intune Plan 1 Microsoft 365 G3 GCC,M365_G3_GOV,e823ca47-49c4-46b3-b38d-ca11d5abe3d2,CDS_O365_P2_GCC,a70bbf38-cdda-470d-adb8-5804b8770f41,Common Data Service for Teams Microsoft 365 G3 GCC,M365_G3_GOV,e823ca47-49c4-46b3-b38d-ca11d5abe3d2,FLOW_O365_P2_GOV,c537f360-6a00-4ace-a7f5-9128d0ac1e4b,Power Automate for Office 365 for Government +Microsoft 365 G3 GCC,M365_G3_GOV,e823ca47-49c4-46b3-b38d-ca11d5abe3d2,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Microsoft 365 GCC G5,M365_G5_GCC,e2be619b-b125-455f-8660-fb503e431a5d,FORMS_GOV_E5,843da3a8-d2cc-4e7a-9e90-dc46019f964c,Microsoft Forms for Government (Plan E5) Microsoft 365 GCC G5,M365_G5_GCC,e2be619b-b125-455f-8660-fb503e431a5d,CDS_O365_P3_GCC,bce5e5ca-c2fd-4d53-8ee2-58dfffed4c10,Common Data Service for Teams Microsoft 365 GCC G5,M365_G5_GCC,e2be619b-b125-455f-8660-fb503e431a5d,LOCKBOX_ENTERPRISE_GOV,89b5d3b1-3855-49fe-b46c-87c66dbc1526,Customer Lockbox for Government @@ -4507,6 +4513,7 @@ Office 365 E1 (no Teams),Office_365_E1_(no_Teams),f8ced641-8e17-4dc5-b014-f5a2d5 Office 365 E1 (no Teams),Office_365_E1_(no_Teams),f8ced641-8e17-4dc5-b014-f5a2d53f6ac8,DYN365_CDS_O365_P1,40b010bb-0b69-4654-ac5e-ba161433f4b4,Common Data Service Office 365 E1 (no Teams),Office_365_E1_(no_Teams),f8ced641-8e17-4dc5-b014-f5a2d53f6ac8,MICROSOFTBOOKINGS,199a5c09-e0ca-4e37-8f7c-b05d533e1ea2,Microsoft Bookings Office 365 E1 (no Teams),Office_365_E1_(no_Teams),f8ced641-8e17-4dc5-b014-f5a2d53f6ac8,SHAREPOINTWAC,e95bec33-7c88-4a70-8e19-b10bd9d0c014,Office for the Web +Office 365 E1 (no Teams),Office_365_E1_(no_Teams),f8ced641-8e17-4dc5-b014-f5a2d53f6ac8,MDOLITE_ENTERPRISE,c6675fa4-68fe-415f-aec1-a44520f0c3a3,Microsoft 365 built-in email and collaboration security Office 365 E1 EEA (no Teams),Office_365_w/o_Teams_Bundle_E1,b57282e3-65bd-4252-9502-c0eae1e5ab7f,SHAREPOINTWAC,e95bec33-7c88-4a70-8e19-b10bd9d0c014,Office for the Web Office 365 E1 EEA (no Teams),Office_365_w/o_Teams_Bundle_E1,b57282e3-65bd-4252-9502-c0eae1e5ab7f,YAMMER_ENTERPRISE,7547a3fe-08ee-4ccb-b430-5077c5041653,Yammer Enterprise Office 365 E1 EEA (no Teams),Office_365_w/o_Teams_Bundle_E1,b57282e3-65bd-4252-9502-c0eae1e5ab7f,VIVAENGAGE_CORE,a82fbf69-b4d7-49f4-83a6-915b2cf354f4,Viva Engage Core @@ -4603,6 +4610,7 @@ Office 365 E3,ENTERPRISEPACK,6fd2c87f-b296-42f0-b197-1e91e994b900,FLOW_O365_P2,7 Office 365 E3,ENTERPRISEPACK,6fd2c87f-b296-42f0-b197-1e91e994b900,POWERAPPS_O365_P2,c68f8d98-5534-41c8-bf36-22fa496fa792,Power Apps for Office 365 Office 365 E3,ENTERPRISEPACK,6fd2c87f-b296-42f0-b197-1e91e994b900,YAMMER_ENTERPRISE,7547a3fe-08ee-4ccb-b430-5077c5041653,Yammer Enterprise Office 365 E3,ENTERPRISEPACK,6fd2c87f-b296-42f0-b197-1e91e994b900,POWER_VIRTUAL_AGENTS_O365_P2,041fe683-03e4-45b6-b1af-c0cdc516daee,Power Virtual Agents for Office 365 +Office 365 E3,ENTERPRISEPACK,6fd2c87f-b296-42f0-b197-1e91e994b900,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Office 365 E3 (no Teams),Office_365_E3_(no_Teams),46c3a859-c90d-40b3-9551-6178a48d5c18,MESH_AVATARS_FOR_TEAMS,dcf9d2f4-772e-4434-b757-77a453cfbc02,Avatars for Teams Office 365 E3 (no Teams),Office_365_E3_(no_Teams),46c3a859-c90d-40b3-9551-6178a48d5c18,KAIZALA_O365_P3,aebd3021-9f8f-4bf8-bbe3-0ed2f4f047a1,Microsoft Kaizala Pro Office 365 E3 (no Teams),Office_365_E3_(no_Teams),46c3a859-c90d-40b3-9551-6178a48d5c18,FORMS_PLAN_E3,2789c901-c14e-48ab-a76a-be334d9d793a,Microsoft Forms (Plan E3) @@ -4693,6 +4701,7 @@ Office 365 E3_USGOV_DOD,ENTERPRISEPACK_USGOV_DOD,b107e5a3-3e60-4c0d-a184-a7e4395 Office 365 E3_USGOV_DOD,ENTERPRISEPACK_USGOV_DOD,b107e5a3-3e60-4c0d-a184-a7e4395eb44c,SHAREPOINTENTERPRISE,5dbe027f-2339-4123-9542-606e4d348a72,SharePoint Online (Plan 2) Office 365 E3_USGOV_DOD,ENTERPRISEPACK_USGOV_DOD,b107e5a3-3e60-4c0d-a184-a7e4395eb44c,MCOSTANDARD,0feaeb32-d00e-4d66-bd5a-43b5b83db82c,Skype for Business Online (Plan 2) Office 365 E3_USGOV_DOD,ENTERPRISEPACK_USGOV_DOD,b107e5a3-3e60-4c0d-a184-a7e4395eb44c,OFFICESUBSCRIPTION,43de0ff5-c92c-492b-9116-175376d08c38,Office 365 ProPlus +Office 365 E3_USGOV_DOD,ENTERPRISEPACK_USGOV_DOD,b107e5a3-3e60-4c0d-a184-a7e4395eb44c,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,MCOSTANDARD,0feaeb32-d00e-4d66-bd5a-43b5b83db82c,Skype for Business Online (Plan 2) Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,SHAREPOINTENTERPRISE,5dbe027f-2339-4123-9542-606e4d348a72,SharePoint Online (Plan 2) Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,SHAREPOINTWAC,e95bec33-7c88-4a70-8e19-b10bd9d0c014,Office Online @@ -4702,6 +4711,7 @@ Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00 Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,RMS_S_ENTERPRISE,bea4c11e-220a-4e6d-8eb8-8ea15d019f90,Microsoft Microsoft Entra Rights Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,EXCHANGE_S_ENTERPRISE,efb87545-963c-4e0d-99df-69c6916d9eb0,Exchange Online (Plan 2) Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,TEAMS_AR_GCCHIGH,9953b155-8aef-4c56-92f3-72b0487fce41,Microsoft Teams for GCCHigh (AR) +Office 365 E3_USGOV_GCCHIGH,ENTERPRISEPACK_USGOV_GCCHIGH,aea38a85-9bd5-4981-aa00-616b411205bf,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Office 365 E4,ENTERPRISEWITHSCAL,1392051d-0cb9-4b7a-88d5-621fee5e8711,BPOS_S_TODO_2,c87f142c-d1e9-4363-8630-aaea9c4d9ae5,BPOS_S_TODO_2 Office 365 E4,ENTERPRISEWITHSCAL,1392051d-0cb9-4b7a-88d5-621fee5e8711,Deskless,8c7d2df8-86f0-4902-b2ed-a0458298f3b3,MICROSOFT STAFFHUB Office 365 E4,ENTERPRISEWITHSCAL,1392051d-0cb9-4b7a-88d5-621fee5e8711,FLOW_O365_P2,76846ad7-7776-4c40-a281-a386362dd1b9,FLOW FOR OFFICE 365 @@ -5058,6 +5068,7 @@ Office 365 G3 GCC,ENTERPRISEPACK_GOV,535a3a29-c5f0-42fe-8215-d3b9e1f38c4a,MIP_S_ Office 365 G3 GCC,ENTERPRISEPACK_GOV,535a3a29-c5f0-42fe-8215-d3b9e1f38c4a,ContentExplorer_Standard,2b815d45-56e4-4e3a-b65c-66cb9175b560,Information Protection and Governance Analytics – Standard Office 365 G3 GCC,ENTERPRISEPACK_GOV,535a3a29-c5f0-42fe-8215-d3b9e1f38c4a,PROJECT_O365_P2_GOV,e7d09ae4-099a-4c34-a2a2-3e166e95c44a,Project for Government (Plan E3) Office 365 G3 GCC,ENTERPRISEPACK_GOV,535a3a29-c5f0-42fe-8215-d3b9e1f38c4a,MYANALYTICS_P2_GOV,6e5b7995-bd4f-4cbd-9d19-0e32010c72f0,Insights by MyAnalytics for Government +Office 365 G3 GCC,ENTERPRISEPACK_GOV,535a3a29-c5f0-42fe-8215-d3b9e1f38c4a,ATP_ENTERPRISE,f20fedf3-f3c3-43c3-8267-2bfdd51c0939,Microsoft Defender for Office 365 (Plan 1) Office 365 G3 without Microsoft 365 Apps GCC,ENTERPRISEPACKWITHOUTPROPLUS_GOV,24aebea8-7fac-48d0-8750-de4ee1fde205,CDS_O365_P2_GCC,a70bbf38-cdda-470d-adb8-5804b8770f41,Common Data Service for Teams Office 365 G3 without Microsoft 365 Apps GCC,ENTERPRISEPACKWITHOUTPROPLUS_GOV,24aebea8-7fac-48d0-8750-de4ee1fde205,EXCHANGE_S_ENTERPRISE_GOV,8c3069c0-ccdb-44be-ab77-986203a67df2,Exchange Online (Plan 2) for Government Office 365 G3 without Microsoft 365 Apps GCC,ENTERPRISEPACKWITHOUTPROPLUS_GOV,24aebea8-7fac-48d0-8750-de4ee1fde205,MIP_S_CLP1,5136a095-5cf0-4aff-bec3-e84448b38ea5,Information Protection for Office 365 - Standard diff --git a/Config/SAMManifest.json b/Config/SAMManifest.json index b3e135310de40..e9d5640e8f554 100644 --- a/Config/SAMManifest.json +++ b/Config/SAMManifest.json @@ -626,6 +626,10 @@ { "id": "e2edbde8-4448-4e49-8ebb-d53ba72df0f3", "type": "Scope" + }, + { + "id": "3bc15058-7858-4141-b24f-ae43b4e80b52", + "type": "Scope" } ] }, diff --git a/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 b/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 index 54f1dc0f3c168..e8e0f8b66205f 100644 --- a/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 +++ b/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 @@ -26,13 +26,8 @@ function Push-AuditLogTenantDownload { $CIPPURL = $LegacyUrl } else { if (!$CippConfig) { - $CippConfig = @{ - PartitionKey = 'InstanceProperties' - RowKey = 'CIPPURL' - Value = [string]([System.Uri]$Request.Headers.'x-ms-original-url').Host - } - Add-AzDataTableEntity @CippConfigTable -Entity $CippConfig -Force - $CIPPURL = 'https://{0}' -f $CippConfig.Value + # No request context here, so this resolves from the platform hostname and stores it + $CIPPURL = 'https://{0}' -f (Get-CIPPHostname -Save) } else { $CIPPURL = 'https://{0}' -f $CippConfig.Value } } diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 index ecff809d72b2f..585f07a90fdce 100644 --- a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1 @@ -28,6 +28,8 @@ function Get-CippAuditLogSearchResults { $GraphRequest.CountOnly = $true } - New-GraphGetRequest @GraphRequest -ErrorAction Stop | Sort-Object -Property createdDateTime -Descending + # Deliberately unsorted: Sort-Object blocks until the entire window is in memory, and + # no caller depends on the order. + New-GraphGetRequest @GraphRequest -Stream -ErrorAction Stop } } diff --git a/Modules/CIPPCore/Public/Functions/Get-CIPPHostname.ps1 b/Modules/CIPPCore/Public/Functions/Get-CIPPHostname.ps1 new file mode 100644 index 0000000000000..9a6b7d84f27d3 --- /dev/null +++ b/Modules/CIPPCore/Public/Functions/Get-CIPPHostname.ps1 @@ -0,0 +1,86 @@ +function Get-CIPPHostname { + <# + .SYNOPSIS + Resolves the hostname the CIPP instance is currently being served from + .DESCRIPTION + Works out the host of the running instance, preferring the inbound request so the value always + reflects the URL the user is on at the time of the call. Falls back to the stored instance + property and then the platform hostname for contexts where no request is available. + Returns the host only (no scheme, no port), matching the format stored in Config/InstanceProperties/CIPPURL. + .PARAMETER Headers + The request headers, usually $Request.Headers. Optional - omit for non-HTTP contexts. + .PARAMETER Save + Persist the resolved hostname to Config/InstanceProperties/CIPPURL so background jobs pick up the current URL. + .FUNCTIONALITY + Internal + .EXAMPLE + Get-CIPPHostname -Headers $Request.Headers -Save + #> + [CmdletBinding()] + param( + $Headers, + [switch]$Save + ) + + $Hostname = $null + + if ($Headers) { + # x-ms-original-url carries the full URL the client requested, including any custom domain + $Candidates = @( + $Headers.'x-ms-original-url' + $Headers.origin + $Headers.referer + ) + foreach ($Candidate in $Candidates) { + if ([string]::IsNullOrWhiteSpace($Candidate)) { continue } + try { + $Parsed = [System.Uri]$Candidate + if ($Parsed.Host) { + $Hostname = $Parsed.Host + break + } + } catch { + continue + } + } + + # Proxied deployments may only expose the host, not a full URL + if (!$Hostname) { + $HostHeader = $Headers.'x-forwarded-host' ?? $Headers.host + if (![string]::IsNullOrWhiteSpace($HostHeader)) { + $Hostname = (($HostHeader -split ',')[0]).Trim().Split(':')[0] + } + } + } + + # Only touch storage when we actually need it: as a fallback, or to persist the resolved value + $StoredConfig = $null + if (!$Hostname -or $Save.IsPresent) { + $ConfigTable = Get-CIPPTable -TableName 'Config' + $StoredConfig = Get-CIPPAzDataTableEntity @ConfigTable -Filter "PartitionKey eq 'InstanceProperties' and RowKey eq 'CIPPURL'" + } + + if (!$Hostname -and ![string]::IsNullOrWhiteSpace($StoredConfig.Value)) { + $Hostname = $StoredConfig.Value + } + + if (!$Hostname -and ![string]::IsNullOrWhiteSpace($env:WEBSITE_HOSTNAME)) { + $Hostname = $env:WEBSITE_HOSTNAME + } + + if ($Save.IsPresent -and ![string]::IsNullOrWhiteSpace($Hostname) -and $StoredConfig.Value -ne $Hostname) { + try { + $AddObject = @{ + PartitionKey = 'InstanceProperties' + RowKey = 'CIPPURL' + Value = [string]$Hostname + } + Add-CIPPAzDataTableEntity @ConfigTable -Entity $AddObject -Force + Write-Information "Get-CIPPHostname: updated stored CIPPURL from '$($StoredConfig.Value)' to '$Hostname'" + } catch { + Write-Information "Get-CIPPHostname: failed to store CIPPURL: $($_.Exception.Message)" + } + } + + return $Hostname +} diff --git a/Modules/CIPPCore/Public/GraphHelper/New-CIPPAzRestRequest.ps1 b/Modules/CIPPCore/Public/GraphHelper/New-CIPPAzRestRequest.ps1 index 248a94ee56ef9..dced5ab0193b8 100644 --- a/Modules/CIPPCore/Public/GraphHelper/New-CIPPAzRestRequest.ps1 +++ b/Modules/CIPPCore/Public/GraphHelper/New-CIPPAzRestRequest.ps1 @@ -105,6 +105,35 @@ function New-CIPPAzRestRequest { [int]$MaxRetries = 3 ) + # Confirm the Uri targets a known Azure service host that is consistent with the token audience (ResourceUrl). + # This prevents the Managed Identity bearer token from being sent to an arbitrary/attacker-controlled host. + # Validated before token acquisition so an invalid service never triggers a token request. + $AzureServiceHosts = @{ + 'https://management.azure.com' = @('management.azure.com') + 'https://vault.azure.net' = @('*.vault.azure.net') + 'https://api.loganalytics.io' = @('api.loganalytics.io') + 'https://storage.azure.com' = @('*.blob.core.windows.net', '*.table.core.windows.net', '*.queue.core.windows.net', '*.file.core.windows.net') + } + + $UriHost = $Uri.Host + $NormalizedResource = $ResourceUrl.TrimEnd('/') + $AllowedHosts = $AzureServiceHosts[$NormalizedResource] + + if (-not $AllowedHosts) { + $ValidationError = "ResourceUrl '$ResourceUrl' is not a recognized Azure service. Allowed resources: $($AzureServiceHosts.Keys -join ', ')" + Write-LogMessage -API 'New-CIPPAzRestRequest' -message $ValidationError -Sev 'Error' -LogData @{ Uri = $Uri.ToString(); ResourceUrl = $ResourceUrl } + Write-Error -Message $ValidationError -ErrorAction $ErrorActionPreference + return + } + + $HostAllowed = $AllowedHosts | Where-Object { $UriHost -eq $_ -or ($_ -like '*.*' -and $UriHost -like $_) } + if (-not $HostAllowed) { + $ValidationError = "Uri host '$UriHost' is not valid for resource '$ResourceUrl'. Allowed hosts: $($AllowedHosts -join ', ')" + Write-LogMessage -API 'New-CIPPAzRestRequest' -message $ValidationError -Sev 'Error' -LogData @{ Uri = $Uri.ToString(); ResourceUrl = $ResourceUrl; UriHost = $UriHost } + Write-Error -Message $ValidationError -ErrorAction $ErrorActionPreference + return + } + # Resolve bearer token: prefer manually-supplied AccessToken, otherwise fetch via Managed Identity $Token = $null if ($AccessToken) { diff --git a/Modules/CIPPCore/Public/Tools/Get-CIPPSchedulerBlockedCommands.ps1 b/Modules/CIPPCore/Public/Tools/Get-CIPPSchedulerBlockedCommands.ps1 index be6aeed06a6ad..bed47db6b34b0 100644 --- a/Modules/CIPPCore/Public/Tools/Get-CIPPSchedulerBlockedCommands.ps1 +++ b/Modules/CIPPCore/Public/Tools/Get-CIPPSchedulerBlockedCommands.ps1 @@ -23,6 +23,17 @@ function Get-CIPPSchedulerBlockedCommands { 'New-GraphGetRequest' 'New-GraphBulkRequest' 'New-ExoRequest' + 'New-ExoBulkRequest' + 'New-TeamsRequestV2' + 'New-ClassicAPIGetRequest' + 'Invoke-CIPPRestMethod' + 'Invoke-GitHubApiRequest' + 'New-CIPPAzStorageRequest' + 'New-CippCoreRequest' + 'New-CIPPDbRequest' + 'New-DeviceLogin' + 'Clear-CippTokenCache' + 'Remove-CIPPDirectTenantToken' # Env 'Set-CIPPEnvVarBackup' @@ -31,6 +42,7 @@ function Get-CIPPSchedulerBlockedCommands { 'Get-CIPPAzFunctionAppSetting' 'Get-CIPPAzFunctionAppSubId' 'Update-CIPPAzFunctionAppSetting' + 'New-CIPPAzRestRequest' # Extension authentication tokens 'Get-GradientToken' @@ -50,6 +62,34 @@ function Get-CIPPSchedulerBlockedCommands { 'Get-ExtensionAPIKey' 'Set-ExtensionAPIKey' 'Remove-ExtensionAPIKey' + 'Get-CIPPOmaSettingDecryptedValue' + + # End-tenant secret material - recovery keys & admin passwords, PostExecution would exfiltrate them + 'Get-CIPPLAPSPassword' + 'Get-CIPPBitlockerKey' + 'Search-CIPPBitlockerKeys' + 'Get-CIPPFileVaultKey' + + # SAM/CPV & app registration configuration - privilege escalation / token theft vectors + 'Set-CIPPCPVConsent' + 'Add-CIPPApplicationPermission' + 'Add-CIPPDelegatedPermission' + 'Update-CippSamPermissions' + 'Set-CIPPSAMAdminRoles' + 'Update-CIPPSAMRedirectUri' + 'Update-CIPPSAMCertificateEnvCache' + 'Add-CIPPSSOAppSecret' + 'Set-CIPPSSOEasyAuth' + 'Set-CIPPSSOStoredCredentials' + 'Update-CIPPSSORedirectUri' + 'New-CIPPAPIConfig' + 'Get-CippApiAuth' + 'Set-CippApiAuth' + 'Repair-CippApiIdentifierUri' + + # CIPP RBAC - would allow privilege escalation within CIPP + 'Get-CIPPAccessRole' + 'Set-CIPPAccessRole' # Tenant enumeration - would reveal full tenant list 'Get-Tenants' @@ -68,6 +108,15 @@ function Get-CIPPSchedulerBlockedCommands { 'Update-AzDataTableEntity' 'Remove-AzDataTableEntity' 'Remove-AzDataTable' + 'Get-CIPPAzStorageContainer' + 'Remove-CIPPAzStorageContainer' + 'Get-CIPPAzStorageQueue' + 'Get-CIPPAzStorageQueueMessage' + 'Get-CIPPAzStorageQueueAnalysis' + 'Clear-CIPPAzStorageQueue' + + # Infrastructure control - denial of service + 'Request-CIPPRestart' # Backup & restore 'Get-CIPPBackup' @@ -77,5 +126,6 @@ function Get-CIPPSchedulerBlockedCommands { 'New-CippQueueEntry' 'Set-CippQueueTask' 'Update-CippQueueEntry' + 'Add-CIPPScheduledTask' ) } diff --git a/Modules/CIPPCore/Public/Webhooks/New-CIPPGraphSubscription.ps1 b/Modules/CIPPCore/Public/Webhooks/New-CIPPGraphSubscription.ps1 index 75d8b4bd6365e..078ad1eca77bd 100644 --- a/Modules/CIPPCore/Public/Webhooks/New-CIPPGraphSubscription.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/New-CIPPGraphSubscription.ps1 @@ -32,8 +32,15 @@ function New-CIPPGraphSubscription { } } + if ([string]::IsNullOrWhiteSpace($BaseURL)) { + Write-LogMessage -headers $Headers -API $APIName -message 'Failed to create Partner Center Webhook subscription: could not determine the CIPP URL' -Sev 'Error' -tenant 'PartnerTenant' + return 'Failed to create Partner Center Webhook subscription: could not determine the CIPP URL' + } + + # The URL we want registered right now, based on the URL CIPP is currently served from + $DesiredWebhookUrl = "https://$BaseURL/API/PublicWebhooks?CIPPID=$($CIPPID)&Type=PartnerCenter" $Body = [PSCustomObject]@{ - WebhookUrl = "https://$BaseURL/API/PublicWebhooks?CIPPID=$($CIPPID)&Type=PartnerCenter" + WebhookUrl = $DesiredWebhookUrl WebhookEvents = @($EventList) } try { @@ -46,7 +53,9 @@ function New-CIPPGraphSubscription { try { $Existing = New-GraphGetRequest -NoAuthCheck $true -uri $Uri -tenantid $env:TenantID -scope 'https://api.partnercenter.microsoft.com/.default' } catch { $Existing = $false } - if (!$Existing -or $Existing.webhookUrl -ne $MatchedWebhook.WebhookNotificationUrl -or $EventCompare) { + # Compare against the URL we want registered, not against our own stored copy - otherwise a + # change of CIPP hostname is never detected because both sides still hold the old URL. + if (!$Existing -or $Existing.webhookUrl -ne $DesiredWebhookUrl -or $EventCompare) { if ($Existing.WebhookUrl) { $Action = 'Updated' $Method = 'PUT' @@ -71,6 +80,19 @@ function New-CIPPGraphSubscription { Write-LogMessage -headers $Headers -API $APIName -message "$Action Partner Center Webhook subscription" -Sev 'Info' -tenant 'PartnerTenant' return "$Action Partner Center Webhook subscription" } else { + # Partner Center already points at the right URL - make sure our own record agrees + if ($MatchedWebhook.WebhookNotificationUrl -ne $DesiredWebhookUrl) { + $WebhookRow = @{ + PartitionKey = [string]$CIPPID + RowKey = [string]$CIPPID + EventType = [string](ConvertTo-Json -InputObject $EventList) + Resource = [string]'PartnerCenter' + SubscriptionID = [string]$MatchedWebhook.SubscriptionID + Expiration = 'Does Not Expire' + WebhookNotificationUrl = [string]$DesiredWebhookUrl + } + $null = Add-CIPPAzDataTableEntity @WebhookTable -Entity $WebhookRow -Force + } Write-LogMessage -headers $Headers -API $APIName -message 'Existing Partner Center Webhook subscription found' -Sev 'Info' -tenant 'PartnerTenant' return 'Existing Partner Center Webhook subscription found' } diff --git a/Modules/CIPPCore/Public/Webhooks/Push-AuditLogDownloadV2.ps1 b/Modules/CIPPCore/Public/Webhooks/Push-AuditLogDownloadV2.ps1 index 8a729938833ae..ded32cbe209dd 100644 --- a/Modules/CIPPCore/Public/Webhooks/Push-AuditLogDownloadV2.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Push-AuditLogDownloadV2.ps1 @@ -54,25 +54,28 @@ function Push-AuditLogDownloadV2 { if ($Status -eq 'succeeded') { try { - $Results = @(Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId) - foreach ($SearchResult in $Results) { + # Streamed, not collected: each record is written to CacheWebhooks and + # dropped, so the window never needs to be resident. + $WindowCount = 0 + Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId | ForEach-Object { Add-CIPPAzDataTableEntity @CacheTable -Entity @{ - RowKey = [string]$SearchResult.id + RowKey = [string]$_.id PartitionKey = [string]$TenantFilter SearchId = $SearchId - JSON = [string]($SearchResult | ConvertTo-Json -Depth 10 -Compress) + JSON = [string]($_ | ConvertTo-Json -Depth 10 -Compress) CippProcessing = $false CippProcessingStarted = '' } -Force + $WindowCount++ } - $Downloaded += $Results.Count + $Downloaded += $WindowCount # Empty windows have nothing to process - mark them Processed directly so they # don't sit at Downloaded forever. Windows with records go to Downloaded and are # advanced to Processed by Push-AuditLogTenantProcessV2 once their rows are drained. - $DownloadState = if ($Results.Count -eq 0) { 'Processed' } else { 'Downloaded' } + $DownloadState = if ($WindowCount -eq 0) { 'Processed' } else { 'Downloaded' } $LedgerUpdate = @{ PartitionKey = $TenantFilter; RowKey = $Row.RowKey; State = $DownloadState - RecordCount = [int]$Results.Count; DownloadedUtc = $Now; Attempts = 0 + RecordCount = [int]$WindowCount; DownloadedUtc = $Now; Attempts = 0 SearchStatus = 'succeeded'; LastPolledUtc = $Now } if ($DownloadState -eq 'Processed') { @@ -80,7 +83,7 @@ function Push-AuditLogDownloadV2 { $LedgerUpdate.MatchedCount = 0 } Add-CIPPAzDataTableEntity @Ledger -Entity $LedgerUpdate -OperationType UpsertMerge - Write-Information "AuditLogV2: downloaded $($Results.Count) record(s) for $TenantFilter window $($Row.RowKey)" + Write-Information "AuditLogV2: downloaded $WindowCount record(s) for $TenantFilter window $($Row.RowKey)" } catch { $Attempts = [int]$Row.Attempts + 1 $RetryTotal = [int]$Row.RetryCount + 1 diff --git a/Modules/CIPPCore/Public/Webhooks/Push-AuditLogTenantProcessV2.ps1 b/Modules/CIPPCore/Public/Webhooks/Push-AuditLogTenantProcessV2.ps1 index e04ec02ca7c5e..4dae1f69ce11c 100644 --- a/Modules/CIPPCore/Public/Webhooks/Push-AuditLogTenantProcessV2.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Push-AuditLogTenantProcessV2.ps1 @@ -25,22 +25,69 @@ function Push-AuditLogTenantProcessV2 { $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' $SearchIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) - $Rows = foreach ($RowId in $RowIds) { - $CacheEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$RowId'" - if ($CacheEntity) { - if ($CacheEntity.SearchId) { [void]$SearchIds.Add([string]$CacheEntity.SearchId) } - $CacheEntity.JSON | ConvertFrom-Json -ErrorAction SilentlyContinue + # Chunked so peak memory tracks $ChunkSize, not batch size. Don't raise much above + # 100: each chunk builds one `RowKey eq ''` predicate per row, and an over-long + # filter is rejected (Azure ~520 predicates, Azurite ~250) and swallowed by the catch. + $ChunkSize = 100 + $ProcessedCount = 0 + $MatchedLogs = 0 + + for ($Offset = 0; $Offset -lt $RowIds.Count; $Offset += $ChunkSize) { + $Slice = @($RowIds[$Offset..([Math]::Min($Offset + $ChunkSize - 1, $RowIds.Count - 1))]) + + # Raw cmdlet: the wrapper merges split parts and reports the logical RowKey. + $KeyFilter = "PartitionKey eq '$TenantFilter' and (" + + (($Slice | ForEach-Object { "RowKey eq '$_'" }) -join ' or ') + ')' + $Keys = @(Get-AzDataTableEntity @CacheWebhooksTable -Filter $KeyFilter ` + -Property 'PartitionKey', 'RowKey', 'OriginalEntityId') + if ($Keys.Count -eq 0) { continue } + + # Split records span X / X-part1 / X-part2 and only reassemble when every part + # arrives in one call, so select on OriginalEntityId rather than RowKey. + $Predicates = [System.Collections.Generic.List[string]]::new() + $SeenLogical = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($Key in $Keys) { + if ($Key.PSObject.Properties.Name -contains 'OriginalEntityId' -and $Key.OriginalEntityId) { + if ($SeenLogical.Add([string]$Key.OriginalEntityId)) { + $Predicates.Add("OriginalEntityId eq '$($Key.OriginalEntityId)'") + } + } else { + $Predicates.Add("RowKey eq '$($Key.RowKey)'") + } + } + if ($Predicates.Count -eq 0) { continue } + + # No -Property: a projection must list every JSON_Part* column or split rows + # come back empty. + $RowFilter = "PartitionKey eq '$TenantFilter' and (" + ($Predicates -join ' or ') + ')' + $Entities = @(Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter $RowFilter) + + $Chunk = [System.Collections.Generic.List[object]]::new() + foreach ($Entity in $Entities) { + if ($Entity.SearchId) { [void]$SearchIds.Add([string]$Entity.SearchId) } + $Parsed = $Entity.JSON | ConvertFrom-Json -ErrorAction SilentlyContinue + if ($null -eq $Parsed) { + Write-Information "AuditLogV2: unparseable cached JSON for RowKey $($Entity.RowKey) ($TenantFilter)" + continue + } + $Chunk.Add($Parsed) + } + + if ($Chunk.Count -gt 0) { + $Result = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -Rows $Chunk + $MatchedLogs += [int]($Result.MatchedLogs ?? 0) + $ProcessedCount += $Chunk.Count } + $Chunk.Clear() + $Entities = $null } - if ($Rows.Count -eq 0) { + if ($ProcessedCount -eq 0) { Write-Information "AuditLogV2: no rows found in cache for the provided row IDs ($TenantFilter)" return $false } - Write-Information "AuditLogV2: processing $($Rows.Count) row(s) for $TenantFilter" - $Result = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -Rows $Rows - $MatchedLogs = [int]($Result.MatchedLogs ?? 0) + Write-Information "AuditLogV2: processed $ProcessedCount row(s) for $TenantFilter" # Advance the ledger to Processed for any SearchId now fully drained from the cache. if ($SearchIds.Count -gt 0) { diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index 19053df0c8d85..e85995c2a37f3 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -455,6 +455,14 @@ function Test-CIPPAuditLogRules { } } + # Deletes are flushed in small batches, not per record. The point is forward + # progress on a poison batch - a crash re-runs at most $DeleteFlushSize records, + # so the run always converges instead of looping on the same rows - and a batch + # takes minutes, so that window is real. Per-record calls cost ~13x more, since + # AzBobbyTables wraps each one in its own $batch transaction. + $DeleteFlushSize = 25 + $PendingDeletes = [System.Collections.Generic.List[object]]::new() + $ProcessedData = foreach ($AuditRecord in $SearchResults) { $RecordStartTime = Get-Date Write-Information "Processing RowKey $($AuditRecord.id) - $($TenantFilter)." @@ -462,42 +470,57 @@ function Test-CIPPAuditLogRules { $Data = $AuditRecord.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties, AuditRecord -ErrorAction SilentlyContinue try { # Attempt to locate GUIDs in $Data and match them with their corresponding user, group, device, or service principal using O(1) hashtable lookups - Write-Information 'Checking Data for GUIDs to map to users, groups, devices, or service principals' + # Write-Information 'Checking Data for GUIDs to map to users, groups, devices, or service principals' Add-CIPPGuidMappings -DataObject $Data -UserLookup $UserLookup -GroupLookup $GroupLookup -DeviceLookup $DeviceLookup -ServicePrincipalLookup $ServicePrincipalLookup -PartnerUserLookup $PartnerUserLookup -PropertyPrefix 'CIPP' # Also check root properties for GUIDs and partner UPNs - Write-Information 'Checking RootProperties for GUIDs to map to users, groups, devices, or service principals' + # Write-Information 'Checking RootProperties for GUIDs to map to users, groups, devices, or service principals' Add-CIPPGuidMappings -DataObject $RootProperties -UserLookup $UserLookup -GroupLookup $GroupLookup -DeviceLookup $DeviceLookup -ServicePrincipalLookup $ServicePrincipalLookup -PartnerUserLookup $PartnerUserLookup + # Flattened onto $Data so rules can match the property names directly. One + # Add-Member per sub-object: per-property calls rebuild the property bag each time. if ($Data.ExtendedProperties) { $Data.CIPPExtendedProperties = ($Data.ExtendedProperties | ConvertTo-Json -Compress -Depth 10) - $Data.ExtendedProperties | ForEach-Object { - if ($_.Value -in $ExtendedPropertiesIgnoreList) { - #write-warning "No need to process this operation as its in our ignore list. Some extended information: $($data.operation):$($_.Value) - $($TenantFilter)" - continue - } - $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue + $Flattened = @{} + foreach ($Prop in $Data.ExtendedProperties) { + # Must be a real loop: `continue` inside ForEach-Object unwinds to the + # enclosing foreach and drops the whole record. + if ($Prop.Value -in $ExtendedPropertiesIgnoreList) { continue } + if ([string]::IsNullOrEmpty($Prop.Name)) { continue } + $Flattened[$Prop.Name] = $Prop.Value } + if ($Flattened.Count -gt 0) { $Data | Add-Member -NotePropertyMembers $Flattened -Force -ErrorAction SilentlyContinue } } if ($Data.DeviceProperties) { $Data.CIPPDeviceProperties = ($Data.DeviceProperties | ConvertTo-Json -Compress -Depth 10) - $Data.DeviceProperties | ForEach-Object { $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue } + $Flattened = @{} + foreach ($Prop in $Data.DeviceProperties) { + if ([string]::IsNullOrEmpty($Prop.Name)) { continue } + $Flattened[$Prop.Name] = $Prop.Value + } + if ($Flattened.Count -gt 0) { $Data | Add-Member -NotePropertyMembers $Flattened -Force -ErrorAction SilentlyContinue } } if ($Data.parameters) { $Data.CIPPParameters = ($Data.parameters | ConvertTo-Json -Compress -Depth 10) - $Data.parameters | ForEach-Object { $Data | Add-Member -NotePropertyName $_.Name -NotePropertyValue $_.Value -Force -ErrorAction SilentlyContinue } + $Flattened = @{} + foreach ($Prop in $Data.parameters) { + if ([string]::IsNullOrEmpty($Prop.Name)) { continue } + $Flattened[$Prop.Name] = $Prop.Value + } + if ($Flattened.Count -gt 0) { $Data | Add-Member -NotePropertyMembers $Flattened -Force -ErrorAction SilentlyContinue } } if ($Data.ModifiedProperties) { $Data.CIPPModifiedProperties = ($Data.ModifiedProperties | ConvertTo-Json -Compress -Depth 10) try { - $Data.ModifiedProperties | ForEach-Object { $Data | Add-Member -NotePropertyName "$($_.Name)" -NotePropertyValue "$($_.NewValue)" -Force -ErrorAction SilentlyContinue } - } catch { - ##write-warning ($Data.ModifiedProperties | ConvertTo-Json -Depth 10) - } - try { - $Data.ModifiedProperties | ForEach-Object { $Data | Add-Member -NotePropertyName $("Previous_Value_$($_.Name)") -NotePropertyValue "$($_.OldValue)" -Force -ErrorAction SilentlyContinue } + $Flattened = @{} + foreach ($Prop in $Data.ModifiedProperties) { + if ([string]::IsNullOrEmpty($Prop.Name)) { continue } + $Flattened["$($Prop.Name)"] = "$($Prop.NewValue)" + $Flattened["Previous_Value_$($Prop.Name)"] = "$($Prop.OldValue)" + } + if ($Flattened.Count -gt 0) { $Data | Add-Member -NotePropertyMembers $Flattened -Force -ErrorAction SilentlyContinue } } catch { - ##write-warning ($Data.ModifiedProperties | ConvertTo-Json -Depth 10) + Write-Information "Error flattening ModifiedProperties for $($AuditRecord.id): $($_.Exception.Message)" } } @@ -545,18 +568,31 @@ function Test-CIPPAuditLogRules { Write-LogMessage -API 'Webhooks' -message 'Error Processing Audit Log Data' -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter } - try { - $null = Remove-AzDataTableEntity -Force @CacheWebhooksTable -Entity ([pscustomobject]@{ - PartitionKey = $TenantFilter - RowKey = [string]$AuditRecord.id - }) - } catch { - Write-Information "Error removing row $($AuditRecord.id) from cache: $($_.Exception.Message)" + $PendingDeletes.Add([PSCustomObject]@{ + PartitionKey = $TenantFilter + RowKey = [string]$AuditRecord.id + }) + if ($PendingDeletes.Count -ge $DeleteFlushSize) { + try { + $null = Remove-AzDataTableEntity -Force @CacheWebhooksTable -Entity $PendingDeletes.ToArray() + } catch { + Write-Information "Error removing $($PendingDeletes.Count) processed row(s) from cache: $($_.Exception.Message)" + } + $PendingDeletes.Clear() } $RecordEndTime = Get-Date $RecordSeconds = ($RecordEndTime - $RecordStartTime).TotalSeconds Write-Warning "Task took $RecordSeconds seconds for RowKey $($AuditRecord.id)" } + + if ($PendingDeletes.Count -gt 0) { + try { + $null = Remove-AzDataTableEntity -Force @CacheWebhooksTable -Entity $PendingDeletes.ToArray() + } catch { + Write-Information "Error removing $($PendingDeletes.Count) processed row(s) from cache: $($_.Exception.Message)" + } + $PendingDeletes.Clear() + } #write-warning "Processed Data: $(($ProcessedData | Measure-Object).Count) - This should be higher than 0 in many cases, because the where object has not run yet." #write-warning "Creating filters - $(($ProcessedData.operation | Sort-Object -Unique) -join ',') - $($TenantFilter)" @@ -690,8 +726,24 @@ function Test-CIPPAuditLogRules { try { $RowIds = [System.Collections.Generic.HashSet[string]]::new([string[]]@($Rows.id | Where-Object { $_ })) if ($RowIds.Count -gt 0) { - $CachedRows = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter'" - $RowsToRemove = @($CachedRows | Where-Object { $RowIds.Contains([string]$_.RowKey) }) + # Only the rows being deleted, not a partition scan - this runs once per chunk. + # Raw cmdlet and OriginalEntityId: the wrapper reports a split record's logical + # RowKey, so deleting that left X-part1 / X-part2 orphaned. + $IdList = @($RowIds) + $FilterBatch = 50 + $RowsToRemove = [System.Collections.Generic.List[object]]::new() + + for ($Start = 0; $Start -lt $IdList.Count; $Start += $FilterBatch) { + $Slice = @($IdList[$Start..([Math]::Min($Start + $FilterBatch - 1, $IdList.Count - 1))]) + $Predicate = ($Slice | ForEach-Object { "RowKey eq '$_' or OriginalEntityId eq '$_'" }) -join ' or ' + $Found = @(Get-AzDataTableEntity @CacheWebhooksTable ` + -Filter "PartitionKey eq '$TenantFilter' and ($Predicate)" ` + -Property 'PartitionKey', 'RowKey') + foreach ($Row in $Found) { + $RowsToRemove.Add([PSCustomObject]@{ PartitionKey = $Row.PartitionKey; RowKey = $Row.RowKey }) + } + } + if ($RowsToRemove.Count -gt 0) { Remove-AzDataTableEntity @CacheWebhooksTable -Entity $RowsToRemove -Force Write-Information "Removed $($RowsToRemove.Count) processed rows from cache" diff --git a/Modules/CIPPHTTP/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecPartnerWebhook.ps1 b/Modules/CIPPHTTP/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecPartnerWebhook.ps1 index 4799774c20253..8edb13057cdee 100644 --- a/Modules/CIPPHTTP/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecPartnerWebhook.ps1 +++ b/Modules/CIPPHTTP/Public/Entrypoints/HTTP Functions/CIPP/Core/Invoke-ExecPartnerWebhook.ps1 @@ -29,19 +29,27 @@ function Invoke-ExecPartnerWebhook { } catch {} if (!$Results) { $Results = [PSCustomObject]@{ - webhoookUrl = 'None' + webhookUrl = 'None' lastModifiedTimestamp = 'Never' webhookEvents = @() enabled = $false } } + + # The URL that would be registered if the subscription were saved right now, so the UI can + # flag a subscription still pointing at a previous CIPP URL. + $CurrentHostname = Get-CIPPHostname -Headers $Request.Headers + if ($CurrentHostname) { + $Results | Add-Member -MemberType NoteProperty -Name 'expectedWebhookUrl' -Value "https://$CurrentHostname/API/PublicWebhooks?CIPPID=$($env:TenantID)&Type=PartnerCenter" -Force + } } 'CreateSubscription' { if ($Request.Body.EventType.value) { $Request.Body.EventType = $Request.Body.EventType.value } - $BaseURL = ([System.Uri]$Request.Headers.'x-ms-original-url').Host + # Resolve the URL CIPP is served from at the time of submit, and store it for background jobs + $BaseURL = Get-CIPPHostname -Headers $Request.Headers -Save $Webhook = @{ TenantFilter = $env:TenantID PartnerCenter = $true diff --git a/Tests/Standards/Invoke-CIPPStandardReusableSettingsTemplate.Tests.ps1 b/Tests/Standards/Invoke-CIPPStandardReusableSettingsTemplate.Tests.ps1 index 82e43e1d0d4b7..82f3fbf7b12f5 100644 --- a/Tests/Standards/Invoke-CIPPStandardReusableSettingsTemplate.Tests.ps1 +++ b/Tests/Standards/Invoke-CIPPStandardReusableSettingsTemplate.Tests.ps1 @@ -8,7 +8,10 @@ BeforeAll { Select-Object -First 1 -ExpandProperty FullName if (-not $StandardPath) { throw 'Could not locate Invoke-CIPPStandardReusableSettingsTemplate.ps1 under Modules/' } - function Test-CIPPStandardLicense { param($StandardName, $TenantFilter, $RequiredCapabilities) } + # Stubs mirror the real signatures and are advanced functions on purpose: strict + # parameter binding makes signature drift in the standard fail loudly here instead + # of silently landing in $args and leaving the captured value $null. + function Test-CIPPStandardLicense { [CmdletBinding()] param($StandardName, $TenantFilter, $RequiredCapabilities, $Preset, [switch]$SkipLog) } function Get-CippTable { param($tablename) } function New-GraphGETRequest { param($uri, $tenantid) } function Get-CippAzDataTableEntity { param($Table, $Filter) } @@ -16,7 +19,7 @@ BeforeAll { function New-GraphPOSTRequest { param($uri, $tenantid, $type, $body) } function Write-LogMessage { param($API, $tenant, $message, $sev) } function Write-StandardsAlert { param($message, $object, $tenant, $standardName, $standardId) } - function Set-CIPPStandardsCompareField { param($FieldName, $FieldValue, $TenantFilter) } + function Set-CIPPStandardsCompareField { [CmdletBinding()] param($FieldName, $FieldValue, $CurrentValue, $ExpectedValue, $TenantFilter, [bool]$LicenseAvailable = $true, [array]$BulkFields) } function Get-NormalizedError { param($Message) $Message } . $StandardPath @@ -57,8 +60,15 @@ Describe 'Invoke-CIPPStandardReusableSettingsTemplate' { $script:alerts += @{ Message = $message; Object = $object; Standard = $standardName; Id = $standardId } } Mock -CommandName Set-CIPPStandardsCompareField -MockWith { - param($FieldName, $FieldValue, $TenantFilter) - $script:compareFields += @{ Field = $FieldName; Value = $FieldValue; Tenant = $TenantFilter } + param($FieldName, $FieldValue, $CurrentValue, $ExpectedValue, $TenantFilter, $LicenseAvailable) + $script:compareFields += @{ + Field = $FieldName + Value = $FieldValue + Current = $CurrentValue + Expected = $ExpectedValue + Tenant = $TenantFilter + LicenseAvailable = $LicenseAvailable + } } } @@ -149,7 +159,10 @@ Describe 'Invoke-CIPPStandardReusableSettingsTemplate' { $logs.Where({ $_.Message -like '*is compliant.*' }).Count | Should -Be 1 $compareFields | Should -HaveCount 1 - $compareFields[0].Value | Should -BeTrue + # The report branch reports through CurrentValue/ExpectedValue, not the legacy FieldValue. + $compareFields[0].Current.isCompliant | Should -BeTrue + $compareFields[0].Current.displayName | Should -Be 'Reusable A' + $compareFields[0].Expected.isCompliant | Should -BeTrue Should -Invoke -CommandName Write-StandardsAlert -Times 0 } } diff --git a/Tests/Test-SchedulerBlocklist.ps1 b/Tests/Test-SchedulerBlocklist.ps1 index 1402faed65a21..7af0d141aeacf 100644 --- a/Tests/Test-SchedulerBlocklist.ps1 +++ b/Tests/Test-SchedulerBlocklist.ps1 @@ -48,7 +48,7 @@ function Assert-Contains { # --------------------------------------------------------------------------- # Load blocklist function from source # --------------------------------------------------------------------------- -$blocklistPath = Join-Path $PSScriptRoot '../Modules/CIPPCore/Private/Get-CIPPSchedulerBlockedCommands.ps1' +$blocklistPath = Join-Path $PSScriptRoot '../Modules/CIPPCore/Public/Tools/Get-CIPPSchedulerBlockedCommands.ps1' if (-not (Test-Path $blocklistPath)) { Write-Error "Get-CIPPSchedulerBlockedCommands.ps1 not found at: $blocklistPath" exit 1 diff --git a/Tests/Webhooks/Get-CippAuditLogSearchResults.Tests.ps1 b/Tests/Webhooks/Get-CippAuditLogSearchResults.Tests.ps1 new file mode 100644 index 0000000000000..e3cb47c385f90 --- /dev/null +++ b/Tests/Webhooks/Get-CippAuditLogSearchResults.Tests.ps1 @@ -0,0 +1,81 @@ +# Pester tests for Get-CippAuditLogSearchResults. +# +# Covers the request shape and that every record comes back. Order is explicitly not part +# of the contract - see the last test. + +BeforeAll { + $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath)) + $FunctionPath = Join-Path $RepoRoot 'Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearchResults.ps1' + + function New-GraphGetRequest { + param($Uri, $tenantid, $AsApp, $CountOnly, [switch]$Stream, $ComplexFilter, $NoPagination) + } + + . $FunctionPath +} + +Describe 'Get-CippAuditLogSearchResults' { + + BeforeEach { + $script:CapturedUri = $null + $script:CapturedTenant = $null + $script:CapturedAsApp = $null + $script:CapturedCountOnly = $null + + Mock -CommandName New-GraphGetRequest -MockWith { + param($Uri, $tenantid, $AsApp, $CountOnly, [switch]$Stream, $ComplexFilter, $NoPagination) + $script:CapturedUri = $Uri + $script:CapturedTenant = $tenantid + $script:CapturedAsApp = $AsApp + $script:CapturedCountOnly = $CountOnly + + # deliberately unsorted so ordering assumptions surface + @( + [pscustomobject]@{ id = 'b'; createdDateTime = '2026-07-29T10:00:00Z' } + [pscustomobject]@{ id = 'a'; createdDateTime = '2026-07-29T12:00:00Z' } + [pscustomobject]@{ id = 'c'; createdDateTime = '2026-07-29T11:00:00Z' } + ) + } + } + + It 'targets the records endpoint for the given query id' { + $null = Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123' + $script:CapturedUri | Should -Match 'security/auditLog/queries/query-123/records' + } + + It 'requests the maximum page size and a count' { + $null = Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123' + $script:CapturedUri | Should -Match '\$top=999' + $script:CapturedUri | Should -Match '\$count=true' + } + + It 'runs as the application against the requested tenant' { + $null = Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123' + $script:CapturedTenant | Should -Be 'contoso.com' + $script:CapturedAsApp | Should -BeTrue + } + + It 'returns every record from graph' { + $result = @(Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123') + $result.Count | Should -Be 3 + ($result.id | Sort-Object) | Should -Be @('a', 'b', 'c') + } + + It 'accepts the query id from the pipeline by property name' { + $result = @([pscustomobject]@{ id = 'query-123' } | Get-CippAuditLogSearchResults -TenantFilter 'contoso.com') + $result.Count | Should -Be 3 + $script:CapturedUri | Should -Match 'query-123' + } + + It 'passes CountOnly through when requested' { + $null = Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123' -CountOnly + $script:CapturedCountOnly | Should -BeTrue + } + + It 'does not promise any particular record order' { + # Documented deliberately: the caller keys each record by id, so sorting here would + # mean holding the whole result set in memory for no downstream benefit. + $result = @(Get-CippAuditLogSearchResults -TenantFilter 'contoso.com' -QueryId 'query-123') + ($result | Measure-Object).Count | Should -Be 3 + } +} diff --git a/Tests/Webhooks/Push-AuditLogDownloadV2.Tests.ps1 b/Tests/Webhooks/Push-AuditLogDownloadV2.Tests.ps1 new file mode 100644 index 0000000000000..4bdf680d12e29 --- /dev/null +++ b/Tests/Webhooks/Push-AuditLogDownloadV2.Tests.ps1 @@ -0,0 +1,204 @@ +# Pester tests for Push-AuditLogDownloadV2 - the audit log V2 download stage. +# +# Pins the ledger state machine and the CacheWebhooks writes. Record order is deliberately +# not asserted; nothing downstream depends on it. + +BeforeAll { + $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath)) + $FunctionPath = Join-Path $RepoRoot 'Modules/CIPPCore/Public/Webhooks/Push-AuditLogDownloadV2.ps1' + + function Get-CippTable { param($TableName) } + function Get-CIPPAzDataTableEntity { param($TableName, $Context, $Filter, $Property, $First, $Skip, $Sort, $Count) } + # -Force must be [switch]: as a plain parameter the caller's trailing -Force binds no + # argument, and the resulting error is swallowed by the function's own catch. + function Add-CIPPAzDataTableEntity { param($TableName, $Context, $Entity, [switch]$Force, $OperationType) } + function New-GraphBulkRequest { param($Requests, $AsApp, $TenantId) } + function Get-CippAuditLogSearchResults { param($TenantFilter, $QueryId, [switch]$CountOnly) } + function Get-CippAuditLogNextAttempt { param($Attempts) } + + function New-AuditRecord { + param([string]$Id, [string]$Created = '2026-07-29T09:00:00Z') + [pscustomobject]@{ + id = $Id + createdDateTime = $Created + operation = 'Set-Mailbox' + auditData = [pscustomobject]@{ ResultStatus = 'Success' } + } + } + + . $FunctionPath +} + +Describe 'Push-AuditLogDownloadV2' { + + BeforeEach { + $script:CacheWrites = [System.Collections.Generic.List[object]]::new() + $script:LedgerWrites = [System.Collections.Generic.List[object]]::new() + $script:SearchResults = @() + $script:SearchStatus = 'succeeded' + $script:ThrowOnDownload = $false + + Mock -CommandName Get-CippTable -MockWith { + param($TableName) + @{ TableName = $TableName } + } + + # one ledger row in state Created, due now + Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { + @( + [pscustomobject]@{ + PartitionKey = 'contoso.com' + RowKey = 'window-1' + State = 'Created' + SearchId = 'search-1' + CreatedUtc = (Get-Date).ToUniversalTime().AddMinutes(-5).ToString('o') + Attempts = 0 + RetryCount = 0 + } + ) + } + + Mock -CommandName Add-CIPPAzDataTableEntity -MockWith { + param($TableName, $Context, $Entity, $Force, $OperationType) + if ($TableName -eq 'CacheWebhooks') { $script:CacheWrites.Add($Entity) } + else { $script:LedgerWrites.Add($Entity) } + } + + Mock -CommandName New-GraphBulkRequest -MockWith { + @([pscustomobject]@{ body = [pscustomobject]@{ id = 'search-1'; status = $script:SearchStatus } }) + } + + Mock -CommandName Get-CippAuditLogSearchResults -MockWith { + if ($script:ThrowOnDownload) { throw 'graph exploded' } + $script:SearchResults + } + + Mock -CommandName Get-CippAuditLogNextAttempt -MockWith { (Get-Date).ToUniversalTime().AddMinutes(10).ToString('o') } + } + + Context 'succeeded search with records' { + BeforeEach { + $script:SearchResults = @(New-AuditRecord 'rec-1'), (New-AuditRecord 'rec-2'), (New-AuditRecord 'rec-3') + } + + It 'writes one CacheWebhooks row per record' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $script:CacheWrites.Count | Should -Be 3 + } + + It 'keys each cache row by the record id and stores the record as JSON' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + ($script:CacheWrites.RowKey | Sort-Object) | Should -Be @('rec-1', 'rec-2', 'rec-3') + $first = $script:CacheWrites | Where-Object { $_.RowKey -eq 'rec-1' } + $first.PartitionKey | Should -Be 'contoso.com' + $first.SearchId | Should -Be 'search-1' + ($first.JSON | ConvertFrom-Json).id | Should -Be 'rec-1' + $first.CippProcessing | Should -BeFalse + } + + It 'advances the ledger to Downloaded with the record count' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.State | Should -Be 'Downloaded' + $ledger.RecordCount | Should -Be 3 + $ledger.Attempts | Should -Be 0 + } + + It 'reports the downloaded count in its result' { + $result = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $result.Success | Should -BeTrue + $result.Downloaded | Should -Be 3 + } + } + + Context 'succeeded search with no records' { + BeforeEach { $script:SearchResults = @() } + + It 'marks an empty window Processed rather than leaving it Downloaded' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.State | Should -Be 'Processed' + $ledger.RecordCount | Should -Be 0 + $ledger.MatchedCount | Should -Be 0 + } + + It 'writes no cache rows' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $script:CacheWrites.Count | Should -Be 0 + } + } + + Context 'single record' { + # Guards the PowerShell unrolling trap: one item must still count as a collection. + BeforeEach { $script:SearchResults = @(New-AuditRecord 'only-1') } + + It 'writes exactly one cache row and counts it as one' { + $result = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $script:CacheWrites.Count | Should -Be 1 + $result.Downloaded | Should -Be 1 + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.State | Should -Be 'Downloaded' + $ledger.RecordCount | Should -Be 1 + } + } + + Context 'download throws' { + BeforeEach { $script:ThrowOnDownload = $true } + + It 'increments Attempts and schedules a retry without dead-lettering' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.Attempts | Should -Be 1 + $ledger.State | Should -BeNullOrEmpty + $ledger.NextAttemptUtc | Should -Not -BeNullOrEmpty + $ledger.LastError | Should -Match 'graph exploded' + } + + It 'still reports success for the activity as a whole' { + $result = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $result.Success | Should -BeTrue + $result.Downloaded | Should -Be 0 + } + } + + Context 'graph reports the search failed' { + BeforeEach { $script:SearchStatus = 'failed' } + + It 're-plans the window and clears the search id' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.State | Should -Be 'Planned' + $ledger.SearchId | Should -Be '' + $ledger.Attempts | Should -Be 1 + } + } + + Context 'search still running' { + BeforeEach { $script:SearchStatus = 'running' } + + It 'leaves the window Created and records the live status' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $ledger = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' } | Select-Object -Last 1 + $ledger.State | Should -BeNullOrEmpty + $ledger.SearchStatus | Should -Be 'running' + } + + It 'writes no cache rows' { + $null = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $script:CacheWrites.Count | Should -Be 0 + } + } + + Context 'no due ledger rows' { + BeforeEach { + Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { @() } + } + + It 'returns early without polling graph' { + $result = Push-AuditLogDownloadV2 -Item @{ TenantFilter = 'contoso.com' } + $result.Success | Should -BeTrue + $result.Downloaded | Should -Be 0 + Should -Invoke New-GraphBulkRequest -Times 0 + } + } +} diff --git a/Tests/Webhooks/Push-AuditLogTenantProcessV2.Tests.ps1 b/Tests/Webhooks/Push-AuditLogTenantProcessV2.Tests.ps1 new file mode 100644 index 0000000000000..6bb87ac4c243c --- /dev/null +++ b/Tests/Webhooks/Push-AuditLogTenantProcessV2.Tests.ps1 @@ -0,0 +1,233 @@ +# Pester tests for Push-AuditLogTenantProcessV2 - the audit log V2 processing stage. +# +# Pins the ledger transitions and the rows handed to the rules engine. The cases that +# matter are split records, stored across rows X / X-part1 / X-part2 and only reassembled +# when every part is fetched in one call. + +BeforeAll { + $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath)) + $FunctionPath = Join-Path $RepoRoot 'Modules/CIPPCore/Public/Webhooks/Push-AuditLogTenantProcessV2.ps1' + + function Get-CippTable { param($TableName) } + function Get-AzDataTableEntity { param($Context, $Filter, $Property, $First) } + function Get-CIPPAzDataTableEntity { param($Context, $Filter, $Property, $First) } + function Add-CIPPAzDataTableEntity { param($Context, $Entity, [switch]$Force, $OperationType) } + function Test-CIPPAuditLogRules { param($TenantFilter, $Rows) } + + function Get-WantedFromFilter { + param([string]$Filter) + $ids = [regex]::Matches($Filter, "RowKey eq '([^']*)'") | ForEach-Object { $_.Groups[1].Value } + if ($ids) { @($ids) } else { $null } + } + + . $FunctionPath +} + +Describe 'Push-AuditLogTenantProcessV2' { + + BeforeEach { + $script:RulesRows = $null + # AllRulesRows accumulates across chunks; RulesRows holds only the last chunk. + $script:AllRulesRows = [System.Collections.Generic.List[object]]::new() + $script:SeenFilters = [System.Collections.Generic.List[string]]::new() + $script:LedgerWrites = [System.Collections.Generic.List[object]]::new() + $script:MatchedLogs = 2 + + # Physical rows; split records carry OriginalEntityId on every part. + $script:CacheRows = @( + [PSCustomObject]@{ RowKey = 'rec-1'; OriginalEntityId = $null; SearchId = 'search-1'; JSON = '{"id":"rec-1"}' } + [PSCustomObject]@{ RowKey = 'rec-2'; OriginalEntityId = $null; SearchId = 'search-1'; JSON = '{"id":"rec-2"}' } + ) + $script:LedgerRows = @( + [PSCustomObject]@{ PartitionKey = 'contoso.com'; RowKey = 'window-1'; SearchId = 'search-1'; State = 'Downloaded' } + ) + $script:RemainingAfterProcess = @() + $script:DownloadedSweepRows = @() + + # Real Get-CippTable returns only @{ Context = ... }; mirror that so splatting @Table + # behaves as it does in production. + Mock -CommandName Get-CippTable -MockWith { param($TableName) @{ Context = "ctx:$TableName" } } + + Mock -CommandName Get-AzDataTableEntity -MockWith { + param($Context, $Filter, $Property, $First) + $script:SeenFilters.Add([string]$Filter) + $rows = $script:CacheRows + if ($Filter -match "RowKey gt '([^']*)'") { $rows = @($rows | Where-Object { $_.RowKey -gt $Matches[1] }) } + $wanted = Get-WantedFromFilter -Filter $Filter + if ($wanted) { $rows = @($rows | Where-Object { $wanted -contains $_.RowKey }) } + $rows = @($rows | Sort-Object RowKey) + if ($First) { $rows = @($rows | Select-Object -First $First) } + $rows | ForEach-Object { + [PSCustomObject]@{ PartitionKey = 'contoso.com'; RowKey = $_.RowKey; OriginalEntityId = $_.OriginalEntityId } + } + } + + Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { + param($Context, $Filter, $Property, $First) + if ($Context -like '*AuditLogCoverage*') { + if ($Filter -match "State eq 'Downloaded'") { return $script:DownloadedSweepRows } + return $script:LedgerRows + } + # CacheWebhooks. A SearchId query is the "is this search drained yet" probe. + if ($Filter -match "SearchId eq") { return $script:RemainingAfterProcess } + + # Selected by RowKey (simple records) or OriginalEntityId (every part of a split + # record, regardless of which parts this batch claimed). + $wantedRow = Get-WantedFromFilter -Filter $Filter + $wantedOrig = @([regex]::Matches($Filter, "OriginalEntityId eq '([^']*)'") | ForEach-Object { $_.Groups[1].Value }) + $rows = if ($wantedRow -or $wantedOrig) { + @($script:CacheRows | Where-Object { + ($wantedRow -contains $_.RowKey) -or + ($_.OriginalEntityId -and $wantedOrig -contains $_.OriginalEntityId) + }) + } else { + @($script:CacheRows) + } + # rejoin parts sharing a logical id, exactly like the real wrapper + $rows | Group-Object { if ($_.OriginalEntityId) { $_.OriginalEntityId } else { $_.RowKey } } | ForEach-Object { + $ordered = @($_.Group | Sort-Object RowKey) + [PSCustomObject]@{ + PartitionKey = 'contoso.com' + RowKey = $_.Name + SearchId = $ordered[0].SearchId + JSON = ($ordered.JSON -join '') + } + } + } + + Mock -CommandName Add-CIPPAzDataTableEntity -MockWith { + param($Context, $Entity, [switch]$Force, $OperationType) + $script:LedgerWrites.Add($Entity) + } + + Mock -CommandName Test-CIPPAuditLogRules -MockWith { + param($TenantFilter, $Rows) + $script:RulesRows = @($Rows) + foreach ($r in @($Rows)) { $script:AllRulesRows.Add($r) } + [PSCustomObject]@{ MatchedLogs = $script:MatchedLogs } + } + } + + Context 'simple single-row records' { + + It 'passes every cached record to the rules engine' { + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1', 'rec-2') } + @($script:RulesRows).Count | Should -Be 2 + ($script:RulesRows.id | Sort-Object) | Should -Be @('rec-1', 'rec-2') + } + + It 'deserialises the cached JSON before handing it over' { + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1') } + @($script:RulesRows)[0].id | Should -Be 'rec-1' + } + + It 'returns true on success' { + Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1', 'rec-2') } | Should -BeTrue + } + + It 'marks the ledger window Processed once the search is drained' { + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1', 'rec-2') } + $processed = $script:LedgerWrites | Where-Object { $_.State -eq 'Processed' -and $_.RowKey -eq 'window-1' } + $processed | Should -Not -BeNullOrEmpty + $processed.MatchedCount | Should -Be 2 + } + + It 'leaves the window alone while cache rows for the search remain' { + $script:RemainingAfterProcess = @([PSCustomObject]@{ PartitionKey = 'contoso.com'; RowKey = 'rec-9' }) + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1') } + ($script:LedgerWrites | Where-Object { $_.RowKey -eq 'window-1' }) | Should -BeNullOrEmpty + } + } + + Context 'no rows found for the batch' { + BeforeEach { $script:CacheRows = @() } + + It 'returns false without invoking the rules engine' { + Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('gone-1') } | Should -BeFalse + Should -Invoke Test-CIPPAuditLogRules -Times 0 + } + } + + Context 'a record split across multiple rows' { + BeforeEach { + # One logical record stored across three physical rows. + $script:CacheRows = @( + [PSCustomObject]@{ RowKey = 'rec-1'; OriginalEntityId = $null; SearchId = 'search-1'; JSON = '{"id":"rec-1"}' } + [PSCustomObject]@{ RowKey = 'big'; OriginalEntityId = 'big'; SearchId = 'search-1'; JSON = '{"id":"big","pad":"AAA' } + [PSCustomObject]@{ RowKey = 'big-part1'; OriginalEntityId = 'big'; SearchId = 'search-1'; JSON = 'BBB' } + [PSCustomObject]@{ RowKey = 'big-part2'; OriginalEntityId = 'big'; SearchId = 'search-1'; JSON = 'CCC"}' } + ) + } + + It 'reassembles the split record into valid JSON' { + # Fetching one RowKey at a time leaves the reassembler with a fragment. + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1', 'big', 'big-part1', 'big-part2') } + $big = @($script:RulesRows) | Where-Object { $_.id -eq 'big' } + $big | Should -Not -BeNullOrEmpty + $big.pad | Should -Be 'AAABBBCCC' + } + + It 'yields the split record exactly once, not once per physical row' { + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1', 'big', 'big-part1', 'big-part2') } + @($script:RulesRows).Count | Should -Be 2 + @($script:RulesRows | Where-Object { $_.id -eq 'big' }).Count | Should -Be 1 + } + } + + Context 'a batch larger than one chunk' { + BeforeEach { + # 250 rows against a chunk size of 100 forces three passes. Smaller fixtures only + # execute the loop body once, hiding any off-by-one in the slice arithmetic. + $script:CacheRows = @( + 1..250 | ForEach-Object { + [PSCustomObject]@{ + RowKey = ('rec-{0:D3}' -f $_) + OriginalEntityId = $null + SearchId = 'search-1' + JSON = ('{{"id":"rec-{0:D3}"}}' -f $_) + } + } + ) + } + + It 'processes every row across all chunks exactly once' { + $ids = @(1..250 | ForEach-Object { 'rec-{0:D3}' -f $_ }) + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = $ids } + @($script:AllRulesRows).Count | Should -Be 250 + (@($script:AllRulesRows).id | Sort-Object -Unique).Count | Should -Be 250 + } + + It 'invokes the rules engine once per chunk, not once per batch' { + $ids = @(1..250 | ForEach-Object { 'rec-{0:D3}' -f $_ }) + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = $ids } + # 250 / 100 = 3 calls. This is what bounds peak memory - the whole batch is + # never resident at once. + Should -Invoke Test-CIPPAuditLogRules -Times 3 -Exactly + } + + It 'never builds a filter long enough to trip the request size limit' { + # Azure rejects ~27kb of filter with HTTP 414 and Azurite ~13kb with HTTP 431, + # and the outer catch swallows both. Stay well inside the stricter one. + $ids = @(1..250 | ForEach-Object { 'rec-{0:D3}' -f $_ }) + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = $ids } + $worst = ($script:SeenFilters | Measure-Object -Property Length -Maximum).Maximum + $worst | Should -BeLessThan 11000 + } + } + + Context 'orphaned Downloaded windows' { + BeforeEach { + $script:DownloadedSweepRows = @( + [PSCustomObject]@{ PartitionKey = 'contoso.com'; RowKey = 'orphan-1'; SearchId = 'search-orphan'; State = 'Downloaded' } + ) + } + + It 'sweeps a Downloaded window whose search has no cache rows left' { + $null = Push-AuditLogTenantProcessV2 -Item @{ TenantFilter = 'contoso.com'; RowIds = @('rec-1') } + $swept = $script:LedgerWrites | Where-Object { $_.RowKey -eq 'orphan-1' } + $swept | Should -Not -BeNullOrEmpty + $swept.State | Should -Be 'Processed' + $swept.MatchedCount | Should -Be 0 + } + } +} diff --git a/Tests/Webhooks/Test-CIPPAuditLogRules.Tests.ps1 b/Tests/Webhooks/Test-CIPPAuditLogRules.Tests.ps1 new file mode 100644 index 0000000000000..2e2f81f212583 --- /dev/null +++ b/Tests/Webhooks/Test-CIPPAuditLogRules.Tests.ps1 @@ -0,0 +1,358 @@ +# Pester tests for Test-CIPPAuditLogRules - record shaping and cache cleanup only. +# +# This function is large; these cover two seams. First, how an audit record is reshaped +# before rule matching: ExtendedProperties, DeviceProperties, parameters and +# ModifiedProperties are flattened onto the record, and rules match on those flattened +# names. Second, the post-processing cleanup that removes drained rows from CacheWebhooks. + +BeforeAll { + $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath)) + $FunctionPath = Join-Path $RepoRoot 'Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1' + + # Called as both -TableName and -tablename; binding is case-insensitive so one covers both. + function Get-CIPPTable { param($TableName) } + function Get-CIPPAzDataTableEntity { param($TableName, $Context, $Filter, $Property, $First) } + function Get-AzDataTableEntity { param($TableName, $Context, $Filter, $Property, $First) } + function Add-CIPPAzDataTableEntity { param($TableName, $Context, $Entity, [switch]$Force, $OperationType) } + function Remove-AzDataTableEntity { param($TableName, $Context, $Entity, [switch]$Force) } + function Expand-CIPPTenantGroups { param($TenantFilter) } + function Test-CIPPConditionFilter { param($Condition) } + function Invoke-CippWebhookProcessing { param($Data, $CIPPURL, $TenantFilter, $AlertComment) } + function Get-CIPPGeoIPLocationBatch { param($IPs) } + function Write-LogMessage { param($API, $tenant, $message, $sev, $LogData) } + function Get-CippException { param($Exception) [pscustomobject]@{ NormalizedError = "$Exception" } } + function New-CIPPDbRequest { param($TenantFilter, $Type, $Endpoint) } + function New-GraphBulkRequest { param($Requests, $AsApp, $TenantId) } + function New-GraphGetRequest { param($uri, $tenantid, $AsApp, [switch]$Stream, $ComplexFilter, $NoPagination) } + function Add-CIPPApplicationPermission { param($RequiredResourceAccess, $ApplicationId, $TenantFilter) } + + # Lookup blob in the 'hashtable' cache format, so no Graph refresh is attempted. + function New-LookupRow { + param([string]$RowKey) + [pscustomobject]@{ + PartitionKey = 'contoso.com' + RowKey = $RowKey + Format = 'hashtable' + Data = (@{} | ConvertTo-Json -Compress) + } + } + + function New-AuditRow { + param([string]$Id = 'rec-1', [string]$Operation = 'Set-Mailbox') + [pscustomobject]@{ + id = $Id + createdDateTime = '2026-07-29T09:00:00Z' + operation = $Operation + auditData = [pscustomobject]@{ + Operation = $Operation + ResultStatus = 'Success' + clientip = '203.0.113.10' + ExtendedProperties = @( + [pscustomobject]@{ Name = 'ExtProp'; Value = 'ExtValue' } + ) + DeviceProperties = @( + [pscustomobject]@{ Name = 'DevProp'; Value = 'DevValue' } + ) + parameters = @( + [pscustomobject]@{ Name = 'ParamProp'; Value = 'ParamValue' } + ) + ModifiedProperties = @( + [pscustomobject]@{ Name = 'ModProp'; NewValue = 'NewVal'; OldValue = 'OldVal' } + ) + } + } + } + + . $FunctionPath +} + +Describe 'Test-CIPPAuditLogRules record shaping' { + + BeforeEach { + Mock -CommandName Get-CIPPTable -MockWith { + param($TableName) + @{ TableName = $TableName } + } + + Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { + param($TableName, $Context, $Filter, $Property, $First) + switch ($TableName) { + 'WebhookRules' { + [pscustomobject]@{ + PartitionKey = 'WebhookRules' + RowKey = 'rule-1' + Tenants = (@('AllTenants') | ConvertTo-Json -Compress) + excludedTenants = $null + Conditions = (@( + @{ + Property = @{ label = 'Operation' } + Operator = @{ label = 'eq' } + Input = @{ value = 'Set-Mailbox' } + } + ) | ConvertTo-Json -Compress -Depth 5) + Actions = (@('generatemail') | ConvertTo-Json -Compress) + Type = 'Audit' + AlertComment = 'test comment' + CustomSubject = '' + } + } + 'cacheauditloglookups' { + @( + New-LookupRow 'users' + New-LookupRow 'groups' + New-LookupRow 'devices' + New-LookupRow 'servicePrincipals' + ) + } + 'Config' { [pscustomobject]@{ Value = 'cipp.contoso.com' } } + default { @() } + } + } + + # Physical CacheWebhooks rows used by the post-processing cleanup. + $script:PhysicalCacheRows = @( + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'rec-1'; OriginalEntityId = $null } + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'other'; OriginalEntityId = $null } + ) + $script:RemovedRows = [System.Collections.Generic.List[object]]::new() + $script:CleanupProperty = $null + $script:CleanupFilter = $null + + # Applies the filter rather than returning everything: the cleanup relies on the + # service to select rows, so a mock that ignores the predicate would prove nothing. + Mock -CommandName Get-AzDataTableEntity -MockWith { + param($TableName, $Context, $Filter, $Property, $First) + $script:CleanupProperty = $Property + $script:CleanupFilter = $Filter + + $wantRow = @([regex]::Matches($Filter, "RowKey eq '([^']*)'") | ForEach-Object { $_.Groups[1].Value }) + $wantOrig = @([regex]::Matches($Filter, "OriginalEntityId eq '([^']*)'") | ForEach-Object { $_.Groups[1].Value }) + + @($script:PhysicalCacheRows | Where-Object { + ($wantRow -contains $_.RowKey) -or + ($_.OriginalEntityId -and $wantOrig -contains $_.OriginalEntityId) + }) + } + + Mock -CommandName Remove-AzDataTableEntity -MockWith { + param($TableName, $Context, $Entity, [switch]$Force) + foreach ($e in @($Entity)) { $script:RemovedRows.Add($e) } + } + + Mock -CommandName Expand-CIPPTenantGroups -MockWith { [pscustomobject]@{ value = @('AllTenants') } } + # Always-true predicate; rule matching is not what these tests cover. + Mock -CommandName Test-CIPPConditionFilter -MockWith { '$_.Operation -eq ''Set-Mailbox''' } + Mock -CommandName Invoke-CippWebhookProcessing -MockWith { } + Mock -CommandName Add-CIPPAzDataTableEntity -MockWith { } + # Remove-AzDataTableEntity is mocked above with a capturing body; a second mock here + # would win and silently capture nothing. + Mock -CommandName Get-CIPPGeoIPLocationBatch -MockWith { @{} } + Mock -CommandName Write-LogMessage -MockWith { } + Mock -CommandName New-CIPPDbRequest -MockWith { @() } + Mock -CommandName New-GraphBulkRequest -MockWith { @() } + Mock -CommandName New-GraphGetRequest -MockWith { @() } + } + + Context 'a record that matches a rule' { + + It 'returns the matched record' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $result.MatchedLogs | Should -Be 1 + @($result.DataToProcess).Count | Should -Be 1 + } + + It 'flattens ExtendedProperties onto the record' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.ExtProp | Should -Be 'ExtValue' + } + + It 'flattens DeviceProperties onto the record' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.DevProp | Should -Be 'DevValue' + } + + It 'flattens parameters onto the record' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.ParamProp | Should -Be 'ParamValue' + } + + It 'flattens ModifiedProperties new and old values onto the record' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.ModProp | Should -Be 'NewVal' + $data.Previous_Value_ModProp | Should -Be 'OldVal' + } + + It 'keeps ModifiedProperties available for lazy rendering' { + # New-CIPPAlertTemplate serialises this at render time once the eager + # CIPPModifiedProperties copy is gone, so the raw collection must survive. + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.ModifiedProperties | Should -Not -BeNullOrEmpty + @($data.ModifiedProperties)[0].NewValue | Should -Be 'NewVal' + } + + It 'drops the raw sub-objects that were flattened' { + # The output projection excludes these three once flattened, so the record does + # not carry both representations. ModifiedProperties is not excluded. + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.ExtendedProperties | Should -BeNullOrEmpty + $data.DeviceProperties | Should -BeNullOrEmpty + $data.parameters | Should -BeNullOrEmpty + } + + It 'must keep CIPPParameters because its source is dropped' { + # New-CIPPAlertTemplate renders this. It cannot become lazy like + # CIPPModifiedProperties - `parameters` is excluded by the projection above, so + # removing the eager copy would silently blank the parameters table in alerts. + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.CIPPParameters | Should -Not -BeNullOrEmpty + (@($data.CIPPParameters | ConvertFrom-Json)[0]).Name | Should -Be 'ParamProp' + } + + It 'sets the action and clause metadata used by alerting' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $data = @($result.DataToProcess)[0] + $data.CIPPAction | Should -Not -BeNullOrEmpty + $data.CIPPClause | Should -Not -BeNullOrEmpty + $data.CIPPAlertComment | Should -Be 'test comment' + } + + It 'dispatches the matched record to webhook processing' { + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + Should -Invoke Invoke-CippWebhookProcessing -Times 1 + } + } + + Context 'a record carrying an ignore-listed extended property' { + BeforeEach { + # 'Consent:Set' is on $ExtendedPropertiesIgnoreList inside the function. + $script:IgnoreRow = New-AuditRow + $script:IgnoreRow.auditData.ExtendedProperties = @( + [pscustomobject]@{ Name = 'IgnoredProp'; Value = 'Consent:Set' } + [pscustomobject]@{ Name = 'KeptProp'; Value = 'KeptValue' } + ) + } + + It 'still processes the record instead of dropping it' { + # Previously the ignore-list `continue` sat inside a ForEach-Object, where it + # unwound to the enclosing foreach and abandoned the whole record - so a record + # with an ignored property never reached $ProcessedData at all. + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @($script:IgnoreRow) + $result.MatchedLogs | Should -Be 1 + } + + It 'skips only the ignored property and keeps the rest' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @($script:IgnoreRow) + $data = @($result.DataToProcess)[0] + $data.KeptProp | Should -Be 'KeptValue' + $data.IgnoredProp | Should -BeNullOrEmpty + } + + It 'still flattens the later sub-objects that used to be skipped' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @($script:IgnoreRow) + $data = @($result.DataToProcess)[0] + $data.ModProp | Should -Be 'NewVal' + $data.ParamProp | Should -Be 'ParamValue' + } + } + + Context 'cache cleanup after processing' { + + It 'reads only key columns, not the JSON payloads' { + # This read exists solely to pick RowKeys to delete. + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $script:CleanupProperty | Should -Contain 'RowKey' + $script:CleanupProperty | Should -Not -Contain 'JSON' + } + + It 'asks only for the rows being deleted, never the whole partition' { + # The caller processes in chunks, so this runs once per chunk. A bare + # "PartitionKey eq X" scan here would be repeated for every chunk of every batch. + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow -Id 'rec-1') + $script:CleanupFilter | Should -Match "PartitionKey eq 'contoso.com'" + $script:CleanupFilter | Should -Match "RowKey eq 'rec-1'" + $script:CleanupFilter | Should -Match "OriginalEntityId eq 'rec-1'" + } + + It 'removes the processed record and leaves unrelated rows alone' { + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow -Id 'rec-1') + @($script:RemovedRows).RowKey | Should -Contain 'rec-1' + @($script:RemovedRows).RowKey | Should -Not -Contain 'other' + } + + It 'removes every physical part of a split record, not just the first' { + # Matching on the logical id but deleting the physical rows is what stops a split + # record's tail parts being orphaned in the cache. + $script:PhysicalCacheRows = @( + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'big'; OriginalEntityId = 'big' } + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'big-part1'; OriginalEntityId = 'big' } + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'big-part2'; OriginalEntityId = 'big' } + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'other'; OriginalEntityId = $null } + ) + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow -Id 'big') + + $removed = @($script:RemovedRows).RowKey + $removed | Should -Contain 'big' + $removed | Should -Contain 'big-part1' + $removed | Should -Contain 'big-part2' + $removed | Should -Not -Contain 'other' + } + + It 'flushes deletes in batches rather than one call per record' { + $rows = @(1..5 | ForEach-Object { New-AuditRow -Id "rec-$_" }) + $script:PhysicalCacheRows = @( + 1..5 | ForEach-Object { [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = "rec-$_"; OriginalEntityId = $null } } + ) + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows $rows + + # Under the flush size, so one flush after the loop plus the end-of-run sweep. + Should -Invoke Remove-AzDataTableEntity -Times 2 -Exactly + @($script:RemovedRows).RowKey | Should -Contain 'rec-1' + @($script:RemovedRows).RowKey | Should -Contain 'rec-5' + } + + It 'flushes mid-loop so a poison batch still makes forward progress' { + # The reason deletes are not deferred to the end: if a record kills the worker, + # everything already flushed is gone from the cache, so the retry starts further + # in and the run converges instead of looping on the same rows forever. + $rows = @(1..60 | ForEach-Object { New-AuditRow -Id "rec-$_" }) + $script:PhysicalCacheRows = @( + 1..60 | ForEach-Object { [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = "rec-$_"; OriginalEntityId = $null } } + ) + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows $rows + + # 60 records at a flush size of 25: two mid-loop flushes, a remainder flush, + # and the sweep - not 60 individual calls. + Should -Invoke Remove-AzDataTableEntity -Times 4 -Exactly + @($script:RemovedRows).RowKey.Count | Should -Be 120 # 60 flushed + 60 swept + } + + It 'never removes a cached row belonging to another record' { + # Deletion runs in two stages - a per-record delete by id, then this sweep. + # Neither may touch an unrelated row. + $script:PhysicalCacheRows = @( + [pscustomobject]@{ PartitionKey = 'contoso.com'; RowKey = 'unrelated'; OriginalEntityId = $null } + ) + $null = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow -Id 'rec-1') + @($script:RemovedRows).RowKey | Should -Not -Contain 'unrelated' + } + } + + Context 'a record that matches no rule' { + BeforeEach { + Mock -CommandName Test-CIPPConditionFilter -MockWith { '$_.Operation -eq ''Never-Matches''' } + } + + It 'matches nothing and dispatches nothing' { + $result = Test-CIPPAuditLogRules -TenantFilter 'contoso.com' -Rows @(New-AuditRow) + $result.MatchedLogs | Should -Be 0 + Should -Invoke Invoke-CippWebhookProcessing -Times 0 + } + } +} diff --git a/version_latest.txt b/version_latest.txt index 6ab00fa0dcd21..a04a9079b9283 100644 --- a/version_latest.txt +++ b/version_latest.txt @@ -1 +1 @@ -10.7.3 \ No newline at end of file +10.7.4 \ No newline at end of file