diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5971015..c682a15 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ defaults: jobs: build: name: ๐Ÿ‘ท Build - runs-on: ubuntu-24.04 + runs-on: windows-2025 outputs: nupkg_artifact_id: ${{ steps.upload_nupkg.outputs.artifact-id }} steps: @@ -51,18 +51,18 @@ jobs: } - name: ๐Ÿงช Test run: | - $config = New-PesterConfiguration - $config.run.throw = $true - $config.output.Verbosity = 'Detailed' - $config.TestResult.Enabled = $true - $config.TestResult.OutputFormat = 'NUnitXml' - Invoke-Pester -Configuration $config - # - name: ๐Ÿ“ˆ Report Test Results - # uses: dorny/test-reporter@v2 - # with: - # name: Pester Tests - # path: testResults.xml - # reporter: dotnet-nunit + try {Invoke-Build Pester} + catch { + "::error file=$PSScriptRoot/build.ps1::Build failed with error: $($_.Exception.Message) `n $(Get-Error | Out-String)" -replace "`n","%0A" + throw + } + - name: ๐Ÿงช Test - WinPS + run: | + try {Invoke-Build Pester-WinPS} + catch { + "::error file=$PSScriptRoot/build.ps1::Build failed with error: $($_.Exception.Message) `n $(Get-Error | Out-String)" -replace "`n","%0A" + throw + } - name: ๐Ÿ“ค Upload Module Zip uses: actions/upload-artifact@v7 with: diff --git a/ExcelFast.build.ps1 b/ExcelFast.build.ps1 index 4eb9792..19c31b6 100644 --- a/ExcelFast.build.ps1 +++ b/ExcelFast.build.ps1 @@ -25,6 +25,11 @@ param( [switch]$SkipPS51 ) + +if ($ENV:CI -or $ENV:GITHUB_ACTIONS) { + +} + Set-BuildHeader { param($Path) "๐Ÿ‘ท $Path $(Get-BuildSynopsis $Task)" @@ -263,7 +268,14 @@ Task Docs Build, { Task Pester { #Run in a separate job so as not to lock the assemblies - Start-Job -ScriptBlock { Invoke-Pester } | Receive-Job -Wait -AutoRemoveJob + Start-Job -ScriptBlock { + $config = New-PesterConfiguration + $config.run.throw = $true + $config.output.Verbosity = 'Detailed' + $config.TestResult.Enabled = $true + $config.TestResult.OutputFormat = 'NUnitXml' + Invoke-Pester -Configuration $config + } | Receive-Job -Wait -AutoRemoveJob } Task Pester-WinPS { @@ -277,10 +289,15 @@ Task Pester-WinPS { Write-Host -ForegroundColor Cyan 'Pester not found. Installing Pester...' Install-Module Pester -MinimumVersion 5.0 -Force -Scope CurrentUser -ErrorAction Stop } - Invoke-Pester + $config = New-PesterConfiguration + $config.run.throw = $true + $config.output.Verbosity = 'Detailed' + $config.TestResult.Enabled = $true + $config.TestResult.OutputFormat = 'NUnitXml' + Invoke-Pester -Configuration $config } } Task CompileAll CompilePS7, CompilePS51 Task Test Pester, Pester-WinPS -Task . Clean, Build, Docs, Test \ No newline at end of file +Task . Clean, Build, Docs \ No newline at end of file diff --git a/Source/PowerShell/Cmdlets/ExportCommand.cs b/Source/PowerShell/Cmdlets/ExportCommand.cs index 30af59d..114bc50 100644 --- a/Source/PowerShell/Cmdlets/ExportCommand.cs +++ b/Source/PowerShell/Cmdlets/ExportCommand.cs @@ -2,11 +2,11 @@ namespace ExcelFast.PowerShell.Cmdlets; using System; using System.Threading.Channels; -using System.Collections.Concurrent; using ExcelFast.Extensions; -using MiniExcelLibs; +using MiniExcelLib; +using MiniExcelLib.OpenXml; using FilePath = Path; @@ -57,19 +57,19 @@ public class ExportCommand : TaskCmdlet private int rowsExported; -#if NET472 - private readonly BlockingCollection> exportQueue; -#else + // #if NET472 + // private readonly BlockingCollection> exportQueue; + // #else private readonly Channel> exportQueue; -#endif + // #endif public ExportCommand() { -#if NET472 - exportQueue = []; -#else + // #if NET472 + // exportQueue = []; + // #else exportQueue = Channel.CreateBounded>(new BoundedChannelOptions(InputQueueSize)); -#endif + // #endif } protected override async Task Begin() @@ -149,11 +149,11 @@ protected override async Task End() try { -#if NET472 - exportQueue.CompleteAdding(); -#else + // #if NET472 + // exportQueue.CompleteAdding(); + // #else exportQueue.Writer.TryComplete(); -#endif + // #endif Debug("Waiting for export task to complete."); @@ -193,7 +193,16 @@ private async Task ProcessInputObjects() continue; } - Dictionary row = inputObject.ToFlatDictionary(); + await Post(() => + { + var row = inputObject.ToFlatDictionary(); + // #if NET472 + // exportQueue.Add(row, PipelineStopToken); + // #else + exportQueue.Writer.TryWrite(row); + // #endif + }); + try { rowsExported++; @@ -202,11 +211,6 @@ private async Task ProcessInputObjects() continue; } -#if NET472 - exportQueue.Add(row, PipelineStopToken); -#else - await exportQueue.Writer.WriteAsync(row, PipelineStopToken); -#endif StartExportTaskIfNeeded(); } catch (OperationCanceledException) @@ -224,19 +228,20 @@ private void StartExportTaskIfNeeded() return; } -#if NET472 - var queue = exportQueue.GetConsumingEnumerable(PipelineStopToken); -#else + // #if NET472 + // var queue = exportQueue.GetConsumingEnumerable(PipelineStopToken); + // #else var queue = exportQueue.Reader.ReadAllAsync(PipelineStopToken); -#endif + // #endif Debug("Starting MiniExcel export task."); - exportTask = Task.Run(async () => await MiniExcel.SaveAsAsync( + var exporter = MiniExcel.Exporters.GetOpenXmlExporter(); + + exportTask = Task.Run(async () => await exporter.ExportAsync( Destination, queue, sheetName: SheetName, - excelType: ExcelType.XLSX, overwriteFile: Force.IsPresent, cancellationToken: PipelineStopToken ), PipelineStopToken); @@ -245,11 +250,11 @@ private void StartExportTaskIfNeeded() protected override async Task Clean() { Debug("Stopping export process due to pipeline stop request."); -#if NET472 - exportQueue.CompleteAdding(); -#else + // #if NET472 + // exportQueue.CompleteAdding(); + // #else exportQueue.Writer.TryComplete(); -#endif + // #endif if (exportTask is null) { @@ -272,4 +277,4 @@ private async void AssertExportTaskNotFaulted() { if (exportTask?.IsFaulted == true) await exportTask; // This will re-throw the exception from the export task to be handled by the cmdlet's error handling } -} +} \ No newline at end of file diff --git a/Source/PowerShell/Cmdlets/ImportCommand.cs b/Source/PowerShell/Cmdlets/ImportCommand.cs index ae18bb5..1ccf749 100644 --- a/Source/PowerShell/Cmdlets/ImportCommand.cs +++ b/Source/PowerShell/Cmdlets/ImportCommand.cs @@ -10,9 +10,10 @@ namespace ExcelFast.PowerShell.Cmdlets; using DocumentFormat.OpenXml.Drawing.Diagrams; -using MiniExcelLibs; +using MiniExcelLib; using FilePath = Path; +using MiniExcelLib.OpenXml; [Cmdlet(VerbsData.Import, CmdletDefaultName)] [Alias("iwb")] @@ -162,10 +163,11 @@ internal void ImportWorkbook(string workbookPath) IEnumerable sheetNamesToImport = []; - IReadOnlyCollection availableSheetNames = MiniExcel.GetSheetNames(providerPath).ToArray(); - if (SheetName == null || SheetName.Length == 0) - { - Debug($"No sheet name provided. Importing the first sheet from '{providerPath}'."); + var importer = MiniExcel.Importers.GetOpenXmlImporter(); + IReadOnlyCollection availableSheetNames = importer.GetSheetNames(providerPath).ToArray(); + if (SheetName == null || SheetName.Length == 0) + { + Debug($"No sheet name provided. Importing the first sheet from '{providerPath}'."); sheetNamesToImport = [availableSheetNames.FirstOrDefault() ?? "Sheet1"]; } else @@ -265,39 +267,40 @@ private void ImportSheetData(string providerPath, string currentSheetName) { IEnumerable rows; - ICollection columns = MiniExcel.GetColumns( - providerPath, - !NoHeaders.IsPresent, - currentSheetName, - startCell: StartCell - ); + var importer = MiniExcel.Importers.GetOpenXmlImporter(); + ICollection columns = importer.GetColumnNames( + providerPath, + hasHeaderRow: !NoHeaders.IsPresent, + sheetName: currentSheetName, + startCell: StartCell + ).ToArray(); - bool duplicateColumnSet = !columnSets.Add(columns); + bool duplicateColumnSet = !columnSets.Add(columns); - if (duplicateColumnSet || !columnSets.Any(c => c.SequenceEqual(columns))) - { - Warning($"Sheet '{currentSheetName}' in '{providerPath}' has different columns than previously imported sheets. The resultant object output may be different and not displayed correctly."); - } + if (duplicateColumnSet || !columnSets.Any(c => c.SequenceEqual(columns))) + { + Warning($"Sheet '{currentSheetName}' in '{providerPath}' has different columns than previously imported sheets. The resultant object output may be different and not displayed correctly."); + } - rows = string.IsNullOrEmpty(EndCell) - ? MiniExcel.Query( - providerPath, - useHeaderRow: !NoHeaders.IsPresent, - sheetName: currentSheetName, - startCell: StartCell - ) - : MiniExcel.QueryRange( - providerPath, - !NoHeaders.IsPresent, - currentSheetName, - startCell: StartCell, - endCell: EndCell - ); - - if (Raw.IsPresent) - { - // Return the raw enumerable as-is so the consumer can stream/transform using their preferred method. - WriteObject(rows, false); + rows = string.IsNullOrEmpty(EndCell) + ? importer.Query( + providerPath, + hasHeaderRow: !NoHeaders.IsPresent, + sheetName: currentSheetName, + startCell: StartCell + ) + : importer.QueryRange( + providerPath, + hasHeaderRow: !NoHeaders.IsPresent, + sheetName: currentSheetName, + startCell: StartCell, + endCell: EndCell + ); + + if (Raw.IsPresent) + { + // Return the raw enumerable as-is so the consumer can stream/transform using their preferred method. + WriteObject(rows, false); return; } @@ -327,8 +330,12 @@ private void ImportSheetData(string providerPath, string currentSheetName) static string GetWorkbookPath(IXLWorkbook workbook) { - string path = workbook.ToString(); - path = Regex.Replace(path, @"^XLWorkbook\((.*)\)$", "$1"); - return path; + string? path = workbook.ToString(); + if (string.IsNullOrEmpty(path)) + { + throw new ArgumentException("The provided workbook does not have a valid path. Ensure the workbook is saved and has a valid file path."); + } + path = Regex.Replace(path, @"^XLWorkbook\((.*)\)$", "$1"); + return path; } } \ No newline at end of file diff --git a/Source/PowerShell/Cmdlets/TaskCmdlet.cs b/Source/PowerShell/Cmdlets/TaskCmdlet.cs index 4137793..84141bc 100644 --- a/Source/PowerShell/Cmdlets/TaskCmdlet.cs +++ b/Source/PowerShell/Cmdlets/TaskCmdlet.cs @@ -2,6 +2,7 @@ namespace System.Management.Automation; using System.Collections; using System.Collections.Concurrent; +using System.Threading; public abstract class TaskCmdlet : BetterPSCmdlet, IDisposable { @@ -11,6 +12,12 @@ public abstract class TaskCmdlet : BetterPSCmdlet, IDisposable protected virtual Task End() => Task.CompletedTask; protected virtual Task Clean() => Task.CompletedTask; + // Hooks we can run against the main thread before we start the async pipeline. This can be useful to run things that need to be on the main thread before we start processing output, like setting up the output collection, etc. + protected virtual void BeginSync() { } + protected virtual void ProcessSync() { } + protected virtual void EndSync() { } + protected virtual void CleanSync() { } + /// /// Queues output for the cmdlet pipeline. This is the primary method used to send data through the async pipeline, /// and the other output helpers build on top of it. @@ -24,6 +31,30 @@ internal void AddOutput(object item, bool raw = false, CancellationToken? cancel _output.Add(new(item, raw), cancelToken ?? PipelineStopToken); } + /// + /// Queues an action to be executed on the main thread of the cmdlet. This is useful for evaluating script properties, etc. that need to be run on the main thread to avoid marshalling issues. Your function can return a result and it will be brought back to the calling thread, but it must be serializable across threads (i.e. no PSObjects, etc.) + /// + internal async Task Exec(Func action) + { + TaskCompletionSource response = new(); + AddOutput(new MainAction(() => action(), response)); + return (T?)await response.Task ?? default!; + } + + /// + /// Queues an action to be executed on the main thread of the cmdlet. This is useful for evaluating script properties, etc. that need to be run on the main thread to avoid marshalling issues. + /// + internal async Task Post(Action action) + { + TaskCompletionSource response = new(); + AddOutput(new MainAction(() => + { + action(); + return null; + }, response)); + await response.Task; + } + /// /// Buffers output from asynchronous pipeline steps on separate threads before writing it to the pipeline. /// @@ -33,9 +64,9 @@ internal void AddOutput(object item, bool raw = false, CancellationToken? cancel private BlockingCollection? _output; // Override the pscmdlet entrypoints to execute our async methods - protected sealed override void BeginProcessing() => ExecuteAsyncPipelineStep(Begin); - protected sealed override void ProcessRecord() => ExecuteAsyncPipelineStep(Process); - protected sealed override void EndProcessing() => ExecuteAsyncPipelineStep(End); + protected sealed override void BeginProcessing() => ExecuteAsyncPipelineStep(Begin, BeginSync); + protected sealed override void ProcessRecord() => ExecuteAsyncPipelineStep(Process, ProcessSync); + protected sealed override void EndProcessing() => ExecuteAsyncPipelineStep(End, EndSync); // PowerShell 7.6 introduces a builtin PipelineStopToken, for older versions we // implement our own cancellation stop trigger and its cleanup. Once 7.6 is the @@ -78,19 +109,20 @@ protected virtual void Dispose(bool disposing) } #endif - int methodCounter = 1; /// /// Executes an asynchronous cmdlet step and routes its output and errors through the PowerShell pipeline. /// - private void ExecuteAsyncPipelineStep(Func cmdletMethod) + private void ExecuteAsyncPipelineStep(Func cmdletMethod, Action? syncMethod = null) { _output = []; + // Run our synchronous pre-step hook to allow the derived class to set up any necessary state before we initialize the output collection and start processing output + syncMethod?.Invoke(); + var task = Task.Run(async () => { try { await cmdletMethod(); - methodCounter++; } catch (Exception ex) { @@ -156,6 +188,26 @@ private void ProcessOutput(OutputItem inputObject) case ProgressRecord progressRecord: base.WriteProgress(progressRecord); break; + case MainAction mainAction: + try + { + object? result = mainAction.Action(); + mainAction.Response?.TrySetResult(result); + } + catch (Exception ex) + { + if (mainAction.Response == null) throw; + mainAction.Response.TrySetException(ex); + } + finally + { + // Ensure we don't have any deadlocks by waiting on the main thread for a response that will never come because of an exception, etc. + if (!mainAction.Response?.Task?.IsCompleted ?? false) + { + mainAction.Response?.TrySetCanceled(); + } + } + break; default: base.WriteObject(item); break; @@ -275,8 +327,6 @@ private void ProcessOutput(OutputItem inputObject) protected new void WriteInformation(InformationRecord informationRecord) => AddOutput(informationRecord); protected new void WriteInformation(object messageData, string[] tags) => AddOutput(new TaggedInformationInfo(messageData, tags)); - protected new bool ShouldProcess(string target, string action = "") - => ShouldProcessAsync(target, action).GetAwaiter().GetResult(); protected async Task ShouldProcessAsync(string target, string action = "") { @@ -403,4 +453,6 @@ internal record ShouldProcessCustomPrompt( string confirmMessage, TaskCompletionSource Response ); -internal record TerminatingError(ErrorRecord Error); \ No newline at end of file +internal record TerminatingError(ErrorRecord Error); +/** Send this via the pipeline to execute an action on the main thread **/ +internal record MainAction(Func Action, TaskCompletionSource? Response = null); diff --git a/Source/PowerShell/Extensions/PSObject.cs b/Source/PowerShell/Extensions/PSObject.cs index 1a477a6..c65624f 100644 --- a/Source/PowerShell/Extensions/PSObject.cs +++ b/Source/PowerShell/Extensions/PSObject.cs @@ -7,86 +7,51 @@ public static class PSObjectExtensions { extension(PSObject psobject) { - public Dictionary ToFlatDictionary() - { - Dictionary dictionary = new Dictionary(); + public Dictionary ToFlatDictionary() + { + Dictionary dictionary = new Dictionary(); - object baseObject = psobject.BaseObject; + object baseObject = psobject.BaseObject; - // Handle IDictionary (Hashtable, OrderedDictionary, generic Dictionary, etc.) - if (baseObject is IDictionary dict) - { - foreach (DictionaryEntry entry in dict) - { - string key = entry.Key?.ToString() ?? string.Empty; - if (!string.IsNullOrWhiteSpace(key)) - dictionary[key] = entry.Value ?? string.Empty; - } - return dictionary; - } + // Handle IDictionary (Hashtable, OrderedDictionary, generic Dictionary, etc.) + if (baseObject is IDictionary dict) + { + foreach (DictionaryEntry entry in dict) + { + string key = entry.Key?.ToString() ?? string.Empty; + if (!string.IsNullOrWhiteSpace(key)) + dictionary[key] = entry.Value ?? string.Empty; + } + return dictionary; + } - // Handle arrays/lists (produce indexed Column1, Column2, ... columns) - if (baseObject is not string && baseObject is IList list) - { - for (int i = 0; i < list.Count; i++) - dictionary[$"Column{i + 1}"] = list[i]?.ToString() ?? string.Empty; - return dictionary; - } + // Handle arrays/lists (produce indexed Column1, Column2, ... columns) + if (baseObject is not string && baseObject is IList list) + { + for (int i = 0; i < list.Count; i++) + dictionary[$"Column{i + 1}"] = list[i]?.ToString() ?? string.Empty; + return dictionary; + } // Handle scalars (primitives, strings, DateTime, etc.) - map to "Value" column Type baseType = baseObject.GetType(); if (baseType.IsPrimitive || baseObject is string || baseObject is decimal || baseObject is DateTime) - { - dictionary["Value"] = baseObject; - return dictionary; - } - - // Standard PSObject properties (PSCustomObject, .NET objects, etc.) - foreach (PSPropertyInfo property in psobject.Properties) - { - if (string.IsNullOrWhiteSpace(property.Name)) - { - continue; - } - - string value; - object? propertyValue = GetPropertyValue(psobject, property); - value = propertyValue is not string && propertyValue is not IDictionary && propertyValue is IEnumerable enumerable - ? string.Join(", ", enumerable.Select(x => x?.ToString() ?? string.Empty)) - : propertyValue?.ToString() ?? string.Empty; - - dictionary[property.Name] = value; - } - - return dictionary; - } - - private static object? GetPropertyValue(PSObject targetObject, PSPropertyInfo property) - { - if (property is not (PSScriptProperty or PSCodeProperty)) { - return property.Value; + dictionary["Value"] = baseObject; + return dictionary; } - Runspace? originalRunspace = Runspace.DefaultRunspace; - Runspace targetRunspace = originalRunspace ?? RunspaceFactory.CreateRunspace(); - if (targetRunspace.RunspaceStateInfo.State == RunspaceState.BeforeOpen) + // Standard PSObject properties (PSCustomObject, .NET objects, etc.) + foreach (PSPropertyInfo property in psobject.Properties) { - targetRunspace.Open(); - } - try - { - Runspace.DefaultRunspace = targetRunspace; - return property.Value; - } - finally - { - Runspace.DefaultRunspace = originalRunspace; - if (originalRunspace is null) + if (string.IsNullOrWhiteSpace(property.Name)) { - targetRunspace.Dispose(); + continue; } + dictionary[property.Name] = LanguagePrimitives.ConvertTo(property.Value); } + + return dictionary; } } } \ No newline at end of file diff --git a/Source/PowerShell/PowerShell.csproj b/Source/PowerShell/PowerShell.csproj index e8d0048..a1a4345 100644 --- a/Source/PowerShell/PowerShell.csproj +++ b/Source/PowerShell/PowerShell.csproj @@ -24,7 +24,7 @@ - +