Traitement par lots des PDF en C# : Automatiser les flux de documents à grande échelle

Install-Package IronPdf

dotnet add package IronPdf

using IronPdf;
using System;
using System.IO;
using System.Linq;
using System.Collections.Generic;

string inputFolder = "documents/";
string outputFolder = "merged/";

Directory.CreateDirectory(outputFolder);

// Group PDFs by prefix (e.g., "invoice-2026-01-*.pdf" -> one merged file)
var pdfFiles = Directory.GetFiles(inputFolder, "*.pdf");
var groups = pdfFiles
    .GroupBy(f => Path.GetFileName(f).Split('-').Take(3).Aggregate((a, b) => $"{a}-{b}"))
    .Where(g => g.Count() > 1);

Console.WriteLine($"Found {groups.Count()} groups to merge");

foreach (var group in groups)
{
    string groupName = group.Key;
    var filesToMerge = group.OrderBy(f => f).ToList();

    Console.WriteLine($"Merging {filesToMerge.Count} files into {groupName}.pdf");

    try
    {
        // Load all PDFs for this group
        var pdfDocs = new List<PdfDocument>();
        foreach (string filePath in filesToMerge)
        {
            pdfDocs.Add(PdfDocument.FromFile(filePath));
        }

        // Merge all documents
        using var merged = PdfDocument.Merge(pdfDocs);
        merged.SaveAs(Path.Combine(outputFolder, $"{groupName}-merged.pdf"));

        // Dispose source documents
        foreach (var doc in pdfDocs)
        {
            doc.Dispose();
        }

        Console.WriteLine($"  [OK] Created {groupName}-merged.pdf ({merged.PageCount} pages)");
    }
    catch (Exception ex)
    {
        Console.WriteLine($"  [ERROR] {groupName}: {ex.Message}");
    }
}

Console.WriteLine("\nMerge complete");

Imports IronPdf
Imports System
Imports System.IO
Imports System.Linq
Imports System.Collections.Generic

Module Program
    Sub Main()
        Dim inputFolder As String = "documents/"
        Dim outputFolder As String = "merged/"

        Directory.CreateDirectory(outputFolder)

        ' Group PDFs by prefix (e.g., "invoice-2026-01-*.pdf" -> one merged file)
        Dim pdfFiles = Directory.GetFiles(inputFolder, "*.pdf")
        Dim groups = pdfFiles _
            .GroupBy(Function(f) Path.GetFileName(f).Split("-"c).Take(3).Aggregate(Function(a, b) $"{a}-{b}")) _
            .Where(Function(g) g.Count() > 1)

        Console.WriteLine($"Found {groups.Count()} groups to merge")

        For Each group In groups
            Dim groupName As String = group.Key
            Dim filesToMerge = group.OrderBy(Function(f) f).ToList()

            Console.WriteLine($"Merging {filesToMerge.Count} files into {groupName}.pdf")

            Try
                ' Load all PDFs for this group
                Dim pdfDocs As New List(Of PdfDocument)()
                For Each filePath As String In filesToMerge
                    pdfDocs.Add(PdfDocument.FromFile(filePath))
                Next

                ' Merge all documents
                Using merged = PdfDocument.Merge(pdfDocs)
                    merged.SaveAs(Path.Combine(outputFolder, $"{groupName}-merged.pdf"))
                End Using

                ' Dispose source documents
                For Each doc In pdfDocs
                    doc.Dispose()
                Next

                Console.WriteLine($"  [OK] Created {groupName}-merged.pdf ({merged.PageCount} pages)")
            Catch ex As Exception
                Console.WriteLine($"  [ERROR] {groupName}: {ex.Message}")
            End Try
        Next

        Console.WriteLine(vbCrLf & "Merge complete")
    End Sub
End Module

using IronPdf;
using System;
using System.IO;
using System.Threading.Tasks;
using System.Threading;

string inputFolder = "originals/";
string outputFolder = "compressed/";

Directory.CreateDirectory(outputFolder);

string[] pdfFiles = Directory.GetFiles(inputFolder, "*.pdf");
Console.WriteLine($"Found {pdfFiles.Length} PDFs to compress");

long totalOriginalSize = 0;
long totalCompressedSize = 0;
int processed = 0;

var options = new ParallelOptions
{
    MaxDegreeOfParallelism = Environment.ProcessorCount / 2
};

Parallel.ForEach(pdfFiles, options, pdfFile =>
{
    string fileName = Path.GetFileName(pdfFile);
    string outputPath = Path.Combine(outputFolder, fileName);

    try
    {
        long originalSize = new FileInfo(pdfFile).Length;
        Interlocked.Add(ref totalOriginalSize, originalSize);

        using var pdf = PdfDocument.FromFile(pdfFile);

        // Apply compression with JPEG quality setting (0-100, lower = more compression)
        pdf.CompressAndSaveAs(outputPath, 60);

        long compressedSize = new FileInfo(outputPath).Length;
        Interlocked.Add(ref totalCompressedSize, compressedSize);
        Interlocked.Increment(ref processed);

        double reduction = (1 - (double)compressedSize / originalSize) * 100;
        Console.WriteLine($"[OK] {fileName}: {originalSize / 1024}KB → {compressedSize / 1024}KB ({reduction:F1}% reduction)");
    }
    catch (Exception ex)
    {
        Console.WriteLine($"[ERROR] {fileName}: {ex.Message}");
    }
});

double totalReduction = (1 - (double)totalCompressedSize / totalOriginalSize) * 100;
Console.WriteLine($"\nCompression complete:");
Console.WriteLine($"  Files processed: {processed}");
Console.WriteLine($"  Total original: {totalOriginalSize / 1024 / 1024}MB");
Console.WriteLine($"  Total compressed: {totalCompressedSize / 1024 / 1024}MB");
Console.WriteLine($"  Overall reduction: {totalReduction:F1}%");

Imports IronPdf
Imports System
Imports System.IO
Imports System.Threading.Tasks
Imports System.Threading

Module Program
    Sub Main()
        Dim inputFolder As String = "originals/"
        Dim outputFolder As String = "compressed/"

        Directory.CreateDirectory(outputFolder)

        Dim pdfFiles As String() = Directory.GetFiles(inputFolder, "*.pdf")
        Console.WriteLine($"Found {pdfFiles.Length} PDFs to compress")

        Dim totalOriginalSize As Long = 0
        Dim totalCompressedSize As Long = 0
        Dim processed As Integer = 0

        Dim options As New ParallelOptions With {
            .MaxDegreeOfParallelism = Environment.ProcessorCount \ 2
        }

        Parallel.ForEach(pdfFiles, options, Sub(pdfFile)
                                                Dim fileName As String = Path.GetFileName(pdfFile)
                                                Dim outputPath As String = Path.Combine(outputFolder, fileName)

                                                Try
                                                    Dim originalSize As Long = New FileInfo(pdfFile).Length
                                                    Interlocked.Add(totalOriginalSize, originalSize)

                                                    Using pdf = PdfDocument.FromFile(pdfFile)
                                                        ' Apply compression with JPEG quality setting (0-100, lower = more compression)
                                                        pdf.CompressAndSaveAs(outputPath, 60)
                                                    End Using

                                                    Dim compressedSize As Long = New FileInfo(outputPath).Length
                                                    Interlocked.Add(totalCompressedSize, compressedSize)
                                                    Interlocked.Increment(processed)

                                                    Dim reduction As Double = (1 - CDbl(compressedSize) / originalSize) * 100
                                                    Console.WriteLine($"[OK] {fileName}: {originalSize \ 1024}KB → {compressedSize \ 1024}KB ({reduction:F1}% reduction)")
                                                Catch ex As Exception
                                                    Console.WriteLine($"[ERROR] {fileName}: {ex.Message}")
                                                End Try
                                            End Sub)

        Dim totalReduction As Double = (1 - CDbl(totalCompressedSize) / totalOriginalSize) * 100
        Console.WriteLine(vbCrLf & "Compression complete:")
        Console.WriteLine($"  Files processed: {processed}")
        Console.WriteLine($"  Total original: {totalOriginalSize \ 1024 \ 1024}MB")
        Console.WriteLine($"  Total compressed: {totalCompressedSize \ 1024 \ 1024}MB")
        Console.WriteLine($"  Overall reduction: {totalReduction:F1}%")
    End Sub
End Module

using IronPdf;
using System;
using System.IO;
using System.Threading.Tasks;
using System.Threading;
using System.Collections.Concurrent;

string inputFolder = "input/";
string outputFolder = "output/";
string errorLogPath = "error-log.txt";

Directory.CreateDirectory(outputFolder);

string[] htmlFiles = Directory.GetFiles(inputFolder, "*.html");
var renderer = new ChromePdfRenderer();
var errorLog = new ConcurrentBag<string>();

int processed = 0;
int failed = 0;
int retried = 0;

const int maxRetries = 3;

var options = new ParallelOptions
{
    MaxDegreeOfParallelism = Environment.ProcessorCount / 2
};

Parallel.ForEach(htmlFiles, options, htmlFile =>
{
    string fileName = Path.GetFileNameWithoutExtension(htmlFile);
    string outputPath = Path.Combine(outputFolder, $"{fileName}.pdf");
    int attempt = 0;
    bool success = false;

    while (attempt < maxRetries && !success)
    {
        attempt++;
        try
        {
            using var pdf = renderer.RenderHtmlFileAsPdf(htmlFile);
            pdf.SaveAs(outputPath);
            success = true;
            Interlocked.Increment(ref processed);

            if (attempt > 1)
            {
                Interlocked.Increment(ref retried);
                Console.WriteLine($"[OK] {fileName}.pdf (succeeded on attempt {attempt})");
            }
            else
            {
                Console.WriteLine($"[OK] {fileName}.pdf");
            }
        }
        catch (Exception ex) when (IsTransientException(ex) && attempt < maxRetries)
        {
            // Transient error - wait and retry with exponential backoff
            int delayMs = (int)Math.Pow(2, attempt) * 500;
            Console.WriteLine($"[RETRY] {fileName}: {ex.Message} (attempt {attempt}, waiting {delayMs}ms)");
            Thread.Sleep(delayMs);
        }
        catch (Exception ex)
        {
            // Non-transient error or max retries exceeded
            Interlocked.Increment(ref failed);
            string errorMessage = $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} | {fileName} | {ex.GetType().Name} | {ex.Message}";
            errorLog.Add(errorMessage);
            Console.WriteLine($"[ERROR] {fileName}: {ex.Message}");
        }
    }
});

// Write error log
if (errorLog.Count > 0)
{
    File.WriteAllLines(errorLogPath, errorLog);
}

Console.WriteLine($"\nBatch complete:");
Console.WriteLine($"  Processed: {processed}");
Console.WriteLine($"  Failed: {failed}");
Console.WriteLine($"  Retried: {retried}");
if (failed > 0)
{
    Console.WriteLine($"  Error log: {errorLogPath}");
}

// Helper to identify transient exceptions worth retrying
bool IsTransientException(Exception ex)
{
    return ex is IOException ||
           ex is OutOfMemoryException ||
           ex.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase) ||
           ex.Message.Contains("locked", StringComparison.OrdinalIgnoreCase);
}

Imports IronPdf
Imports System
Imports System.IO
Imports System.Threading
Imports System.Collections.Concurrent

Module Program
    Sub Main()
        Dim inputFolder As String = "input/"
        Dim outputFolder As String = "output/"
        Dim errorLogPath As String = "error-log.txt"

        Directory.CreateDirectory(outputFolder)

        Dim htmlFiles As String() = Directory.GetFiles(inputFolder, "*.html")
        Dim renderer As New ChromePdfRenderer()
        Dim errorLog As New ConcurrentBag(Of String)()

        Dim processed As Integer = 0
        Dim failed As Integer = 0
        Dim retried As Integer = 0

        Const maxRetries As Integer = 3

        Dim options As New ParallelOptions With {
            .MaxDegreeOfParallelism = Environment.ProcessorCount \ 2
        }

        Parallel.ForEach(htmlFiles, options, Sub(htmlFile)
                                                 Dim fileName As String = Path.GetFileNameWithoutExtension(htmlFile)
                                                 Dim outputPath As String = Path.Combine(outputFolder, $"{fileName}.pdf")
                                                 Dim attempt As Integer = 0
                                                 Dim success As Boolean = False

                                                 While attempt < maxRetries AndAlso Not success
                                                     attempt += 1
                                                     Try
                                                         Using pdf = renderer.RenderHtmlFileAsPdf(htmlFile)
                                                             pdf.SaveAs(outputPath)
                                                             success = True
                                                             Interlocked.Increment(processed)

                                                             If attempt > 1 Then
                                                                 Interlocked.Increment(retried)
                                                                 Console.WriteLine($"[OK] {fileName}.pdf (succeeded on attempt {attempt})")
                                                             Else
                                                                 Console.WriteLine($"[OK] {fileName}.pdf")
                                                             End If
                                                         End Using
                                                     Catch ex As Exception When IsTransientException(ex) AndAlso attempt < maxRetries
                                                         ' Transient error - wait and retry with exponential backoff
                                                         Dim delayMs As Integer = CInt(Math.Pow(2, attempt)) * 500
                                                         Console.WriteLine($"[RETRY] {fileName}: {ex.Message} (attempt {attempt}, waiting {delayMs}ms)")
                                                         Thread.Sleep(delayMs)
                                                     Catch ex As Exception
                                                         ' Non-transient error or max retries exceeded
                                                         Interlocked.Increment(failed)
                                                         Dim errorMessage As String = $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} | {fileName} | {ex.GetType().Name} | {ex.Message}"
                                                         errorLog.Add(errorMessage)
                                                         Console.WriteLine($"[ERROR] {fileName}: {ex.Message}")
                                                     End Try
                                                 End While
                                             End Sub)

        ' Write error log
        If errorLog.Count > 0 Then
            File.WriteAllLines(errorLogPath, errorLog)
        End If

        Console.WriteLine(vbCrLf & "Batch complete:")
        Console.WriteLine($"  Processed: {processed}")
        Console.WriteLine($"  Failed: {failed}")
        Console.WriteLine($"  Retried: {retried}")
        If failed > 0 Then
            Console.WriteLine($"  Error log: {errorLogPath}")
        End If
    End Sub

    ' Helper to identify transient exceptions worth retrying
    Function IsTransientException(ex As Exception) As Boolean
        Return TypeOf ex Is IOException OrElse
               TypeOf ex Is OutOfMemoryException OrElse
               ex.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase) OrElse
               ex.Message.Contains("locked", StringComparison.OrdinalIgnoreCase)
    End Function
End Module

using IronPdf;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Collections.Generic;

string inputFolder = "input/";
string outputFolder = "output/";
string checkpointPath = "checkpoint.txt";
string errorLogPath = "errors.txt";

Directory.CreateDirectory(outputFolder);

// Load checkpoint - files already processed successfully
var completedFiles = new HashSet<string>();
if (File.Exists(checkpointPath))
{
    completedFiles = new HashSet<string>(File.ReadAllLines(checkpointPath));
    Console.WriteLine($"Resuming from checkpoint: {completedFiles.Count} files already processed");
}

// Get files to process (excluding already completed)
string[] allFiles = Directory.GetFiles(inputFolder, "*.html");
string[] filesToProcess = allFiles
    .Where(f => !completedFiles.Contains(Path.GetFileName(f)))
    .ToArray();

Console.WriteLine($"Files to process: {filesToProcess.Length} (skipping {completedFiles.Count} already done)");

var renderer = new ChromePdfRenderer();
var checkpointLock = new object();
int processed = 0;
int failed = 0;

var options = new ParallelOptions
{
    MaxDegreeOfParallelism = Environment.ProcessorCount / 2
};

Parallel.ForEach(filesToProcess, options, htmlFile =>
{
    string fileName = Path.GetFileName(htmlFile);
    string baseName = Path.GetFileNameWithoutExtension(htmlFile);
    string outputPath = Path.Combine(outputFolder, $"{baseName}.pdf");

    try
    {
        using var pdf = renderer.RenderHtmlFileAsPdf(htmlFile);
        pdf.SaveAs(outputPath);

        // Record success in checkpoint (thread-safe)
        lock (checkpointLock)
        {
            File.AppendAllText(checkpointPath, fileName + Environment.NewLine);
        }

        Interlocked.Increment(ref processed);
        Console.WriteLine($"[OK] {baseName}.pdf");
    }
    catch (Exception ex)
    {
        Interlocked.Increment(ref failed);

        // Log error for review
        lock (checkpointLock)
        {
            File.AppendAllText(errorLogPath,
                $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} | {fileName} | {ex.Message}{Environment.NewLine}");
        }

        Console.WriteLine($"[ERROR] {fileName}: {ex.Message}");
    }
});

Console.WriteLine($"\nBatch complete:");
Console.WriteLine($"  Newly processed: {processed}");
Console.WriteLine($"  Failed: {failed}");
Console.WriteLine($"  Total completed: {completedFiles.Count + processed}");
Console.WriteLine($"  Checkpoint saved to: {checkpointPath}");

Imports IronPdf
Imports System
Imports System.IO
Imports System.Linq
Imports System.Threading.Tasks
Imports System.Threading
Imports System.Collections.Generic

Module Program
    Sub Main()
        Dim inputFolder As String = "input/"
        Dim outputFolder As String = "output/"
        Dim checkpointPath As String = "checkpoint.txt"
        Dim errorLogPath As String = "errors.txt"

        Directory.CreateDirectory(outputFolder)

        ' Load checkpoint - files already processed successfully
        Dim completedFiles As New HashSet(Of String)()
        If File.Exists(checkpointPath) Then
            completedFiles = New HashSet(Of String)(File.ReadAllLines(checkpointPath))
            Console.WriteLine($"Resuming from checkpoint: {completedFiles.Count} files already processed")
        End If

        ' Get files to process (excluding already completed)
        Dim allFiles As String() = Directory.GetFiles(inputFolder, "*.html")
        Dim filesToProcess As String() = allFiles _
            .Where(Function(f) Not completedFiles.Contains(Path.GetFileName(f))) _
            .ToArray()

        Console.WriteLine($"Files to process: {filesToProcess.Length} (skipping {completedFiles.Count} already done)")

        Dim renderer As New ChromePdfRenderer()
        Dim checkpointLock As New Object()
        Dim processed As Integer = 0
        Dim failed As Integer = 0

        Dim options As New ParallelOptions With {
            .MaxDegreeOfParallelism = Environment.ProcessorCount \ 2
        }

        Parallel.ForEach(filesToProcess, options, Sub(htmlFile)
                                                      Dim fileName As String = Path.GetFileName(htmlFile)
                                                      Dim baseName As String = Path.GetFileNameWithoutExtension(htmlFile)
                                                      Dim outputPath As String = Path.Combine(outputFolder, $"{baseName}.pdf")

                                                      Try
                                                          Using pdf = renderer.RenderHtmlFileAsPdf(htmlFile)
                                                              pdf.SaveAs(outputPath)
                                                          End Using

                                                          ' Record success in checkpoint (thread-safe)
                                                          SyncLock checkpointLock
                                                              File.AppendAllText(checkpointPath, fileName & Environment.NewLine)
                                                          End SyncLock

                                                          Interlocked.Increment(processed)
                                                          Console.WriteLine($"[OK] {baseName}.pdf")
                                                      Catch ex As Exception
                                                          Interlocked.Increment(failed)

                                                          ' Log error for review
                                                          SyncLock checkpointLock
                                                              File.AppendAllText(errorLogPath,
                                                                                 $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} | {fileName} | {ex.Message}{Environment.NewLine}")
                                                          End SyncLock

                                                          Console.WriteLine($"[ERROR] {fileName}: {ex.Message}")
                                                      End Try
                                                  End Sub)

        Console.WriteLine(vbCrLf & "Batch complete:")
        Console.WriteLine($"  Newly processed: {processed}")
        Console.WriteLine($"  Failed: {failed}")
        Console.WriteLine($"  Total completed: {completedFiles.Count + processed}")
        Console.WriteLine($"  Checkpoint saved to: {checkpointPath}")
    End Sub
End Module

using IronPdf;
using System;
using System.IO;
using System.Threading.Tasks;
using System.Threading;
using System.Diagnostics;

string inputFolder = "input/";
string outputFolder = "output/";

Directory.CreateDirectory(outputFolder);

string[] htmlFiles = Directory.GetFiles(inputFolder, "*.html");
var renderer = new ChromePdfRenderer();

Console.WriteLine($"Processing {htmlFiles.Length} files with {Environment.ProcessorCount} CPU cores");

int processed = 0;
var stopwatch = Stopwatch.StartNew();

// Configure parallelism based on system resources
// Rule of thumb: ProcessorCount / 2 for memory-intensive operations
var options = new ParallelOptions
{
    MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2)
};

Console.WriteLine($"Max parallelism: {options.MaxDegreeOfParallelism}");

// Use Parallel.ForEach for CPU-bound batch operations
Parallel.ForEach(htmlFiles, options, htmlFile =>
{
    string fileName = Path.GetFileNameWithoutExtension(htmlFile);
    string outputPath = Path.Combine(outputFolder, $"{fileName}.pdf");

    try
    {
        // Render HTML to PDF
        using var pdf = renderer.RenderHtmlFileAsPdf(htmlFile);
        pdf.SaveAs(outputPath);

        int current = Interlocked.Increment(ref processed);

        // Progress reporting every 10 files
        if (current % 10 == 0)
        {
            double elapsed = stopwatch.Elapsed.TotalSeconds;
            double rate = current / elapsed;
            double remaining = (htmlFiles.Length - current) / rate;
            Console.WriteLine($"Progress: {current}/{htmlFiles.Length} ({rate:F1} files/sec, ~{remaining:F0}s remaining)");
        }
    }
    catch (Exception ex)
    {
        Console.WriteLine($"[ERROR] {fileName}: {ex.Message}");
    }
});

stopwatch.Stop();
double totalRate = processed / stopwatch.Elapsed.TotalSeconds;

Console.WriteLine($"\nComplete:");
Console.WriteLine($"  Files processed: {processed}/{htmlFiles.Length}");
Console.WriteLine($"  Total time: {stopwatch.Elapsed.TotalSeconds:F1}s");
Console.WriteLine($"  Average rate: {totalRate:F1} files/sec");
Console.WriteLine($"  Time per file: {stopwatch.Elapsed.TotalMilliseconds / processed:F0}ms");

// Memory monitoring helper (call between chunks for large batches)
void CheckMemoryPressure()
{
    const long memoryThreshold = 4L * 1024 * 1024 * 1024; // 4 GB
    long currentMemory = GC.GetTotalMemory(forceFullCollection: false);

    if (currentMemory > memoryThreshold)
    {
        Console.WriteLine($"Memory pressure detected ({currentMemory / 1024 / 1024}MB), forcing GC...");
        GC.Collect();
        GC.WaitForPendingFinalizers();
        GC.Collect();
    }
}

Imports IronPdf
Imports System
Imports System.IO
Imports System.Threading.Tasks
Imports System.Threading
Imports System.Diagnostics

Module Program
    Sub Main()
        Dim inputFolder As String = "input/"
        Dim outputFolder As String = "output/"

        Directory.CreateDirectory(outputFolder)

        Dim htmlFiles As String() = Directory.GetFiles(inputFolder, "*.html")
        Dim renderer As New ChromePdfRenderer()

        Console.WriteLine($"Processing {htmlFiles.Length} files with {Environment.ProcessorCount} CPU cores")

        Dim processed As Integer = 0
        Dim stopwatch As Stopwatch = Stopwatch.StartNew()

        ' Configure parallelism based on system resources
        ' Rule of thumb: ProcessorCount / 2 for memory-intensive operations
        Dim options As New ParallelOptions With {
            .MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount \ 2)
        }

        Console.WriteLine($"Max parallelism: {options.MaxDegreeOfParallelism}")

        ' Use Parallel.ForEach for CPU-bound batch operations
        Parallel.ForEach(htmlFiles, options, Sub(htmlFile)
                                                 Dim fileName As String = Path.GetFileNameWithoutExtension(htmlFile)
                                                 Dim outputPath As String = Path.Combine(outputFolder, $"{fileName}.pdf")

                                                 Try
                                                     ' Render HTML to PDF
                                                     Using pdf = renderer.RenderHtmlFileAsPdf(htmlFile)
                                                         pdf.SaveAs(outputPath)
                                                     End Using

                                                     Dim current As Integer = Interlocked.Increment(processed)

                                                     ' Progress reporting every 10 files
                                                     If current Mod 10 = 0 Then
                                                         Dim elapsed As Double = stopwatch.Elapsed.TotalSeconds
                                                         Dim rate As Double = current / elapsed
                                                         Dim remaining As Double = (htmlFiles.Length - current) / rate
                                                         Console.WriteLine($"Progress: {current}/{htmlFiles.Length} ({rate:F1} files/sec, ~{remaining:F0}s remaining)")
                                                     End If
                                                 Catch ex As Exception
                                                     Console.WriteLine($"[ERROR] {fileName}: {ex.Message}")
                                                 End Try
                                             End Sub)

        stopwatch.Stop()
        Dim totalRate As Double = processed / stopwatch.Elapsed.TotalSeconds

        Console.WriteLine(vbCrLf & "Complete:")
        Console.WriteLine($"  Files processed: {processed}/{htmlFiles.Length}")
        Console.WriteLine($"  Total time: {stopwatch.Elapsed.TotalSeconds:F1}s")
        Console.WriteLine($"  Average rate: {totalRate:F1} files/sec")
        Console.WriteLine($"  Time per file: {stopwatch.Elapsed.TotalMilliseconds / processed:F0}ms")

        ' Memory monitoring helper (call between chunks for large batches)
        CheckMemoryPressure()
    End Sub

    Sub CheckMemoryPressure()
        Const memoryThreshold As Long = 4L * 1024 * 1024 * 1024 ' 4 GB
        Dim currentMemory As Long = GC.GetTotalMemory(forceFullCollection:=False)

        If currentMemory > memoryThreshold Then
            Console.WriteLine($"Memory pressure detected ({currentMemory \ 1024 \ 1024}MB), forcing GC...")
            GC.Collect()
            GC.WaitForPendingFinalizers()
            GC.Collect()
        End If
    End Sub
End Module

using IronPdf;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text.Json;

// Configuration
var config = new PipelineConfig
{
    InputFolder = "input/",
    OutputFolder = "output/",
    ArchiveFolder = "archive/",
    ErrorFolder = "errors/",
    CheckpointPath = "pipeline-checkpoint.json",
    ReportPath = "pipeline-report.json",
    MaxConcurrency = Math.Max(1, Environment.ProcessorCount / 2),
    MaxRetries = 3,
    JpegQuality = 70
};

// Initialize folders
Directory.CreateDirectory(config.OutputFolder);
Directory.CreateDirectory(config.ArchiveFolder);
Directory.CreateDirectory(config.ErrorFolder);

// Load checkpoint for resume capability
var checkpoint = LoadCheckpoint(config.CheckpointPath);
var results = new ConcurrentBag<ProcessingResult>();
var stopwatch = Stopwatch.StartNew();

// Get files to process
string[] allFiles = Directory.GetFiles(config.InputFolder, "*.html");
string[] filesToProcess = allFiles
    .Where(f => !checkpoint.CompletedFiles.Contains(Path.GetFileName(f)))
    .ToArray();

Console.WriteLine($"Pipeline starting:");
Console.WriteLine($"  Total files: {allFiles.Length}");
Console.WriteLine($"  Already processed: {checkpoint.CompletedFiles.Count}");
Console.WriteLine($"  To process: {filesToProcess.Length}");
Console.WriteLine($"  Concurrency: {config.MaxConcurrency}");

var renderer = new ChromePdfRenderer();
var checkpointLock = new object();

var options = new ParallelOptions
{
    MaxDegreeOfParallelism = config.MaxConcurrency
};

Parallel.ForEach(filesToProcess, options, inputFile =>
{
    var result = new ProcessingResult
    {
        FileName = Path.GetFileName(inputFile),
        StartTime = DateTime.UtcNow
    };

    try
    {
        // Stage: Pre-validation
        if (!ValidateInput(inputFile))
        {
            result.Status = "PreValidationFailed";
            result.Error = "Input file failed validation";
            results.Add(result);
            return;
        }

        string baseName = Path.GetFileNameWithoutExtension(inputFile);
        string tempPath = Path.Combine(config.OutputFolder, $"{baseName}.pdf");
        string archivePath = Path.Combine(config.ArchiveFolder, $"{baseName}.pdf");

        // Stage: Process with retry
        PdfDocument pdf = null;
        int attempt = 0;
        bool success = false;

        while (attempt < config.MaxRetries && !success)
        {
            attempt++;
            try
            {
                pdf = renderer.RenderHtmlFileAsPdf(inputFile);
                success = true;
            }
            catch (Exception ex) when (IsTransient(ex) && attempt < config.MaxRetries)
            {
                Thread.Sleep((int)Math.Pow(2, attempt) * 500);
            }
        }

        if (!success || pdf == null)
        {
            result.Status = "ProcessingFailed";
            result.Error = "Max retries exceeded";
            results.Add(result);
            return;
        }

        using (pdf)
        {
            // Stage: Compress and convert to PDF/A-3b for archival
            pdf.SaveAsPdfA(tempPath, PdfAVersions.PdfA3b);
        }

        // Stage: Post-validation
        if (!ValidateOutput(tempPath))
        {
            File.Move(tempPath, Path.Combine(config.ErrorFolder, $"{baseName}.pdf"), overwrite: true);
            result.Status = "PostValidationFailed";
            result.Error = "Output file failed validation";
            results.Add(result);
            return;
        }

        // Stage: Archive
        File.Move(tempPath, archivePath, overwrite: true);

        // Update checkpoint
        lock (checkpointLock)
        {
            checkpoint.CompletedFiles.Add(result.FileName);
            SaveCheckpoint(config.CheckpointPath, checkpoint);
        }

        result.Status = "Success";
        result.OutputSize = new FileInfo(archivePath).Length;
        result.EndTime = DateTime.UtcNow;
        results.Add(result);

        Console.WriteLine($"[OK] {baseName}.pdf ({result.OutputSize / 1024}KB)");
    }
    catch (Exception ex)
    {
        result.Status = "Error";
        result.Error = ex.Message;
        result.EndTime = DateTime.UtcNow;
        results.Add(result);
        Console.WriteLine($"[ERROR] {result.FileName}: {ex.Message}");
    }
});

stopwatch.Stop();

// Generate report
var report = new PipelineReport
{
    TotalFiles = allFiles.Length,
    ProcessedThisRun = results.Count,
    Succeeded = results.Count(r => r.Status == "Success"),
    PreValidationFailed = results.Count(r => r.Status == "PreValidationFailed"),
    ProcessingFailed = results.Count(r => r.Status == "ProcessingFailed"),
    PostValidationFailed = results.Count(r => r.Status == "PostValidationFailed"),
    Errors = results.Count(r => r.Status == "Error"),
    TotalDuration = stopwatch.Elapsed,
    AverageFileTime = results.Any() ? TimeSpan.FromMilliseconds(stopwatch.Elapsed.TotalMilliseconds / results.Count) : TimeSpan.Zero
};

string reportJson = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true });
File.WriteAllText(config.ReportPath, reportJson);

Console.WriteLine($"\n=== Pipeline Complete ===");
Console.WriteLine($"Succeeded: {report.Succeeded}");
Console.WriteLine($"Failed: {report.PreValidationFailed + report.ProcessingFailed + report.PostValidationFailed + report.Errors}");
Console.WriteLine($"Duration: {report.TotalDuration.TotalMinutes:F1} minutes");
Console.WriteLine($"Report: {config.ReportPath}");

// Helper methods
bool ValidateInput(string path)
{
    try
    {
        var info = new FileInfo(path);
        if (!info.Exists || info.Length == 0 || info.Length > 50 * 1024 * 1024) return false;
        string content = File.ReadAllText(path);
        return content.Contains("<html", StringComparison.OrdinalIgnoreCase) ||
               content.Contains("<!DOCTYPE", StringComparison.OrdinalIgnoreCase);
    }
    catch { return false; }
}

bool ValidateOutput(string path)
{
    try
    {
        using var pdf = PdfDocument.FromFile(path);
        return pdf.PageCount > 0 && new FileInfo(path).Length > 1024;
    }
    catch { return false; }
}

bool IsTransient(Exception ex) =>
    ex is IOException || ex is OutOfMemoryException ||
    ex.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase);

Checkpoint LoadCheckpoint(string path)
{
    if (File.Exists(path))
    {
        string json = File.ReadAllText(path);
        return JsonSerializer.Deserialize<Checkpoint>(json) ?? new Checkpoint();
    }
    return new Checkpoint();
}

void SaveCheckpoint(string path, Checkpoint cp) =>
    File.WriteAllText(path, JsonSerializer.Serialize(cp));

// Data classes
class PipelineConfig
{
    public string InputFolder { get; set; } = "";
    public string OutputFolder { get; set; } = "";
    public string ArchiveFolder { get; set; } = "";
    public string ErrorFolder { get; set; } = "";
    public string CheckpointPath { get; set; } = "";
    public string ReportPath { get; set; } = "";
    public int MaxConcurrency { get; set; }
    public int MaxRetries { get; set; }
    public int JpegQuality { get; set; }
}

class Checkpoint
{
    public HashSet<string> CompletedFiles { get; set; } = new();
}

class ProcessingResult
{
    public string FileName { get; set; } = "";
    public string Status { get; set; } = "";
    public string Error { get; set; } = "";
    public long OutputSize { get; set; }
    public DateTime StartTime { get; set; }
    public DateTime EndTime { get; set; }
}

class PipelineReport
{
    public int TotalFiles { get; set; }
    public int ProcessedThisRun { get; set; }
    public int Succeeded { get; set; }
    public int PreValidationFailed { get; set; }
    public int ProcessingFailed { get; set; }
    public int PostValidationFailed { get; set; }
    public int Errors { get; set; }
    public TimeSpan TotalDuration { get; set; }
    public TimeSpan AverageFileTime { get; set; }
}

Imports IronPdf
Imports System
Imports System.IO
Imports System.Linq
Imports System.Threading.Tasks
Imports System.Threading
Imports System.Collections.Concurrent
Imports System.Diagnostics
Imports System.Text.Json

' Configuration
Dim config As New PipelineConfig With {
    .InputFolder = "input/",
    .OutputFolder = "output/",
    .ArchiveFolder = "archive/",
    .ErrorFolder = "errors/",
    .CheckpointPath = "pipeline-checkpoint.json",
    .ReportPath = "pipeline-report.json",
    .MaxConcurrency = Math.Max(1, Environment.ProcessorCount \ 2),
    .MaxRetries = 3,
    .JpegQuality = 70
}

' Initialize folders
Directory.CreateDirectory(config.OutputFolder)
Directory.CreateDirectory(config.ArchiveFolder)
Directory.CreateDirectory(config.ErrorFolder)

' Load checkpoint for resume capability
Dim checkpoint = LoadCheckpoint(config.CheckpointPath)
Dim results As New ConcurrentBag(Of ProcessingResult)()
Dim stopwatch = Stopwatch.StartNew()

' Get files to process
Dim allFiles As String() = Directory.GetFiles(config.InputFolder, "*.html")
Dim filesToProcess As String() = allFiles.
    Where(Function(f) Not checkpoint.CompletedFiles.Contains(Path.GetFileName(f))).
    ToArray()

Console.WriteLine("Pipeline starting:")
Console.WriteLine($"  Total files: {allFiles.Length}")
Console.WriteLine($"  Already processed: {checkpoint.CompletedFiles.Count}")
Console.WriteLine($"  To process: {filesToProcess.Length}")
Console.WriteLine($"  Concurrency: {config.MaxConcurrency}")

Dim renderer As New ChromePdfRenderer()
Dim checkpointLock As New Object()

Dim options As New ParallelOptions With {
    .MaxDegreeOfParallelism = config.MaxConcurrency
}

Parallel.ForEach(filesToProcess, options, Sub(inputFile)
    Dim result As New ProcessingResult With {
        .FileName = Path.GetFileName(inputFile),
        .StartTime = DateTime.UtcNow
    }

    Try
        ' Stage: Pre-validation
        If Not ValidateInput(inputFile) Then
            result.Status = "PreValidationFailed"
            result.Error = "Input file failed validation"
            results.Add(result)
            Return
        End If

        Dim baseName As String = Path.GetFileNameWithoutExtension(inputFile)
        Dim tempPath As String = Path.Combine(config.OutputFolder, $"{baseName}.pdf")
        Dim archivePath As String = Path.Combine(config.ArchiveFolder, $"{baseName}.pdf")

        ' Stage: Process with retry
        Dim pdf As PdfDocument = Nothing
        Dim attempt As Integer = 0
        Dim success As Boolean = False

        While attempt < config.MaxRetries AndAlso Not success
            attempt += 1
            Try
                pdf = renderer.RenderHtmlFileAsPdf(inputFile)
                success = True
            Catch ex As Exception When IsTransient(ex) AndAlso attempt < config.MaxRetries
                Thread.Sleep(CInt(Math.Pow(2, attempt)) * 500)
            End Try
        End While

        If Not success OrElse pdf Is Nothing Then
            result.Status = "ProcessingFailed"
            result.Error = "Max retries exceeded"
            results.Add(result)
            Return
        End If

        Using pdf
            ' Stage: Compress and convert to PDF/A-3b for archival
            pdf.SaveAsPdfA(tempPath, PdfAVersions.PdfA3b)
        End Using

        ' Stage: Post-validation
        If Not ValidateOutput(tempPath) Then
            File.Move(tempPath, Path.Combine(config.ErrorFolder, $"{baseName}.pdf"), overwrite:=True)
            result.Status = "PostValidationFailed"
            result.Error = "Output file failed validation"
            results.Add(result)
            Return
        End If

        ' Stage: Archive
        File.Move(tempPath, archivePath, overwrite:=True)

        ' Update checkpoint
        SyncLock checkpointLock
            checkpoint.CompletedFiles.Add(result.FileName)
            SaveCheckpoint(config.CheckpointPath, checkpoint)
        End SyncLock

        result.Status = "Success"
        result.OutputSize = New FileInfo(archivePath).Length
        result.EndTime = DateTime.UtcNow
        results.Add(result)

        Console.WriteLine($"[OK] {baseName}.pdf ({result.OutputSize \ 1024}KB)")
    Catch ex As Exception
        result.Status = "Error"
        result.Error = ex.Message
        result.EndTime = DateTime.UtcNow
        results.Add(result)
        Console.WriteLine($"[ERROR] {result.FileName}: {ex.Message}")
    End Try
End Sub)

stopwatch.Stop()

' Generate report
Dim report As New PipelineReport With {
    .TotalFiles = allFiles.Length,
    .ProcessedThisRun = results.Count,
    .Succeeded = results.Count(Function(r) r.Status = "Success"),
    .PreValidationFailed = results.Count(Function(r) r.Status = "PreValidationFailed"),
    .ProcessingFailed = results.Count(Function(r) r.Status = "ProcessingFailed"),
    .PostValidationFailed = results.Count(Function(r) r.Status = "PostValidationFailed"),
    .Errors = results.Count(Function(r) r.Status = "Error"),
    .TotalDuration = stopwatch.Elapsed,
    .AverageFileTime = If(results.Any(), TimeSpan.FromMilliseconds(stopwatch.Elapsed.TotalMilliseconds / results.Count), TimeSpan.Zero)
}

Dim reportJson As String = JsonSerializer.Serialize(report, New JsonSerializerOptions With {.WriteIndented = True})
File.WriteAllText(config.ReportPath, reportJson)

Console.WriteLine(vbCrLf & "=== Pipeline Complete ===")
Console.WriteLine($"Succeeded: {report.Succeeded}")
Console.WriteLine($"Failed: {report.PreValidationFailed + report.ProcessingFailed + report.PostValidationFailed + report.Errors}")
Console.WriteLine($"Duration: {report.TotalDuration.TotalMinutes:F1} minutes")
Console.WriteLine($"Report: {config.ReportPath}")

' Helper methods
Function ValidateInput(path As String) As Boolean
    Try
        Dim info As New FileInfo(path)
        If Not info.Exists OrElse info.Length = 0 OrElse info.Length > 50 * 1024 * 1024 Then Return False
        Dim content As String = File.ReadAllText(path)
        Return content.Contains("<html", StringComparison.OrdinalIgnoreCase) OrElse
               content.Contains("<!DOCTYPE", StringComparison.OrdinalIgnoreCase)
    Catch
        Return False
    End Try
End Function

Function ValidateOutput(path As String) As Boolean
    Try
        Using pdf = PdfDocument.FromFile(path)
            Return pdf.PageCount > 0 AndAlso New FileInfo(path).Length > 1024
        End Using
    Catch
        Return False
    End Try
End Function

Function IsTransient(ex As Exception) As Boolean
    Return TypeOf ex Is IOException OrElse TypeOf ex Is OutOfMemoryException OrElse
           ex.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase)
End Function

Function LoadCheckpoint(path As String) As Checkpoint
    If File.Exists(path) Then
        Dim json As String = File.ReadAllText(path)
        Return JsonSerializer.Deserialize(Of Checkpoint)(json) OrElse New Checkpoint()
    End If
    Return New Checkpoint()
End Function

Sub SaveCheckpoint(path As String, cp As Checkpoint)
    File.WriteAllText(path, JsonSerializer.Serialize(cp))
End Sub

' Data classes
Class PipelineConfig
    Public Property InputFolder As String = ""
    Public Property OutputFolder As String = ""
    Public Property ArchiveFolder As String = ""
    Public Property ErrorFolder As String = ""
    Public Property CheckpointPath As String = ""
    Public Property ReportPath As String = ""
    Public Property MaxConcurrency As Integer
    Public Property MaxRetries As Integer
    Public Property JpegQuality As Integer
End Class

Class Checkpoint
    Public Property CompletedFiles As HashSet(Of String) = New HashSet(Of String)()
End Class

Class ProcessingResult
    Public Property FileName As String = ""
    Public Property Status As String = ""
    Public Property Error As String = ""
    Public Property OutputSize As Long
    Public Property StartTime As DateTime
    Public Property EndTime As DateTime
End Class

Class PipelineReport
    Public Property TotalFiles As Integer
    Public Property ProcessedThisRun As Integer
    Public Property Succeeded As Integer
    Public Property PreValidationFailed As Integer
    Public Property ProcessingFailed As Integer
    Public Property PostValidationFailed As Integer
    Public Property Errors As Integer
    Public Property TotalDuration As TimeSpan
    Public Property AverageFileTime As TimeSpan
End Class