I have written several programs that read data from raw input text files,extract required fields and write to tab delimited output text files.The files are usually few GB in size.I wanted to test the performance of the StreamWriter class and below are the results.
StreamWriter when used with a buffer size greater than the default buffer size of 1024 KB performed better.
Run 1: 350 MB Input file with default buffer
using System;
using System.Diagnostics;
using System.IO;
using System.Text;
namespace FileWriterPerf
{
class Program
{
static void Main(string[] args)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
try
{
using (StreamWriter oWriter = new StreamWriter(@"c:\File1.txt"))
{
foreach (string sLine in File.ReadLines(@"c:\File.txt"))
oWriter.WriteLine(sLine);
oWriter.Flush();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
stopWatch.Stop();
TimeSpan ts = stopWatch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds,
ts.Milliseconds);
Console.WriteLine("RunTime " + elapsedTime);
Console.Read();
}
}
}
Run 2: 350 MB Input file with 64MB buffer size
using (StreamWriter oWriter = new StreamWriter(@"c:\File1.txt",true,Encoding,ASCII,65536))
Run 3: 2 GB Input file with default buffer size
using (StreamWriter oWriter = new StreamWriter(@"c:\RawFile1.txt"))
Run 4: 2 GB Input file with 64MB buffer size
using (StreamWriter oWriter = new StreamWriter(@"c:\RawFile1.txt",true,Encoding,ASCII,65536))
