Monday, August 4, 2014

c# - Merge multiple gzip files

In this example I am taking a set of gzip files in multiple folders and merging them to larger gzip files.The total number of files to merge are determined by the key from the configuration file.The process is multithreaded.

The output of the following program is merged gzip files Output_1.gz,Output_2.gz,Output_3.gz and so on


 try
 {
        int iBufferSize = int.Parse(ConfigurationManager.AppSettings["BUFFER_SIZE"])
        int iFileMergeCounter = int.Parse(ConfigurationManager.AppSettings["FILE_MERGE_COUNT"]);
        int iFileCounter = 0;
        string[] sFileList = Directory.GetFiles(@"E:\YourDirectoryPath\", "*.gz");
                        
         var rangePartitioner = Partitioner.Create(0, sFileList.Length,iFileMergeCounter);
         Parallel.ForEach(rangePartitioner,new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount}, (range,loopState) =>
         {
            try
            {
               byte[] bBuffer = new byte[iBufferSize];

               using (FileStream oFile = new FileStream(@"E:\Output_" + (iFileCounter++).ToString() + ".gz" , FileAccess.Write, iBufferSize))

               using(GZipStream oCompressStream = new GZipStream(oFile, CompressionMode.Compress))
               {
                   for(int i = range.Item1; i < range.Item2; i++)
                   {
                      using (FileStream oFileStream = new FileStream(sFileList[i], FileMode.Open, FileAccess.Read, FileShare.Read))
                      using (GZipStream oDecompressStream = new GZipStream(oFileStream, CompressionMode.Decompress))
                      {
                       oDecompressStream.CopyTo(oCompressStream);
                      }
                   }
              }
           }
           catch (Exception ex)
           {
                oLog.Error(ex.Message);
                oLog.Error(ex.InnerException);
           }
        });
}
catch (Exception ex)
{
      oLog.Error(ex.Message);
      oLog.Error(ex.InnerException);
}