c# - Send alerts when database table row value is changed

Requirement:
Send an alert when any column value in a given list of rows in a table changes.

There are different ways to build an alert system when a row's column value changes but in this case I am using C# to write a program that tracks these changes and send an email report with the changes.
The C# program is scheduled to run every minute to check whether the row value changed.

Few things to note.The table has primary key so all rows are unique.The table rows do not change often  and the size of the table is small with only a few thousand rows.Also we want to only track a subset of rows in the table.

Step 1: Create a file or table that holds the rows from the original table that will be tracked.The primary key list that will be tracked is stored in the configuration file.In this case I will be using a file to store these rows.

Step 2: Extract the rows based on the primary key list in configuration file from the original table and compare with the rows in the file or table created in the Step 1.

Step 3: If the column values are different add the primary key,column name,old value,new value to the report.

Step 4: Send an email with the report created in Step 3

Step 5: If the primary key count in the configuration file has changed or does not match with the rows in the file created in Step 1,then overwrite the file with the new list of rows that will be tracked.

Below is the entire code

using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Data.SqlClient;
using System.IO;
using System.Linq;
using System.Net.Mail;
using System.Text;
using NLog;

namespace MyCompany.MyDept.MyProject.TableRowEditAlert
{
    class Program
    {
        static void Main(string[] args)
        {
            Logger oLog = LogManager.GetCurrentClassLogger();
            try
            {
                string sConnectionString = ConfigurationManager.AppSettings["MY_DB"];
                string sIdList = ConfigurationManager.AppSettings["ID_LIST"];
                string sFilePath = ConfigurationManager.AppSettings["FILE_PATH"];

                DataSet ds = GetIdDetails(sConnectionString, sIdList);
                if (ds != null)
                {
                    if (!File.Exists(sFilePath))
                    {
                        CreateIdDetailsFile(sFilePath, ds,true);
                    }
                    else
                    {
                        string [] sFileLines = File.ReadAllLines(sFilePath);
                        if ((sFileLines.Length - 1) != ds.Tables[0].Rows.Count)
                        {
                            CreateIdDetailsFile(sFilePath, ds, false);
                        }
                        else
                        {
                            StringBuilder sDiff = new StringBuilder();
                            string [] sCols = sFileLines[0].Split('\t');
                           
                            for (int iIndex = 1; iIndex < sFileLines.Length; iIndex++)
                            {
                                string[] sFileLineItems = sFileLines[iIndex].Split('\t');
                                for (int jIndex = 0; jIndex < sFileLineItems.Length; jIndex++)
                                {
                                    if (ds.Tables[0].Rows[iIndex -1][jIndex] != null && ds.Tables[0].Rows[iIndex-1][jIndex].ToString() != sFileLineItems[jIndex])
                                    {
                                        sDiff.Append(sFileLineItems[0]);
                                        sDiff.Append('\t');
                                        sDiff.Append(sFileLineItems[6]);
                                        sDiff.Append('\t');
                                        sDiff.Append(sFileLineItems[1]);
                                        sDiff.Append('\t');
                                        sDiff.Append(sCols[jIndex]);
                                        sDiff.Append('\t');
                                        sDiff.Append(sFileLineItems[jIndex]);
                                        sDiff.Append('\t');
                                        sDiff.Append(ds.Tables[0].Rows[iIndex-1][jIndex].ToString());
                                        sDiff.Append('\n');
                                    }
                                }
                            }
                             if (!string.IsNullOrEmpty(sDiff.ToString()))
                            {
                                SendMail(GenerateReport(sDiff.ToString()).ToString(), ConfigurationManager.AppSettings["EMAIL_SUBJECT"], true, ConfigurationManager.AppSettings["NOTIFICATION_LIST"].Split(','));

                                CreateIdDetailsFile(sFilePath, ds, false);
                            }
                        }
                    }
                }
                
            }
            catch (Exception ex)
            {
                oLog.Error(ex.Message);
                oLog.Error(ex.StackTrace);
            }
        }

        private static void CreateIdDetailsFile(string sFilePath, DataSet ds,bool bAppend)
        {
            using (StreamWriter oWriter = new StreamWriter(sFilePath,bAppend))
            {
                StringBuilder sb = new StringBuilder();
                foreach (DataColumn col in ds.Tables[0].Columns)
                {
                    sb.Append(col.ColumnName);
                    if (col.Ordinal != ds.Tables[0].Columns.Count-1)
                        sb.Append('\t');
                }
                sb.Append(Environment.NewLine);

                foreach (DataRow row in ds.Tables[0].Rows)
                {
                    foreach (DataColumn col in ds.Tables[0].Columns)
                    {
                        sb.Append(row[col] != null ? row[col].ToString() : "");
                        if(col.Ordinal != ds.Tables[0].Columns.Count - 1)
                            sb.Append('\t');
                    }
                    sb.Append(Environment.NewLine);
                }
                oWriter.Write(sb.ToString());
            }
        }

        private static DataSet GetIdDetails(string sConnectionString, string sIdList)
        {
            using (SqlConnection oConn = new SqlConnection(sConnectionString))
            {
                oConn.Open();
                SqlCommand oCmd = new SqlCommand("select * from dbo.my_table where id in (" + sIdList + ") order by id", oConn);
                oCmd.CommandType = CommandType.Text;
                SqlDataAdapter oDataAdapter = new SqlDataAdapter(oCmd);
                DataSet ds = new DataSet();
                oDataAdapter.Fill(ds);
                return ds;
            }
        }

        public static void SendMail(String message, String subject, Boolean p_bIsHtml, String[] notificationList)
        {
            try
            {
                MailMessage MyMessage = new MailMessage();
                MyMessage.From = new MailAddress(ConfigurationManager.AppSettings["EMAIL_FROM"]);
                foreach (String email in notificationList)
                    MyMessage.To.Add(new MailAddress(email));
                MyMessage.IsBodyHtml = p_bIsHtml;
                MyMessage.Subject = subject;
                MyMessage.Body = message;

                SmtpClient emailClient = new SmtpClient("smtp.office.mycompany.com");
                emailClient.Send(MyMessage);
            }
            catch
            {
            }
        }

        private static StringBuilder GenerateReport(string sDiff)
        {
            string [] sLines = sDiff.Split('\n');
            StringBuilder sbHtml = new StringBuilder();
            sbHtml.AppendLine(@"<style type='text/css'>td{font-size: 9px;font-family: verdana} </style>");
            sbHtml.AppendLine("<table><border=1><tr bgcolor=Orange align=center><td colspan=6><b>Table Row Edit Alert</b></td></tr><tr bgcolor=LightBlue align=center><td><b>ID</b></td><td><b> Name</b></td><td><b> URL</b></td><td><b>Column</b></td><td><b>Old Value</b></td><td><b>New Value</b></td></tr>");
            foreach (string  s in sLines)
            {
                if(!string.IsNullOrEmpty(s))
                {
                    string[] sLineItems = s.Split('\t');
                    sbHtml.AppendLine("<tr bgcolor=LightGreen align=center><td><b>" + sLineItems[0] + "</b></td><td><b>" + sLineItems[1] + "</b></td><td><b>" + sLineItems[2] + "</b></td><td><b>" + sLineItems[3] + "</b></td><td><b>" + sLineItems[4] + "</b></td><td><b>" + sLineItems[5] + "</b></td></tr>");
                }
            }
            sbHtml.AppendLine("</table");
            sbHtml.AppendLine("<br/>");
            sbHtml.AppendLine("<br/>");
            return sbHtml;
        }
    }
}

Configuration File

<?xml version="1.0" encoding="utf-8" ?>

<configuration>

  <appSettings>

    <add key="MY_DB" value="server=server_name;database=database_name;uid='my_id';pwd='my_password';"/>

    <add key="ID_LIST" value="100,101,102,103,500,501,501,701,703,704" />

    <add key="FILE_PATH" value="C:\Data\Id.txt"/>

    <add key="EMAIL_FROM" value="admin@mycompany.com"/>

    <add key="EMAIL_SUBJECT" value="Table Row Edit Alert"/>

    <add key="NOTIFICATION_LIST" value="my_email@mycompany.com"/>

  </appSettings>

</configuration>

c# - Read data from IPhone Backup file

There are two different types of file formats used by IPhone iOS to store data.Binary and SqLite.In this post I am reading data from a SQLite file.

Download the dll (http://system.data.sqlite.org/index.html/doc/trunk/www/downloads.wiki) and add it as a reference in your project.

Add the following using statement after adding the reference

using System.Data.SQLite;

try
{
         using (SQLiteConnection oSqlLiteConnection = new SQLiteConnection("Data Source=" + @"D:\MyBackups\MyIphoneBackupFile.sqlite"))
        {
               oSqlLiteConnection.Open();
               SQLiteCommand cmd = new SQLiteCommand("Select * from Scalars", oSqlLiteConnection);
               SQLiteDataReader dr = cmd.ExecuteReader();

               while (dr.Read())
                     Console.WriteLine(String.Format("{0}\t{1}\t{2}\t{3}", dr.GetValue(0), dr.GetValue(1), dr.GetValue(2)));
         }
}
catch(Exception ex)
{
    Console.WriteLine(ex.Message);
    Console.WriteLine(ex.Stacktrace);
}

c# - Editing/Updating confluence wiki page using c#, Atlassian Developer WebService

Program to update the company's intranet wiki page using Atlassian Developer WebService.

To be able to do this using a c# program you would need the web reference in the project that points to the web service.Add the web service url as the web reference in your project.

Right click on your project and select Add Service Reference

Once the service is added to the project the next step is to use the service methods to update the pages on your intranet.

Below is the entire code.You would need login,password and page id.

using System;
using System.Configuration;
using MyCompany.MyDepartment.ConfluenceWikiUpdate.MyCompany.Atlassian.Developer;

namespace MyCompany.MyDepartment.ConfluenceWikiUpdate
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                ConfluenceSoapServiceService oWikiUpdateService = new ConfluenceSoapServiceService();
                String sLoginToken = String.Empty;

                try
                {
                    sLoginToken = oWikiUpdateService.login(ConfigurationManager.AppSettings["CONFLUENCE_LOGIN"], ConfigurationManager.AppSettings["CONFLUENCE_PASSWD"]);
                }
                catch
                {
                    Console.WriteLine("Login attempt failed");
                }

                RemotePage oPage = oWikiUpdateService.getPage(sLoginToken, Convert.ToInt64(ConfigurationManager.AppSettings["CONFLUENCE_PAGEID"]));
                oPage.content = ConfigurationManager.AppSettings["TEXT"];
                oWikiUpdateService.updatePage(sLoginToken, oPage, new RemotePageUpdateOptions());

                oWikiUpdateService.logout(sLoginToken);

            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.WriteLine(ex.StackTrace);
            }
        }
    }
}

The Configuration file is below

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
  <appSettings>
    <add key="CONFLUENCE_LOGIN" value="MyDepartment@MyCompany.com"/>
    <add key="CONFLUENCE_PASSWD" value="Something"/>
    <add key="CONFLUENCE_PAGEID" value="12345678"/>
    <add key="TEXT" value="This is the table generated from a daily process."/>
  </appSettings>
    <system.serviceModel>
        <bindings />
        <client />
    </system.serviceModel>
    <applicationSettings>
        <MyCompany.MyDepartment.ConfluenceWikiUpdate.Properties.Settings>
            <setting name="MyCompany_MyDepartment_ConfluenceWikiUpdate_com_atlassian_developer_ConfluenceSoapServiceService"
                serializeAs="String">
                <value>https://wiki.office.mycompany.com/plugins/servlet/soap-axis1/confluenceservice-v2</value>
            </setting>
        </MyCompany.MyDepartment.ConfluenceWikiUpdate.Properties.Settings>
    </applicationSettings>
</configuration>

PIG,Hadoop,Java UDF - Custom gzip loader for handling gzip files in Pig,Java UDF,Hadoop

One of the requirement for a PIG/Hadoop job is that the files are in the right format.The entire job fails when a single corrupt line or file is encountered. 

While converting one of the processes we encountered the following errors

ERROR 2998: Unhandled internal error. Java heap space
ERROR 1071: Cannot convert a generic_writablecomparable to a String

One way to handle this is in a external process that would cleanup all the compressed files and then copy them over to hdfs and process it using pig script or we can extend the LoadFunc in Pig using a java udf. 

Below is the code for the CustomGzipHandler.jar source code

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;


public class CustomGzipLoader extends LoadFunc {

@SuppressWarnings("rawtypes")
private RecordReader reader;
private TupleFactory tupleFactory;

public CustomGzipLoader() 
{
tupleFactory = TupleFactory.getInstance();
}
 
@SuppressWarnings("rawtypes")
@Override
public InputFormat getInputFormat() throws IOException 
{
return new TextInputFormat();
}

@Override
public Tuple getNext() throws IOException 
{
try 
{
if (!reader.nextKeyValue()) 
return null;
 
Text value = (Text)reader.getCurrentValue();
 
  if(value != null  && !value.toString().isEmpty() && value.toString().length() < 9999) 
  return tupleFactory.newTuple(value.toString());
}
catch(Exception e){}
return null;  
}

@Override
public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit pigSplit)
  throws IOException 
{
this.reader = reader; 
}

@Override
public void setLocation(String location, Job job) throws IOException 
{
FileInputFormat.setInputPaths(job, location); 
}
} 
 
After writing my own custom loader the load statement was transformed from 

rawdata = LOAD '/output/mydir/*.gz' USING PigStorage('\t') AS (mystring:chararray);

to this

set mapred.linerecordreader.maxlength 100000;//Ignore any lines more than 100000 in length

REGISTER CustomGzipHandler.jar; 
DEFINE CustomGzipLoader com.mycompany.package.CustomGzipLoader() ;

rawdata = LOAD '/output/mydir/*.gz' USING CustomGzipLoader();

c# - Find the largest interval between successive timestamps

Yesterday we had a network outage issue and some of the servers were down for a while.So today when I got to work, I wanted to check how bad the effect was on a few systems that I maintain.These systems record/collect data all year long so I wanted to see if there were any gaps between two successive records.

I wrote a small c# program to load all the timestamps and compare them to get the largest gap between timestamps.

using System;
using System.Collections.Generic;
using System.Data.SqlClient;

namespace LargestGapInTimeStamp
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                List<DateTime> TimeStampList = new List<DateTime>();
                DateTime oTimeStamp1 = new DateTime();
                DateTime oTimeStamp2 = new DateTime();
                double dMaxSeconds = 0;
                using (SqlConnection oConn = new SqlConnection("server=myserver;database=mydatabase;uid='myid';pwd='mypassword';"))
                {
                    oConn.Open();
                    SqlCommand oCmd = new SqlCommand("select event_time from dbo.mytable with (nolock) where event_time > \'2014-10-01\' and event_time < \'2014-10-09\' order by event_time", oConn);
                    oCmd.CommandTimeout = 30000;
                    SqlDataReader oReader = oCmd.ExecuteReader();
                    while (oReader != null && oReader.Read())
                    {
                        object oTimeStamp = oReader.GetValue(0);
                        if(oTimeStamp != null)
                            TimeStampList.Add(Convert.ToDateTime(oTimeStamp));
                    }

                    for(int iIndex = 0;iIndex < TimeStampList.Count -1 ; iIndex++)
                    {
                        if(dMaxSeconds < (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds)
                        {
                            dMaxSeconds = (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds;
                            oTimeStamp1 = TimeStampList[iIndex + 1];
                            oTimeStamp2 = TimeStampList[iIndex];
                        }
                    }
                }

                Console.WriteLine(oTimeStamp1.ToString());
                Console.WriteLine(oTimeStamp2.ToString());
                Console.WriteLine("The largest timestamp gap is: " + dMaxSeconds.ToString());
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.WriteLine(ex.StackTrace);
            }
            Console.Read();
        }
    }
}

Output

Extending this further I wanted to list out all the timestamp gaps in the last 8 days since the beginning of the month to see if there were any similar outages.Below program lists all timestamp gaps which are greater than a second.

using System;

using System.Collections.Generic;

using System.Data.SqlClient;

namespace LargestGapInTimeStamp

{

    class Program

    {

        static void Main(string[] args)

        {

            try

            {

                List<TimeStampGap> TimeStampGapList = new List<TimeStampGap>();

                List<DateTime> TimeStampList = new List<DateTime>();

                DateTime oTimeStamp1 = new DateTime();

                DateTime oTimeStamp2 = new DateTime();

                double dMaxSeconds = 0;

                using (SqlConnection oConn = new SqlConnection("server=myserver;database=mydatabase;uid='myid';pwd='mypassword';"))

                {

                    oConn.Open();

                    SqlCommand oCmd = new SqlCommand("select create_ts from dbo.mytable with (nolock) where create_ts > \'2014-10-01\' and create_ts < \'2014-10-09\' order by create_ts", oConn);

                    oCmd.CommandTimeout = 30000;

                    SqlDataReader oReader = oCmd.ExecuteReader();

                    while (oReader != null && oReader.Read())

                    {

                        object oTimeStamp = oReader.GetValue(0);

                        if(oTimeStamp != null)

                            TimeStampList.Add(Convert.ToDateTime(oTimeStamp));

                    }

                    for(int iIndex = 0;iIndex < TimeStampList.Count -1 ; iIndex++)

                    {

                        if (dMaxSeconds < (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds && (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds > 1)

                        {

                            dMaxSeconds = (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds;

                            oTimeStamp1 = TimeStampList[iIndex + 1];

                            oTimeStamp2 = TimeStampList[iIndex];

                            TimeStampGapList.Add(new TimeStampGap(dMaxSeconds,oTimeStamp1,oTimeStamp2));

                        }

                    }

                }

                foreach(TimeStampGap item in TimeStampGapList)

                {

                    Console.WriteLine(item.FirstTimeStamp.ToString());

                    Console.WriteLine(item.SecondTimeStamp.ToString());

                    Console.WriteLine("Timestamp Gap:" + item.Seconds.ToString());

                    Console.WriteLine("************************************");

                }

            }

            catch (Exception ex)

            {

                Console.WriteLine(ex.Message);

                Console.WriteLine(ex.StackTrace);

            }

            Console.Read();

        }

    }

    public class TimeStampGap

    {

        double _dSeconds;

        public double Seconds

        {

            get { return _dSeconds; }

            set { _dSeconds = value; }

        }

        DateTime _oFirstTimeStamp = new DateTime();

        public DateTime FirstTimeStamp

        {

            get { return _oFirstTimeStamp; }

            set { _oFirstTimeStamp = value; }

        }

        DateTime _oSecondTimeStamp = new DateTime();

        public DateTime SecondTimeStamp

        {

            get { return _oSecondTimeStamp; }

            set { _oSecondTimeStamp = value; }

        }

        public TimeStampGap(){}

        public TimeStampGap(double p_Seconds, DateTime p_FirstTimeStamp, DateTime p_SecondTimeStamp)

        {

            Seconds = p_Seconds;

            FirstTimeStamp = p_FirstTimeStamp;

            SecondTimeStamp = p_SecondTimeStamp;

        }

    }

}

Output

Output confirms that yesterday's outage was significant ! I'll try to post a sql solution soon.

An error has occurred while trying to access the log file. Logging may not function properly - VMware VS 2012

VMware installation on my machine caused this error to pop up every time I opened Visual Studio 2012. To fix it I had to Modify/Change VMware installation and remove to check to add VS plugin.

c# - Merge multiple gzip files

In this example I am taking a set of gzip files in multiple folders and merging them to larger gzip files.The total number of files to merge are determined by the key from the configuration file.The process is multithreaded.

The output of the following program is merged gzip files Output_1.gz,Output_2.gz,Output_3.gz and so on

 try
 {
        int iBufferSize = int.Parse(ConfigurationManager.AppSettings["BUFFER_SIZE"])
        int iFileMergeCounter = int.Parse(ConfigurationManager.AppSettings["FILE_MERGE_COUNT"]);
        int iFileCounter = 0;
        string[] sFileList = Directory.GetFiles(@"E:\YourDirectoryPath\", "*.gz");
                        
         var rangePartitioner = Partitioner.Create(0, sFileList.Length,iFileMergeCounter);
         Parallel.ForEach(rangePartitioner,new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount}, (range,loopState) =>
         {
            try
            {
               byte[] bBuffer = new byte[iBufferSize];

               using (FileStream oFile = new FileStream(@"E:\Output_" + (iFileCounter++).ToString() + ".gz" , FileAccess.Write, iBufferSize))
               using(GZipStream oCompressStream = new GZipStream(oFile, CompressionMode.Compress))
               {
                   for(int i = range.Item1; i < range.Item2; i++)
                   {
                      using (FileStream oFileStream = new FileStream(sFileList[i], FileMode.Open, FileAccess.Read, FileShare.Read))
                      using (GZipStream oDecompressStream = new GZipStream(oFileStream, CompressionMode.Decompress))
                      {
                       oDecompressStream.CopyTo(oCompressStream);
                      }
                   }
              }
           }
           catch (Exception ex)
           {
                oLog.Error(ex.Message);
                oLog.Error(ex.InnerException);
           }
        });
}
catch (Exception ex)
{
      oLog.Error(ex.Message);
      oLog.Error(ex.InnerException);
}
              

The process cannot access the file because it is being used by another process Task Parallel Library

In one of the projects I am converting to .Net 4.0, I started getting the following error 'The process cannot access the file because it is being used by another process'. The program is processing a bunch of files in a set of folders. I am using .Net 4.0 Task Parallel Library features to handle these folders across different threads on the server. The server has 16 GB RAM and 24 cores.

The code that was causing the error is below

Parallel.ForEach(rangePartitioner,new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount}, (range,loopState) =>
{
     try
     {

         using (FileStream oFileStream = new FileStream(oFileInfo.FullName, FileMode.Open))

         using (GZipStream oGzipStream = new GZipStream(oFileStream, CompressionMode.Decompress))

         using (StreamReader oStreamReader = new StreamReader(oGzipStream, Encoding.ASCII))

         {

                   String sRawDataLine = oStreamReader.ReadLine();

                   while (sRawDataLine != null)

                   {

                           //Do Something();

                            sRawDataLine = oStreamReader.ReadLine();

                    }

           }

     }

     catch (Exception ex)

     {

       oLog.WriteLine("Error processing file name = {0} Exception {1}.", oFileInfo.FullName, ex.ToString());

     }

});

The "key" for the "locks" was to add the following options highlighted in the below code to the FileStream object so the file access is for read operation and that the file could be shared while reading.

Parallel.ForEach(rangePartitioner,new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount}, (range,loopState) =>
{
     try
     {
         using (FileStream oFileStream = new FileStream(oFileInfo.FullName, FileMode.Open,FileAccess.Read, FileShare.Read)))
         using (GZipStream oGzipStream = new GZipStream(oFileStream, CompressionMode.Decompress))
         using (StreamReader oStreamReader = new StreamReader(oGzipStream, Encoding.ASCII))
         {
                   String sRawDataLine = oStreamReader.ReadLine();
                   while (sRawDataLine != null)
                   {
                           //Do Something();
                            sRawDataLine = oStreamReader.ReadLine();
                    }
           }
     }
     catch (Exception ex)
     {
       oLog.WriteLine("Error processing file name = {0} Exception {1}.", oFileInfo.FullName, ex.ToString());
     }
});

I have not seen the error since adding these two options.The process no longer fails and executes on all the 24 cores on the server without any issues.

c# - Remove duplicate lines

A lot of the programs I have written are related to parsing files and converting them to tab delimited files for the data warehouse.The files range from few megabytes to gigabytes in size.In one of the processes I noticed that the output files had duplicate lines.I wanted to remove the duplicate lines from these files.The file sizes ranged from 2 GB to 6 GB.

Below is the c# program I wrote to remove the duplicates.Fortunately I have Windows 2008 server with 16 GB RAM so that helps :)

using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;

namespace DeDupeTextFiles
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                string sInputFilePath = ConfigurationManager.AppSettings["INPUT_FILEPATH"];
                HashSet<string> oDistinctLines = new HashSet<string>();
                using (StreamReader oReader = new StreamReader(sInputFilePath))
                {
                    string sLine = oReader.ReadLine();
                    while (sLine != null)
                    {
                        if (!oDistinctLines.Contains(sLine))
                            oDistinctLines.Add(sLine);

                        sLine = oReader.ReadLine();
                    }
                }
                            File.WriteAllLines(sInputFilePath.Replace(Path.GetFileName(sInputFilePath), Path.GetFileNameWithoutExtension(sInputFilePath) + "_Distinct.txt"), oDistinctLines.ToArray<string>());
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.WriteLine(ex.StackTrace);
            }
            Console.Read();
        }
    }
}

App.config

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
  <appSettings>
    <add key="INPUT_FILEPATH" value="E:\test_data\myfolder\file\lookups\201401\0.txt"/>
  </appSettings>
</configuration>

There are other ways to achieve the same using DOS batch commands or load files to database and distinct it out.The program took less than a minute to remove duplicates from 3 GB file.The final file was reduced to 180 MB !