c# - Read data from IPhone Backup file

There are two different types of file formats used by IPhone iOS to store data.Binary and SqLite.In this post I am reading data from a SQLite file.

Download the dll (http://system.data.sqlite.org/index.html/doc/trunk/www/downloads.wiki) and add it as a reference in your project.

Add the following using statement after adding the reference

using System.Data.SQLite;

try
{
         using (SQLiteConnection oSqlLiteConnection = new SQLiteConnection("Data Source=" + @"D:\MyBackups\MyIphoneBackupFile.sqlite"))
        {
               oSqlLiteConnection.Open();
               SQLiteCommand cmd = new SQLiteCommand("Select * from Scalars", oSqlLiteConnection);
               SQLiteDataReader dr = cmd.ExecuteReader();

               while (dr.Read())
                     Console.WriteLine(String.Format("{0}\t{1}\t{2}\t{3}", dr.GetValue(0), dr.GetValue(1), dr.GetValue(2)));
         }
}
catch(Exception ex)
{
    Console.WriteLine(ex.Message);
    Console.WriteLine(ex.Stacktrace);
}

c# - Editing/Updating confluence wiki page using c#, Atlassian Developer WebService

Program to update the company's intranet wiki page using Atlassian Developer WebService.

To be able to do this using a c# program you would need the web reference in the project that points to the web service.Add the web service url as the web reference in your project.

Right click on your project and select Add Service Reference

Once the service is added to the project the next step is to use the service methods to update the pages on your intranet.

Below is the entire code.You would need login,password and page id.

using System;
using System.Configuration;
using MyCompany.MyDepartment.ConfluenceWikiUpdate.MyCompany.Atlassian.Developer;

namespace MyCompany.MyDepartment.ConfluenceWikiUpdate
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                ConfluenceSoapServiceService oWikiUpdateService = new ConfluenceSoapServiceService();
                String sLoginToken = String.Empty;

                try
                {
                    sLoginToken = oWikiUpdateService.login(ConfigurationManager.AppSettings["CONFLUENCE_LOGIN"], ConfigurationManager.AppSettings["CONFLUENCE_PASSWD"]);
                }
                catch
                {
                    Console.WriteLine("Login attempt failed");
                }

                RemotePage oPage = oWikiUpdateService.getPage(sLoginToken, Convert.ToInt64(ConfigurationManager.AppSettings["CONFLUENCE_PAGEID"]));
                oPage.content = ConfigurationManager.AppSettings["TEXT"];
                oWikiUpdateService.updatePage(sLoginToken, oPage, new RemotePageUpdateOptions());

                oWikiUpdateService.logout(sLoginToken);

            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.WriteLine(ex.StackTrace);
            }
        }
    }
}

The Configuration file is below

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
  <appSettings>
    <add key="CONFLUENCE_LOGIN" value="MyDepartment@MyCompany.com"/>
    <add key="CONFLUENCE_PASSWD" value="Something"/>
    <add key="CONFLUENCE_PAGEID" value="12345678"/>
    <add key="TEXT" value="This is the table generated from a daily process."/>
  </appSettings>
    <system.serviceModel>
        <bindings />
        <client />
    </system.serviceModel>
    <applicationSettings>
        <MyCompany.MyDepartment.ConfluenceWikiUpdate.Properties.Settings>
            <setting name="MyCompany_MyDepartment_ConfluenceWikiUpdate_com_atlassian_developer_ConfluenceSoapServiceService"
                serializeAs="String">
                <value>https://wiki.office.mycompany.com/plugins/servlet/soap-axis1/confluenceservice-v2</value>
            </setting>
        </MyCompany.MyDepartment.ConfluenceWikiUpdate.Properties.Settings>
    </applicationSettings>
</configuration>

PIG,Hadoop,Java UDF - Custom gzip loader for handling gzip files in Pig,Java UDF,Hadoop

One of the requirement for a PIG/Hadoop job is that the files are in the right format.The entire job fails when a single corrupt line or file is encountered. 

While converting one of the processes we encountered the following errors

ERROR 2998: Unhandled internal error. Java heap space
ERROR 1071: Cannot convert a generic_writablecomparable to a String

One way to handle this is in a external process that would cleanup all the compressed files and then copy them over to hdfs and process it using pig script or we can extend the LoadFunc in Pig using a java udf. 

Below is the code for the CustomGzipHandler.jar source code

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;


public class CustomGzipLoader extends LoadFunc {

@SuppressWarnings("rawtypes")
private RecordReader reader;
private TupleFactory tupleFactory;

public CustomGzipLoader() 
{
tupleFactory = TupleFactory.getInstance();
}
 
@SuppressWarnings("rawtypes")
@Override
public InputFormat getInputFormat() throws IOException 
{
return new TextInputFormat();
}

@Override
public Tuple getNext() throws IOException 
{
try 
{
if (!reader.nextKeyValue()) 
return null;
 
Text value = (Text)reader.getCurrentValue();
 
  if(value != null  && !value.toString().isEmpty() && value.toString().length() < 9999) 
  return tupleFactory.newTuple(value.toString());
}
catch(Exception e){}
return null;  
}

@Override
public void prepareToRead(@SuppressWarnings("rawtypes") RecordReader reader, PigSplit pigSplit)
  throws IOException 
{
this.reader = reader; 
}

@Override
public void setLocation(String location, Job job) throws IOException 
{
FileInputFormat.setInputPaths(job, location); 
}
} 
 
After writing my own custom loader the load statement was transformed from 

rawdata = LOAD '/output/mydir/*.gz' USING PigStorage('\t') AS (mystring:chararray);

to this

set mapred.linerecordreader.maxlength 100000;//Ignore any lines more than 100000 in length

REGISTER CustomGzipHandler.jar; 
DEFINE CustomGzipLoader com.mycompany.package.CustomGzipLoader() ;

rawdata = LOAD '/output/mydir/*.gz' USING CustomGzipLoader();

c# - Find the largest interval between successive timestamps

Yesterday we had a network outage issue and some of the servers were down for a while.So today when I got to work, I wanted to check how bad the effect was on a few systems that I maintain.These systems record/collect data all year long so I wanted to see if there were any gaps between two successive records.

I wrote a small c# program to load all the timestamps and compare them to get the largest gap between timestamps.

using System;
using System.Collections.Generic;
using System.Data.SqlClient;

namespace LargestGapInTimeStamp
{
    class Program
    {
        static void Main(string[] args)
        {
            try
            {
                List<DateTime> TimeStampList = new List<DateTime>();
                DateTime oTimeStamp1 = new DateTime();
                DateTime oTimeStamp2 = new DateTime();
                double dMaxSeconds = 0;
                using (SqlConnection oConn = new SqlConnection("server=myserver;database=mydatabase;uid='myid';pwd='mypassword';"))
                {
                    oConn.Open();
                    SqlCommand oCmd = new SqlCommand("select event_time from dbo.mytable with (nolock) where event_time > \'2014-10-01\' and event_time < \'2014-10-09\' order by event_time", oConn);
                    oCmd.CommandTimeout = 30000;
                    SqlDataReader oReader = oCmd.ExecuteReader();
                    while (oReader != null && oReader.Read())
                    {
                        object oTimeStamp = oReader.GetValue(0);
                        if(oTimeStamp != null)
                            TimeStampList.Add(Convert.ToDateTime(oTimeStamp));
                    }

                    for(int iIndex = 0;iIndex < TimeStampList.Count -1 ; iIndex++)
                    {
                        if(dMaxSeconds < (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds)
                        {
                            dMaxSeconds = (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds;
                            oTimeStamp1 = TimeStampList[iIndex + 1];
                            oTimeStamp2 = TimeStampList[iIndex];
                        }
                    }
                }

                Console.WriteLine(oTimeStamp1.ToString());
                Console.WriteLine(oTimeStamp2.ToString());
                Console.WriteLine("The largest timestamp gap is: " + dMaxSeconds.ToString());
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.WriteLine(ex.StackTrace);
            }
            Console.Read();
        }
    }
}

Output

Extending this further I wanted to list out all the timestamp gaps in the last 8 days since the beginning of the month to see if there were any similar outages.Below program lists all timestamp gaps which are greater than a second.

using System;

using System.Collections.Generic;

using System.Data.SqlClient;

namespace LargestGapInTimeStamp

{

    class Program

    {

        static void Main(string[] args)

        {

            try

            {

                List<TimeStampGap> TimeStampGapList = new List<TimeStampGap>();

                List<DateTime> TimeStampList = new List<DateTime>();

                DateTime oTimeStamp1 = new DateTime();

                DateTime oTimeStamp2 = new DateTime();

                double dMaxSeconds = 0;

                using (SqlConnection oConn = new SqlConnection("server=myserver;database=mydatabase;uid='myid';pwd='mypassword';"))

                {

                    oConn.Open();

                    SqlCommand oCmd = new SqlCommand("select create_ts from dbo.mytable with (nolock) where create_ts > \'2014-10-01\' and create_ts < \'2014-10-09\' order by create_ts", oConn);

                    oCmd.CommandTimeout = 30000;

                    SqlDataReader oReader = oCmd.ExecuteReader();

                    while (oReader != null && oReader.Read())

                    {

                        object oTimeStamp = oReader.GetValue(0);

                        if(oTimeStamp != null)

                            TimeStampList.Add(Convert.ToDateTime(oTimeStamp));

                    }

                    for(int iIndex = 0;iIndex < TimeStampList.Count -1 ; iIndex++)

                    {

                        if (dMaxSeconds < (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds && (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds > 1)

                        {

                            dMaxSeconds = (TimeStampList[iIndex + 1] - TimeStampList[iIndex]).TotalSeconds;

                            oTimeStamp1 = TimeStampList[iIndex + 1];

                            oTimeStamp2 = TimeStampList[iIndex];

                            TimeStampGapList.Add(new TimeStampGap(dMaxSeconds,oTimeStamp1,oTimeStamp2));

                        }

                    }

                }

                foreach(TimeStampGap item in TimeStampGapList)

                {

                    Console.WriteLine(item.FirstTimeStamp.ToString());

                    Console.WriteLine(item.SecondTimeStamp.ToString());

                    Console.WriteLine("Timestamp Gap:" + item.Seconds.ToString());

                    Console.WriteLine("************************************");

                }

            }

            catch (Exception ex)

            {

                Console.WriteLine(ex.Message);

                Console.WriteLine(ex.StackTrace);

            }

            Console.Read();

        }

    }

    public class TimeStampGap

    {

        double _dSeconds;

        public double Seconds

        {

            get { return _dSeconds; }

            set { _dSeconds = value; }

        }

        DateTime _oFirstTimeStamp = new DateTime();

        public DateTime FirstTimeStamp

        {

            get { return _oFirstTimeStamp; }

            set { _oFirstTimeStamp = value; }

        }

        DateTime _oSecondTimeStamp = new DateTime();

        public DateTime SecondTimeStamp

        {

            get { return _oSecondTimeStamp; }

            set { _oSecondTimeStamp = value; }

        }

        public TimeStampGap(){}

        public TimeStampGap(double p_Seconds, DateTime p_FirstTimeStamp, DateTime p_SecondTimeStamp)

        {

            Seconds = p_Seconds;

            FirstTimeStamp = p_FirstTimeStamp;

            SecondTimeStamp = p_SecondTimeStamp;

        }

    }

}

Output

Output confirms that yesterday's outage was significant ! I'll try to post a sql solution soon.

An error has occurred while trying to access the log file. Logging may not function properly - VMware VS 2012

VMware installation on my machine caused this error to pop up every time I opened Visual Studio 2012. To fix it I had to Modify/Change VMware installation and remove to check to add VS plugin.