Android : ListView Scrolling Issue: on Item Selection

I have Listview with which I'm trying to display my custom Adapter.Everything works fine except when I select list item and scroll it,the items which were not selected are already being selected.I don't really understand what's the issue with my listview.

Here's my class:

@Override
protected void onCreate(Bundle savedInstanceState) 
{
    // TODO Auto-generated method stub
    super.onCreate(savedInstanceState);

    setContentView(R.layout.select_contact_layout);

    getActionBar().setHomeButtonEnabled(true);
    getActionBar().setDisplayHomeAsUpEnabled(true);
    getActionBar().setTitle("Select Contact");

    mArrayAdapter = new CustomAdapter(this,getContacts());

    setListAdapter(mArrayAdapter);

    contactPreferences = getSharedPreferences("contactPref", MODE_PRIVATE);

    mListView = getListView();

}



@Override
protected void onListItemClick(ListView l, View v, int position, long id) 
{
    super.onListItemClick(l, v, position, id);

    String name = mArrayAdapter.getItem(position).getName();

    v.setBackgroundColor(Color.parseColor("#88dfdf"));

    Toast.makeText(getApplicationContext(), "Items Pos: " + position +"and Name : "+ name, 0).show();

}

and My Custom Adapter:

 class CustomAdapter extends ArrayAdapter<Contacts>
{
     LayoutInflater layoutInflater;
    private List<Contacts> conctactList;


    public CustomAdapter(Context context, List<Contacts> mList)
    {
        super(context, R.layout.single_contact_layout,mList);
        this.conctactList = mList;
        layoutInflater = LayoutInflater.from(context);
    }

        @Override
        public int getCount() {
            // TODO Auto-generated method stub
            return conctactList.size();
        }

        @Override
        public Contacts getItem(int position) {
            // TODO Auto-generated method stub
            return conctactList.get(position);
        }

        @Override
        public long getItemId(int position) {
            // TODO Auto-generated method stub
            return position;
        }

        @Override
        public View getView(final int position, View convertView, ViewGroup parent)
        {
            final Holder holder;
            Bitmap bitmap = null;
            Bitmap scaleBitmap = null;

            if(convertView == null)
            {
                holder = new Holder();


                convertView = layoutInflater.inflate(R.layout.single_contact_layout, null);
                holder.name = (TextView) convertView.findViewById(R.id.contact_name);
                holder.number = (TextView) convertView.findViewById(R.id.contact_number);
                holder.contact_img = (ImageView)convertView.findViewById(R.id.contact_img);


                convertView.setTag(holder);
                convertView.setTag(R.id.contact_name, holder.name);
            }

            else{
                holder = (Holder) convertView.getTag();
            }

            holder.name.setText(conctactList.get(position).getName());
            holder.number.setText(conctactList.get(position).getNumber());
            try {
                bitmap = MediaStore.Images.Media.getBitmap(getContentResolver(), Uri.parse(contactsList.get(position).getImgUri()));

                scaleBitmap = Bitmap.createScaledBitmap(bitmap, 100, 100, true);

            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            holder.contact_img.setImageBitmap(ImageHelper.getRoundedCornerBitmap(scaleBitmap, 100));

            return convertView;
        }


}

private static class Holder{
     TextView name;
     TextView number;
     ImageView contact_img;
    }