Android : Android Listview ArrayAdapter with two layouts

I am quite new to Android. In the following class the data is retrieved from the database and displayed in a ListView which has two different layouts. Though it works as expected, the problem is that scrolling is not smooth because the textviews are assigned again and again. I couldn't figure out how to have them assigned only once. Please somebody help me out with this. Thanks in advance. My apology for the code, I know it looks bad.

public class FragmentVerses extends ListFragment {

Typeface font;
ViewHolder viewHolder = new ViewHolder();
ViewHolderHeader viewHolderHeader = new ViewHolderHeader();
DatabaseHelper db;

public List<VersesModel> verses;
public List<ChapterModel> chapterName;
ArrayAdapter<VersesModel> adapter;

public FragmentVerses() {
}

@Nullable
@Override
public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
    View view = inflater.inflate(R.layout.verses_fragment, container, false);

    db = new DatabaseHelper(getActivity());
    try {
        db.createDatabase();
    } catch (IOException e) {
        Toast.makeText(getActivity(), "Error Creating Database", Toast.LENGTH_LONG)
                .show();
    }

    verses = db.getVerses(" WHERE " + getActivity().getIntent().getStringExtra(MainActivity.CONDITION));
    chapterName = db.getChapter();
    adapter = new MyListAdapter();
    setListAdapter(adapter);
    return view;
}

private class MyListAdapter extends ArrayAdapter<VersesModel> {
    public MyListAdapter() {
        super(getActivity(), R.layout.verses_custom_list, verses);
    }

    @Override
    public View getView(int position, View convertView, ViewGroup parent) {
        VersesModel currentVerse = verses.get(position);
        if (convertView == null) {
            convertView = getActivity().getLayoutInflater().inflate(
                    R.layout.verses_custom_list, parent, false);
            font = Typeface.createFromAsset(convertView.getContext().getAssets(), "Quran_Taha.ttf");
            viewHolder.textView = (TextViewEx) convertView.findViewById(R.id.textView_Verse);
            viewHolder.textViewTranslation = (TextView) convertView.findViewById(R.id.textView_VerseTranslation);
            viewHolder.nView = (TextView) convertView.findViewById(R.id.textView_verseNumber);
            viewHolder.textView.setTypeface(font);
            viewHolder.imageView = (ImageView) convertView.findViewById(R.id.versesImageView);
            convertView.setTag(viewHolder);
        } else {
            if (currentVerse.getVerseNumber() != 0) {
                convertView = getActivity().getLayoutInflater().inflate(
                        R.layout.verses_custom_list, parent, false);
                viewHolder.textView = (TextViewEx) convertView.findViewById(R.id.textView_Verse);
                viewHolder.nView = (TextView) convertView.findViewById(R.id.textView_verseNumber);
                viewHolder.textViewTranslation = (TextView) convertView.findViewById(R.id.textView_VerseTranslation);
                viewHolder.textView.setTypeface(font);
                viewHolder.textView.setText(currentVerse.getVerseText() + "", true);
                viewHolder.textViewTranslation.setText(currentVerse.getVerseTranslation());
                viewHolder.nView.setText(currentVerse.getVerseNumber() + "");
                convertView.setTag(viewHolder);
            } else {
                convertView = getActivity().getLayoutInflater().inflate(
                        R.layout.verses_custom_list_header, parent, false);
                ChapterModel chapterModel = chapterName.get(currentVerse.getChapterNumber() - 1);
                if (viewHolderHeader.textViewChapter == null)    viewHolderHeader.textViewBismillah = (TextView) convertView.findViewById(R.id.textView_Verse_Bismillah);
                    viewHolderHeader.textViewChapter = (TextView) convertView.findViewById(R.id.textView_Verse_ChapterName);
                    viewHolderHeader.textViewChapter.setText("سورة  " + chapterModel.getChapterText());
                    viewHolderHeader.textViewBismillah.setTypeface(font);
                    viewHolderHeader.textViewChapter.setTypeface(font);
                } else {
                    viewHolderHeader = (ViewHolderHeader) convertView.getTag();
                }
                convertView.setTag(viewHolderHeader);

            }
        }
        return convertView;
    }
}