kexi

kexicsvimportdialog.cpp

00001 /* This file is part of the KDE project
00002    Copyright (C) 2005-2006 Jaroslaw Staniek <js@iidea.pl>
00003 
00004    This work is based on kspread/dialogs/kspread_dlg_csv.cc
00005    and will be merged back with KOffice libraries.
00006 
00007    Copyright (C) 2002-2003 Norbert Andres <nandres@web.de>
00008    Copyright (C) 2002-2003 Ariya Hidayat <ariya@kde.org>
00009    Copyright (C) 2002 Laurent Montel <montel@kde.org>
00010    Copyright (C) 1999 David Faure <faure@kde.org>
00011 
00012    This library is free software; you can redistribute it and/or
00013    modify it under the terms of the GNU Library General Public
00014    License as published by the Free Software Foundation; either
00015    version 2 of the License, or (at your option) any later version.
00016 
00017    This library is distributed in the hope that it will be useful,
00018    but WITHOUT ANY WARRANTY; without even the implied warranty of
00019    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00020    Library General Public License for more details.
00021 
00022    You should have received a copy of the GNU Library General Public License
00023    along with this library; see the file COPYING.LIB.  If not, write to
00024    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
00025  * Boston, MA 02110-1301, USA.
00026 */
00027 
00028 #include <qbuttongroup.h>
00029 #include <qcheckbox.h>
00030 #include <qclipboard.h>
00031 #include <qlabel.h>
00032 #include <qlineedit.h>
00033 #include <qmime.h>
00034 #include <qpushbutton.h>
00035 #include <qradiobutton.h>
00036 #include <qtable.h>
00037 #include <qlayout.h>
00038 #include <qfiledialog.h>
00039 #include <qpainter.h>
00040 #include <qtextcodec.h>
00041 #include <qtimer.h>
00042 #include <qfontmetrics.h>
00043 #include <qtooltip.h>
00044 
00045 #include <kapplication.h>
00046 #include <kdebug.h>
00047 #include <kdialogbase.h>
00048 #include <kfiledialog.h>
00049 #include <klocale.h>
00050 #include <kmessagebox.h>
00051 #include <kglobalsettings.h>
00052 #include <kiconloader.h>
00053 #include <kcharsets.h>
00054 #include <knuminput.h>
00055 #include <kprogress.h>
00056 #include <kactivelabel.h>
00057 
00058 #include <kexiutils/identifier.h>
00059 #include <kexiutils/utils.h>
00060 #include <core/kexi.h>
00061 #include <core/kexiproject.h>
00062 #include <core/kexipart.h>
00063 #include <core/kexipartinfo.h>
00064 #include <core/keximainwindow.h>
00065 #include <core/kexiguimsghandler.h>
00066 #include <kexidb/connection.h>
00067 #include <kexidb/tableschema.h>
00068 #include <kexidb/transaction.h>
00069 #include <widget/kexicharencodingcombobox.h>
00070 
00071 #include "kexicsvimportdialog.h"
00072 #include "kexicsvwidgets.h"
00073 
00074 #ifdef Q_WS_WIN
00075 #include <krecentdirs.h>
00076 #include <windows.h>
00077 #endif
00078 
00079 #if 0
00080 #include <kspread_cell.h>
00081 #include <kspread_doc.h>
00082 #include <kspread_sheet.h>
00083 #include <kspread_undo.h>
00084 #include <kspread_view.h>
00085 #endif
00086 
00087 #define _IMPORT_ICON "table" /*todo: change to "file_import" or so*/
00088 #define _TEXT_TYPE 0
00089 #define _NUMBER_TYPE 1
00090 #define _DATE_TYPE 2
00091 #define _TIME_TYPE 3
00092 #define _DATETIME_TYPE 4
00093 #define _PK_FLAG 5
00094 
00095 //extra:
00096 #define _NO_TYPE_YET -1 //allows to accept a number of empty cells, before something non-empty
00097 #define _FP_NUMBER_TYPE 255 //_NUMBER_TYPE variant
00098 #define MAX_ROWS_TO_PREVIEW 100 //max 100 rows is reasonable
00099 #define MAX_BYTES_TO_PREVIEW 10240 //max 10KB is reasonable
00100 #define MAX_CHARS_TO_SCAN_WHILE_DETECTING_DELIMITER 4096
00101 
00102 class KexiCSVImportDialogTable : public QTable
00103 {
00104 public:
00105     KexiCSVImportDialogTable( QWidget * parent = 0, const char * name = 0 )
00106     : QTable(parent, name) {
00107         f = font();
00108         f.setBold(true);
00109     }
00110     virtual void paintCell( QPainter * p, int row, int col, const QRect & cr, bool selected, const QColorGroup & cg ) {
00111         if (row==0)
00112             p->setFont(f);
00113         else
00114             p->setFont(font());
00115         QTable::paintCell(p, row, col, cr, selected, cg);
00116     }
00117     virtual void setColumnWidth( int col, int w ) {
00118         //make columns a bit wider
00119         QTable::setColumnWidth( col, w + 16 );
00120     }
00121     QFont f;
00122 };
00123 
00125 void installRecursiveEventFilter(QObject *filter, QObject *object)
00126 {
00127     object->installEventFilter(filter);
00128 
00129     if (!object->children())
00130         return;
00131 
00132     QObjectList list = *object->children();
00133     for(QObject *obj = list.first(); obj; obj = list.next())
00134         installRecursiveEventFilter(filter, obj);
00135 }
00136 
00137 KexiCSVImportDialog::KexiCSVImportDialog( Mode mode, KexiMainWindow* mainWin, 
00138     QWidget * parent, const char * name
00139 )
00140  : KDialogBase( 
00141     KDialogBase::Plain, 
00142     i18n( "Import CSV Data File" )
00144     ,
00145     (mode==File ? User1 : (ButtonCode)0) |Ok|Cancel, 
00146     Ok,
00147     parent, 
00148     name ? name : "KexiCSVImportDialog", 
00149     true, 
00150     false,
00151     KGuiItem( i18n("&Options"))
00152   ),
00153     m_mainWin(mainWin),
00154     m_cancelled( false ),
00155     m_adjustRows( true ),
00156     m_startline( 0 ),
00157     m_textquote( QString(KEXICSV_DEFAULT_FILE_TEXT_QUOTE)[0] ),
00158     m_mode(mode),
00159     m_prevSelectedCol(-1),
00160     m_columnsAdjusted(false),
00161     m_1stRowForFieldNamesDetected(false),
00162     m_firstFillTableCall(true),
00163     m_blockUserEvents(false),
00164     m_primaryKeyColumn(-1),
00165     m_dialogCancelled(false),
00166     m_conn(0),
00167     m_destinationTableSchema(0),
00168     m_allRowsLoadedInPreview(false),
00169     m_stoppedAt_MAX_BYTES_TO_PREVIEW(false)
00170 {
00171     setWFlags(getWFlags() | Qt::WStyle_Maximize | Qt::WStyle_SysMenu);
00172     hide();
00173     setButtonOK(KGuiItem( i18n("&Import..."), _IMPORT_ICON));
00174 
00175     m_typeNames.resize(5);
00176     m_typeNames[0] = i18n("text");
00177     m_typeNames[1] = i18n("number");
00178     m_typeNames[2] = i18n("date");
00179     m_typeNames[3] = i18n("time");
00180     m_typeNames[4] = i18n("date/time");
00181 
00182     kapp->config()->setGroup("ImportExport");
00183     m_maximumRowsForPreview = kapp->config()->readNumEntry("MaximumRowsForPreviewInImportDialog", MAX_ROWS_TO_PREVIEW);
00184     m_maximumBytesForPreview = kapp->config()->readNumEntry("MaximumBytesForPreviewInImportDialog", MAX_BYTES_TO_PREVIEW);
00185 
00186     m_pkIcon = SmallIcon("key");
00187 
00188     m_uniquenessTest.setAutoDelete(true);
00189 
00190     setIcon(DesktopIcon(_IMPORT_ICON));
00191     setSizeGripEnabled( TRUE );
00192 
00193 //  m_encoding = QString::fromLatin1(KGlobal::locale()->encoding());
00194 //  m_stripWhiteSpaceInTextValuesChecked = true;
00195     m_file = 0;
00196     m_inputStream = 0;
00197     
00198     QVBoxLayout *lyr = new QVBoxLayout(plainPage(), 0, KDialogBase::spacingHint(), "lyr");
00199 
00200     m_infoLbl = new KexiCSVInfoLabel(
00201         m_mode==File ? i18n("Preview of data from file:")
00202         : i18n("Preview of data from clipboard:"),
00203         plainPage()
00204     );
00205     lyr->addWidget( m_infoLbl );
00206 
00207     QWidget* page = new QFrame( plainPage(), "page" );
00208     QGridLayout *glyr= new QGridLayout( page, 4, 5, 0, KDialogBase::spacingHint(), "glyr");
00209     lyr->addWidget( page );
00210 
00211     // Delimiter: comma, semicolon, tab, space, other
00212     m_delimiterWidget = new KexiCSVDelimiterWidget(true /*lineEditOnBottom*/, page);
00213     m_detectDelimiter = true;
00214     glyr->addMultiCellWidget( m_delimiterWidget, 1, 2, 0, 0 );
00215 
00216     QLabel *delimiterLabel = new QLabel(m_delimiterWidget, i18n("Delimiter:"), page);
00217     delimiterLabel->setAlignment(Qt::AlignAuto | Qt::AlignBottom);
00218     glyr->addMultiCellWidget( delimiterLabel, 0, 0, 0, 0 );
00219 
00220     // Format: number, text, currency,
00221     m_formatComboText = i18n( "Format for column %1:" );
00222     m_formatCombo = new KComboBox(page, "m_formatCombo");
00223     m_formatCombo->insertItem(i18n("Text"));
00224     m_formatCombo->insertItem(i18n("Number"));
00225     m_formatCombo->insertItem(i18n("Date"));
00226     m_formatCombo->insertItem(i18n("Time"));
00227     m_formatCombo->insertItem(i18n("Date/Time"));
00228     glyr->addMultiCellWidget( m_formatCombo, 1, 1, 1, 1 );
00229 
00230     m_formatLabel = new QLabel(m_formatCombo, "", page);
00231     m_formatLabel->setAlignment(Qt::AlignAuto | Qt::AlignBottom);
00232     glyr->addWidget( m_formatLabel, 0, 1 );
00233 
00234     m_primaryKeyField = new QCheckBox( i18n( "Primary key" ), page, "m_primaryKeyField" );
00235     glyr->addWidget( m_primaryKeyField, 2, 1 );
00236     connect(m_primaryKeyField, SIGNAL(toggled(bool)), this, SLOT(slotPrimaryKeyFieldToggled(bool)));
00237 
00238     m_comboQuote = new KexiCSVTextQuoteComboBox( page );
00239     glyr->addWidget( m_comboQuote, 1, 2 );
00240 
00241     TextLabel2 = new QLabel( m_comboQuote, i18n( "Text quote:" ), page, "TextLabel2" );
00242     TextLabel2->setSizePolicy( QSizePolicy::Fixed, QSizePolicy::Preferred );
00243     TextLabel2->setAlignment(Qt::AlignAuto | Qt::AlignBottom);
00244     glyr->addWidget( TextLabel2, 0, 2 );
00245 
00246     m_startAtLineSpinBox = new KIntSpinBox( page, "m_startAtLineSpinBox" );
00247     m_startAtLineSpinBox->setMinValue(1);
00248     m_startAtLineSpinBox->setSizePolicy( QSizePolicy::Fixed, QSizePolicy::Fixed );
00249     m_startAtLineSpinBox->setMinimumWidth(QFontMetrics(m_startAtLineSpinBox->font()).width("8888888"));
00250     glyr->addWidget( m_startAtLineSpinBox, 1, 3 );
00251 
00252     m_startAtLineLabel = new QLabel( m_startAtLineSpinBox, "", 
00253         page, "TextLabel3" );
00254     m_startAtLineLabel->setSizePolicy( QSizePolicy::Fixed, QSizePolicy::Preferred );
00255     m_startAtLineLabel->setAlignment(Qt::AlignAuto | Qt::AlignBottom);
00256     glyr->addWidget( m_startAtLineLabel, 0, 3 );
00257 
00258     QSpacerItem* spacer_2 = new QSpacerItem( 0, 0, QSizePolicy::Minimum, QSizePolicy::Preferred );
00259     glyr->addItem( spacer_2, 0, 4 );
00260 
00261     m_ignoreDuplicates = new QCheckBox( page, "m_ignoreDuplicates" );
00262     m_ignoreDuplicates->setText( i18n( "Ignore duplicated delimiters" ) );
00263     glyr->addMultiCellWidget( m_ignoreDuplicates, 2, 2, 2, 4 );
00264 
00265     m_1stRowForFieldNames = new QCheckBox( page, "m_1stRowForFieldNames" );
00266     m_1stRowForFieldNames->setText( i18n( "First row contains column names" ) );
00267     glyr->addMultiCellWidget( m_1stRowForFieldNames, 3, 3, 2, 4 );
00268 
00269     m_table = new KexiCSVImportDialogTable( plainPage(), "m_table" );
00270     lyr->addWidget( m_table );
00271 
00272     m_table->setSizePolicy( QSizePolicy(QSizePolicy::MinimumExpanding, QSizePolicy::MinimumExpanding, 1, 1) );
00273     m_table->setNumRows( 0 );
00274     m_table->setNumCols( 0 );
00275 
00277 /*
00278 if ( m_mode == Clipboard )
00279   {
00280     setCaption( i18n( "Inserting From Clipboard" ) );
00281     QMimeSource * mime = QApplication::clipboard()->data();
00282     if ( !mime )
00283     {
00284       KMessageBox::information( this, i18n("There is no data in the clipboard.") );
00285       m_cancelled = true;
00286       return;
00287     }
00288 
00289     if ( !mime->provides( "text/plain" ) )
00290     {
00291       KMessageBox::information( this, i18n("There is no usable data in the clipboard.") );
00292       m_cancelled = true;
00293       return;
00294     }
00295     m_fileArray = QByteArray(mime->encodedData( "text/plain" ) );
00296   }
00297   else if ( mode == File )
00298   {*/
00299     m_dateRegExp = QRegExp("(\\d{1,4})([/\\-\\.])(\\d{1,2})([/\\-\\.])(\\d{1,4})");
00300     m_timeRegExp1 = QRegExp("(\\d{1,2}):(\\d{1,2}):(\\d{1,2})");
00301     m_timeRegExp2 = QRegExp("(\\d{1,2}):(\\d{1,2})");
00302     m_fpNumberRegExp = QRegExp("[\\-]{0,1}\\d*[,\\.]\\d+");
00303 
00304     if (m_mode == File) {
00305         QStringList mimetypes( csvMimeTypes() );
00306 #ifdef Q_WS_WIN
00308         QString recentDir = KGlobalSettings::documentPath();
00309         m_fname = QFileDialog::getOpenFileName( 
00310             KFileDialog::getStartURL(":CSVImportExport", recentDir).path(),
00311             KexiUtils::fileDialogFilterStrings(mimetypes, false),
00312             page, "KexiCSVImportDialog", i18n("Open CSV Data File"));
00313         if ( !m_fname.isEmpty() ) {
00314             //save last visited path
00315             KURL url;
00316             url.setPath( m_fname );
00317             if (url.isLocalFile())
00318                 KRecentDirs::add(":CSVImportExport", url.directory());
00319         }
00320 #else
00321         m_fname = KFileDialog::getOpenFileName(":CSVImportExport", mimetypes.join(" "), this);
00322 #endif
00323         //cancel action !
00324         if ( m_fname.isEmpty() )
00325         {
00326             actionButton( Ok )->setEnabled( false );
00327             m_cancelled = true;
00328             if (parentWidget())
00329                 parentWidget()->raise();
00330             return;
00331         }
00332     }
00333     else if (m_mode == Clipboard) {
00334         QCString subtype("plain");
00335         m_clipboardData = QApplication::clipboard()->text(subtype, QClipboard::Clipboard);
00336 /* debug
00337         for (int i=0;QApplication::clipboard()->data(QClipboard::Clipboard)->format(i);i++)
00338             kdDebug() << i << ": " 
00339                 << QApplication::clipboard()->data(QClipboard::Clipboard)->format(i) << endl;
00340 */
00341     }
00342     else {
00343         return;
00344     }
00345 
00346     m_loadingProgressDlg = 0;
00347     m_importingProgressDlg = 0;
00348     if (m_mode == File) {
00349         m_loadingProgressDlg = new KProgressDialog(
00350             this, "m_loadingProgressDlg", i18n("Loading CSV Data"), i18n("Loading CSV Data from \"%1\"...")
00351             .arg(QDir::convertSeparators(m_fname)), true);
00352         m_loadingProgressDlg->progressBar()->setTotalSteps( m_maximumRowsForPreview+1 );
00353         m_loadingProgressDlg->show();
00354     }
00355 
00356     if (m_mode==Clipboard) {
00357         m_infoLbl->setIcon("editpaste");
00358     }
00359     //updateRowCountInfo();
00360 
00361     m_table->setSelectionMode(QTable::NoSelection);
00362 
00363     connect(m_formatCombo, SIGNAL(activated(int)),
00364       this, SLOT(formatChanged(int)));
00365     connect(m_delimiterWidget, SIGNAL(delimiterChanged(const QString&)),
00366       this, SLOT(delimiterChanged(const QString&)));
00367     connect(m_startAtLineSpinBox, SIGNAL(valueChanged ( int )),
00368       this, SLOT(startlineSelected(int)));
00369     connect(m_comboQuote, SIGNAL(activated(int)),
00370       this, SLOT(textquoteSelected(int)));
00371     connect(m_table, SIGNAL(currentChanged(int, int)),
00372       this, SLOT(currentCellChanged(int, int)));
00373     connect(m_table, SIGNAL(valueChanged(int,int)),
00374       this, SLOT(cellValueChanged(int,int)));
00375     connect(m_ignoreDuplicates, SIGNAL(stateChanged(int)),
00376       this, SLOT(ignoreDuplicatesChanged(int)));
00377     connect(m_1stRowForFieldNames, SIGNAL(stateChanged(int)),
00378       this, SLOT(slot1stRowForFieldNamesChanged(int)));
00379 
00380     connect(this, SIGNAL(user1Clicked()), this, SLOT(optionsButtonClicked()));
00381 
00382     installRecursiveEventFilter(this, this);
00383 
00384     initLater();
00385 }
00386 
00387 KexiCSVImportDialog::~KexiCSVImportDialog()
00388 {
00389     delete m_file;
00390 }
00391 
00392 void KexiCSVImportDialog::initLater()
00393 {
00394     if (!openData())
00395         return;
00396 
00397 //  delimiterChanged(detectedDelimiter); // this will cause fillTable()
00398     m_columnsAdjusted = false;
00399     fillTable();
00400     delete m_loadingProgressDlg;
00401     m_loadingProgressDlg = 0;
00402     if (m_dialogCancelled) {
00403 //      m_loadingProgressDlg->hide();
00404     //  m_loadingProgressDlg->close();
00405         QTimer::singleShot(0, this, SLOT(reject()));
00406         return;
00407     }
00408 
00409     currentCellChanged(0, 0);
00410 
00411 //  updateGeometry();
00412     adjustSize();
00413     KDialog::centerOnScreen( this ); 
00414 
00415     if (m_loadingProgressDlg)
00416         m_loadingProgressDlg->hide();
00417     show();
00418     m_table->setFocus();
00419 }
00420 
00421 bool KexiCSVImportDialog::openData()
00422 {
00423     if (m_mode!=File) //data already loaded, no encoding stuff needed
00424         return true;
00425 
00426     delete m_inputStream;
00427     m_inputStream = 0;
00428     if (m_file) {
00429         m_file->close();
00430         delete m_file;
00431     }
00432     m_file = new QFile(m_fname);
00433     if (!m_file->open(IO_ReadOnly))
00434     {
00435         m_file->close();
00436         delete m_file;
00437         m_file = 0;
00438         KMessageBox::sorry( this, i18n("Cannot open input file <nobr>\"%1\"</nobr>.")
00439             .arg(QDir::convertSeparators(m_fname)) );
00440         actionButton( Ok )->setEnabled( false );
00441         m_cancelled = true;
00442         if (parentWidget())
00443             parentWidget()->raise();
00444         return false;
00445     }
00446     return true;
00447 }
00448 
00449 bool KexiCSVImportDialog::cancelled() const
00450 {
00451     return m_cancelled;
00452 }
00453 
00454 void KexiCSVImportDialog::fillTable()
00455 {
00456     KexiUtils::WaitCursor wc(true);
00457     repaint();
00458     m_blockUserEvents = true;
00459     QPushButton *pb = actionButton(KDialogBase::Cancel);
00460     if (pb)
00461         pb->setEnabled(true); //allow to cancel
00462     KexiUtils::WaitCursor wait;
00463 
00464     if (m_table->numRows()>0) //to accept editor
00465         m_table->setCurrentCell(0,0);
00466 
00467     int row, column, maxColumn;
00468     QString field = QString::null;
00469 
00470     for (row = 0; row < m_table->numRows(); ++row)
00471         for (column = 0; column < m_table->numCols(); ++column)
00472             m_table->clearCell(row, column);
00473 
00474     m_detectedTypes.clear();
00475     m_detectedTypes.resize(1024, _NO_TYPE_YET);//_TEXT_TYPE);
00476     m_uniquenessTest.clear();
00477     m_uniquenessTest.resize(1024);
00478     m_1stRowForFieldNamesDetected = true;
00479 
00480     if (true != loadRows(field, row, column, maxColumn, true))
00481         return;
00482 
00483     m_1stRowForFieldNamesDetected = false;
00484 
00485     // file with only one line without '\n'
00486     if (field.length() > 0)
00487     {
00488         setText(row - m_startline, column, field, true);
00489         ++row;
00490         field = QString::null;
00491     }
00492 
00493     adjustRows( row - m_startline - (m_1stRowForFieldNames->isChecked()?1:0) );
00494 
00495     maxColumn = QMAX( maxColumn, column );
00496     m_table->setNumCols(maxColumn);
00497 
00498     for (column = 0; column < m_table->numCols(); ++column)
00499     {
00500 //      QString header = m_table->horizontalHeader()->label(column);
00501 //      if (header != i18n("Text") && header != i18n("Number") &&
00502 //          header != i18n("Date") && header != i18n("Currency"))
00503 //      const int detectedType = m_detectedTypes[column+1];
00504 //      m_table->horizontalHeader()->setLabel(column, m_typeNames[ detectedType ]); //i18n("Text"));
00505         updateColumnText(column);
00506         if (!m_columnsAdjusted)
00507             m_table->adjustColumn(column);
00508     }
00509     m_columnsAdjusted = true;
00510 
00511     if (m_primaryKeyColumn>=0 && m_primaryKeyColumn<m_table->numCols()) {
00512         if (_NUMBER_TYPE != m_detectedTypes[ m_primaryKeyColumn ]) {
00513             m_primaryKeyColumn = -1;
00514         }
00515     }
00516 
00517     m_prevSelectedCol = -1;
00518     m_table->setCurrentCell(0,0);
00519     currentCellChanged(0, 0);
00520     if (m_primaryKeyColumn != -1)
00521         m_table->setPixmap(0, m_primaryKeyColumn, m_pkIcon);
00522 
00523     const int count = QMAX(0, m_table->numRows()-1+m_startline);
00524     m_allRowsLoadedInPreview = count < m_maximumRowsForPreview && !m_stoppedAt_MAX_BYTES_TO_PREVIEW;
00525     if (m_allRowsLoadedInPreview) {
00526         m_startAtLineSpinBox->setMaxValue(count);
00527         m_startAtLineSpinBox->setValue(m_startline+1);
00528     }
00529     m_startAtLineLabel->setText(i18n( "Start at line%1:").arg(
00530             m_allRowsLoadedInPreview ? QString(" (1-%1)").arg(count)
00531             : QString::null //we do not know what's real count
00532     ));
00533     updateRowCountInfo();
00534 
00535     m_blockUserEvents = false;
00536     repaint();
00537     m_table->verticalScrollBar()->repaint();//avoid missing repaint
00538     m_table->horizontalScrollBar()->repaint();//avoid missing repaint
00539 }
00540 
00541 QString KexiCSVImportDialog::detectDelimiterByLookingAtFirstBytesOfFile(QTextStream& inputStream)
00542 {
00543     m_file->at(0);
00544 
00545     // try to detect delimiter
00546     // \t has priority, then ; then ,
00547     const QIODevice::Offset origOffset = inputStream.device()->at();
00548     QChar c, prevChar=0;
00549     int detectedDelimiter = 0;
00550     bool insideQuote = false;
00551 
00552     //characters by priority
00553     const int CH_TAB_AFTER_QUOTE = 500;
00554     const int CH_SEMICOLON_AFTER_QUOTE = 499;
00555     const int CH_COMMA_AFTER_QUOTE = 498;
00556     const int CH_TAB = 200; // \t
00557     const int CH_SEMICOLON = 199; // ;
00558     const int CH_COMMA = 198; // ,
00559 
00560     QValueList<int> tabsPerLine, semicolonsPerLine, commasPerLine;
00561     int tabs = 0, semicolons = 0, commas = 0;
00562     int line = 0;
00563     for (uint i=0; !inputStream.atEnd() && i < MAX_CHARS_TO_SCAN_WHILE_DETECTING_DELIMITER; i++) {
00564         (*m_inputStream) >> c; // read one char
00565         if (prevChar=='"') {
00566             if (c!='"') //real quote (not double "")
00567                 insideQuote = !insideQuote;
00568         }
00569         if (insideQuote) {
00570             prevChar = c;
00571             continue;
00572         }
00573         if (c==' ')
00574             continue;
00575         if (c=='\n') {//end of line
00576             //remember # of tabs/semicolons/commas in this line
00577             tabsPerLine += tabs;
00578             tabs = 0;
00579             semicolonsPerLine += semicolons;
00580             semicolons = 0;
00581             commasPerLine += commas;
00582             commas = 0;
00583             line++;
00584         }
00585         else if (c=='\t') {
00586             tabs++;
00587             detectedDelimiter = QMAX( prevChar=='"' ? CH_TAB_AFTER_QUOTE : CH_TAB, detectedDelimiter );
00588         }
00589         else if (c==';') {
00590             semicolons++;
00591             detectedDelimiter = QMAX( prevChar=='"' ? CH_SEMICOLON_AFTER_QUOTE : CH_SEMICOLON, detectedDelimiter );
00592         }
00593         else if (c==',') {
00594             commas++;
00595             detectedDelimiter = QMAX( prevChar=='"' ? CH_COMMA_AFTER_QUOTE : CH_COMMA, detectedDelimiter );
00596         }
00597         prevChar = c;
00598     }
00599 
00600     inputStream.device()->at(origOffset); //restore orig. offset
00601 
00602     //now, try to find a delimiter character that exists the same number of times in all the checked lines
00603     //this detection method has priority over others
00604     QValueList<int>::ConstIterator it;
00605     if (tabsPerLine.count()>1) {
00606         tabs = tabsPerLine.isEmpty() ? 0 : tabsPerLine.first();
00607         for (it=tabsPerLine.constBegin(); it!=tabsPerLine.constEnd(); ++it) {
00608             if (tabs != *it)
00609                 break;
00610         }
00611         if (tabs>0 && it==tabsPerLine.constEnd())
00612             return "\t";
00613     }
00614     if (semicolonsPerLine.count()>1) {
00615         semicolons = semicolonsPerLine.isEmpty() ? 0 : semicolonsPerLine.first();
00616         for (it=semicolonsPerLine.constBegin(); it!=semicolonsPerLine.constEnd(); ++it) {
00617             if (semicolons != *it)
00618                 break;
00619         }
00620         if (semicolons > 0 && it==semicolonsPerLine.constEnd())
00621             return ";";
00622     }
00623     if (commasPerLine.count()>1) {
00624         commas = commasPerLine.first();
00625         for (it=commasPerLine.constBegin(); it!=commasPerLine.constEnd(); ++it) {
00626             if (commas != *it)
00627                 break;
00628         }
00629         if (commas > 0 && it==commasPerLine.constEnd())
00630             return ",";
00631     }
00632     //now return the winning character by looking at CH_* symbol
00633     if (detectedDelimiter == CH_TAB_AFTER_QUOTE || detectedDelimiter == CH_TAB)
00634         return "\t";
00635     if (detectedDelimiter == CH_SEMICOLON_AFTER_QUOTE || detectedDelimiter == CH_SEMICOLON)
00636         return ";";
00637     if (detectedDelimiter == CH_COMMA_AFTER_QUOTE || detectedDelimiter == CH_COMMA)
00638         return ",";
00639 
00640     return KEXICSV_DEFAULT_FILE_DELIMITER; //<-- default
00641 }
00642 
00643 tristate KexiCSVImportDialog::loadRows(QString &field, int &row, int &column, int &maxColumn, 
00644     bool inGUI)
00645 {
00646     enum { S_START, S_QUOTED_FIELD, S_MAYBE_END_OF_QUOTED_FIELD, S_END_OF_QUOTED_FIELD,
00647          S_MAYBE_NORMAL_FIELD, S_NORMAL_FIELD } state = S_START;
00648     field = QString::null;
00649     const bool ignoreDups = m_ignoreDuplicates->isChecked();
00650     bool lastCharDelimiter = false;
00651     bool nextRow = false;
00652     row = column = 1;
00653     maxColumn = 0;
00654     QChar x;
00655     const bool hadInputStream = m_inputStream!=0;
00656     delete m_inputStream;
00657     if ( m_mode == Clipboard ) {
00658         m_inputStream = new QTextStream(m_clipboardData, IO_ReadOnly);
00659         if (!hadInputStream)
00660             m_delimiterWidget->setDelimiter(KEXICSV_DEFAULT_CLIPBOARD_DELIMITER);
00661     }
00662     else {
00663         m_file->at(0); //always seek at 0 because loadRows() is called many times
00664         m_inputStream = new QTextStream(m_file);
00665         if (m_options.defaultEncodingExplicitySet) {
00666             QTextCodec *codec = KGlobal::charsets()->codecForName(m_options.encoding);
00667             if (codec)
00668                 m_inputStream->setCodec(codec); //QTextCodec::codecForName("CP1250"));
00669         }
00670         if (m_detectDelimiter) {
00671             const QString delimiter( detectDelimiterByLookingAtFirstBytesOfFile(*m_inputStream) );
00672             if (m_delimiterWidget->delimiter() != delimiter)
00673                 m_delimiterWidget->setDelimiter( delimiter );
00674         }
00675     }
00676     const QChar delimiter(m_delimiterWidget->delimiter()[0]);
00677     m_stoppedAt_MAX_BYTES_TO_PREVIEW = false;
00678     int progressStep = 0;
00679     if (m_importingProgressDlg)
00680         progressStep = QMAX( 1, m_importingProgressDlg->progressBar()->totalSteps()/200 );
00681     int offset = 0;
00682     for (;!m_inputStream->atEnd(); offset++)
00683     {
00684 //disabled: this breaks wide spreadsheets
00685 //  if (column >= m_maximumRowsForPreview)
00686 //      return true;
00687 
00688         if (m_importingProgressDlg && ((offset % progressStep) < 5)) {
00689             //update progr. bar dlg on final exporting
00690             m_importingProgressDlg->progressBar()->setValue(offset);
00691             qApp->processEvents();
00692             if (m_importingProgressDlg->wasCancelled()) {
00693                 delete m_importingProgressDlg;
00694                 m_importingProgressDlg = 0;
00695                 return ::cancelled;
00696             }
00697         }
00698 
00699         (*m_inputStream) >> x; // read one char
00700 
00701         if (x == '\r') {
00702             continue; // eat '\r', to handle RFC-compliant files
00703         }
00704         if (offset==0 && x.unicode()==0xfeff) {
00705             // Ignore BOM, the "Byte Order Mark" 
00706             // (http://en.wikipedia.org/wiki/Byte_Order_Mark, // http://www.unicode.org/charts/PDF/UFFF0.pdf)
00707             // Probably fixed in Qt4.
00708             continue;
00709         }
00710 
00711         switch (state)
00712         {
00713         case S_START :
00714             if (x == m_textquote)
00715             {
00716                 state = S_QUOTED_FIELD;
00717             }
00718             else if (x == delimiter)
00719             {
00720                 setText(row - m_startline, column, field, inGUI);
00721                 field = QString::null;
00722                 if ((ignoreDups == false) || (lastCharDelimiter == false))
00723                     ++column;
00724                 lastCharDelimiter = true;
00725             }
00726             else if (x == '\n')
00727             {
00728                 if (!inGUI) {
00729                     //fill remaining empty fields (database wants them explicitly)
00730                     for (int additionalColumn = column; additionalColumn <= maxColumn; additionalColumn++) {
00731                         setText(row - m_startline, additionalColumn, QString::null, inGUI);
00732                     }
00733                 }
00734                 nextRow = true;
00735                 maxColumn = QMAX( maxColumn, column );
00736                 column = 1;
00737             }
00738             else
00739             {
00740                 field += x;
00741                 state = S_MAYBE_NORMAL_FIELD;
00742             }
00743             break;
00744         case S_QUOTED_FIELD :
00745             if (x == m_textquote)
00746             {
00747                 state = S_MAYBE_END_OF_QUOTED_FIELD;
00748             }
00749 /*allow \n inside quoted fields
00750             else if (x == '\n')
00751             {
00752                 setText(row - m_startline, column, field, inGUI);
00753                 field = "";
00754                 if (x == '\n')
00755                 {
00756                     nextRow = true;
00757                     maxColumn = QMAX( maxColumn, column );
00758                     column = 1;
00759                 }
00760                 else
00761                 {
00762                     if ((ignoreDups == false) || (lastCharDelimiter == false))
00763                         ++column;
00764                     lastCharDelimiter = true;
00765                 }
00766                 state = S_START;
00767             }*/
00768             else
00769             {
00770                 field += x;
00771             }
00772             break;
00773         case S_MAYBE_END_OF_QUOTED_FIELD :
00774             if (x == m_textquote)
00775             {
00776                 field += x; //no, this was just escaped quote character
00777                 state = S_QUOTED_FIELD;
00778             }
00779             else if (x == delimiter || x == '\n')
00780             {
00781                 setText(row - m_startline, column, field, inGUI);
00782                 field = QString::null;
00783                 if (x == '\n')
00784                 {
00785                     nextRow = true;
00786                     maxColumn = QMAX( maxColumn, column );
00787                     column = 1;
00788                 }
00789                 else
00790                 {
00791                     if ((ignoreDups == false) || (lastCharDelimiter == false))
00792                         ++column;
00793                     lastCharDelimiter = true;
00794                 }
00795                 state = S_START;
00796             }
00797             else
00798             {
00799                 state = S_END_OF_QUOTED_FIELD;
00800             }
00801             break;
00802         case S_END_OF_QUOTED_FIELD :
00803             if (x == delimiter || x == '\n')
00804             {
00805                 setText(row - m_startline, column, field, inGUI);
00806                 field = QString::null;
00807                 if (x == '\n')
00808                 {
00809                     nextRow = true;
00810                     maxColumn = QMAX( maxColumn, column );
00811                     column = 1;
00812                 }
00813                 else
00814                 {
00815                     if ((ignoreDups == false) || (lastCharDelimiter == false))
00816                         ++column;
00817                     lastCharDelimiter = true;
00818                 }
00819                 state = S_START;
00820             }
00821             else
00822             {
00823                 state = S_END_OF_QUOTED_FIELD;
00824             }
00825             break;
00826         case S_MAYBE_NORMAL_FIELD :
00827             if (x == m_textquote)
00828             {
00829                 field = QString::null;
00830                 state = S_QUOTED_FIELD;
00831                 break;
00832             }
00833         case S_NORMAL_FIELD :
00834             if (x == delimiter || x == '\n')
00835             {
00836                 setText(row - m_startline, column, field, inGUI);
00837                 field = QString::null;
00838                 if (x == '\n')
00839                 {
00840                     nextRow = true;
00841                     maxColumn = QMAX( maxColumn, column );
00842                     column = 1;
00843                 }
00844                 else
00845                 {
00846                     if ((ignoreDups == false) || (lastCharDelimiter == false))
00847                         ++column;
00848                     lastCharDelimiter = true;
00849                 }
00850                 state = S_START;
00851             }
00852             else
00853             {
00854                 field += x;
00855             }
00856         }
00857         if (x != delimiter)
00858             lastCharDelimiter = false;
00859 
00860         if (nextRow) {
00861             if (!inGUI && row==1 && m_1stRowForFieldNames->isChecked()) {
00862                 // do not save to the database 1st row if it contains column names
00863                 m_importingStatement->clearArguments();
00864             }
00865             else if (!saveRow(inGUI))
00866                 return false;
00867     
00868             ++row;
00869         }
00870 
00871         if (m_firstFillTableCall && row==2 
00872             && !m_1stRowForFieldNames->isChecked() && m_1stRowForFieldNamesDetected) 
00873         {
00874             //'1st row for field name' flag detected: reload table
00875             m_1stRowForFieldNamesDetected = false;
00876             m_table->setNumRows( 0 );
00877             m_firstFillTableCall = false; //this trick is allowed only once, on startup
00878             m_1stRowForFieldNames->setChecked(true); //this will reload table
00879             //slot1stRowForFieldNamesChanged(1);
00880             m_blockUserEvents = false;
00881             repaint();
00882             return false;
00883         }
00884 
00885         if (!m_importingProgressDlg && row % 20 == 0) {
00886             qApp->processEvents();
00887             //only for GUI mode:
00888             if (!m_firstFillTableCall && m_loadingProgressDlg && m_loadingProgressDlg->wasCancelled()) {
00889                 delete m_loadingProgressDlg;
00890                 m_loadingProgressDlg = 0;
00891                 m_dialogCancelled = true;
00892                 reject();
00893                 return false;
00894             }
00895         }
00896 
00897         if (!m_firstFillTableCall && m_loadingProgressDlg) {
00898             m_loadingProgressDlg->progressBar()->setValue(QMIN(m_maximumRowsForPreview, row));
00899         }
00900 
00901         if ( inGUI && row > (m_maximumRowsForPreview + (m_1stRowForFieldNamesDetected?1:0)) ) {
00902             kexipluginsdbg << "KexiCSVImportDialog::fillTable() loading stopped at row #" 
00903                 << m_maximumRowsForPreview << endl;
00904             break;
00905         }
00906         if (nextRow) {
00907             nextRow = false;
00908             //additional speedup: stop processing now if too many bytes were loaded for preview
00909             kexipluginsdbg << offset << endl;
00910             if (inGUI && offset >= m_maximumBytesForPreview && row >= 2) {
00911                 m_stoppedAt_MAX_BYTES_TO_PREVIEW = true;
00912                 return true;
00913             }
00914         }
00915     }
00916     return true;
00917 }
00918 
00919 void KexiCSVImportDialog::updateColumnText(int col)
00920 {
00921     QString colName;
00922     if (col<(int)m_columnNames.count() && (m_1stRowForFieldNames->isChecked() || m_changedColumnNames[col]))
00923         colName = m_columnNames[ col ];
00924     if (colName.isEmpty()) {
00925         colName = i18n("Column %1").arg(col+1); //will be changed to a valid identifier on import
00926         m_changedColumnNames[ col ] = false;
00927     }
00928     int detectedType = m_detectedTypes[col];
00929     if (detectedType==_FP_NUMBER_TYPE)
00930         detectedType=_NUMBER_TYPE; //we're simplifying that for now
00931     else if (detectedType==_NO_TYPE_YET) {
00932         m_detectedTypes[col]=_TEXT_TYPE; //entirely empty column
00933         detectedType=_TEXT_TYPE;
00934     }
00935     m_table->horizontalHeader()->setLabel(col, 
00936         i18n("Column %1").arg(col+1) + "  \n(" + m_typeNames[ detectedType ] + ")  ");
00937     m_table->setText(0, col, colName);
00938     m_table->horizontalHeader()->adjustHeaderSize();
00939 
00940     //check uniqueness
00941     QValueList<int> *list = m_uniquenessTest[col];
00942     if (m_primaryKeyColumn==-1 && list && !list->isEmpty()) {
00943         qHeapSort(*list);
00944         QValueList<int>::ConstIterator it=list->constBegin();
00945         int prevValue = *it;
00946         ++it;
00947         for(; it!=list->constEnd() && prevValue!=(*it); ++it)
00948             prevValue=(*it);
00949         if (it!=list->constEnd()) {
00950             //duplicates:
00951             list->clear();
00952         }
00953         else {
00954             //a candidate for PK (autodetected)!
00955             if (-1==m_primaryKeyColumn) {
00956                 m_primaryKeyColumn=col;
00957             }
00958         }
00959     }
00960     if (list) //not needed now: conserve memory
00961         list->clear();
00962 }
00963 
00964 void KexiCSVImportDialog::detectTypeAndUniqueness(int row, int col, const QString& text)
00965 {
00966     int intValue;
00967     const int type = m_detectedTypes[col];
00968     if (row==1 || type!=_TEXT_TYPE) {
00969         bool found = false;
00970         if (text.isEmpty() && type==_NO_TYPE_YET)
00971             found = true; //real type should be found later
00972         //detect type because it's 1st row or all prev. rows were not text
00973         //-FP number? (trying before "number" type is a must)
00974         if (!found && (row==1 || type==_NUMBER_TYPE || type==_FP_NUMBER_TYPE || type==_NO_TYPE_YET)) {
00975             bool ok = text.isEmpty() || m_fpNumberRegExp.exactMatch(text);
00976             //if (!ok)
00977             //  text.toDouble(&ok);
00978             if (ok && (row==1 || type==_NUMBER_TYPE || type==_FP_NUMBER_TYPE || type==_NO_TYPE_YET)) {
00979                 m_detectedTypes[col]=_FP_NUMBER_TYPE;
00980                 found = true; //yes
00981             }
00982         }
00983         //-number?
00984         if (!found && (row==1 || type==_NUMBER_TYPE || type==_NO_TYPE_YET)) {
00985             bool ok = text.isEmpty();//empty values allowed
00986             if (!ok)
00987                 intValue = text.toInt(&ok);
00988             if (ok && (row==1 || type==_NO_TYPE_YET)) {
00989                 m_detectedTypes[col]=_NUMBER_TYPE;
00990                 found = true; //yes
00991             }
00992         }
00993         //-date?
00994         if (!found && (row==1 || type==_DATE_TYPE || type==_NO_TYPE_YET)) {
00995             if ((row==1 || type==_NO_TYPE_YET)
00996                 && (text.isEmpty() || m_dateRegExp.exactMatch(text)))
00997             {
00998                 m_detectedTypes[col]=_DATE_TYPE;
00999                 found = true; //yes
01000             }
01001         }
01002         //-time?
01003         if (!found && (row==1 || type==_TIME_TYPE || type==_NO_TYPE_YET)) {
01004             if ((row==1 || type==_NO_TYPE_YET)
01005                 && (text.isEmpty() || m_timeRegExp1.exactMatch(text) || m_timeRegExp2.exactMatch(text)))
01006             {
01007                 m_detectedTypes[col]=_TIME_TYPE;
01008                 found = true; //yes
01009             }
01010         }
01011         //-date/time?
01012         if (!found && (row==1 || type==_TIME_TYPE || type==_NO_TYPE_YET)) {
01013             if (row==1 || type==_NO_TYPE_YET) {
01014                 bool detected = text.isEmpty();
01015                 if (!detected) {
01016                     const QStringList dateTimeList( QStringList::split(" ", text) );
01017                     bool ok = dateTimeList.count()>=2;
01020                     if (ok) {
01021                         //try all combinations
01022                         QString datePart( dateTimeList[0].stripWhiteSpace() );
01023                         QString timePart( dateTimeList[1].stripWhiteSpace() );
01024                         ok = m_dateRegExp.exactMatch(datePart)
01025                             && (m_timeRegExp1.exactMatch(timePart) || m_timeRegExp2.exactMatch(timePart));
01026                     }
01027                     detected = ok;
01028                 }
01029                 if (detected) {
01030                     m_detectedTypes[col]=_DATETIME_TYPE;
01031                     found = true; //yes
01032                 }
01033             }
01034         }
01035         if (!found && type==_NO_TYPE_YET && !text.isEmpty()) {
01036             //eventually, a non-emptytext after a while
01037             m_detectedTypes[col]=_TEXT_TYPE;
01038             found = true; //yes
01039         }
01040         //default: text type (already set)
01041     }
01042     //check uniqueness for this value
01043     QValueList<int> *list = m_uniquenessTest[col];
01044     if (row==1 && (!list || !list->isEmpty()) && !text.isEmpty() && _NUMBER_TYPE == m_detectedTypes[col]) {
01045         if (!list) {
01046             list = new QValueList<int>();
01047             m_uniquenessTest.insert(col, list);
01048         }
01049         list->append( intValue );
01050     }
01051     else {
01052         //the value is empty or uniqueness test failed in the past
01053         if (list && !list->isEmpty())
01054             list->clear(); //indicate that uniqueness test failed
01055     }
01056 }
01057 
01058 bool KexiCSVImportDialog::parseDate(const QString& text, QDate& date)
01059 {
01060     if (!m_dateRegExp.exactMatch(text))
01061         return false;
01062     //dddd - dd - dddd
01063     //1    2 3  4 5    <- pos
01064     const int d1 = m_dateRegExp.cap(1).toInt(), d3 = m_dateRegExp.cap(3).toInt(), d5 = m_dateRegExp.cap(5).toInt();
01065     if (m_dateRegExp.cap(2)=="/") //probably separator for american format mm/dd/yyyy
01066         date = QDate(d5, d1, d3);
01067     else {
01068         if (d5 > 31) //d5 == year
01069             date = QDate(d5, d3, d1);
01070         else //d1 == year
01071             date = QDate(d1, d3, d5);
01072     }
01073     return date.isValid();
01074 }
01075 
01076 bool KexiCSVImportDialog::parseTime(const QString& text, QTime& time)
01077 {
01078     time = QTime::fromString(text, Qt::ISODate); //same as m_timeRegExp1
01079     if (time.isValid())
01080         return true;
01081     if (m_timeRegExp2.exactMatch(text)) { //hh:mm:ss
01082         time = QTime(m_timeRegExp2.cap(1).toInt(), m_timeRegExp2.cap(3).toInt(), m_timeRegExp2.cap(5).toInt());
01083         return true;
01084     }
01085     return false;
01086 }
01087 
01088 void KexiCSVImportDialog::setText(int row, int col, const QString& text, bool inGUI)
01089 {
01090     if (!inGUI) {
01091         //save text directly to database buffer
01092         if (col==1) { //1st col
01093             m_importingStatement->clearArguments();
01094             if (m_implicitPrimaryKeyAdded)
01095                 *m_importingStatement << QVariant(); //id will be autogenerated here
01096         }
01097         const int detectedType = m_detectedTypes[col-1];
01098         if (detectedType==_NUMBER_TYPE) {
01099             *m_importingStatement << ( text.isEmpty() ? QVariant() : text.toInt() );
01101         }
01102         else if (detectedType==_FP_NUMBER_TYPE) {
01103             //replace ',' with '.'
01104             QCString t(text.latin1());
01105             const int textLen = t.length();
01106             for (int i=0; i<textLen; i++) {
01107                 if (t.at(i)==',') {
01108                     t.at(i) = '.';
01109                     break;
01110                 }
01111             }
01112             *m_importingStatement << ( t.isEmpty() ? QVariant() : t.toDouble() );
01113         }
01114         else if (detectedType==_DATE_TYPE) {
01115             QDate date;
01116             if (parseDate(text, date))
01117                 *m_importingStatement << date;
01118         }
01119         else if (detectedType==_TIME_TYPE) {
01120             QTime time;
01121             if (parseTime(text, time))
01122                 *m_importingStatement << time;
01123         }
01124         else if (detectedType==_DATETIME_TYPE) {
01125             QStringList dateTimeList( QStringList::split(" ", text) );
01126             if (dateTimeList.count()<2)
01127                 dateTimeList = QStringList::split("T", text); //also support ISODateTime's "T" separator
01129             if (dateTimeList.count()>=2) {
01130                 QString datePart( dateTimeList[0].stripWhiteSpace() );
01131                 QDate date;
01132                 if (parseDate(datePart, date)) {
01133                     QString timePart( dateTimeList[1].stripWhiteSpace() );
01134                     QTime time;
01135                     if (parseTime(timePart, time))
01136                         *m_importingStatement << QDateTime(date, time);
01137                 }
01138             }
01139         }
01140         else //_TEXT_TYPE and the rest
01141             *m_importingStatement << (m_options.stripWhiteSpaceInTextValuesChecked ? text.stripWhiteSpace() : text);
01142         return;
01143     }
01144     //save text to GUI (table view)
01145     if (m_table->numCols() < col) {
01146         m_table->setNumCols(col);
01147         if ((int)m_columnNames.size() < m_table->numCols()) {
01148             m_columnNames.resize(m_table->numCols()+10);
01149             m_changedColumnNames.resize(m_table->numCols()+10);
01150         }
01151     }
01152 
01153     if (m_1stRowForFieldNames->isChecked()) {
01154         if ((row+m_startline)==1) {//this is for column name
01155             if ((col-1) < (int)m_changedColumnNames.size() && false==m_changedColumnNames[col-1]) {
01156                 //this column has no custom name entered by a user
01157                 //-get the name from the data cell
01158                 QString colName(text.simplifyWhiteSpace());
01159                 if (!colName.isEmpty()) {
01160                     if (colName.left(1)>="0" && colName.left(1)<="9")
01161                         colName.prepend(i18n("Column")+" ");
01162                     m_columnNames[ col-1 ] = colName;
01163                 }
01164             }
01165             return;
01166         }
01167     }
01168     else {
01169         if ((row+m_startline)==1) {//this row is for column name
01170             if (m_1stRowForFieldNamesDetected && !m_1stRowForFieldNames->isChecked()) {
01171                 QString f( text.simplifyWhiteSpace() );
01172                 if (f.isEmpty() || !f[0].isLetter())
01173                     m_1stRowForFieldNamesDetected = false; //this couldn't be a column name
01174             }
01175         }
01176         row++; //1st row was for column names
01177     }
01178 
01179     if (row < 2) // skipped by the user
01180         return;
01181 
01182     if (m_table->numRows() < row) {
01183 //      if (m_maximumRowsForPreview >= row+100)
01184         m_table->setNumRows(row+100); /* We add more rows at a time to limit recalculations */
01185         //else
01186 //          m_table->setNumRows(m_maximumRowsForPreview);
01187         m_table->verticalHeader()->setLabel(0, i18n("Column name")+"   ");
01188         m_adjustRows=true;
01189     }
01190 
01191     m_table->setText(row - 1, col - 1, (m_options.stripWhiteSpaceInTextValuesChecked ? text.stripWhiteSpace() : text));
01192     m_table->verticalHeader()->setLabel(row-1, QString::number(row-1));
01193 
01194     detectTypeAndUniqueness(row-1, col-1, text);
01195 }
01196 
01197 bool KexiCSVImportDialog::saveRow(bool inGUI)
01198 {
01199     if (inGUI) {
01200         //nothing to do
01201         return true;
01202     }
01203     //save db buffer
01204     bool res = m_importingStatement->execute();
01205 //todo: move
01206     m_importingStatement->clearArguments();
01207     return res;
01208 //  return m_conn->insertRecord(*m_destinationTableSchema, m_dbRowBuffer);
01209 }
01210 
01211 void KexiCSVImportDialog::adjustRows(int iRows)
01212 {
01213     if (m_adjustRows)
01214     {
01215         m_table->setNumRows( iRows );
01216         m_adjustRows=false;
01217         for (int i = 0; i<iRows; i++)
01218             m_table->adjustRow(i);
01219     }
01220 }
01221 
01222 void KexiCSVImportDialog::formatChanged(int id)
01223 {
01224     if (id==_PK_FLAG) {
01225         if (m_primaryKeyColumn>=0 && m_primaryKeyColumn<m_table->numCols()) {
01226             m_table->setPixmap(0, m_primaryKeyColumn, QPixmap());
01227         }
01228         if (m_primaryKeyField->isChecked()) {
01229             m_primaryKeyColumn = m_table->currentColumn();
01230             m_table->setPixmap(0, m_primaryKeyColumn, m_pkIcon);
01231         }
01232         else
01233             m_primaryKeyColumn = -1;
01234         return;
01235     }
01236     else {
01237         m_detectedTypes[m_table->currentColumn()]=id;
01238         m_primaryKeyField->setEnabled( _NUMBER_TYPE == id );
01239         m_primaryKeyField->setChecked( m_primaryKeyColumn == m_table->currentColumn() && m_primaryKeyField->isEnabled() );
01240     }
01241     updateColumnText(m_table->currentColumn());
01242 }
01243 
01244 void KexiCSVImportDialog::delimiterChanged(const QString& delimiter)
01245 {
01246     Q_UNUSED(delimiter);
01247     m_columnsAdjusted = false;
01248     m_detectDelimiter = false; //selected by hand: do not detect in the future
01249     //delayed, otherwise combobox won't be repainted
01250     fillTableLater();
01251 }
01252 
01253 void KexiCSVImportDialog::textquoteSelected(int)
01254 {
01255     const QString tq(m_comboQuote->textQuote());
01256     if (tq.isEmpty())
01257         m_textquote = 0;
01258     else
01259         m_textquote = tq[0];
01260 
01261     kexipluginsdbg << "KexiCSVImportDialog::textquoteSelected(): " << m_textquote << endl;
01262 
01263     //delayed, otherwise combobox won't be repainted
01264     fillTableLater();
01265 }
01266 
01267 void KexiCSVImportDialog::fillTableLater()
01268 {
01269     m_table->setNumRows( 0 );
01270     QTimer::singleShot(10, this, SLOT(fillTable()));
01271 }
01272 
01273 void KexiCSVImportDialog::startlineSelected(int startline)
01274 {
01275 //  const int startline = line.toInt() - 1;
01276     if (m_startline == (startline-1))
01277         return;
01278     m_startline = startline-1;
01279     m_adjustRows=true;
01280     fillTable();
01281     m_table->setFocus();
01282 }
01283 
01284 void KexiCSVImportDialog::currentCellChanged(int, int col)
01285 {
01286     if (m_prevSelectedCol==col)
01287         return;
01288     m_prevSelectedCol = col;
01289     int type = m_detectedTypes[col];
01290     if (type==_FP_NUMBER_TYPE)
01291         type=_NUMBER_TYPE; //we're simplifying that for now
01292 
01293     m_formatCombo->setCurrentItem( type );
01294     m_formatLabel->setText( m_formatComboText.arg(col+1) );
01295     m_primaryKeyField->setEnabled( _NUMBER_TYPE == m_detectedTypes[col]);
01296     m_primaryKeyField->blockSignals(true); //block to disable executing slotPrimaryKeyFieldToggled()
01297      m_primaryKeyField->setChecked( m_primaryKeyColumn == col );
01298     m_primaryKeyField->blockSignals(false);
01299 }
01300 
01301 void KexiCSVImportDialog::cellValueChanged(int row,int col)
01302 {
01303     if (row==0) {//column name has changed
01304         m_columnNames[ col ] = m_table->text(row, col);
01305         m_changedColumnNames.setBit( col );
01306     }
01307 }
01308 
01309 void KexiCSVImportDialog::accept()
01310 {
01312 
01313     KexiGUIMessageHandler msg; 
01314 
01315     const uint numRows( m_table->numRows() );
01316     if (numRows == 0)
01317         return; //impossible
01318 
01319     if (numRows == 1) {
01320         if (KMessageBox::No == KMessageBox::questionYesNo(this, 
01321             i18n("Data set contains no rows. Do you want to import empty table?")))
01322             return;
01323     }
01324 
01325     KexiProject* project = m_mainWin->project();
01326     if (!project) {
01327         msg.showErrorMessage(i18n("No project available."));
01328         return;
01329     }
01330     m_conn = project->dbConnection(); //cache this pointer
01331     if (!m_conn) {
01332         msg.showErrorMessage(i18n("No database connection available."));
01333         return;
01334     }
01335     KexiPart::Part *part = Kexi::partManager().partForMimeType("kexi/table");
01336     if (!part) {
01337         msg.showErrorMessage(&Kexi::partManager());
01338         return;
01339     }
01340 
01341     //get suggested name based on the file name
01342     QString suggestedName;
01343     if (m_mode==File) {
01344         suggestedName = KURL::fromPathOrURL(m_fname).fileName();
01345         //remove extension
01346         if (!suggestedName.isEmpty()) {
01347             const int idx = suggestedName.findRev(".");
01348             if (idx!=-1)
01349                 suggestedName = suggestedName.mid(0, idx ).simplifyWhiteSpace();
01350         }
01351     }
01352 
01353     //-new part item
01354     KexiPart::Item* partItemForSavedTable = project->createPartItem(part->info(), suggestedName);
01355     if (!partItemForSavedTable) {
01356     //      msg.showErrorMessage(project);
01357         return;
01358     }
01359 
01360 #define _ERR \
01361     { project->deleteUnstoredItem(partItemForSavedTable); \
01362       m_conn = 0; \
01363       delete m_destinationTableSchema; \
01364       m_destinationTableSchema = 0; \
01365     return; }
01366 
01367     //-ask for table name/title
01368     // (THIS IS FROM KexiMainWindowImpl::saveObject())
01369     bool allowOverwriting = true;
01370     tristate res = m_mainWin->getNewObjectInfo( partItemForSavedTable, part, allowOverwriting );
01371     if (~res || !res) {
01373         _ERR;
01374     }
01375     //(allowOverwriting is now set to true, if user accepts overwriting, 
01376     // and overwriting will be needed)
01377 
01378 //  KexiDB::SchemaData sdata(part->info()->projectPartID());
01379 //  sdata.setName( partItem->name() );
01380 
01381     //-create table schema (and thus schema object)
01382     //-assign information (THIS IS FROM KexiDialogBase::storeNewData())
01383     m_destinationTableSchema = new KexiDB::TableSchema(partItemForSavedTable->name());
01384     m_destinationTableSchema->setCaption( partItemForSavedTable->caption() );
01385     m_destinationTableSchema->setDescription( partItemForSavedTable->description() );
01386     const uint numCols( m_table->numCols() );
01387 
01388     m_implicitPrimaryKeyAdded = false;
01389     //add PK if user wanted it
01390     int msgboxResult;
01391     if (m_primaryKeyColumn==-1
01392         && KMessageBox::No != (msgboxResult = KMessageBox::questionYesNoCancel(this, 
01393             i18n("No Primary Key (autonumber) has been defined.\n"
01394             "Should it be automatically defined on import (recommended)?\n\n"
01395             "Note: An imported table without a Primary Key may not be editable (depending on database type)."),
01396             QString::null, KGuiItem(i18n("Add Database Primary Key to a Table", "Add Primary Key"), "key"),
01397             KGuiItem(i18n("Do Not Add Database Primary Key to a Table", "Do Not Add")))))
01398     {
01399         if (msgboxResult == KMessageBox::Cancel)
01400             _ERR; //cancel accepting
01401 
01402         //add implicit PK field
01404         m_implicitPrimaryKeyAdded = true;
01405 
01406         QString fieldName("id");
01407         QString fieldCaption("Id");
01408 
01409         QStringList colnames;
01410         for (uint col = 0; col < numCols; col++)
01411             colnames.append( m_table->text(0, col).lower().simplifyWhiteSpace() );
01412 
01413         if (colnames.find(fieldName)!=colnames.end()) {
01414             int num = 1;
01415             while (colnames.find(fieldName+QString::number(num))!=colnames.end())
01416                 num++;
01417             fieldName += QString::number(num);
01418             fieldCaption += QString::number(num);
01419         }
01420         KexiDB::Field *field = new KexiDB::Field(
01421             fieldName,
01422             KexiDB::Field::Integer,
01423             KexiDB::Field::NoConstraints,
01424             KexiDB::Field::NoOptions,
01425             0,0, //uint length=0, uint precision=0,
01426             QVariant(), //QVariant defaultValue=QVariant(),
01427             fieldCaption
01428         ); //no description and width for now
01429         field->setPrimaryKey(true);
01430         field->setAutoIncrement(true);
01431         m_destinationTableSchema->addField( field );
01432     }
01433 
01434     for (uint col = 0; col < numCols; col++) {
01435         QString fieldCaption( m_table->text(0, col).simplifyWhiteSpace() );
01436         QString fieldName( KexiUtils::string2Identifier( fieldCaption ) );
01437         if (m_destinationTableSchema->field(fieldName)) {
01438             QString fixedFieldName;
01439             uint i = 2; //"apple 2, apple 3, etc. if there're many "apple" names
01440             do {
01441                 fixedFieldName = fieldName + "_" + QString::number(i);
01442                 if (!m_destinationTableSchema->field(fixedFieldName))
01443                     break;
01444                 i++;
01445             } while (true);
01446             fieldName = fixedFieldName;
01447             fieldCaption += (" " + QString::number(i));
01448         }
01449         const int detectedType = m_detectedTypes[col];
01450         KexiDB::Field::Type fieldType;
01451         if (detectedType==_DATE_TYPE)
01452             fieldType = KexiDB::Field::Date;
01453         if (detectedType==_TIME_TYPE)
01454             fieldType = KexiDB::Field::Time;
01455         if (detectedType==_DATETIME_TYPE)
01456             fieldType = KexiDB::Field::DateTime;
01457         else if (detectedType==_NUMBER_TYPE)
01458             fieldType = KexiDB::Field::Integer;
01459         else if (detectedType==_FP_NUMBER_TYPE)
01460             fieldType = KexiDB::Field::Double;
01462         else //_TEXT_TYPE and the rest
01463             fieldType = KexiDB::Field::Text;
01465 
01466         KexiDB::Field *field = new KexiDB::Field(
01467             fieldName,
01468             fieldType,
01469             KexiDB::Field::NoConstraints,
01470             KexiDB::Field::NoOptions,
01471             0,0, //uint length=0, uint precision=0,
01472             QVariant(), //QVariant defaultValue=QVariant(),
01473             fieldCaption
01474         ); //no description and width for now
01475 
01476         if ((int)col == m_primaryKeyColumn) {
01477             field->setPrimaryKey(true);
01478             field->setAutoIncrement(true);
01479         }
01480         m_destinationTableSchema->addField( field );
01481     }
01482 
01483     KexiDB::Transaction transaction = m_conn->beginTransaction();
01484     if (transaction.isNull()) {
01485         msg.showErrorMessage(m_conn);
01486         _ERR;
01487     }
01488     KexiDB::TransactionGuard tg(transaction);
01489 
01490     //-create physical table
01491     if (!m_conn->createTable(m_destinationTableSchema, allowOverwriting)) {
01492             msg.showErrorMessage(m_conn);
01493         _ERR;
01494     }
01495 
01496 #define _DROP_DEST_TABLE_AND_RETURN \
01497     { \
01498     if (m_importingProgressDlg) \
01499         m_importingProgressDlg->hide(); \
01500     project->deleteUnstoredItem(partItemForSavedTable); \
01501     m_conn->dropTable(m_destinationTableSchema); /*alsoRemoveSchema*/ \
01502     m_destinationTableSchema = 0; \
01503     m_conn = 0; \
01504     return; \
01505     }
01506 
01507     m_importingStatement = m_conn->prepareStatement(
01508         KexiDB::PreparedStatement::InsertStatement, *m_destinationTableSchema);
01509     if (!m_importingStatement) {
01510         msg.showErrorMessage(m_conn);
01511         _DROP_DEST_TABLE_AND_RETURN;
01512     }
01513 
01514     if (m_file) {
01515         if (!m_importingProgressDlg) {
01516             m_importingProgressDlg = new KProgressDialog( this, "m_importingProgressDlg", 
01517                 i18n("Importing CSV Data"), QString::null, true );
01518         }
01519         m_importingProgressDlg->setLabel(
01520             i18n("Importing CSV Data from <nobr>\"%1\"</nobr> into \"%2\" table...")
01521             .arg(QDir::convertSeparators(m_fname)).arg(m_destinationTableSchema->name()) );
01522         m_importingProgressDlg->progressBar()->setTotalSteps( QFileInfo(*m_file).size() );
01523         m_importingProgressDlg->show();
01524     }
01525 
01526     int row, column, maxColumn;
01527     QString field = QString::null;
01528 
01529     // main job
01530     res = loadRows(field, row, column, maxColumn, false  );
01531 
01532     delete m_importingProgressDlg;
01533   m_importingProgressDlg = 0;
01534     if (true != res) {
01535         //importing cancelled or failed
01536         if (!res) //do not display err msg when res == cancelled
01537             msg.showErrorMessage(m_conn);
01538         _DROP_DEST_TABLE_AND_RETURN;
01539     }
01540 
01541     // file with only one line without '\n'
01542     if (field.length() > 0)
01543     {
01544         setText(row - m_startline, column, field, false );
01545         //fill remaining empty fields (database wants them explicitly)
01546         for (int additionalColumn = column; additionalColumn <= maxColumn; additionalColumn++) {
01547             setText(row - m_startline, additionalColumn, QString::null, false );
01548         }
01549         if (!saveRow(false )) {
01550             msg.showErrorMessage(m_conn);
01551             _DROP_DEST_TABLE_AND_RETURN;
01552         }
01553         ++row;
01554         field = QString::null;
01555     }
01556 
01557     if (!tg.commit()) {
01558         msg.showErrorMessage(m_conn);
01559         _DROP_DEST_TABLE_AND_RETURN;
01560     }
01561 
01562     //-now we can store the item
01563     partItemForSavedTable->setIdentifier( m_destinationTableSchema->id() );
01564     project->addStoredItem( part->info(), partItemForSavedTable );
01565 
01566     QDialog::accept();
01567     KMessageBox::information(this, i18n("Data has been successfully imported to table \"%1\".")
01568         .arg(m_destinationTableSchema->name()));
01569     parentWidget()->raise();
01570     m_conn = 0;
01571 }
01572 
01573 int KexiCSVImportDialog::getHeader(int col)
01574 {
01575     QString header = m_table->horizontalHeader()->label(col);
01576 
01577     if (header == i18n("Text type for column", "Text"))
01578         return TEXT;
01579     else if (header == i18n("Numeric type for column", "Number"))
01580         return NUMBER;
01581     else if (header == i18n("Currency type for column", "Currency"))
01582         return CURRENCY;
01583     else
01584         return DATE;
01585 }
01586 
01587 QString KexiCSVImportDialog::getText(int row, int col)
01588 {
01589     return m_table->text(row, col);
01590 }
01591 
01592 void KexiCSVImportDialog::ignoreDuplicatesChanged(int)
01593 {
01594     fillTable();
01595 }
01596 
01597 void KexiCSVImportDialog::slot1stRowForFieldNamesChanged(int)
01598 {
01599     m_adjustRows=true;
01600     if (m_1stRowForFieldNames->isChecked() && m_startline>0 && m_startline>=(m_startAtLineSpinBox->maxValue()-1))
01601         m_startline--;
01602     fillTable();
01603 }
01604 
01605 void KexiCSVImportDialog::optionsButtonClicked()
01606 {
01607     KexiCSVImportOptionsDialog dlg(m_options, this);
01608     if (QDialog::Accepted != dlg.exec())
01609         return;
01610 
01611     KexiCSVImportOptions newOptions( dlg.options() );
01612     if (m_options != newOptions) {
01613         m_options = newOptions;
01614         if (!openData())
01615             return;
01616         fillTable();
01617     }
01618 }
01619 
01620 bool KexiCSVImportDialog::eventFilter ( QObject * watched, QEvent * e )
01621 {
01622     QEvent::Type t = e->type();
01623     // temporary disable keyboard and mouse events for time-consuming tasks
01624     if (m_blockUserEvents && (t==QEvent::KeyPress || t==QEvent::KeyRelease 
01625         || t==QEvent::MouseButtonPress || t==QEvent::MouseButtonDblClick
01626         || t==QEvent::Paint ))
01627         return true;
01628 
01629     if (watched == m_startAtLineSpinBox && t==QEvent::KeyPress) {
01630         QKeyEvent *ke = static_cast<QKeyEvent*>(e);
01631         if (ke->key()==Qt::Key_Enter || ke->key()==Qt::Key_Return) {
01632             m_table->setFocus();
01633             return true;
01634         }
01635     }
01636     return QDialog::eventFilter( watched, e );
01637 }
01638 
01639 void KexiCSVImportDialog::slotPrimaryKeyFieldToggled(bool on)
01640 {
01641     Q_UNUSED(on);
01642     formatChanged(_PK_FLAG);
01643 }
01644 
01645 void KexiCSVImportDialog::updateRowCountInfo()
01646 {
01647     m_infoLbl->setFileName( m_fname );
01648     if (m_allRowsLoadedInPreview) {
01649         m_infoLbl->setCommentText( 
01650             i18n("row count", "(rows: %1)").arg( m_table->numRows()-1+m_startline ) );
01651         QToolTip::remove( m_infoLbl );
01652     }
01653     else {
01654         m_infoLbl->setCommentText( 
01655             i18n("row count", "(rows: more than %1)").arg( m_table->numRows()-1+m_startline ) );
01656         QToolTip::add( m_infoLbl->commentLabel(), i18n("Not all rows are visible on this preview") );
01657     }
01658 }
01659 
01660 #include "kexicsvimportdialog.moc"
KDE Home | KDE Accessibility Home | Description of Access Keys