From kde-commits Sun May 24 16:40:13 2009 From: Nickolai Shaforostoff Date: Sun, 24 May 2009 16:40:13 +0000 To: kde-commits Subject: KDE/kdesdk/lokalize/src/catalog/gettext Message-Id: <1243183213.357322.30489.nullmailer () svn ! kde ! org> X-MARC-Message: https://marc.info/?l=kde-commits&m=124318322208914 SVN commit 972342 by shaforo: gettext parsing robustness++. i still can't guarantee that lokalize will open any .po file passing msgfmt but I consider current level sufficient. BUG: 183702 BUG: 178290 M +5 -3 catalogitem.cpp M +33 -29 gettextimport.cpp M +1 -1 gettextimport.h --- trunk/KDE/kdesdk/lokalize/src/catalog/gettext/catalogitem.cpp #972341:972342 @@ -83,7 +83,6 @@ const QString& CatalogItem::msgstr(const int form) const { - //kWarning()<_msgstrPlural.size())) return d->_msgstrPlural.at(form); else @@ -113,7 +112,7 @@ void CatalogItem::setMsgid(const QString& msg, const int form) { - if (d->_msgidPlural.size()>=form) + if (form_msgidPlural.size()) d->_msgidPlural[form]=msg; else d->_msgidPlural.append(msg); @@ -135,7 +134,10 @@ void CatalogItem::setMsgstr(const QString& msg, const int form) { - d->_msgstrPlural[form]=msg; + if (form_msgstrPlural.size()) + d->_msgstrPlural[form]=msg; + else + d->_msgstrPlural.append(msg); } void CatalogItem::setMsgstr(const QStringList& msg) --- trunk/KDE/kdesdk/lokalize/src/catalog/gettext/gettextimport.cpp #972341:972342 @@ -3,7 +3,7 @@ This file is based on the one from KBabel Copyright (C) 1999-2000 by Matthias Kiefer - 2001-2003 by Stanislav Visnovsky + 2001-2003 by Stanislav Visnovsky 2006 by Nicolas GOUTTE 2007 by Nick Shaforostoff @@ -104,7 +104,7 @@ // if somethings goes wrong with the parsing, we don't have deleted the old contents CatalogItem tempHeader; - //kDebug() << "start parsing..."; + kDebug() << "start parsing..."; QTime aaa; aaa.start(); // first read header @@ -122,9 +122,10 @@ return status; } + bool reconstructedHeader=!_msgid.isEmpty() && !_msgid.first().isEmpty(); //kWarning() << "HEADER MSGID: " << _msgid; //kWarning() << "HEADER MSGSTR: " << _msgstr; - if (KDE_ISUNLIKELY( !_msgid.isEmpty() && !_msgid.first().isEmpty() )) + if (KDE_ISUNLIKELY( reconstructedHeader )) { // The header must have an empty msgid kWarning() << "Header entry has non-empty msgid. Creating a temporary header! " << _msgid; @@ -166,9 +167,14 @@ bool docbookFile=false; ExtraDataSaver _extraDataSaver; + ConversionStatus success=OK; while( !stream.atEnd() ) { - const ConversionStatus success=readEntry(stream); + if (reconstructedHeader) + reconstructedHeader=false; + else + success=readEntry(stream); + if(KDE_ISLIKELY(success==OK)) { if( _obsolete ) @@ -213,7 +219,7 @@ } else { - kWarning() << "Unknown success status, assumig parse error " << success; + kDebug() << "Unknown success status, assumig parse error " << success; return PARSE_ERROR; } counter++; @@ -221,7 +227,7 @@ } // TODO: can we check that there is no useful entry? - if (KDE_ISUNLIKELY( !counter )) + if (KDE_ISUNLIKELY( !counter && !recoveredErrorInHeader )) { // Empty file? (Otherwise, there would be a try of getting an entry and the count would be 1 !) kDebug() << " Empty file?"; @@ -240,7 +246,7 @@ setErrorIndex(errorIndex); //setFileCodec(codec); //setMimeTypes( "text/x-gettext-translation" ); - +#if 0 if (KDE_ISUNLIKELY( recoveredErrorInHeader )) { kDebug() << " Returning: header error"; @@ -252,6 +258,7 @@ return RECOVERED_PARSE_ERROR; } else +#endif { kDebug() << " Returning: OK! :-)"; return OK; @@ -262,11 +269,12 @@ { QTextStream stream( device ); stream.seek(0); + _errorLine=0; stream.setCodec( "UTF-8" ); stream.setAutoDetectUnicode(true); //this way we can QTextCodec* codec=stream.codec(); //detect UTF-16 - ConversionStatus status = readHeader(stream); + ConversionStatus status = readEntry(stream); if (KDE_ISUNLIKELY( status!=OK && status != RECOVERED_PARSE_ERROR )) { kDebug() << "wasn't able to read header"; @@ -309,22 +317,6 @@ return codec;//UTF-8 } -ConversionStatus GettextImportPlugin::readHeader(QTextStream& stream) -{ - int filePos = stream.device()->pos(); - ConversionStatus status=readEntry(stream); - - if(KDE_ISLIKELY( status==OK || status==RECOVERED_PARSE_ERROR )) - { - // test if this is the header - if(!_msgid.first().isEmpty()) - stream.device()->seek( filePos ); - return status; - } - - return PARSE_ERROR; -} - ConversionStatus GettextImportPlugin::readEntry(QTextStream& stream) { ConversionStatus result=readEntryRaw(stream); @@ -339,7 +331,6 @@ //kDebug() << " START"; enum {Begin,Comment,Msgctxt,Msgid,Msgstr} part=Begin; - //QString line; _trailingNewLines=0; bool error=false; bool recoverableError=false; @@ -354,14 +345,21 @@ _obsolete=false; QStringList::Iterator msgstrIt=_msgstr.begin(); + QString line; while( !stream.atEnd() ) { _errorLine++; //line=stream.readLine(); - QString line(stream.readLine()); + if (!_bufferedLine.isEmpty()) + { + line=_bufferedLine; + _bufferedLine.clear(); + } + else + line=stream.readLine(); - //kWarning() << "Parsing line: " << line; + kDebug() << "Parsing line: " << line; if (KDE_ISUNLIKELY( line.startsWith( "<<<<<<<" ) || line.startsWith( "=======" ) || line.startsWith( ">>>>>>>" ) )) { @@ -713,8 +711,14 @@ msgstrIt=_msgstr.end(); --msgstrIt; } - else if(KDE_ISUNLIKELY(/*_testBorked&&*/ _gettextPluralForm && ( line.contains( _rxMsgStrPluralBorked ) ) )) + else if ( line.startsWith( '#' ) || line.startsWith( "msgid" ) ) { + _errorLine--; + _bufferedLine=line; + break; + } + else if(KDE_ISUNLIKELY(/*_testBorked&&*/ _gettextPluralForm && ( line.contains( _rxMsgStrPluralBorked ) ) )) + { // remove quotes at beginning and the end of the lines line.remove(QRegExp("^msgstr\\[[0-9]\\]\\s*\"?")); line.remove(_rxMsgLineRemEndQuote); @@ -728,7 +732,7 @@ } else if(line.startsWith("msgstr")) { - kDebug() << "Another msgstr found after a msgstr while parsing: " << _msgstr.last(); + kDebug() << "Another msgstr found after a msgstr while parsing: " << line << _msgstr.last(); error=true; break; } --- trunk/KDE/kdesdk/lokalize/src/catalog/gettext/gettextimport.h #972341:972342 @@ -78,7 +78,6 @@ private: QTextCodec* codecForDevice(QIODevice* /*, bool* hadCodec*/); - ConversionStatus readHeader(QTextStream& stream); ConversionStatus readEntryRaw(QTextStream& stream); ConversionStatus readEntry(QTextStream& stream); @@ -113,6 +112,7 @@ QString _obsoleteStart; QString _msgctxtStart; + QString _bufferedLine; }; } #endif