[prev in list] [next in list] [prev in thread] [next in thread] 

List:       kmail-devel
Subject:    Re: Memory usage (Re: QCString construction)
From:       David Faure <dfaure () klaralvdalens-datakonsult ! se>
Date:       2007-02-15 16:10:08
Message-ID: 200702151710.28032.dfaure () klaralvdalens-datakonsult ! se
[Download RAW message or body]

On Wednesday 14 February 2007, Ingo Klöcker wrote:
> On Wednesday 14 February 2007 12:50, David Faure wrote:
> > I found more... In messagecomposer.cpp mNewBodyPart was allocated
> > with new and never deleted ;)
> 
> Ouch.
> 
> > Working on this some more, I realized that the best solution in
> > messagecomposer.cpp is actually to port it to QByteArray, since this
> > is what we want with at the end anyway (for signing/encrypting, and
> > setting into the kmmessage). Of course QByteArray is a bit tricky in
> > qt3, so I wrote utility methods for converting to/from qbytearray,
> > appending, and inserting - all without a trailing nul. It's all
> > unit-tested, and whenever I had a bug my test messages would have an
> > invalid signature, which is good, since it showed that the code was
> > incorrect; so now I'm quite confident about those changes (signatures
> > are correct and unit tests pass ;)
> >
> > But at least sending a 30MB attachment in a signed mail went from
> > VmPeak:  1324284 kB
> > VmSize:   877792 kB
> > VmRSS:    530372 kB
> > VmData:   719364 kB
> > to
> > VmPeak:   968596 kB
> > VmSize:   601560 kB
> > VmRSS:    325900 kB
> > VmData:   458020 kB
> > (data measured after signing, and while sending)
> 
> Looks very promising.

> > There's still too much memory consumption in kmsender (due to
> > KMMessage::asSendableString()

Wow. With the attached patch - note the new solution for asSendableString -
it gets down to:

-obelix- dfaure 17:07 ~>grep Vm'[PRSD][eiSa]' /proc/`pidof kmail`/status
VmPeak:   642944 kB
VmSize:   275912 kB
VmRSS:    129228 kB
VmData:   132368 kB

-- 
David Faure, faure@kde.org, dfaure@klaralvdalens-datakonsult.se
KDE/KOffice developer, Qt consultancy projects
Klarälvdalens Datakonsult AB, Platform-independent software solutions

["kmail_mem_2.diff" (text/x-diff)]

Index: kmmsgpart.cpp
===================================================================
--- kmmsgpart.cpp	(revision 633625)
+++ kmmsgpart.cpp	(working copy)
@@ -123,6 +123,17 @@ void KMMessagePart::setBody(const DwStri
     mBodyDecodedSize = -1; // Can't know the decoded size
 }
 
+void KMMessagePart::setBody(const QByteArray &aStr)
+{
+  mBody = aStr;
+
+  int enc = cte();
+  if (enc == DwMime::kCte7bit || enc == DwMime::kCte8bit || enc == \
DwMime::kCteBinary) +    mBodyDecodedSize = mBody.size();
+  else
+    mBodyDecodedSize = -1; // Can't know the decoded size
+}
+
 void KMMessagePart::setBodyFromUnicode( const QString & str ) {
   QCString encoding = KMMsgBase::autoDetectCharset( charset(), \
KMMessage::preferredCharsets(), str );  if ( encoding.isEmpty() )
Index: kmmessage.h
===================================================================
--- kmmessage.h	(revision 633625)
+++ kmmessage.h	(working copy)
@@ -230,7 +230,7 @@ public:
    * Return the message contents with the headers that should not be
    * sent stripped off.
    */
-  QCString asSendableString() const;
+  QByteArray asSendableString() const;
 
   /**
    * Return the message header with the headers that should not be
Index: kmmessage.cpp
===================================================================
--- kmmessage.cpp	(revision 633625)
+++ kmmessage.cpp	(working copy)
@@ -311,19 +311,30 @@ QCString KMMessage::asString() const {
 }
 
 
-QCString KMMessage::asSendableString() const
+QByteArray KMMessage::asSendableString() const
 {
+#if 0
   KMMessage msg;
   msg.fromDwString(asDwString()); // slow!
+#else
+  //KMMessage msg( *this );
+  KMMessage msg;
+  delete msg.mMsg;
+  msg.mMsg = new DwMessage( *mMsg );
+#endif
   msg.removePrivateHeaderFields();
   msg.removeHeaderField("Bcc");
-  return msg.asString();
+  return KMail::Util::ByteArray( msg.asDwString() ); // and another copy again!
 }
 
 QCString KMMessage::headerAsSendableString() const
 {
+#if 0
   KMMessage msg;
   msg.fromDwString(asDwString());
+#else
+  KMMessage msg( *this );
+#endif
   msg.removePrivateHeaderFields();
   msg.removeHeaderField("Bcc");
   return msg.headerAsString().latin1();
Index: kmcommands.cpp
===================================================================
--- kmcommands.cpp	(revision 633625)
+++ kmcommands.cpp	(working copy)
@@ -876,13 +876,11 @@ void KMSaveMsgCommand::slotSaveDataReq()
 void KMSaveMsgCommand::slotMessageRetrievedForSaving(KMMessage *msg)
 {
   if ( msg ) {
-    QCString str( msg->mboxMessageSeparator() );
-    str += KMFolderMbox::escapeFrom( msg->asString() );
-    str += '\n';
+    mData = KMFolderMbox::escapeFrom( msg->asDwString() );
+    KMail::Util::insert( mData, 0, msg->mboxMessageSeparator() );
+    KMail::Util::append( mData, "\n" );
     msg->setTransferInProgress(false);
 
-    mData = str;
-    mData.resize(mData.size() - 1);
     mOffset = 0;
     QByteArray data;
     int size;
@@ -1176,7 +1174,6 @@ KMCommand::Result KMForwardInlineCommand
   if (msgList.count() >= 2) { // Multiple forward
 
     uint id = 0;
-    // QCString msgText = "";
     QPtrList<KMMessage> linklist;
     for ( KMMessage *msg = msgList.first(); msg; msg = msgList.next() ) {
       // set the identity
@@ -1476,7 +1473,6 @@ KMCommand::Result KMCustomForwardCommand
   if (msgList.count() >= 2) { // Multiple forward
 
     uint id = 0;
-    // QCString msgText = "";
     QPtrList<KMMessage> linklist;
     for ( KMMessage *msg = msgList.first(); msg; msg = msgList.next() ) {
       // set the identity
@@ -2580,9 +2576,7 @@ KMCommand::Result KMSaveAttachmentsComma
   {
     // This does not decode the Message Content-Transfer-Encoding
     // but saves the _original_ content of the message part
-    QCString cstr( node->msgPart().body() );
-    data = cstr;
-    data.resize(data.size() - 1);
+    data = KMail::Util::ByteArray( node->msgPart().dwBody() );
   }
   else
   {
Index: kmsender_p.h
===================================================================
--- kmsender_p.h	(revision 633624)
+++ kmsender_p.h	(working copy)
@@ -6,7 +6,6 @@
 #define __KMAIL_SENDER_P_H__
 #include "kmsender.h"
 
-#include <qcstring.h>
 #include <qstring.h>
 #include <qstringlist.h>
 #include <qobject.h>
@@ -33,7 +32,7 @@ public:
   void start() { emit started( doStart() ); }
 
   /** Send given message. May return before message is sent. */
-  bool send( const QString & sender, const QStringList & to, const QStringList & cc, \
const QStringList & bcc, const QCString & message ) { +  bool send( const QString & \
sender, const QStringList & to, const QStringList & cc, const QStringList & bcc, \
const QByteArray & message ) {  reset(); return doSend( sender, to, cc, bcc, message \
);  }
 
@@ -72,7 +71,7 @@ private:
 
 private:
   virtual void doFinish() = 0;
-  virtual bool doSend( const QString & sender, const QStringList & to, const \
QStringList & cc, const QStringList & bcc, const QCString & message ) = 0; +  virtual \
bool doSend( const QString & sender, const QStringList & to, const QStringList & cc, \
const QStringList & bcc, const QByteArray & message ) = 0;  virtual bool doStart() = \
0;  
 protected:
@@ -102,12 +101,12 @@ private:
   /** implemented from KMSendProc */
   void doFinish();
   /** implemented from KMSendProc */
-  bool doSend( const QString & sender, const QStringList & to, const QStringList & \
cc, const QStringList & bcc, const QCString & message ); +  bool doSend( const \
QString & sender, const QStringList & to, const QStringList & cc, const QStringList & \
bcc, const QByteArray & message );  /** implemented from KMSendProc */
   bool doStart();
 
 private:
-  QCString mMsgStr;
+  QByteArray mMsgStr;
   char* mMsgPos;
   int mMsgRest;
   KProcess* mMailerProc;
@@ -132,14 +131,14 @@ private:
   /** implemented from KMSendProc */
   void doFinish();
   /** implemented from KMSendProc */
-  bool doSend( const QString & sender, const QStringList & to, const QStringList & \
cc, const QStringList & bcc, const QCString & message ); +  bool doSend( const \
QString & sender, const QStringList & to, const QStringList & cc, const QStringList & \
bcc, const QByteArray & message );  /** implemented from KMSendProc */
   bool doStart() { return true; }
 
   void cleanup();
 
 private:
-  QCString mMessage;
+  QByteArray mMessage;
   uint mMessageLength;
   uint mMessageOffset;
 
Index: tests/utiltests.cpp
===================================================================
--- tests/utiltests.cpp	(revision 633625)
+++ tests/utiltests.cpp	(working copy)
@@ -65,8 +65,6 @@ void UtilTester::test_escapeFrom()
 
 }
 
-// TODO port KMSender to DwString, port asSendableString
-
 void UtilTester::test_append()
 {
   QCString test;
Index: kmfoldermbox.cpp
===================================================================
--- kmfoldermbox.cpp	(revision 633624)
+++ kmfoldermbox.cpp	(working copy)
@@ -870,12 +870,12 @@ static size_t unescapeFrom( char* str, s
 }
 
 //static
-QCString KMFolderMbox::escapeFrom( const QCString & str ) {
+QByteArray KMFolderMbox::escapeFrom( const DwString & str ) {
   const unsigned int strLen = str.length();
   if ( strLen <= STRDIM("From ") )
-    return str;
+    return KMail::Util::ByteArray( str );
   // worst case: \nFrom_\nFrom_\nFrom_... => grows to 7/6
-  QCString result( int( strLen + 5 ) / 6 * 7 + 1 );
+  QByteArray result( int( strLen + 5 ) / 6 * 7 + 1 );
 
   const char * s = str.data();
   const char * const e = s + strLen - STRDIM("From ");
@@ -961,7 +961,7 @@ int KMFolderMbox::addMsg( KMMessage* aMs
 {
   if (!canAddMsgNow(aMsg, aIndex_ret)) return 0;
   bool opened = false;
-  QCString msgText;
+  QByteArray msgText;
   char endStr[3];
   int idx = -1, rc;
   KMFolder* msgParent;
@@ -1018,8 +1018,8 @@ if( fileD1.open( IO_WriteOnly ) ) {
     if (aMsg->headerField("Content-Type").isEmpty())  // This might be added by
       aMsg->removeHeaderField("Content-Type");        // the line above
   }
-  msgText = escapeFrom( aMsg->asString() );
-  size_t len = msgText.length();
+  msgText = escapeFrom( aMsg->asDwString() );
+  size_t len = msgText.size();
 
   assert(mStream != 0);
   clearerr(mStream);
@@ -1059,7 +1059,7 @@ if( fileD1.open( IO_WriteOnly ) ) {
   QCString messageSeparator( aMsg->mboxMessageSeparator() );
   fwrite( messageSeparator.data(), messageSeparator.length(), 1, mStream );
   off_t offs = ftell(mStream);
-  fwrite(msgText, len, 1, mStream);
+  fwrite(msgText.data(), len, 1, mStream);
   if (msgText[(int)len-1]!='\n') fwrite("\n\n", 1, 2, mStream);
   fflush(mStream);
   size_t size = ftell(mStream) - offs;
Index: kmmsgpart.h
===================================================================
--- kmmsgpart.h	(revision 633625)
+++ kmmsgpart.h	(working copy)
@@ -48,6 +48,8 @@ public:
   void setBody(const QCString &aStr);
   DwString dwBody() const;
   void setBody(const DwString &aStr);
+  // warning, doesn't detach from 'arr'
+  void setBody(const QByteArray &arr);
 
   /** Sets this body part's content to @p str. @p str is subject to
       automatic charset and CTE detection.
Index: kmfoldermbox.h
===================================================================
--- kmfoldermbox.h	(revision 633624)
+++ kmfoldermbox.h	(working copy)
@@ -109,7 +109,7 @@ public:
 
   void setProcmailLockFileName( const QString& );
 
-  static QCString escapeFrom( const QCString & str );
+  static QByteArray escapeFrom( const DwString & str );
 
   virtual IndexStatus indexStatus();
 
Index: objecttreeparser.cpp
===================================================================
--- objecttreeparser.cpp	(revision 633625)
+++ objecttreeparser.cpp	(working copy)
@@ -970,9 +970,8 @@ namespace KMail {
   }
 
   bool ObjectTreeParser::processTextPlainSubtype( partNode * curNode, ProcessResult \
                & result ) {
-    const QCString cstr = curNode->msgPart().bodyDecoded();
     if ( !mReader ) {
-      mRawReplyString = cstr;
+      mRawReplyString = curNode->msgPart().bodyDecoded();
       if ( curNode->isFirstTextPart() ) {
         mTextualContent += curNode->msgPart().bodyToUnicode();
         mTextualContentCharset = curNode->msgPart().charset();
@@ -980,13 +979,12 @@ namespace KMail {
       return true;
     }
 
-    //resultingRawData += cstr;
     if ( !curNode->isFirstTextPart() &&
          attachmentStrategy()->defaultDisplay( curNode ) != \
                AttachmentStrategy::Inline &&
          !showOnlyOneMimePart() )
       return false;
 
-    mRawReplyString = cstr;
+    mRawReplyString = curNode->msgPart().bodyDecoded();
     if ( curNode->isFirstTextPart() ) {
       mTextualContent += curNode->msgPart().bodyToUnicode();
       mTextualContentCharset = curNode->msgPart().charset();
@@ -1029,7 +1027,7 @@ namespace KMail {
     // enable verification of the embedded messages' signatures
     if ( !isMailmanMessage( curNode ) ||
          !processMailmanMessage( curNode ) )
-      writeBodyString( cstr, curNode->trueFromAddress(),
+      writeBodyString( mRawReplyString, curNode->trueFromAddress(),
                        codecFor( curNode ), result, !bDrawFrame );
     if ( bDrawFrame )
       htmlWriter()->queue( "</td></tr></table>" );
Index: kmsender.cpp
===================================================================
--- kmsender.cpp	(revision 633624)
+++ kmsender.cpp	(working copy)
@@ -605,7 +605,7 @@ void KMSender::doSendMsgAux()
   if ( messageIsDispositionNotificationReport( mCurrentMsg ) && \
GlobalSettings::self()->sendMDNsWithEmptySender() )  sender = "<>";
 
-  const QCString message = mCurrentMsg->asSendableString();
+  const QByteArray message = mCurrentMsg->asSendableString();
   if ( sender.isEmpty() || !mSendProc->send( sender, to, cc, bcc, message ) ) {
     if ( mCurrentMsg )
       mCurrentMsg->setTransferInProgress( false );
@@ -963,7 +963,7 @@ void KMSendSendmail::abort()
   idle();
 }
 
-bool KMSendSendmail::doSend( const QString & sender, const QStringList & to, const \
QStringList & cc, const QStringList & bcc, const QCString & message ) { +bool \
KMSendSendmail::doSend( const QString & sender, const QStringList & to, const \
QStringList & cc, const QStringList & bcc, const QByteArray & message ) {  \
                mMailerProc->clearArguments();
   *mMailerProc << mSender->transportInfo()->host
                << "-i" << "-f" << sender
@@ -977,7 +977,7 @@ bool KMSendSendmail::doSend( const QStri
     return false;
   }
   mMsgPos  = mMsgStr.data();
-  mMsgRest = mMsgStr.length();
+  mMsgRest = mMsgStr.size();
   wroteStdin( mMailerProc );
 
   return true;
@@ -1048,7 +1048,7 @@ KMSendSMTP::~KMSendSMTP()
   if (mJob) mJob->kill();
 }
 
-bool KMSendSMTP::doSend( const QString & sender, const QStringList & to, const \
QStringList & cc, const QStringList & bcc, const QCString & message ) { +bool \
KMSendSMTP::doSend( const QString & sender, const QStringList & to, const QStringList \
& cc, const QStringList & bcc, const QByteArray & message ) {  QString query = \
"headers=0&from=";  query += KURL::encode_string( sender );
 
@@ -1119,7 +1119,7 @@ bool KMSendSMTP::doSend( const QString &
 
   // dotstuffing is now done by the slave (see setting of metadata)
   mMessage = message;
-  mMessageLength = mMessage.length();
+  mMessageLength = mMessage.size();
   mMessageOffset = 0;
 
   if ( mMessageLength )



_______________________________________________
KMail developers mailing list
KMail-devel@kde.org
https://mail.kde.org/mailman/listinfo/kmail-devel


[prev in list] [next in list] [prev in thread] [next in thread]