Changeset 1048

Timestamp:

10/23/06 14:59:59

Author:

pvanhoof

Message:

Memory reduction and bugfix

Files:

• trunk/ChangeLog (modified) (1 diff)
• trunk/libtinymail-camel/camel-lite/camel/camel-folder-summary.c (modified) (7 diffs)
• trunk/libtinymail-camel/camel-lite/camel/camel-folder-summary.h (modified) (1 diff)
• trunk/libtinymail-camel/camel-lite/camel/camel-folder.c (modified) (1 diff)
• trunk/libtinymail-camel/camel-lite/camel/providers/Makefile.am (modified) (1 diff)
• trunk/libtinymail-camel/camel-lite/camel/providers/imap/camel-imap-folder.c (modified) (17 diffs)
• trunk/libtinymail-camel/camel-lite/camel/providers/nntp/camel-nntp-summary.c (modified) (4 diffs)
• trunk/libtinymail-gnome-desktop/tny-gnome-device-priv.h (modified) (1 diff)
• trunk/libtinymail-gnome-desktop/tny-gnome-device.c (modified) (8 diffs)

trunk/ChangeLog

@@ +1 @@
+2006-10-23  Philip Van Hoof <pvanhoof@gnome.org>
+
+        * libtinymail-camel/camel-lite: Drastic reduction of memory
+        consumption on initial download of both IMAP and NNTP folders.
+        * libtinymail-gnome-desktop: Default status of the device is now
+        offline in stead of online (for example in case network manager
+        installation was invalid or not found)
+
-2006-10-20  Sankarshan Mukhopadhyay <sankarshan.mukhopadhyay@gmail.com>
-

trunk/libtinymail-camel/camel-lite/camel/camel-folder-summary.c

@@ -114 +114 @@
-static void camel_folder_summary_finalize   (CamelObject *obj);
-
+
+static void camel_folder_summary_mmap_add(CamelFolderSummary *s, CamelMessageInfo *info);
+static void camel_folder_summary_unload_mmap (CamelFolderSummary *s);
+
-static CamelObjectClass *camel_folder_summary_parent;
-
@@ -162 +166 @@
-}
-
-void
-
-
-
-
-
-
-
+gboolean always_true (gpointer key, gpointer value, gpointer gp)
+
+
+
+
+
+
+
-        camel_folder_summary_clear(s);
-        if (s->file)
@@ -176 +180 @@
-
-        g_ptr_array_foreach (s->messages, foreach_msginfo, (gpointer)s->message_info_size);
+        if (s->messages->len > 0)
+                g_ptr_array_remove_range (s->messages, 0, s->messages->len-1);
+        g_hash_table_foreach_remove (s->messages_uid, always_true, NULL);
+
+}
+
+static void
+camel_folder_summary_finalize (CamelObject *obj)
+{
+        struct _CamelFolderSummaryPrivate *p;
+        CamelFolderSummary *s = (CamelFolderSummary *)obj;
+
+        p = _PRIVATE(obj);
+
+        camel_folder_summary_clear(s);
+        if (s->file)
+                g_mapped_file_free (s->file);
+        s->file = NULL;
+
+        g_ptr_array_foreach (s->messages, foreach_msginfo, (gpointer)s->message_info_size);
-        g_ptr_array_free(s->messages, TRUE);
-        g_hash_table_destroy(s->messages_uid);
+        s->messages_uid = NULL;
-
-        g_hash_table_foreach(p->filter_charset, free_o_name, 0);
@@ -592 +617 @@
-                }
-
-
+mmap_
-        }
-
@@ -808 +833 @@
-
-
+
+void 
+camel_folder_summary_dump_mmap (CamelFolderSummary *s)
+{
+        camel_folder_summary_save (s);
+        camel_folder_summary_unload_mmap (s);
+        camel_folder_summary_load(s);
+
+        return;
+}
+
-/**
- * camel_folder_summary_add:
@@ -846 +882 @@
-}
-
+static void
+camel_folder_summary_mmap_add(CamelFolderSummary *s, CamelMessageInfo *info)
+{
+        CAMEL_SUMMARY_LOCK(s, summary_lock);
+
+/* unnecessary for pooled vectors */
+#ifdef DOESTRV
+        /* this is vitally important, and also if this is ever modified, then
+           the hash table needs to be resynced */
+        info->strings = e_strv_pack(info->strings);
+#endif
+
+        g_ptr_array_add(s->messages, info);
+        g_hash_table_insert(s->messages_uid, (char *)camel_message_info_uid(info), info);
+        s->flags |= CAMEL_SUMMARY_DIRTY;
+
+        CAMEL_SUMMARY_UNLOCK(s, summary_lock);
+}
-
-/**
@@ -1091 +1145 @@
-camel_folder_summary_touch(CamelFolderSummary *s)
-{
-
+//
-        s->flags |= CAMEL_SUMMARY_DIRTY;
-
+//
-}
-

trunk/libtinymail-camel/camel-lite/camel/camel-folder-summary.h

@@ -309 +309 @@
-CamelFolderSummary      *camel_folder_summary_new       (struct _CamelFolder *folder);
-
+void camel_folder_summary_dump_mmap (CamelFolderSummary *s);
+
-unsigned char*
-decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string);

trunk/libtinymail-camel/camel-lite/camel/camel-folder.c

@@ -2117 +2117 @@
-        GPtrArray *olduids;
-        char *olduid;
-
+
-        g_assert(info != NULL);
-        

trunk/libtinymail-camel/camel-lite/camel/providers/Makefile.am

@@ -1 +1 @@
-## Process this file with automake to produce Makefile.in
-
+
+SUBDIRS = pop3 smtp imap local  
+
-if ENABLE_NNTP
-
+
-endif
-
-if ENABLE_IMAPP
-
+
-endif
+
-
-if OS_WIN32
-else
-ENDMAIL_DIR=sendmail
+UBDIRS += sendmail 
-endif
-
-SUBDIRS = pop3 $(SENDMAIL_DIR) smtp imap $(NNTP_DIR) local $(IMAPP_DIR) 
-
-

trunk/libtinymail-camel/camel-lite/camel/providers/imap/camel-imap-folder.c

@@ -2338 +2338 @@
-                     CamelException *ex)
-{
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        /* Figure out if any of the new messages are already cached (which
-         * may be the case if we're re-syncing after disconnected operation).
@@ -2375 +2382 @@
-        } else
-                uidval = 0;
-
+
-        size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE + IMAP_PRETEND_SIZEOF_HEADERS);
-        got = 0;
-        if (!camel_imap_command_start (store, folder, ex,
-*
-
+%d
+uidval + 1 + nextn, 
-                return;
+
+        /* 1, 7, 19, 43, 91, .. until 1000 (the smaller, the more dumps, 
+        the slower. But too large consumes a lot memory, and also takes longer 
+        to fetch from the service before we can dump) */
+
+        if (nextn < 1000)
+                nextn += (nextn+5);
+        else
+                nextn = 1000;
+
-        camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
-
+
-        /* Parse the responses. We can't add a message to the summary
-         * until we've gotten its headers, and there's no guarantee
-         * the server will send the responses in a useful order...
-         */
+
-        fetch_data = g_ptr_array_new ();
-        messages = g_ptr_array_new ();
-        while ((type = camel_imap_command_response (store, &resp, ex)) ==
-
+
+
+
-                data = parse_fetch_response (imap_folder, resp);
-                g_free (resp);
+
-                if (!data)
-                        continue;
@@ -2402 +2423 @@
-                        continue;
-                }
-
+
-                if (g_datalist_get_data (&data, "FLAGS"))
-                        got += IMAP_PRETEND_SIZEOF_FLAGS;
@@ -2417 +2438 @@
-                        g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
-                }
-
-
+
+
+
-                g_ptr_array_add (fetch_data, data);
-        }
-        camel_operation_end (NULL);
-
+
-        if (type == CAMEL_IMAP_RESPONSE_ERROR)
-                goto lose;
-
+
-        /* Free the final tagged response */
-        g_free (resp);
-
+
-        /* Figure out which headers we still need to fetch. */
-        needheaders = g_ptr_array_new ();
-        size = got = 0;
-
+
+
-                data = fetch_data->pdata[i];
-                if (g_datalist_get_data (&data, "BODY_PART_LEN"))
-                        continue;
-
+
-                uid = g_datalist_get_data (&data, "UID");
-                if (uid) {
@@ -2443 +2466 @@
-                }
-        }
-
+
-        /* And fetch them */
-
+
+
-                char *uidset;
-                int uid = 0;
-
+
-                qsort (needheaders->pdata, needheaders->len,
-       
-
+       
+
-                camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
-                
-                while (uid < needheaders->len) {
-                        uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
@@ -2465 +2488 @@
-                        }
-                        g_free (uidset);
-
+
-                        while ((type = camel_imap_command_response (store, &resp, ex))
-                               == CAMEL_IMAP_RESPONSE_UNTAGGED) {
@@ -2477 +2500 @@
-                                        add_message_from_data (folder, messages, first, data);
-                                        got += IMAP_PRETEND_SIZEOF_HEADERS;
-
+
+
-                                }
-                                g_datalist_clear (&data);
@@ -2488 +2512 @@
-                        }
-                }
-                
-                g_ptr_array_free (needheaders, TRUE);
-                camel_operation_end (NULL);
-        }
-
+
-        /* Now finish up summary entries (fix UIDs, set flags and size) */
-        for (i = 0; i < fetch_data->len; i++) {
-                data = fetch_data->pdata[i];
-
+
-                seq = GPOINTER_TO_INT (g_datalist_get_data (&data, "SEQUENCE"));
-                if (seq >= first + messages->len) {
@@ -2502 +2525 @@
-                        continue;
-                }
-
+
-                mi = messages->pdata[seq - first];
-                if (mi == NULL) {
-                        CamelMessageInfo *pmi = NULL;
-                        int j;
-
+
-                        /* This is a kludge around a bug in Exchange
-                         * 5.5 that sometimes claims multiple messages
@@ -2524 +2547 @@
-                                        break;
-                        }
-
+
-                        if (pmi == NULL) {
-                                /* Server response is *really* fucked up,
@@ -2533 +2556 @@
-                        mi = (CamelImapMessageInfo *)camel_message_info_clone(pmi);
-                }
-
+
-                uid = g_datalist_get_data (&data, "UID");
-
-
-
+
-                        mi->info.uid = g_strdup (uid);
+                        mi->info.uid_needs_free = TRUE;
+                }
-
-                flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
@@ -2549 +2572 @@
-                        flags_to_label(folder, mi);
-                }
-
+
-#ifdef NON_TINYMAIL_FEATURES
-                size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
@@ -2555 +2578 @@
-                        mi->info.size = size;
-#endif
+
-                g_datalist_clear (&data);
-        }
-        g_ptr_array_free (fetch_data, TRUE);
-
+
-        /* And add the entries to the summary, etc. */
-
+
+
+
-                mi = messages->pdata[i];
-                if (!mi) {
-
+/* 
-                        camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
-                                              _("Incomplete server response: no information provided for message %d"),
-
-
-
+
+
+
+
-                uid = (char *)camel_message_info_uid(mi);
-                if (uid[0] == 0) {
@@ -2577 +2604 @@
-                        break;
-                }
+
-                info = (CamelImapMessageInfo *)camel_folder_summary_uid(folder->summary, uid);
-                if (info) {
@@ -2592 +2620 @@
-                        break;
-                }
-
+
-                camel_folder_summary_add (folder->summary, (CamelMessageInfo *)mi);
-                camel_folder_change_info_add_uid (changes, camel_message_info_uid (mi));
-
+
-                if ((mi->info.flags & CAMEL_IMAP_MESSAGE_RECENT))
-                        camel_folder_change_info_recent_uid(changes, camel_message_info_uid (mi));
-        }
+
+        /* Dumping each nextn'th header (this is needed for the nextn calc to 
+           be correct), it *must* write the summary cache to disk! */
+
+        camel_folder_summary_dump_mmap (folder->summary);
-
-        for ( ; i < messages->len; i++) {
@@ -2604 +2637 @@
-                        camel_message_info_free(&mi->info);
-        }
-        
-        g_ptr_array_free (messages, TRUE);
-
-return
-
+
+goto endbmore
+
- lose:
-        if (fetch_data) {
@@ -2624 +2656 @@
-                g_ptr_array_free (messages, TRUE);
-        }
+
+        endbmore:
+        count = 0;
+   } /* more */
-}
-

trunk/libtinymail-camel/camel-lite/camel/providers/nntp/camel-nntp-summary.c

@@ -220 +220 @@
-        char *line, *tab;
-        int len, ret;
-
+, acnt=0
-        struct _xover_header *xover;
-
@@ -282 +282 @@
-                        mi = (CamelMessageInfoBase *)camel_folder_summary_uid(s, cns->priv->uid);
-                        if (mi == NULL) {
+
+                                if (acnt > 1000) {
+                                        camel_folder_summary_dump_mmap (s);
+                                        acnt = 0;
+                                }
+                                acnt++;
-                                mi = (CamelMessageInfoBase *)camel_folder_summary_add_from_header(s, headers);
-                                if (mi) {
@@ -315 +321 @@
-        int ret = -1;
-        char *line, *msgid;
-
+, acnt=0
-        CamelMessageInfo *mi;
-        CamelMimeParser *mp;
@@ -349 +355 @@
-                        line[1] = 0;
-                        cns->priv->uid = g_strdup_printf("%u,%s\n", n, msgid);
+
-                        mi = camel_folder_summary_uid(s, cns->priv->uid);
-                        if (mi == NULL) {
-                                if (camel_mime_parser_init_with_stream(mp, (CamelStream *)store->stream) == -1)
-                                        goto error;
+                                if (acnt > 1000) {
+                                        camel_folder_summary_dump_mmap (s);
+                                        acnt = 0;
+                                }
+                                acnt++;
-                                mi = camel_folder_summary_add_from_parser(s, mp);
-                                while (camel_mime_parser_step(mp, NULL, NULL) != CAMEL_MIME_PARSER_STATE_EOF)

trunk/libtinymail-gnome-desktop/tny-gnome-device-priv.h

@@ -28 +28 @@
-    
-#ifdef GNOME
+        gboolean invnm;
-        guint callback_id;
-        libnm_glib_ctx *nm_ctx;

trunk/libtinymail-gnome-desktop/tny-gnome-device.c

@@ -58 +58 @@
-        priv->fset = FALSE;
-        priv->forced = FALSE;
-
+
-#ifdef GNOME
-        nm_callback (priv->nm_ctx, self);
@@ -75 +75 @@
-        nm_callback (priv->nm_ctx, self);
-#endif
-
+
-        return;
-}
@@ -89 +89 @@
-
-#ifdef GNOME
-    
-
-
+       
+
+
-        return;
-}
@@ -118 +118 @@
-
-#ifdef GNOME
-
+ && !priv->invnm
-        {
-                libnm_glib_state state = libnm_glib_get_network_state (priv->nm_ctx);
@@ -124 +124 @@
-                switch (state)
-                {
-NO
-FALS
+ACTIVE
+TRU
-                        break;
-
@@ -131 +131 @@
-                        case LIBNM_NO_NETWORKMANAGER:
-                        case LIBNM_INVALID_CONTEXT:
-
-
+
+
+
-                        default:
-TRU
+FALS
-                        break;
-                }
-        }
-#endif
-
+
-        return retval;
-}
@@ -150 +151 @@
-        TnyGnomeDevicePriv *priv = TNY_GNOME_DEVICE_GET_PRIVATE (self);
-
+        priv->invnm = FALSE;
-        priv->fset = FALSE;
-        priv->forced = FALSE;
@@ -157 +159 @@
-        priv->callback_id = libnm_glib_register_callback 
-                (priv->nm_ctx, nm_callback, self, NULL);
-
-
-
+
+
-        return;
-}