Synchronizing Mail with monotone

I have inboxes across several hosts stored in djb's Maildir format.
One of the things I would like to be able to do is merge/distribute all of those Maildirs to other hosts.
Discounting how much simpler this would be to with rsync, I decided to try monotone for this.
This perl script issues the appropriate monotone add/drop/rename commands, for new/deleted and modified messages.
It does this by keeping track of the inodes and filenames in the Maildirs, and calculating the difference between the last current and last state of the
Maildir.
This script is run from cron. I'm currently using it to sync a few of my inboxes of about 40k messages (~300MiB), and about 1000 add/drop/rename
operations a day. The script typically takes about a minute to run, and a
netsync takes about 2 minutes.
There are a few annoyances:
The inital sync of the maildir branch takes a long time
Each update requires a merge, which can take a few minutes
I'm happy with it. It means that I don't have to worry as much about my mail going away when ice is on the wrong side of a network schism.
<font color="#8080ff"><b>#!/usr/bin/env perl</b></font>
<font color="#ffff00"><b>use warnings</b></font>;
<font color="#ffff00"><b>use strict</b></font>;

<font color="#ffff00"><b>use </b></font>DB_File;
<font color="#ffff00"><b>use </b></font>File::Find;
<font color="#ffff00"><b>use </b></font>File::Basename;
<font color="#ffff00"><b>use </b></font>Data::Dumper;

<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$MAIL_BASE</b></font>=<font color="#ff40ff"><b>&quot;</b></font><font color="#00ffff"><b>$ENV</b></font><font color="#ff40ff"><b>{&#39;HOME&#39;}/Mail</b></font><font color="#ff40ff"><b>&quot;</b></font>;
<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$INODE_CACHE</b></font>=<font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>${MAIL_BASE}/inode_cache.db</b></font><font color="#ff40ff"><b>&quot;</b></font>;
<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@FOLDERS</b></font>=(<font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>inbox</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>sent</b></font><font color="#ff40ff"><b>&#39;</b></font>);

<font color="#ffff00"><b>my</b></font> (<font color="#00ffff"><b>@add</b></font>,<font color="#00ffff"><b>@drop</b></font>,<font color="#00ffff"><b>@rename</b></font>);
<font color="#ffff00"><b>my</b></font> (<font color="#00ffff"><b>%new_inodes</b></font>,<font color="#00ffff"><b>%done</b></font>);

<font color="#ffff00"><b>tie</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>%old_inodes</b></font>, <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>DB_File</b></font><font color="#ff40ff"><b>&quot;</b></font>, <font color="#00ffff"><b>$INODE_CACHE</b></font>, O_CREAT|O_RDWR, <font color="#ff40ff"><b>0644</b></font>, <font color="#00ffff"><b>$DB_HASH</b></font>
        <font color="#ffff00"><b>or</b></font> <font color="#ffff00"><b>die</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>Cannot open file </b></font><font color="#00ffff"><b>$INODE_CACHE</b></font><font color="#ff40ff"><b>: </b></font><font color="#00ffff"><b>$!</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;

<font color="#ffff00"><b>chdir</b></font> <font color="#00ffff"><b>$MAIL_BASE</b></font>;

<font color="#ffff00"><b>print</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>scanning inodes...</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;

<font color="#ffff00"><b>foreach</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$folder</b></font> (<font color="#00ffff"><b>@FOLDERS</b></font>) {
        <font color="#ffff00"><b>print</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>${MAIL_BASE}/${folder}</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;
        File::Find::find({<font color="#ff40ff"><b>no_chdir </b></font>=&gt; <font color="#ff40ff"><b>1</b></font>, <font color="#ff40ff"><b>wanted </b></font>=&gt; <font color="#00ffff"><b>\&amp;wanted</b></font>}, <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>${folder}</b></font><font color="#ff40ff"><b>&quot;</b></font>);
}

<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@keys</b></font>  = <font color="#ffff00"><b>keys</b></font> <font color="#00ffff"><b>%old_inodes</b></font>;
<font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@keys</b></font>, <font color="#ffff00"><b>keys</b></font> <font color="#00ffff"><b>%new_inodes</b></font>;

<font color="#ffff00"><b>print</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>finding differences...</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;

<font color="#ffff00"><b>foreach</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$k</b></font> (<font color="#ffff00"><b>sort</b></font>(<font color="#00ffff"><b>@keys</b></font>)) {
        <font color="#ffff00"><b>next</b></font> <font color="#ffff00"><b>if</b></font> <font color="#ffff00"><b>defined</b></font> <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>} ;

        <font color="#00ffff"><b># In old, but not in new -&gt; drop</b></font>
        <font color="#ffff00"><b>if</b></font> (<font color="#ffff00"><b>not</b></font> <font color="#ffff00"><b>defined</b></font> <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>}) {
                <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@drop</b></font>, <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
                <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>}=<font color="#ff40ff"><b>1</b></font>;
                <font color="#ffff00"><b>next</b></font>;
        }

        <font color="#00ffff"><b># In new, but not in old -&gt; add</b></font>
        <font color="#ffff00"><b>if</b></font> (<font color="#ffff00"><b>not</b></font> <font color="#ffff00"><b>defined</b></font> <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>}) {
                <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@add</b></font>, <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
                <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>}=<font color="#ff40ff"><b>1</b></font>;
                <font color="#ffff00"><b>next</b></font>;
        }

        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$nfile</b></font>=<font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$ofile</b></font>=<font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};

        <font color="#00ffff"><b># inodes same, and files are the same -&gt; do nothing</b></font>
        <font color="#ffff00"><b>if</b></font> (<font color="#00ffff"><b>$nfile</b></font> <font color="#ffff00"><b>eq</b></font> <font color="#00ffff"><b>$ofile</b></font>) {
                <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>}=<font color="#ff40ff"><b>1</b></font>;
                <font color="#ffff00"><b>next</b></font>;
        }

        <font color="#00ffff"><b>$nfile</b></font> =~ <font color="#ffff00"><b>s/</b></font><font color="#ff40ff"><b>:</b></font><font color="#ff6060"><b>.*</b></font><font color="#ffff00"><b>//</b></font>;
        <font color="#00ffff"><b>$ofile</b></font> =~ <font color="#ffff00"><b>s/</b></font><font color="#ff40ff"><b>:</b></font><font color="#ff6060"><b>.*</b></font><font color="#ffff00"><b>//</b></font>;
                <font color="#00ffff"><b># strip off the trailing garbage. </b></font>

        <font color="#00ffff"><b># inodes same, and the msg basename is the same -&gt; move</b></font>
        <font color="#ffff00"><b>if</b></font> ( basename(<font color="#00ffff"><b>$nfile</b></font>) <font color="#ffff00"><b>eq</b></font> basename(<font color="#00ffff"><b>$ofile</b></font>) ) {
                <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@rename</b></font>, [ <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>}, <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>} ] ;
                <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>}=<font color="#ff40ff"><b>1</b></font>;
        } <font color="#ffff00"><b>else</b></font> {
        <font color="#00ffff"><b># inodes same, and the msg basename different. Reused inode. Drop old,</b></font>
        <font color="#00ffff"><b># add new</b></font>
                <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@drop</b></font>, <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
                <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@add</b></font>, <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
                <font color="#00ffff"><b>$done</b></font>{<font color="#00ffff"><b>$k</b></font>}=<font color="#ff40ff"><b>1</b></font>;
        }
}

<font color="#00ffff"><b>#</b></font><span style="background-color: #ffff00"><font color="#808080">XXX</font></span><font color="#00ffff"><b>: Bleech. There&#39;s gotta be a better way to chunk the arrays through to monotone. </b></font>

<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$commit</b></font>=<font color="#ff40ff"><b>0</b></font>;
<font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$work</b></font>=<font color="#ff40ff"><b>0</b></font>;

<font color="#ffff00"><b>print</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>marking differences...</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;

<font color="#ffff00"><b>while</b></font> ( <font color="#00ffff"><b>$#add</b></font> &gt; <font color="#00ffff"><b>$work</b></font> ) {
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@cmd</b></font>=(<font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>monotone</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>add</b></font><font color="#ff40ff"><b>&#39;</b></font>);
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$chunk</b></font>=((<font color="#00ffff"><b>$#add</b></font> - <font color="#00ffff"><b>$work</b></font>) &gt; <font color="#ff40ff"><b>500</b></font>) ? <font color="#ff40ff"><b>500</b></font> : <font color="#00ffff"><b>$#add</b></font>-<font color="#00ffff"><b>$work</b></font>;
        <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@cmd</b></font>, <font color="#00ffff"><b>@add</b></font>[<font color="#00ffff"><b>$work</b></font>..<font color="#00ffff"><b>$work</b></font>+<font color="#00ffff"><b>$chunk</b></font>];

        <font color="#ffff00"><b>system</b></font>(<font color="#00ffff"><b>@cmd</b></font>) == <font color="#ff40ff"><b>0</b></font> <font color="#ffff00"><b>or</b></font> <font color="#ffff00"><b>die</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>system </b></font><font color="#00ffff"><b>@cmd</b></font><font color="#ff40ff"><b> failed: </b></font><font color="#00ffff"><b>$?</b></font><font color="#ff40ff"><b>&quot;</b></font>;
        <font color="#00ffff"><b>$work</b></font>+=<font color="#00ffff"><b>$chunk</b></font>;
        <font color="#00ffff"><b>$commit</b></font>=<font color="#ff40ff"><b>1</b></font>;
}
<font color="#00ffff"><b>$work</b></font>=<font color="#ff40ff"><b>0</b></font>;
<font color="#ffff00"><b>while</b></font> ( <font color="#00ffff"><b>$#drop</b></font> &gt; <font color="#00ffff"><b>$work</b></font> ) {
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@cmd</b></font>=(<font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>monotone</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>drop</b></font><font color="#ff40ff"><b>&#39;</b></font>);
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$chunk</b></font>=((<font color="#00ffff"><b>$#drop</b></font> - <font color="#00ffff"><b>$work</b></font>) &gt; <font color="#ff40ff"><b>500</b></font>) ? <font color="#ff40ff"><b>500</b></font> : <font color="#00ffff"><b>$#drop</b></font>-<font color="#00ffff"><b>$work</b></font>;
        <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@cmd</b></font>, <font color="#00ffff"><b>@drop</b></font>[<font color="#00ffff"><b>$work</b></font>..<font color="#00ffff"><b>$work</b></font>+<font color="#00ffff"><b>$chunk</b></font>];

        <font color="#ffff00"><b>system</b></font>(<font color="#00ffff"><b>@cmd</b></font>) == <font color="#ff40ff"><b>0</b></font> <font color="#ffff00"><b>or</b></font> <font color="#ffff00"><b>die</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>system </b></font><font color="#00ffff"><b>@cmd</b></font><font color="#ff40ff"><b> failed: </b></font><font color="#00ffff"><b>$?</b></font><font color="#ff40ff"><b>&quot;</b></font>;
        <font color="#00ffff"><b>$work</b></font>+=<font color="#00ffff"><b>$chunk</b></font>;
        <font color="#00ffff"><b>$commit</b></font>=<font color="#ff40ff"><b>1</b></font>;
}

<font color="#00ffff"><b># have to iterate through the renames. </b></font>
<font color="#ffff00"><b>foreach</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$e</b></font> (<font color="#00ffff"><b>@rename</b></font>) {
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@cmd</b></font>=(<font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>monotone</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>rename</b></font><font color="#ff40ff"><b>&#39;</b></font>);
        <font color="#ffff00"><b>push</b></font> <font color="#00ffff"><b>@cmd</b></font>, @{<font color="#00ffff"><b>$e</b></font>};
        <font color="#ffff00"><b>system</b></font>(<font color="#00ffff"><b>@cmd</b></font>) == <font color="#ff40ff"><b>0</b></font> <font color="#ffff00"><b>or</b></font> <font color="#ffff00"><b>die</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>system </b></font><font color="#00ffff"><b>@cmd</b></font><font color="#ff40ff"><b> failed: </b></font><font color="#00ffff"><b>$?</b></font><font color="#ff40ff"><b>&quot;</b></font>;
        <font color="#00ffff"><b>$commit</b></font>=<font color="#ff40ff"><b>1</b></font>;
}

<font color="#ffff00"><b>if</b></font> ( <font color="#00ffff"><b>$commit</b></font> != <font color="#ff40ff"><b>0</b></font> ) {
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>@cmd</b></font>=(<font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>monotone</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>commit</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>-m</b></font><font color="#ff40ff"><b>&#39;</b></font>, <font color="#ff40ff"><b>&#39;</b></font><font color="#ff40ff"><b>automatic commit</b></font><font color="#ff40ff"><b>&#39;</b></font>);
        <font color="#ffff00"><b>system</b></font>(<font color="#00ffff"><b>@cmd</b></font>) == <font color="#ff40ff"><b>0</b></font> <font color="#ffff00"><b>or</b></font> <font color="#ffff00"><b>die</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>system </b></font><font color="#00ffff"><b>@cmd</b></font><font color="#ff40ff"><b> failed: </b></font><font color="#00ffff"><b>$?</b></font><font color="#ff40ff"><b>&quot;</b></font>;
}

<font color="#00ffff"><b># </b></font><span style="background-color: #ffff00"><font color="#808080">XXX</font></span><font color="#00ffff"><b>: Bleech, There&#39;s probably a better way of doing this. </b></font>

<font color="#ffff00"><b>print</b></font> <font color="#ff40ff"><b>&quot;</b></font><font color="#ff40ff"><b>updatin inode cache...</b></font><font color="#ff6060"><b>\n</b></font><font color="#ff40ff"><b>&quot;</b></font>;

<font color="#ffff00"><b>foreach</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$k</b></font> (<font color="#ffff00"><b>keys</b></font> <font color="#00ffff"><b>%old_inodes</b></font>) {
        <font color="#ffff00"><b>delete</b></font> <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
}

<font color="#ffff00"><b>foreach</b></font> <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$k</b></font> (<font color="#ffff00"><b>keys</b></font> <font color="#00ffff"><b>%new_inodes</b></font>) {
        <font color="#00ffff"><b>$old_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>} = <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$k</b></font>};
}

<font color="#ffff00"><b>untie</b></font> <font color="#00ffff"><b>%old_inodes</b></font>;

<font color="#00ffff"><b>##############################</b></font>

<font color="#ffff00"><b>sub</b></font><font color="#00ffff"><b> </b></font><font color="#00ffff"><b>wanted</b></font><font color="#00ffff"><b> </b></font>{
        <font color="#ffff00"><b>my</b></font> <font color="#00ffff"><b>$f</b></font>=<font color="#00ffff"><b>$_</b></font>;
        ! <font color="#ffff00"><b>-f</b></font> <font color="#00ffff"><b>$f</b></font> &amp;&amp; <font color="#ffff00"><b>return</b></font> ;
        <font color="#ffff00"><b>my</b></font> (<font color="#00ffff"><b>$dev</b></font>,<font color="#00ffff"><b>$ino</b></font>) = <font color="#ffff00"><b>lstat</b></font>(<font color="#00ffff"><b>$f</b></font>);


        <font color="#00ffff"><b>$new_inodes</b></font>{<font color="#00ffff"><b>$ino</b></font>}=<font color="#00ffff"><b>$File::Find::name</b></font>;
}
jack | posted Tue Sep 20 17:00:32 2005 | #
category: projects/mail_sync