Yet An Other Convert Script
Well, we used above examples, but added a lot of things which were missing in above versions:
- Added support for seen/unseen status
- Added support for system flags (flagged/replied/draft/trashed)
- Added support for courier user flags
- Added support for folder subscription
Hope it's usefull for someone.
#!/usr/bin/perl
# Licence: GPL 2006, Samage
# Author: Mart van Santen
#
# This scrips converts maildir boxes to cyrus boxes,
# please BE VERY CAREFULL WITH EXECUTION OF THIS SCRIPT,
# it is made for our internal use and some options
# are a littlebit dangerous.
#
# What it does is this:
#
# - Takes a Maildir box en copies its email to a cyrus box
# - Read flags from user Maildir boxes and write them in
# the cyrus database
# - Creates cyrus mailboxes
#
# Some code works on the cyrus databases directly, without
# using the cyrus commands. Please be very carefull if
# you are running an other version then we did
# (that was, 2.2.13-8 from debian etch/testing)
# It is possible that the database format has been changed.
#
# Also be sure cyrus is not running when converting
#
# It alters these files directly (some files also indirectly):
# 1. /var/lib/cyrus/mailboxes.db (cyrus skiplist file)
# 2. /var/lib/cyrus/user/$char/$user.seen (skiplist file)
# 3. /var/lib/cyrus/user/$char/$user.sub (plaintext file)
# 4. /var/spool/cyrus/mail/$char/user/$mailbox/cyrus.header
# 5. /var/spool/cyrus/mail/$char/user/$mailbox/cyrus.index
#
# This is what is done:
# For file 1. Add user mailboxes and subboxes
# For file 2. Write list of 'seen' messages
# For file 3. Write list of subscribed mailboxes
# For file 4. Read mailbox id & add custom flags
# For file 5. Write per message system & custom flags
#
# ALL FILES CREATED BY THIS SCRIPT ARE WORLD READABLE/WRITABLE,
# CHANGE FILE MODES AFTER CONVERSION
#
# Thanks to & parts used from:
# http://www.codepoets.co.uk/docs/courier2cyrus (David Goodwin)
# http://loophole.morpheus.net/linux/ (index_dump_pl.txt)
#
# This file is distributed under GPL, please send all
# changes/additions to me.
#
# usage: maildir2cyrus maildir_username cyrus_username
#
# be sure cyrus is NOT running when executing
# Usefull include for debugging
use Data::Dumper;
# More output, use fast to
$DEBUG = 1;
$FAST = 0;
$user = $ARGV[0];
$to = $ARGV[1];
$c = substr($to, 0, 1);
#####################################################
# Change these items according your local config
$source = "/mail/$user/Maildir";
$destination = "/mail/cyrus/mail/$c/user/$to";
$CYRUS_UID = 501;
$CYRUS_GID = 8;
$CYRUS_USER = "cyrus";
$CYRUS_RECONSTRUCT = "/usr/sbin/cyrreconstruct";
$CYRUS_DBCONVERT = "/usr/sbin/cvt_cyrusdb";
$CYRUS_ROOT = "/var/lib/cyrus";
# End: Local Config
#####################################################
print "Handling user: $user => $to\n";
# Some ugly global, used to map maildirfile to their
# cyrus index number
%map;
# Handel all
handleall($source, $destination, $to, $to);
# Main function, takes a userbox and convert it,
# also handle subfolders
sub handleall {
my $src = $_[0];
my $dst = $_[1];
my $user = $_[2];
my $user2 = $_[3];
my %flags;
my $c = 1;
# Flush GLOBAL map
undef(%map);
$c = 1;
$t = 1;
print " * Processing mainfolder:\n" if $DEBUG;
# Create userbox for this user
createmb($user2, "");
# Create destination dir
$tmp = $dst;
$tmp =~ s/'/'\\''/g;
system("mkdir -p '$tmp'");
chown $CYRUS_UID, $CYRUS_GID, $dst;
chmod 0777, $dst;
# Handele cur/new dirs of maildir
$c = handledir("$src/cur",$dst, $c);
$c = handledir("$src/new",$dst, $c);
# Reconstruct folder
system("su $CYRUS_USER -c '$CYRUS_RECONSTRUCT user.$user'");
# Add userflags
$userflags = getuserflags($src);
my $mailbox_id = handlecyrusheader($dst, $userflags);
# Handle index: write flags
handlecyrusindex($dst);
# Handle seen database
handlecyrusseen($mailbox_id, $user2);
# Read subdits
opendir(DIR, $src);
@files = readdir(DIR);
closedir(DIR);
$t = ($c - 1);
foreach $dir (@files) {
next if $dir eq ".";
next if $dir eq "..";
next if (! -d "$src/$dir");
next if (substr($dir, 0, 1) ne ".");
print " * Processing subfolder: $dir\n" if $DEBUG;
$c = 1;
undef(%map);
$src2 = $src . "/$dir";
$dir = substr($dir, 1);
$dst_dir = $dir;
$folder = $dir;
$dst_dir =~ s/\./\//g;
$dst2 = $dst . "/" . $dst_dir;
$tmp = $dst2;
$tmp =~ s/'/'\\''/g;
system("mkdir -p '$tmp'");
chmod 0777, $dst2;
chown $CYRUS_UID, $CYRUS_GID, $dst2;
createmb($user, $folder);
$c = handledir("$src2/cur",$dst2, $c);
$c = handledir("$src2/new",$dst2, $c);
$t += ($c - 1);
$userflags = getuserflags($src2);
$folder =~ s/'/'\\''/g;
open(OUT, ">/tmp/cyrus-convert.sh");
print OUT "#!/bin/sh\n";
print OUT "$CYRUS_RECONSTRUCT 'user.$user.$folder'\n";
close(OUT);
chmod 0777, "/tmp/cyrus-convert.sh";
system("su $CYRUS_USER -c /tmp/cyrus-convert.sh");
my $mailbox_id = handlecyrusheader($dst2, $userflags);
print "MAILBOX: $mailbox_id\n";
handlecyrusindex($dst2);
handlecyrusseen($mailbox_id, $user);
}
# New, add junk mailbox to every account
createmb($user, "Junk");
system("mkdir -p $dst/Junk");
chmod 0777, "$dst/Junk";
chown $CYRUS_UID, $CYRUS_GID, "$dst/Junk";
open(OUT, ">/tmp/cyrus-convert.sh");
print OUT "#!/bin/sh\n";
print OUT "$CYRUS_RECONSTRUCT 'user.$user.Junk'\n";
close(OUT);
system("su $CYRUS_USER -c /tmp/cyrus-convert.sh");
updatesubscriptions($user, $src);
print " * Handled $t mails, finished\n";
}
sub updatesubscriptions {
my $user = $_[0];
my $src = $_[1];
if (! -f "$src/courierimapsubscribed") {
return;
}
print " * Updating subscribtions\n" if $DEBUG;
$l = substr($user, 0, 1);
open(IN, "<$src/courierimapsubscribed");
open(OUT, ">$CYRUS_ROOT/user/$l/${user}.sub");
$junk = 0;
while() {
$_ =~ s/^INBOX/user.$user/;
$_ =~ s/$/\t/;
print OUT $_;
if ($_ =~ m/^user.$user.Junk/) {
$junk = 1;
}
}
if ($junk == 0) {
print OUT "user.$user.Junk\t\n";
}
close(IN);
close(OUT);
chown $CYRUS_UID, $CYRUS_GID, "$CYRUS_ROOT/user/$l/${user}.sub";
chmod 0666, "$CYRUS_ROOT/user/$l/${user}.sub";
return;
}
sub createmb {
my $user = $_[0];
my $folder = $_[1];
my $mb = $user;
if ($folder ne "") {
$mb .= "." . $folder;
}
unlink("/tmp/cyrus-mailbox");
unlink("/tmp/cyrus-mailbox.out");
system("$CYRUS_DBCONVERT $CYRUS_ROOT/mailboxes.db skiplist /tmp/cyrus-mailbox flat");
open(IN, "/tmp/cyrus-mailbox.out");
$found = 0;
while() {
if (m/^user.$mb\t/) {
$found = 1;
}
print OUT $_;
}
if ($found == 0) {
print " - Add mailbox $mb\n" if $DEBUG;
print OUT "user.$mb\t0 default $user\tlrswipcda\t\n";
}
close(IN);
close(OUT);
if ($found == 0) {
unlink("$CYRUS_ROOT/mailboxes.db");
system("$CYRUS_DBCONVERT /tmp/cyrus-mailbox.out flat $CYRUS_ROOT/mailboxes.db skiplist");
chown $CYRUS_UID, $CYRUS_GID, "$CYRUS_ROOT/mailboxes.db";
chmod 0666, "$CYRUS_ROOT/mailboxes.db";
}
}
sub handlecyrusseen {
my $mailbox = $_[0];
my $user = $_[1];
print " - Fixing seen index file ($user)...\n" if $DEBUG;
my @seen;
foreach $file_id (keys %map) {
if ($map{$file_id}{'systemflags'}{'seen'}) {
push @seen, $map{$file_id}{'cyrusid'};
}
}
@seen = sort {$a <=> $b} @seen;
$seen_string = "";
$last = -1;
$range = 0;
foreach $cyrus_id (@seen) {
if ($cyrus_id == $last + 1) {
$range = 1;
}
else {
if ($range == 1) {
$seen_string .= ":$last";
}
$range = 0;
$seen_string .= ",$cyrus_id";
}
$last = $cyrus_id;
}
if ($range == 1) {
$seen_string .= ":$last";
}
if ($seen_string eq "") {
print " - Empty mailbox, skipping\n" if $DEBUG;
return;
}
$seen_string = substr($seen_string, 1);
$stamp = time;
$line = "$mailbox\t1 $stamp $last $stamp $seen_string";
open(OUT, ">/tmp/cyrus-seen.out");
$l = substr($user, 0, 1);
$src = "$CYRUS_ROOT/user/$l/$user.seen";
unlink("/tmp/cyrus-seen");
my $written = 0;
if (-f $src) {
system("$CYRUS_DBCONVERT $src skiplist /tmp/cyrus-seen flat");
open(IN, ") {
if ($_ =~ m/^$mailbox/) {
print OUT $line . "\n";
$written = 1;
}
else {
print OUT $_;
}
}
close(IN);
}
if ($written == 0) { print OUT $line . "\n"; }
close(OUT);
system("$CYRUS_DBCONVERT /tmp/cyrus-seen.out flat $src skiplist");
chown $CYRUS_UID, $CYRUS_GID, $src;
chmod 0666, $src;
# DB format:
# SPSPSPSP
}
sub handlecyrusindex {
print " - Fixing mailbox index file...\n" if $DEBUG;
my $dst = $_[0];
my $index_file = $dst . '/cyrus.index';
my @field_names = ( "uid", "internal date", "sent date", "size",
"header size", "content offset", "cache offset", "last updated",
"system flags", "user 1", "user 2", "user 3", "user 4", "unknown 1", "unknown 2");
my %flags = ( "answered" => 1<<0, "flagged" => 1<<1, "deleted" => 1<<2, "draft" => 1<<3 );
sysopen(IDX,$index_file, 2) or return "Failed to open index.\n";
# Skip header..
return "unable to seek $!" unless sysseek(IDX,76,0);
my $seek = 76;
my $i = 1;
my $r = sysread(IDX,$doi,60);
while($r){
# First 8, system fields,
# Next 5, flags,
# Last 2, unknown
my @data = unpack("N8N5N2",$doi);
my $j = 0;
foreach $field_name (@field_names){
$record{$field_name} = $data[$j];
$j++;
}
# Seek position
$record{'seek'} = $seek;
$records{$record{'uid'}} = { %record };
$seek += 60;
$r = sysread(IDX,$doi,60);
$i++;
}
# Index file read (cyrus)
foreach $file_id (keys %map) {
$cyrus_id = $map{$file_id}{'cyrusid'};
if ($records{$cyrus_id}) {
$offset = $records{$cyrus_id}{'seek'} + 32;
$data = 0;
$data = $data | 1<<0 if $map{$file_id}{'systemflags'}{'replied'};
$data = $data | 1<<1 if $map{$file_id}{'systemflags'}{'flagged'};
# $data = $data | 1<<2 if $map{$file_id}{'systemflags'}{'deleted'};
$data = $data | 1<<3 if $map{$file_id}{'systemflags'}{'draft'};
# Write system flags
$data = pack("N1", $data);
die "unable to seek!" unless sysseek(IDX, $offset, 0);
die "unable to write!" unless syswrite(IDX, $data, 4);
# Write custom flags
$flag_string = $map{$file_id}{'customflags'};
@custom_flags = split /\ /, $flag_string;
$data = 0;
foreach $flag (@custom_flags) {
$data = $data | 1<<$flag
}
$data = pack("N4", $data);
die "unable to seek!" unless sysseek(IDX, $offset + 4, 0);
die "unable to write!" unless syswrite(IDX, $data, 16);
}
else {
print "No cyrus-index entry found for maildir file: " . $file_id . "\n";
}
}
close(IDX);
}
sub handlecyrusheader {
my $dst = $_[0];
my $customflags = $_[1];
my $header = $dst . '/cyrus.header';
print " - Fixing mailbox header file...\n" if $DEBUG;
open(IN, "<$header");
$data = "";
while() {
$data .= $_;
}
close(IN);
# Header = 115 chars
# Header + tab = 116
my $offset = 116;
my $eol = index($data, "\n",$offset);
my $neol = index($data, "\n",$eol+1);
my $mailbox = substr($data, $offset, $eol - $offset);
my $start = substr($data, 0, $eol+1);
my $end = substr($data, $neol);
open(OUT, ">$header");
print OUT $start;
foreach my $key (@{$customflags}) {
print OUT "$key ";
}
print OUT $end;
close(OUT);
return $mailbox;
}
sub getuserflags {
my $src = $_[0];
my @flags;
return \@flags if (! -d "$src/courierimapkeywords");
return \@flags if (! -f "$src/courierimapkeywords/:list");
open(IN, "<$src/courierimapkeywords/:list") or die ("Error opening");
my $header_done = 0;
my $k = 0;
while() {
if ($_ eq "\n") { $header_done = 1; next; };
chop;
if ($header_done == 0) {
$flags[$k] = $_;
$k++;
}
else {
my $file_id_pos = index($_, ":");
my $file_id = substr($_, 0, $file_id_pos);
# Check if file is in map, else no need to add to index
if ($map{$file_id}) {
$flag_string = substr($_, $file_id_pos + 1);
$map{$file_id}{'customflags'} = $flag_string;
}
}
}
close(IN);
return \@flags;
}
sub handledir {
my $src = $_[0];
my $dst = $_[1];
my $i = $_[2];
my $file;
$j = 0;
if (! -d $src) { return $i; }
opendir(DIR, $src);
my @files = readdir(DIR);
closedir(DIR);
foreach $file (@files) {
next if $file eq ".";
next if $file eq "..";
$j++;
next if ($FAST && $j > 25);
if (-f "$src/$file") {
$file_src = "$src/$file";
$file_dst = $dst . "/" . $i . ".";
# If file already exists, increase number
while (-f $file_dst) {
$i++;
$file_dst = $dst . "/" . $i . ".";
}
open(IN, "<$file_src");
$data = "";
while() {
$data .= $_;
}
$data =~ s/\n/\r\n/g;
close(IN);
open(OUT, ">$file_dst");
print OUT $data;
close(OUT);
chown $CYRUS_UID, $CYRUS_GID, $file_dst;
chmod 0666, $file_dst;
my $file_id_pos = index($file, ":2,");
my $file_id = substr($file, 0, $file_id_pos);
$map{$file_id}{'cyrusid'} = $i;
$systemflags = getsystemflags($file_src);
# TODO: Add dovecot custom flags
$map{$file_id}{'cyrusid'} = $i;
$map{$file_id}{'systemflags'} = $systemflags;
$i++;
}
if ($i % 250 == 0) { print "Handled $i messages so far...\n" if $DEBUG; }
}
return $i;
}
sub getsystemflags {
# R: replied
# S: seen
# T: deleted/junk
# D: draft
# F: flagged
# P: Not used, implemented as user-flag
my $file = $_[0];
my $fs = index($file, ":2,");
my $flag_string = substr($file, $fs+3);
my %flags;
$flags{'seen'} = 1 if index($flag_string, 'S') >= 0;
$flags{'replied'} = 1 if index($flag_string, 'R') >= 0;
$flags{'junk'} = 1 if index($flag_string, 'T') >= 0;
$flags{'draft'} = 1 if index($flag_string, 'D') >= 0;
$flags{'flagged'} = 1 if index($flag_string, 'F') >= 0;
return \%flags;
}
Recent comments
5 days 2 hours ago
5 days 4 hours ago
5 days 14 hours ago
1 week 3 days ago
1 week 3 days ago
1 week 5 days ago
1 week 5 days ago
2 weeks 1 day ago
2 weeks 2 days ago
2 weeks 2 days ago