From 8e85be38ac747a47d2857a576a0f8790e93f5238 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 3 Jul 2015 23:43:05 -0500 Subject: [PATCH 001/331] GmailDB: tables for calendars and contacts --- JMAP/GmailDB.pm | 43 +++++++++++++++++++++++++++++++++++++++++++ JMAP/Sync/Gmail.pm | 4 ++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 452b34f..f8986bc 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -766,6 +766,49 @@ CREATE TABLE IF NOT EXISTS imessages ( size INTEGER, mtime DATE NOT NULL ); +EOF + + $dbh->do(<do(<do(<do(<{calendars}) { + if ($Self->{calendars}) { $Self->{lastused} = time(); return $Self->{calendars}; } @@ -50,7 +50,7 @@ sub get_calendars { sub get_contacts { my $Self = shift; - if ($self->{contacts}) { + if ($Self->{contacts}) { $Self->{lastused} = time(); return $Self->{contacts}; } From fe6fba4c87792e50646c25c0c2585969b4f99e5e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 00:20:39 -0500 Subject: [PATCH 002/331] syncserver: handle contacts and calendars --- JMAP/Sync/Gmail.pm | 42 ++++++++++++++++++++++++++++++++++++++++-- bin/syncserver.pl | 5 +++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 8ffaed3..60646ae 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -29,7 +29,7 @@ sub DESTROY { } } -sub get_calendars { +sub connect_calendars { my $Self = shift; if ($Self->{calendars}) { @@ -47,7 +47,7 @@ sub get_calendars { return $Self->{calendars}; } -sub get_contacts { +sub connect_contacts { my $Self = shift; if ($Self->{contacts}) { @@ -107,6 +107,44 @@ sub connect_imap { die "Could not connect to IMAP server: $@"; } +sub get_calendars { + my $Self = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetCalendars(); + + return $data; +} + +sub get_events { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetEvents($Args->{href}); + + return $data; +} + +sub get_abooks { + my $Self = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetAdddressBooks(); + + return $data; +} + +sub get_contacts { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetContacts($Args->{href}); + + return $data; +} + sub send_email { my $Self = shift; my $rfc822 = shift; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index aeef23b..e83dfeb 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -104,9 +104,10 @@ sub mk_handler { my ($cmd, $args, $tag) = @$json; my $res = eval { - if (SyncServer->can("handle_$cmd")) { + my $fn = "handle_$cmd"; + if (SyncServer->can($fn)) { no strict 'refs'; - return ${"handle_$cmd"}->($args); + return $fn->($args); } die "Unknown command $cmd"; }; From a72464f8ddb3aa655cccda3365d44d8f62abe2b7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 01:19:47 -0500 Subject: [PATCH 003/331] all the rest of the sync backend support for calendar/contacts (in theory) --- JMAP/Sync/Gmail.pm | 4 ++-- bin/syncserver.pl | 34 ++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 60646ae..98b5352 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -130,7 +130,7 @@ sub get_abooks { my $Self = shift; my $talk = $Self->connect_contacts(); - my $data = $talk->GetAdddressBooks(); + my $data = $talk->GetAddressBooks(); return $data; } @@ -140,7 +140,7 @@ sub get_contacts { my $Args = shift; my $talk = $Self->connect_contacts(); - my $data = $talk->GetContacts($Args->{href}); + my $data = $talk->GetContacts($Args->{path}); return $data; } diff --git a/bin/syncserver.pl b/bin/syncserver.pl index e83dfeb..1e22209 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -73,23 +73,45 @@ sub handle_ping { sub handle_status { my $args = shift; - my $status = $backend->fetch_status(@$args); - return ['status', $status]; + my $data = $backend->fetch_status(@$args); + return ['status', $data]; } sub handle_folder { my $args = shift; - my $folder = $backend->fetch_folder(@$args); - return ['folder', $folder]; + my $data = $backend->fetch_folder(@$args); + return ['folder', $data]; } sub handle_folders { my $args = shift; - my $folders = $backend->folders(@$args); - return ['folders', $folders]; + my $data = $backend->folders(@$args); + return ['folders', $data]; } +sub handle_calendars { + my $args = shift; + my $data = $backend->get_calendars(@$args); + return ['calendars', $data]; +} + +sub handle_events { + my $args = shift; + my $data = $backend->get_events(@$args); + return ['events', $data]; +} +sub handle_abooks { + my $args = shift; + my $data = $backend->get_abooks(@$args); + return ['abooks', $data]; +} + +sub handle_contacts { + my $args = shift; + my $data = $backend->get_contacts(@$args); + return ['contacts', $data]; +} sub mk_handler { my ($db) = @_; From a9382885cfcd55bac4be2583d116e81bbbeb31a4 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 01:30:29 -0500 Subject: [PATCH 004/331] ICloud backend --- JMAP/Sync/ICloud.pm | 271 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 JMAP/Sync/ICloud.pm diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm new file mode 100644 index 0000000..710853c --- /dev/null +++ b/JMAP/Sync/ICloud.pm @@ -0,0 +1,271 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Sync::ICloud; +use base qw(JMAP::DB); + +use Mail::IMAPTalk; +use JSON::XS qw(encode_json decode_json); +use Email::Simple; +use Email::Sender::Simple qw(sendmail); +use Email::Sender::Transport::GmailSMTP; +use Net::CalDAVTalk; +use Net::CardDAVTalk; + +my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); + +sub new { + my $Class = shift; + my $auth = shift; + return bless { auth => $auth }, ref($Class) || $Class; +} + +sub DESTROY { + my $Self = shift; + if ($Self->{imap}) { + $Self->{imap}->logout(); + } +} + +sub connect_calendars { + my $Self = shift; + + if ($Self->{calendars}) { + $Self->{lastused} = time(); + return $Self->{calendars}; + } + + $Self->{calendars} = Net::CalDAVTalk->new( + user => $Self->{auth}{username}, + password => $Self->{auth}{password}, + url => "https://caldav.icloud.com/", + expandurl => 1, + ); + + return $Self->{calendars}; +} + +sub connect_contacts { + my $Self = shift; + + if ($Self->{contacts}) { + $Self->{lastused} = time(); + return $Self->{contacts}; + } + + $Self->{contacts} = Net::CardDAVTalk->new( + user => $Self->{auth}{username}, + password => $Self->{auth}{password}, + url => "https://contacts.icloud.com/", + expandurl => 1, + ); + + return $Self->{contacts}; +} + +sub connect_imap { + my $Self = shift; + + if ($Self->{imap}) { + $Self->{lastused} = time(); + return $Self->{imap}; + } + + for (1..3) { + $Self->log('debug', "Looking for server for $Self->{auth}{username}"); + my $port = 993; + my $usessl = $port != 143; # we use SSL for anything except default + $Self->log('debug', "getting imaptalk"); + $Self->{imap} = Mail::IMAPTalk->new( + Server => 'imap.mail.me.com', + Port => $port, + Username => $Self->{auth}{username}, + Password => $Self->{auth}{password}, + # not configurable right now... + UseSSL => $usessl, + UseBlocking => $usessl, + ); + next unless $Self->{imap}; + $Self->log('debug', "Connected as $Self->{auth}{username}"); + $Self->{lastused} = time(); + my @folders = $Self->{imap}->xlist('', '*'); + + delete $Self->{folders}; + delete $Self->{labels}; + foreach my $folder (@folders) { + my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; + my $name = $folder->[2]; + my $label = $role || $folder->[2]; + $Self->{folders}{$name} = $label; + $Self->{labels}{$label} = $name; + } + return $Self->{imap}; + } + + die "Could not connect to IMAP server: $@"; +} + +sub get_calendars { + my $Self = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetCalendars(); + + return $data; +} + +sub get_events { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetEvents($Args->{href}); + + return $data; +} + +sub get_abooks { + my $Self = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetAddressBooks(); + + return $data; +} + +sub get_contacts { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetContacts($Args->{path}); + + return $data; +} + +sub send_email { + my $Self = shift; + my $rfc822 = shift; + + my $email = Email::Simple->new($rfc822); + sendmail($email, { + from => $Self->{auth}{username}, + transport => Email::Sender::Transport::GmailSMTP->new({ + host => 'smtp.gmail.com', + port => 465, + ssl => 1, + sasl_username => $Self->{auth}{username}, + access_token => $Self->{auth}{access_token}, + }) + }); +} + +# read folder list from the server +sub folders { + my $Self = shift; + $Self->connect_imap(); + return $Self->{folders}; +} + +sub labels { + my $Self = shift; + $Self->connect_imap(); + return $Self->{labels}; +} + +sub fetch_status { + my $Self = shift; + my $justfolders = shift; + + my $imap = $Self->connect_imap(); + + my $folders = $Self->folders; + if ($justfolders) { + my %data = map { $_ => $folders->{$_} } + grep { exists $folders->{$_} } + @$justfolders; + $folders = \%data; + } + + my $fields = "(uidvalidity uidnext highestmodseq messages)"; + my $data = $imap->multistatus($fields, sort keys %$folders); + + return $data; +} + +sub fetch_folder { + my $Self = shift; + my $imapname = shift; + my $state = shift || { uidvalidity => 0 }; + + my $imap = $Self->connect_imap(); + + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + + my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidnext = $imap->get_response_code('uidnext'); + my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; + my $exists = $imap->get_response_code('exists') || 0; + + if ($state->{uidvalidity} != $uidvalidity) { + # force a delete/recreate and resync + $state = { + uidvalidity => $uidvalidity. + highestmodseq => 0, + uidnext => 0, + exists => 0, + }; + } + + if ($highestmodseq and $highestmodseq == $state->{highestmodseq}) { + $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); + return {}; # yay, nothing to do + } + + my $changed = {}; + if ($state->{uidnext} > 1) { + my $from = 1; + my $to = $state->{uidnext} - 1; + my @extra; + push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; + $Self->log('debug', "UPDATING $imapname: $from:$to"); + $changed = $imap->fetch("$from:$to", "(uid flags)", @extra) || {}; + } + + my $new = {}; + if ($uidnext > $state->{uidnext}) { + my $from = $state->{uidnext}; + my $to = $uidnext - 1; # or just '*' + $Self->log('debug', "FETCHING $imapname: $from:$to"); + $new = $imap->fetch("$from:$to", '(uid flags internaldate envelope rfc822.size)') || {}; + } + + my $alluids = undef; + if ($state->{exists} + scalar(keys %$new) > $exists) { + # some messages were deleted + my $from = 1; + my $to = $uidnext - 1; + # XXX - you could do some clever UID vs position queries to bisect this out, but it + # would need more data than we have here + $Self->log('debug', "COUNTING $imapname: $from:$to (something deleted)"); + $alluids = $imap->search("UID", "$from:$to"); + } + + return { + oldstate => $state, + newstate => { + highestmodseq => $highestmodseq, + uidvalidity => $uidvalidity. + uidnext => $uidnext, + exists => $exists, + }, + changed => $changed, + new => $new, + ($alluids ? (alluids => $alluids) : ()), + }; +} + +1; From a924a5bec9caad3aca1c8afa49b0ac15c70c4e05 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 02:10:01 -0500 Subject: [PATCH 005/331] ICloud support and working --- JMAP/Sync/ICloud.pm | 16 +++++++++++++--- bin/syncserver.pl | 9 ++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index 710853c..73a4f4f 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -121,9 +121,14 @@ sub get_events { my $Args = shift; my $talk = $Self->connect_calendars(); - my $data = $talk->GetEvents($Args->{href}); + my $data = $talk->GetEvents($Args->{href}, Full => 1); - return $data; + my %res; + foreach my $item (@$data) { + $res{$item->{id}} = $item->{_raw}; + } + + return \%res; } sub get_abooks { @@ -142,7 +147,12 @@ sub get_contacts { my $data = $talk->GetContacts($Args->{path}); - return $data; + my %res; + foreach my $item (@$data) { + $res{$item->{CPath}} = $item->{_raw}; + } + + return \%res; } sub send_email { diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 1e22209..6fe3e22 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -11,6 +11,7 @@ package SyncServer; use JSON::XS qw(encode_json decode_json); use Net::Server::PreFork; use JMAP::Sync::Gmail; +use JMAP::Sync::ICloud; use base qw(Net::Server::PreFork); @@ -24,7 +25,13 @@ package SyncServer; sub setup { my $config = shift; - $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; + if ($id =~ m/gmail\.com/) { + $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; + } elsif ($id =~ m/icloud\.com/) { + $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; + } else { + die "UNKNOWN ID $id"; + } warn "Connected $id"; $0 = "[jmap proxy imapsync] $id"; $hdl->push_write(json => [ 'setup', $id ]); From 51ba3df123f0cd3131a1f7d3bca95a56d5d47785 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 06:57:33 -0500 Subject: [PATCH 006/331] ICloud fixes --- JMAP/Sync/ICloud.pm | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index 73a4f4f..82a46b8 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -10,11 +10,11 @@ use Mail::IMAPTalk; use JSON::XS qw(encode_json decode_json); use Email::Simple; use Email::Sender::Simple qw(sendmail); -use Email::Sender::Transport::GmailSMTP; +use Email::Sender::Transport::SMTPS; use Net::CalDAVTalk; use Net::CardDAVTalk; -my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); +my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); sub new { my $Class = shift; @@ -90,7 +90,8 @@ sub connect_imap { next unless $Self->{imap}; $Self->log('debug', "Connected as $Self->{auth}{username}"); $Self->{lastused} = time(); - my @folders = $Self->{imap}->xlist('', '*'); + my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; + my @folders = $Self->{imap}->$list('', '*'); delete $Self->{folders}; delete $Self->{labels}; @@ -162,13 +163,13 @@ sub send_email { my $email = Email::Simple->new($rfc822); sendmail($email, { from => $Self->{auth}{username}, - transport => Email::Sender::Transport::GmailSMTP->new({ - host => 'smtp.gmail.com', - port => 465, - ssl => 1, + transport => Email::Sender::Transport::SMTPS->new({ + host => 'smtp.mail.me.com' + port => 587, + ssl => 'starttls', sasl_username => $Self->{auth}{username}, - access_token => $Self->{auth}{access_token}, - }) + sasl_password => $Self->{auth}{password}, + }), }); } From b6e03c2f428952be6904180b6b797d0eeceada86 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 07:00:18 -0500 Subject: [PATCH 007/331] server: sending thing --- bin/syncserver.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 6fe3e22..51bf995 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -120,6 +120,12 @@ sub handle_contacts { return ['contacts', $data]; } +sub handle_send { + my $args = shift; + my $data = $backend->send_email(@$args); + return ['sent', $data]; +} + sub mk_handler { my ($db) = @_; From bdcf7a50c74585ec6deb4946060bca2a14c3024e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 07:08:10 -0500 Subject: [PATCH 008/331] backend helo --- JMAP/Sync/Gmail.pm | 1 + JMAP/Sync/ICloud.pm | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 98b5352..81f4bb0 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -153,6 +153,7 @@ sub send_email { sendmail($email, { from => $Self->{auth}{username}, transport => Email::Sender::Transport::GmailSMTP->new({ + helo => 'proxy.jmap.io', host => 'smtp.gmail.com', port => 465, ssl => 1, diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index 82a46b8..b983617 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -164,7 +164,8 @@ sub send_email { sendmail($email, { from => $Self->{auth}{username}, transport => Email::Sender::Transport::SMTPS->new({ - host => 'smtp.mail.me.com' + helo => 'proxy.jmap.io', + host => 'smtp.mail.me.com', port => 587, ssl => 'starttls', sasl_username => $Self->{auth}{username}, From 1ccfaecf8ffeb02e918ab7da3249e2966e1e5660 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 07:27:50 -0500 Subject: [PATCH 009/331] Factor out common Sync code --- JMAP/Sync/Common.pm | 136 ++++++++++++++++++++++++++++++++++++++++++++ JMAP/Sync/Gmail.pm | 86 +--------------------------- JMAP/Sync/ICloud.pm | 96 +------------------------------ 3 files changed, 138 insertions(+), 180 deletions(-) create mode 100644 JMAP/Sync/Common.pm diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm new file mode 100644 index 0000000..76b927b --- /dev/null +++ b/JMAP/Sync/Common.pm @@ -0,0 +1,136 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Sync::Common; + +use Mail::IMAPTalk; +use JSON::XS qw(encode_json decode_json); +use Email::Simple; +use Email::Sender::Simple qw(sendmail); +use Email::Sender::Transport::SMTPS; +use Net::CalDAVTalk; +use Net::CardDAVTalk; + +my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); + +sub new { + my $Class = shift; + my $auth = shift; + return bless { auth => $auth }, ref($Class) || $Class; +} + +sub DESTROY { + my $Self = shift; + if ($Self->{imap}) { + $Self->{imap}->logout(); + } +} + +sub get_calendars { + my $Self = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetCalendars(); + + return $data; +} + +sub get_events { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_calendars(); + + my $data = $talk->GetEvents($Args->{href}, Full => 1); + + my %res; + foreach my $item (@$data) { + $res{$item->{id}} = $item->{_raw}; + } + + return \%res; +} + +sub get_abooks { + my $Self = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetAddressBooks(); + + return $data; +} + +sub get_contacts { + my $Self = shift; + my $Args = shift; + my $talk = $Self->connect_contacts(); + + my $data = $talk->GetContacts($Args->{path}); + + my %res; + foreach my $item (@$data) { + $res{$item->{CPath}} = $item->{_raw}; + } + + return \%res; +} + +# read folder list from the server +sub folders { + my $Self = shift; + $Self->connect_imap(); + return $Self->{folders}; +} + +sub labels { + my $Self = shift; + $Self->connect_imap(); + return $Self->{labels}; +} + +sub fetch_status { + my $Self = shift; + my $justfolders = shift; + + my $imap = $Self->connect_imap(); + + my $folders = $Self->folders; + if ($justfolders) { + my %data = map { $_ => $folders->{$_} } + grep { exists $folders->{$_} } + @$justfolders; + $folders = \%data; + } + + my $fields = "(uidvalidity uidnext highestmodseq messages)"; + my $data = $imap->multistatus($fields, sort keys %$folders); + + return $data; +} + +sub fetch_bodies { + my $Self = shift; + my $request = shift; + + my $imap = $Self->connect_imap(); + + my %res; + foreach my $item (@$request) { + my $name = $item->[0]; + my $uids = $item->[1]; + + my $r = $imap->examine($name); + die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + + my $messages = $imap->fetch(join(',', @$uids), "rfc822"); + + foreach my $uid (keys %$messages) { + $res{$name}{$uid} = $messages->{$uid}{rfc822}; + } + } + + return \%res; +} + +1; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 81f4bb0..636522c 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -4,7 +4,7 @@ use strict; use warnings; package JMAP::Sync::Gmail; -use base qw(JMAP::DB); +use base qw(JMAP::Sync::Common); use Mail::GmailTalk; use JSON::XS qw(encode_json decode_json); @@ -16,19 +16,6 @@ use Net::GmailContacts; my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); -sub new { - my $Class = shift; - my $auth = shift; - return bless { auth => $auth }, ref($Class) || $Class; -} - -sub DESTROY { - my $Self = shift; - if ($Self->{imap}) { - $Self->{imap}->logout(); - } -} - sub connect_calendars { my $Self = shift; @@ -107,44 +94,6 @@ sub connect_imap { die "Could not connect to IMAP server: $@"; } -sub get_calendars { - my $Self = shift; - my $talk = $Self->connect_calendars(); - - my $data = $talk->GetCalendars(); - - return $data; -} - -sub get_events { - my $Self = shift; - my $Args = shift; - my $talk = $Self->connect_calendars(); - - my $data = $talk->GetEvents($Args->{href}); - - return $data; -} - -sub get_abooks { - my $Self = shift; - my $talk = $Self->connect_contacts(); - - my $data = $talk->GetAddressBooks(); - - return $data; -} - -sub get_contacts { - my $Self = shift; - my $Args = shift; - my $talk = $Self->connect_contacts(); - - my $data = $talk->GetContacts($Args->{path}); - - return $data; -} - sub send_email { my $Self = shift; my $rfc822 = shift; @@ -163,39 +112,6 @@ sub send_email { }); } -# read folder list from the server -sub folders { - my $Self = shift; - $Self->connect_imap(); - return $Self->{folders}; -} - -sub labels { - my $Self = shift; - $Self->connect_imap(); - return $Self->{labels}; -} - -sub fetch_status { - my $Self = shift; - my $justfolders = shift; - - my $imap = $Self->connect_imap(); - - my $folders = $Self->folders; - if ($justfolders) { - my %data = map { $_ => $folders->{$_} } - grep { exists $folders->{$_} } - @$justfolders; - $folders = \%data; - } - - my $fields = "(uidvalidity uidnext highestmodseq messages)"; - my $data = $imap->multistatus($fields, sort keys %$folders); - - return $data; -} - sub fetch_folder { my $Self = shift; my $imapname = shift; diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index b983617..e499261 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -4,7 +4,7 @@ use strict; use warnings; package JMAP::Sync::ICloud; -use base qw(JMAP::DB); +use base qw(JMAP::Sync::Common); use Mail::IMAPTalk; use JSON::XS qw(encode_json decode_json); @@ -16,19 +16,6 @@ use Net::CardDAVTalk; my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); -sub new { - my $Class = shift; - my $auth = shift; - return bless { auth => $auth }, ref($Class) || $Class; -} - -sub DESTROY { - my $Self = shift; - if ($Self->{imap}) { - $Self->{imap}->logout(); - } -} - sub connect_calendars { my $Self = shift; @@ -108,54 +95,6 @@ sub connect_imap { die "Could not connect to IMAP server: $@"; } -sub get_calendars { - my $Self = shift; - my $talk = $Self->connect_calendars(); - - my $data = $talk->GetCalendars(); - - return $data; -} - -sub get_events { - my $Self = shift; - my $Args = shift; - my $talk = $Self->connect_calendars(); - - my $data = $talk->GetEvents($Args->{href}, Full => 1); - - my %res; - foreach my $item (@$data) { - $res{$item->{id}} = $item->{_raw}; - } - - return \%res; -} - -sub get_abooks { - my $Self = shift; - my $talk = $Self->connect_contacts(); - - my $data = $talk->GetAddressBooks(); - - return $data; -} - -sub get_contacts { - my $Self = shift; - my $Args = shift; - my $talk = $Self->connect_contacts(); - - my $data = $talk->GetContacts($Args->{path}); - - my %res; - foreach my $item (@$data) { - $res{$item->{CPath}} = $item->{_raw}; - } - - return \%res; -} - sub send_email { my $Self = shift; my $rfc822 = shift; @@ -174,39 +113,6 @@ sub send_email { }); } -# read folder list from the server -sub folders { - my $Self = shift; - $Self->connect_imap(); - return $Self->{folders}; -} - -sub labels { - my $Self = shift; - $Self->connect_imap(); - return $Self->{labels}; -} - -sub fetch_status { - my $Self = shift; - my $justfolders = shift; - - my $imap = $Self->connect_imap(); - - my $folders = $Self->folders; - if ($justfolders) { - my %data = map { $_ => $folders->{$_} } - grep { exists $folders->{$_} } - @$justfolders; - $folders = \%data; - } - - my $fields = "(uidvalidity uidnext highestmodseq messages)"; - my $data = $imap->multistatus($fields, sort keys %$folders); - - return $data; -} - sub fetch_folder { my $Self = shift; my $imapname = shift; From 79760b25d29e799fa113eb3ee19dd707250e9fe2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 07:38:27 -0500 Subject: [PATCH 010/331] factor out update_folder --- JMAP/Sync/Common.pm | 83 +++++++++++++++++++++++++++++++++++++++++++++ JMAP/Sync/Gmail.pm | 73 ++------------------------------------- JMAP/Sync/ICloud.pm | 73 --------------------------------------- 3 files changed, 86 insertions(+), 143 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 76b927b..50bfe24 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -83,6 +83,12 @@ sub folders { return $Self->{folders}; } +sub capability { + my $Self = shift; + my $imap = $Self->connect_imap(); + return $imap->capability(); +} + sub labels { my $Self = shift; $Self->connect_imap(); @@ -133,4 +139,81 @@ sub fetch_bodies { return \%res; } +sub update_folder { + my $Self = shift; + my $imapname = shift; + my $state = shift || { uidvalidity => 0 }; + my $dynamic_extra = shift || []; + my $static_extra = shift || []; + + my $imap = $Self->connect_imap(); + + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + + my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidnext = $imap->get_response_code('uidnext'); + my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; + my $exists = $imap->get_response_code('exists') || 0; + + if ($state->{uidvalidity} != $uidvalidity) { + # force a delete/recreate and resync + $state = { + uidvalidity => $uidvalidity. + highestmodseq => 0, + uidnext => 0, + exists => 0, + }; + } + + if ($highestmodseq and $highestmodseq == $state->{highestmodseq}) { + $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); + return {}; # yay, nothing to do + } + + my $changed = {}; + if ($state->{uidnext} > 1) { + my $from = 1; + my $to = $state->{uidnext} - 1; + $Self->log('debug', "UPDATING $imapname: $from:$to"); + my @flags = qw(uid flags), @$dynamic_extra; + my @extra; + push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; + $changed = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; + } + + my $new = {}; + if ($uidnext > $state->{uidnext}) { + my $from = $state->{uidnext}; + my $to = $uidnext - 1; # or just '*' + $Self->log('debug', "FETCHING $imapname: $from:$to"); + my @flags = qw(uid flags internaldate envelope rfc822.size), @$dynamic_extra, @$static_extra; + $new = $imap->fetch("$from:$to", "(@flags)") || {}; + } + + my $alluids = undef; + if ($state->{exists} + scalar(keys %$new) > $exists) { + # some messages were deleted + my $from = 1; + my $to = $uidnext - 1; + # XXX - you could do some clever UID vs position queries to bisect this out, but it + # would need more data than we have here + $Self->log('debug', "COUNTING $imapname: $from:$to (something deleted)"); + $alluids = $imap->search("UID", "$from:$to"); + } + + return { + oldstate => $state, + newstate => { + highestmodseq => $highestmodseq, + uidvalidity => $uidvalidity. + uidnext => $uidnext, + exists => $exists, + }, + changed => $changed, + new => $new, + ($alluids ? (alluids => $alluids) : ()), + }; +} + 1; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 636522c..689c5c3 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -112,77 +112,10 @@ sub send_email { }); } -sub fetch_folder { +sub update_folder { my $Self = shift; my $imapname = shift; - my $state = shift || { uidvalidity => 0 }; - - my $imap = $Self->connect_imap(); - - my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); - - my $uidvalidity = $imap->get_response_code('uidvalidity'); - my $uidnext = $imap->get_response_code('uidnext'); - my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; - my $exists = $imap->get_response_code('exists') || 0; - - if ($state->{uidvalidity} != $uidvalidity) { - # force a delete/recreate and resync - $state = { - uidvalidity => $uidvalidity. - highestmodseq => 0, - uidnext => 0, - exists => 0, - }; - } - - if ($highestmodseq and $highestmodseq == $state->{highestmodseq}) { - $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); - return {}; # yay, nothing to do - } - - my $changed = {}; - if ($state->{uidnext} > 1) { - my $from = 1; - my $to = $state->{uidnext} - 1; - my @extra; - push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; - $Self->log('debug', "UPDATING $imapname: $from:$to"); - $changed = $imap->fetch("$from:$to", "(uid flags x-gm-labels)", @extra) || {}; - } + my $state = shift; - my $new = {}; - if ($uidnext > $state->{uidnext}) { - my $from = $state->{uidnext}; - my $to = $uidnext - 1; # or just '*' - $Self->log('debug', "FETCHING $imapname: $from:$to"); - $new = $imap->fetch("$from:$to", '(uid flags internaldate envelope rfc822.size x-gm-msgid x-gm-thrid x-gm-labels)') || {}; - } - - my $alluids = undef; - if ($state->{exists} + scalar(keys %$new) > $exists) { - # some messages were deleted - my $from = 1; - my $to = $uidnext - 1; - # XXX - you could do some clever UID vs position queries to bisect this out, but it - # would need more data than we have here - $Self->log('debug', "COUNTING $imapname: $from:$to (something deleted)"); - $alluids = $imap->search("UID", "$from:$to"); - } - - return { - oldstate => $state, - newstate => { - highestmodseq => $highestmodseq, - uidvalidity => $uidvalidity. - uidnext => $uidnext, - exists => $exists, - }, - changed => $changed, - new => $new, - ($alluids ? (alluids => $alluids) : ()), - }; + return $Self->SUPER::update_folder($imapname, $state, ['x-gm-labels'], ['x-gm-msgid', 'x-gm-thrid']); } - -1; diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index e499261..3b06736 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -113,77 +113,4 @@ sub send_email { }); } -sub fetch_folder { - my $Self = shift; - my $imapname = shift; - my $state = shift || { uidvalidity => 0 }; - - my $imap = $Self->connect_imap(); - - my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); - - my $uidvalidity = $imap->get_response_code('uidvalidity'); - my $uidnext = $imap->get_response_code('uidnext'); - my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; - my $exists = $imap->get_response_code('exists') || 0; - - if ($state->{uidvalidity} != $uidvalidity) { - # force a delete/recreate and resync - $state = { - uidvalidity => $uidvalidity. - highestmodseq => 0, - uidnext => 0, - exists => 0, - }; - } - - if ($highestmodseq and $highestmodseq == $state->{highestmodseq}) { - $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); - return {}; # yay, nothing to do - } - - my $changed = {}; - if ($state->{uidnext} > 1) { - my $from = 1; - my $to = $state->{uidnext} - 1; - my @extra; - push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; - $Self->log('debug', "UPDATING $imapname: $from:$to"); - $changed = $imap->fetch("$from:$to", "(uid flags)", @extra) || {}; - } - - my $new = {}; - if ($uidnext > $state->{uidnext}) { - my $from = $state->{uidnext}; - my $to = $uidnext - 1; # or just '*' - $Self->log('debug', "FETCHING $imapname: $from:$to"); - $new = $imap->fetch("$from:$to", '(uid flags internaldate envelope rfc822.size)') || {}; - } - - my $alluids = undef; - if ($state->{exists} + scalar(keys %$new) > $exists) { - # some messages were deleted - my $from = 1; - my $to = $uidnext - 1; - # XXX - you could do some clever UID vs position queries to bisect this out, but it - # would need more data than we have here - $Self->log('debug', "COUNTING $imapname: $from:$to (something deleted)"); - $alluids = $imap->search("UID", "$from:$to"); - } - - return { - oldstate => $state, - newstate => { - highestmodseq => $highestmodseq, - uidvalidity => $uidvalidity. - uidnext => $uidnext, - exists => $exists, - }, - changed => $changed, - new => $new, - ($alluids ? (alluids => $alluids) : ()), - }; -} - 1; From f0e30ed8da88badf2d9f8fcf0d0eb08c8510f290 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 07:44:04 -0500 Subject: [PATCH 011/331] ICloud: just wrap standard with some magic ports --- JMAP/Sync/ICloud.pm | 118 +++++--------------------------------------- 1 file changed, 12 insertions(+), 106 deletions(-) diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index 3b06736..c54172f 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -4,113 +4,19 @@ use strict; use warnings; package JMAP::Sync::ICloud; -use base qw(JMAP::Sync::Common); - -use Mail::IMAPTalk; -use JSON::XS qw(encode_json decode_json); -use Email::Simple; -use Email::Sender::Simple qw(sendmail); -use Email::Sender::Transport::SMTPS; -use Net::CalDAVTalk; -use Net::CardDAVTalk; - -my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); - -sub connect_calendars { - my $Self = shift; - - if ($Self->{calendars}) { - $Self->{lastused} = time(); - return $Self->{calendars}; - } - - $Self->{calendars} = Net::CalDAVTalk->new( - user => $Self->{auth}{username}, - password => $Self->{auth}{password}, - url => "https://caldav.icloud.com/", - expandurl => 1, - ); - - return $Self->{calendars}; -} - -sub connect_contacts { - my $Self = shift; - - if ($Self->{contacts}) { - $Self->{lastused} = time(); - return $Self->{contacts}; - } - - $Self->{contacts} = Net::CardDAVTalk->new( - user => $Self->{auth}{username}, - password => $Self->{auth}{password}, - url => "https://contacts.icloud.com/", - expandurl => 1, +use base qw(JMAP::Sync::Standard); + +sub new { + my $class = shift; + my $auth = shift; + my %a = ( + imapserver => 'imap.mail.me.com', + smtpserver => 'smtp.mail.me.com', + calurl => 'https://caldav.icloud.com/', + aburl => 'https://contacts.icloud.com/', + %$auth, ); - - return $Self->{contacts}; -} - -sub connect_imap { - my $Self = shift; - - if ($Self->{imap}) { - $Self->{lastused} = time(); - return $Self->{imap}; - } - - for (1..3) { - $Self->log('debug', "Looking for server for $Self->{auth}{username}"); - my $port = 993; - my $usessl = $port != 143; # we use SSL for anything except default - $Self->log('debug', "getting imaptalk"); - $Self->{imap} = Mail::IMAPTalk->new( - Server => 'imap.mail.me.com', - Port => $port, - Username => $Self->{auth}{username}, - Password => $Self->{auth}{password}, - # not configurable right now... - UseSSL => $usessl, - UseBlocking => $usessl, - ); - next unless $Self->{imap}; - $Self->log('debug', "Connected as $Self->{auth}{username}"); - $Self->{lastused} = time(); - my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; - my @folders = $Self->{imap}->$list('', '*'); - - delete $Self->{folders}; - delete $Self->{labels}; - foreach my $folder (@folders) { - my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; - my $name = $folder->[2]; - my $label = $role || $folder->[2]; - $Self->{folders}{$name} = $label; - $Self->{labels}{$label} = $name; - } - return $Self->{imap}; - } - - die "Could not connect to IMAP server: $@"; -} - -sub send_email { - my $Self = shift; - my $rfc822 = shift; - - my $email = Email::Simple->new($rfc822); - sendmail($email, { - from => $Self->{auth}{username}, - transport => Email::Sender::Transport::SMTPS->new({ - helo => 'proxy.jmap.io', - host => 'smtp.mail.me.com', - port => 587, - ssl => 'starttls', - sasl_username => $Self->{auth}{username}, - sasl_password => $Self->{auth}{password}, - }), - }); + return JMAP::Sync::Standard->new(\%a); } 1; From 32f484112bea48bfa2658e8aea4b06a74e924ec8 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 09:05:43 -0500 Subject: [PATCH 012/331] better notice messages --- JMAP/Sync/Common.pm | 4 ++-- bin/syncserver.pl | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 50bfe24..deb9c53 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -176,7 +176,7 @@ sub update_folder { my $from = 1; my $to = $state->{uidnext} - 1; $Self->log('debug', "UPDATING $imapname: $from:$to"); - my @flags = qw(uid flags), @$dynamic_extra; + my @flags = (qw(uid flags), @$dynamic_extra); my @extra; push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; $changed = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; @@ -187,7 +187,7 @@ sub update_folder { my $from = $state->{uidnext}; my $to = $uidnext - 1; # or just '*' $Self->log('debug', "FETCHING $imapname: $from:$to"); - my @flags = qw(uid flags internaldate envelope rfc822.size), @$dynamic_extra, @$static_extra; + my @flags = (qw(uid flags internaldate envelope rfc822.size), @$dynamic_extra, @$static_extra); $new = $imap->fetch("$from:$to", "(@flags)") || {}; } diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 51bf995..8212319 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -32,7 +32,7 @@ sub setup { } else { die "UNKNOWN ID $id"; } - warn "Connected $id"; + warn "$$ Connected $id"; $0 = "[jmap proxy imapsync] $id"; $hdl->push_write(json => [ 'setup', $id ]); $hdl->push_write("\n"); @@ -130,7 +130,7 @@ sub mk_handler { my ($db) = @_; # don't last forever - $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $id ON TIMEOUT\n"; undef $hdl; $cv->send }); + $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "$$ SHUTTING DOWN $id ON TIMEOUT\n"; undef $hdl; $cv->send }); return sub { my ($hdl, $json) = @_; @@ -150,12 +150,10 @@ sub mk_handler { $res = ['error', "$@"] } $res->[2] = $tag; - use Data::Dumper; - warn Dumper($res); $hdl->push_write(json => $res); $hdl->push_write("\n"); - warn "HANDLED $cmd ($tag) => $res->[0] ($id)\n"; + warn "$$ HANDLED $cmd ($tag) => $res->[0] ($id)\n"; $hdl->push_read(json => mk_handler($db)); }; } From 852682adc02ed3d8c818d25da191986cb0e5cf65 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 09:43:47 -0500 Subject: [PATCH 013/331] more general IMAP commands --- JMAP/Sync/Common.pm | 151 ++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 76 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index deb9c53..e7dc4ce 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -95,56 +95,79 @@ sub labels { return $Self->{labels}; } -sub fetch_status { +sub imap_status { my $Self = shift; - my $justfolders = shift; + my $folders = shift; my $imap = $Self->connect_imap(); - my $folders = $Self->folders; - if ($justfolders) { - my %data = map { $_ => $folders->{$_} } - grep { exists $folders->{$_} } - @$justfolders; - $folders = \%data; - } - my $fields = "(uidvalidity uidnext highestmodseq messages)"; - my $data = $imap->multistatus($fields, sort keys %$folders); + my $data = $imap->multistatus($fields, @$folders); return $data; } -sub fetch_bodies { +# no newname == delete +sub imap_move { my $Self = shift; - my $request = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uids = shift; + my $newname = shift; my $imap = $Self->connect_imap(); - my %res; - foreach my $item (@$request) { - my $name = $item->[0]; - my $uids = $item->[1]; + my $r = $imap->select($imapname); + die "SELECT FAILED $r" unless lc($r) eq 'ok'; - my $r = $imap->examine($name); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + my $uidvalidity = $imap->get_response_code('uidvalidity'); + + my %res = { + imapname => $imapname, + newname => $newname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + }; - my $messages = $imap->fetch(join(',', @$uids), "rfc822"); + if ($olduidvalidity != $uidvalidity) { + return \%res; + } - foreach my $uid (keys %$messages) { - $res{$name}{$uid} = $messages->{$uid}{rfc822}; + if ($newname) { + # move + if ($imap->capability->{move}) { + my $res = $imap->move($uids, $newname); + unless ($res) { + $res{notMoved} = $uids; + return \%res; + } } + else { + my $res = $imap->copy($uids, $newname); + unless ($res) { + $res{notMoved} = $uids; + return \%res; + } + $imap->store($uids, "+flags", "(\\seen \\deleted)"); + $imap->uidexpunge($uids); + } + } + else { + $imap->store($uids, "+flags", "(\\seen \\deleted)"); + $imap->uidexpunge($uids); } + $res{moved} = $uids; + return \%res; } -sub update_folder { + +sub imap_fetch { my $Self = shift; my $imapname = shift; - my $state = shift || { uidvalidity => 0 }; - my $dynamic_extra = shift || []; - my $static_extra = shift || []; + my $state = shift || {}; + my $fetch = shift || {}; my $imap = $Self->connect_imap(); @@ -156,64 +179,40 @@ sub update_folder { my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; my $exists = $imap->get_response_code('exists') || 0; - if ($state->{uidvalidity} != $uidvalidity) { - # force a delete/recreate and resync - $state = { - uidvalidity => $uidvalidity. - highestmodseq => 0, - uidnext => 0, - exists => 0, - }; + my %res = ( + imapname => $imapname, + oldstate => $state, + newstate => { + uidvalidity => $uidvalidity, + uidnext => $uidnext, + highestmodseq => $highestmodseq, + exists => $exists, + }, + ); + + if (($state->{uidvalidity} || 0) != $uidvalidity) { + return \%res; } - if ($highestmodseq and $highestmodseq == $state->{highestmodseq}) { + if ($highestmodseq and $highestmodseq == ($state->{highestmodseq} || 0)) { $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); - return {}; # yay, nothing to do + return \%res; } - my $changed = {}; - if ($state->{uidnext} > 1) { - my $from = 1; - my $to = $state->{uidnext} - 1; - $Self->log('debug', "UPDATING $imapname: $from:$to"); - my @flags = (qw(uid flags), @$dynamic_extra); + foreach my $key (keys %$fetch) { + my $item = $fetch->{$key}; + my $from = $item->[0]; + my $to = $item->[1]; + my @flags = qw(uid flags); + push @flags, @{$item->[2]} if $item->[2]; my @extra; - push @extra, "(changedsince $state->{highestmodseq})" if $state->{highestmodseq}; - $changed = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; - } - - my $new = {}; - if ($uidnext > $state->{uidnext}) { - my $from = $state->{uidnext}; - my $to = $uidnext - 1; # or just '*' - $Self->log('debug', "FETCHING $imapname: $from:$to"); - my @flags = (qw(uid flags internaldate envelope rfc822.size), @$dynamic_extra, @$static_extra); - $new = $imap->fetch("$from:$to", "(@flags)") || {}; - } - - my $alluids = undef; - if ($state->{exists} + scalar(keys %$new) > $exists) { - # some messages were deleted - my $from = 1; - my $to = $uidnext - 1; - # XXX - you could do some clever UID vs position queries to bisect this out, but it - # would need more data than we have here - $Self->log('debug', "COUNTING $imapname: $from:$to (something deleted)"); - $alluids = $imap->search("UID", "$from:$to"); + push @extra, "(changedsince $item->[3])" if $item->[3]; + $Self->log('debug', "FETCHING $imapname: $from:$to @flags @extra"); + my $data = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; + $res{$key} = [$item, $data]; } - return { - oldstate => $state, - newstate => { - highestmodseq => $highestmodseq, - uidvalidity => $uidvalidity. - uidnext => $uidnext, - exists => $exists, - }, - changed => $changed, - new => $new, - ($alluids ? (alluids => $alluids) : ()), - }; + return \%res; } 1; From a82d97372a66c6b4a4d26d1714439cddeeebdbe2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 4 Jul 2015 20:10:44 -0500 Subject: [PATCH 014/331] imap append support --- JMAP/Sync/Common.pm | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index e7dc4ce..1614e22 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -162,7 +162,6 @@ sub imap_move { return \%res; } - sub imap_fetch { my $Self = shift; my $imapname = shift; @@ -215,4 +214,24 @@ sub imap_fetch { return \%res; } +sub imap_append { + my $Self = shift; + my $imapname = shift; + my $flags = shift; + my $internaldate = shift; + my $rfc822 = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->append($imapname, $flags, $internaldate, ['Literal', $rfc822]); + die "APPEND FAILED $r" unless lc($r) eq 'ok'; + + my $uid = $Mailbox->get_response_code('appenduid'); + + # XXX - fetch the x-gm-msgid or envelope from the server so we know the + # the ID that the server gave this message + + return ['append', $imapname, $uid]; +} + 1; From 75b4fb10d345e71b9061e139a3e3cee585837479 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 5 Jul 2015 06:47:24 -0500 Subject: [PATCH 015/331] standard --- JMAP/Sync/Standard.pm | 116 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 JMAP/Sync/Standard.pm diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm new file mode 100644 index 0000000..63897f0 --- /dev/null +++ b/JMAP/Sync/Standard.pm @@ -0,0 +1,116 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Sync::Standard; +use base qw(JMAP::Sync::Common); + +use Mail::IMAPTalk; +use JSON::XS qw(encode_json decode_json); +use Email::Simple; +use Email::Sender::Simple qw(sendmail); +use Email::Sender::Transport::SMTPS; +use Net::CalDAVTalk; +use Net::CardDAVTalk; + +my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); + +sub connect_calendars { + my $Self = shift; + + if ($Self->{calendars}) { + $Self->{lastused} = time(); + return $Self->{calendars}; + } + + $Self->{calendars} = Net::CalDAVTalk->new( + user => $Self->{auth}{username}, + password => $Self->{auth}{password}, + url => $Self->{auth}{calurl}, + expandurl => 1, + ); + + return $Self->{calendars}; +} + +sub connect_contacts { + my $Self = shift; + + if ($Self->{contacts}) { + $Self->{lastused} = time(); + return $Self->{contacts}; + } + + $Self->{contacts} = Net::CardDAVTalk->new( + user => $Self->{auth}{username}, + password => $Self->{auth}{password}, + url => $Self->{auth}{abookurl}, + expandurl => 1, + ); + + return $Self->{contacts}; +} + +sub connect_imap { + my $Self = shift; + + if ($Self->{imap}) { + $Self->{lastused} = time(); + return $Self->{imap}; + } + + for (1..3) { + $Self->log('debug', "Looking for server for $Self->{auth}{username}"); + my $port = 993; + my $usessl = $port != 143; # we use SSL for anything except default + $Self->log('debug', "getting imaptalk"); + $Self->{imap} = Mail::IMAPTalk->new( + Server => $Self->{auth}{imapserver}, + Port => $port, + Username => $Self->{auth}{username}, + Password => $Self->{auth}{password}, + # not configurable right now... + UseSSL => $usessl, + UseBlocking => $usessl, + ); + next unless $Self->{imap}; + $Self->log('debug', "Connected as $Self->{auth}{username}"); + $Self->{lastused} = time(); + my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; + my @folders = $Self->{imap}->$list('', '*'); + + delete $Self->{folders}; + delete $Self->{labels}; + foreach my $folder (@folders) { + my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; + my $name = $folder->[2]; + my $label = $role || $folder->[2]; + $Self->{folders}{$name} = $label; + $Self->{labels}{$label} = $name; + } + return $Self->{imap}; + } + + die "Could not connect to IMAP server: $@"; +} + +sub send_email { + my $Self = shift; + my $rfc822 = shift; + + my $email = Email::Simple->new($rfc822); + sendmail($email, { + from => $Self->{auth}{username}, + transport => Email::Sender::Transport::SMTPS->new({ + helo => 'proxy.jmap.io', + host => $Self->{auth}{smtpserver}, + port => 587, + ssl => 'starttls', + sasl_username => $Self->{auth}{username}, + sasl_password => $Self->{auth}{password}, + }), + }); +} + +1; From 3f1304ad01075a644469c5f8de8548341f44dc04 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 6 Jul 2015 00:21:40 +1000 Subject: [PATCH 016/331] apiendpoint: way to delay responses --- bin/apiendpoint.pl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 0f134e3..66fa0df 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -102,7 +102,6 @@ sub process_request { set_accountid(shift); warn "Connected $accountid\n"; $handle->push_read(json => mk_handler($accountid)); - }); $cv->recv; @@ -172,16 +171,16 @@ sub mk_handler { return handle_ping(); } if ($cmd eq 'upload') { - return handle_upload(getdb(), $args); + return handle_upload(getdb(), $args, $tag); } if ($cmd eq 'download') { - return handle_download(getdb(), $args); + return handle_download(getdb(), $args, $tag); } if ($cmd eq 'raw') { - return handle_raw(getdb(), $args); + return handle_raw(getdb(), $args, $tag); } if ($cmd eq 'jmap') { - return handle_jmap(getdb(), $args); + return handle_jmap(getdb(), $args, $tag); } if ($cmd eq 'cb_google') { return handle_cb_google($args); @@ -193,26 +192,27 @@ sub mk_handler { return handle_delete(); } if ($cmd eq 'gettoken') { - return handle_gettoken(getdb(), $args); + return handle_gettoken(getdb(), $args, $tag); } if ($cmd eq 'getstate') { - return handle_getstate(getdb(), $args); + return handle_getstate(getdb(), $args, $tag); } if ($cmd eq 'sync') { - return handle_sync(getdb(), $args); + return handle_sync(getdb(), $args, $tag); } if ($cmd eq 'getinfo') { - return handle_getinfo(); + return handle_getinfo(); } die "Unknown command $cmd"; }; unless ($res) { $res = ['error', "$@"] } - $res->[2] = $tag; - $hdl->push_write(json => $res); - - warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n"; + if ($res->[0]) { + $res->[2] = $tag; + $hdl->push_write(json => $res) if $res->[0]; + warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; + } $hdl->push_read(json => mk_handler($db)); }; } From be384a7979507b604f75d69be92800cabceb60d9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 19:57:43 +1000 Subject: [PATCH 017/331] JMAP API and DB - contacts and calendars support --- JMAP/API.pm | 578 ++++++++++++++++++++++++++++++++++++++++++++++++++++ JMAP/DB.pm | 85 ++++++++ 2 files changed, 663 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 3e9939e..902a433 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1130,4 +1130,582 @@ sub _prop_wanted { return 0; } +sub getCalendars { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $data = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, isVisible, mayReadFreeBusy, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jcalendars WHERE active = 1"); + + my %ids; + if ($args->{ids}) { + %ids = map { $_ => 1 } @{$args->{ids}}; + } + else { + %ids = map { $_->[0] => 1 } @$data; + } + + my @list; + + foreach my $item (@$data) { + next unless delete $ids{$item->[0]}; + + my %rec = ( + id => "$item->[0]", + name => $item->[1], + colour => $item->[2], + isVisible => $item->[3] ? $JSON::true : $JSON::false, + mayReadFreeBusy => $item->[4] ? $JSON::true : $JSON::false, + mayReadItems => $item->[5] ? $JSON::true : $JSON::false, + mayAddItems => $item->[6] ? $JSON::true : $JSON::false, + mayModifyItems => $item->[7] ? $JSON::true : $JSON::false, + mayRemoveItems => $item->[8] ? $JSON::true : $JSON::false, + mayDelete => $item->[9] ? $JSON::true : $JSON::false, + mayRename => $item->[10] ? $JSON::true : $JSON::false, + ); + + foreach my $key (keys %rec) { + delete $rec{$key} unless _prop_wanted($args, $key); + } + + push @list, \%rec; + } + my %missingids = %ids; + + return ['calendars', { + list => \@list, + accountId => $accountid, + state => "$user->{jhighestmodseq}", + notFound => (%missingids ? [map { "$_" } keys %missingids] : undef), + }]; +} + +sub getCalendarUpdates { + my $Self = shift; + my $args = shift; + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $sinceState = $args->{sinceState}; + return ['error', {type => 'invalidArguments'}] + if not $args->{sinceState}; + return ['error', {type => 'cannotCalculateChanges'}] + if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); + + my $data = $dbh->selectall_arrayref("SELECT jcalendarid, jmodseq, active FROM jcalendars ORDER BY jcalendarid"); + + my @changed; + my @removed; + my $onlyCounts = 1; + foreach my $item (@$data) { + if ($item->[1] > $sinceState) { + if ($item->[3]) { + push @changed, $item->[0]; + $onlyCounts = 0; + } + else { + push @removed, $item->[0]; + } + } + elsif (($item->[2] || 0) > $sinceState) { + if ($item->[3]) { + push @changed, $item->[0]; + } + else { + push @removed, $item->[0]; + } + } + } + + my @res = (['calendarUpdates', { + accountId => $accountid, + oldState => "$sinceState", + newState => "$user->{jhighestmodseq}", + changed => [map { "$_" } @changed], + removed => [map { "$_" } @removed], + }]); + + if (@changed and $args->{fetchRecords}) { + my %items = ( + accountid => $accountid, + ids => \@changed, + ); + push @res, $Self->getCalendars(\%items); + } + + return @res; +} + +sub _event_match { + my $Self = shift; + my ($item, $condition, $storage) = @_; + + # XXX - condition handling code + if ($condition->{inCalendars}) { + my $match = 0; + foreach my $id (@{$condition->{inCalendars}}) { + next unless $item->[1] eq $id; + $match = 1; + } + return 0 unless $match; + } + + return 1; +} + +sub _event_filter { + my $Self = shift; + my ($data, $filter, $storage) = @_; + my @res; + foreach my $item (@$data) { + next unless $Self->_event_match($item, $filter, $storage); + push @res, $item; + } + return \@res; +} + +sub getCalendarEventList { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $start = $args->{position} || 0; + return ['error', {type => 'invalidArguments'}] if $start < 0; + + my $data = $dbh->selectall_arrayref("SELECT jeventid,jcalendarid FROM jevents WHERE active = 1 ORDER BY jeventid"); + + $data = $Self->_event_filter($data, $args->{filter}, {}) if $args->{filter}; + + my $end = $args->{limit} ? $start + $args->{limit} - 1 : $#$data; + $end = $#$data if $end > $#$data; + + my @result = map { $data->[$_][0] } $start..$end; + + my @res; + push @res, ['calendarEventList', { + accountId => $accountid, + filter => $args->{filter}, + state => "$user->{jhighestmodseq}", + position => $start, + total => scalar(@$data), + calendarEventIds => [map { "$_" } @result], + }]; + + if ($args->{fetchCalendarEvents}) { + push @res, $Self->getCalendarEvents({ + ids => \@result, + properties => $args->{fetchCalendarEventProperties}, + }) if @result; + } + + return @res; +} + +sub getCalendarEvents { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + return ['error', {type => 'invalidArguments'}] unless $args->{ids}; + #properties: String[] A list of properties to fetch for each message. + + my %seenids; + my %missingids; + my @list; + foreach my $eventid (@{$args->{ids}}) { + next if $seenids{$eventid}; + $seenids{$eventid} = 1; + my $data = $dbh->selectrow_hashref("SELECT * FROM jevents WHERE jeventid = ?", {}, $eventid); + unless ($data) { + $missingids{$eventid} = 1; + next; + } + + my $item = json_decode($data->{payload}); + + foreach my $key (keys %$item) { + delete $item->{$key} unless _prop_wanted($args, $key); + } + + $item->{id} = $eventid; + + push @list, $item; + } + + return ['calendarEvents', { + list => \@list, + accountId => $accountid, + state => "$user->{jhighestmodseq}", + notFound => (%missingids ? [keys %missingids] : undef), + }]; +} + +sub getCalendarEventUpdates { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + return ['error', {type => 'invalidArguments'}] + if not $args->{sinceState}; + return ['error', {type => 'cannotCalculateChanges'}] + if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); + + my $sql = "SELECT jeventid,active FROM jevents WHERE jmodseq > ?"; + + my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); + + if ($args->{maxChanges} and @$data > $args->{maxChanges}) { + return ['error', {type => 'tooManyChanges'}]; + } + + my @changed; + my @removed; + + foreach my $row (@$data) { + if ($row->[1]) { + push @changed, $row->[0]; + } + else { + push @removed, $row->[0]; + } + } + + my @res; + push @res, ['calendarEventUpdates', { + accountId => $accountid, + oldState => "$args->{sinceState}", + newState => "$user->{jhighestmodseq}", + changed => [map { "$_" } @changed], + removed => [map { "$_" } @removed], + }]; + + if ($args->{fetchCalendarEvents}) { + push @res, $Self->getCalendarEvents({ + accountid => $accountid, + ids => \@changed, + properties => $args->{fetchCalendarEventProperties}, + }) if @changed; + } + + return @res; +} + +sub getAddressbooks { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, name, isVisible, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jaddressbooks WHERE active = 1"); + + my %ids; + if ($args->{ids}) { + %ids = map { $_ => 1 } @{$args->{ids}}; + } + else { + %ids = map { $_->[0] => 1 } @$data; + } + + my @list; + + foreach my $item (@$data) { + next unless delete $ids{$item->[0]}; + + my %rec = ( + id => "$item->[0]", + name => $item->[1], + isVisible => $item->[2] ? $JSON::true : $JSON::false, + mayReadItems => $item->[3] ? $JSON::true : $JSON::false, + mayAddItems => $item->[4] ? $JSON::true : $JSON::false, + mayModifyItems => $item->[5] ? $JSON::true : $JSON::false, + mayRemoveItems => $item->[6] ? $JSON::true : $JSON::false, + mayDelete => $item->[7] ? $JSON::true : $JSON::false, + mayRename => $item->[8] ? $JSON::true : $JSON::false, + ); + + foreach my $key (keys %rec) { + delete $rec{$key} unless _prop_wanted($args, $key); + } + + push @list, \%rec; + } + my %missingids = %ids; + + return ['addressbooks', { + list => \@list, + accountId => $accountid, + state => "$user->{jhighestmodseq}", + notFound => (%missingids ? [map { "$_" } keys %missingids] : undef), + }]; +} + +sub getAddressbookUpdates { + my $Self = shift; + my $args = shift; + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $sinceState = $args->{sinceState}; + return ['error', {type => 'invalidArguments'}] + if not $args->{sinceState}; + return ['error', {type => 'cannotCalculateChanges'}] + if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); + + my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, jmodseq, active FROM jaddressbooks ORDER BY jaddressbookid"); + + my @changed; + my @removed; + my $onlyCounts = 1; + foreach my $item (@$data) { + if ($item->[1] > $sinceState) { + if ($item->[3]) { + push @changed, $item->[0]; + $onlyCounts = 0; + } + else { + push @removed, $item->[0]; + } + } + elsif (($item->[2] || 0) > $sinceState) { + if ($item->[3]) { + push @changed, $item->[0]; + } + else { + push @removed, $item->[0]; + } + } + } + + my @res = (['addressbookUpdates', { + accountId => $accountid, + oldState => "$sinceState", + newState => "$user->{jhighestmodseq}", + changed => [map { "$_" } @changed], + removed => [map { "$_" } @removed], + }]); + + if (@changed and $args->{fetchRecords}) { + my %items = ( + accountid => $accountid, + ids => \@changed, + ); + push @res, $Self->getAddressbooks(\%items); + } + + return @res; +} + +sub _contact_match { + my $Self = shift; + my ($item, $condition, $storage) = @_; + + # XXX - condition handling code + if ($condition->{inAddressbooks}) { + my $match = 0; + foreach my $id (@{$condition->{inAddressbooks}}) { + next unless $item->[1] eq $id; + $match = 1; + } + return 0 unless $match; + } + + return 1; +} + +sub _contact_filter { + my $Self = shift; + my ($data, $filter, $storage) = @_; + my @res; + foreach my $item (@$data) { + next unless $Self->_contact_match($item, $filter, $storage); + push @res, $item; + } + return \@res; +} + +sub getContactList { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + my $start = $args->{position} || 0; + return ['error', {type => 'invalidArguments'}] if $start < 0; + + my $data = $dbh->selectall_arrayref("SELECT jcontactid,jaddressbookid FROM jcontacts WHERE active = 1 ORDER BY jcontactid"); + + $data = $Self->_event_filter($data, $args->{filter}, {}) if $args->{filter}; + + my $end = $args->{limit} ? $start + $args->{limit} - 1 : $#$data; + $end = $#$data if $end > $#$data; + + my @result = map { $data->[$_][0] } $start..$end; + + my @res; + push @res, ['contactList', { + accountId => $accountid, + filter => $args->{filter}, + state => "$user->{jhighestmodseq}", + position => $start, + total => scalar(@$data), + contactIds => [map { "$_" } @result], + }]; + + if ($args->{fetchContacts}) { + push @res, $Self->getContacts({ + ids => \@result, + properties => $args->{fetchContactProperties}, + }) if @result; + } + + return @res; +} + +sub getContacts { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + return ['error', {type => 'invalidArguments'}] unless $args->{ids}; + #properties: String[] A list of properties to fetch for each message. + + my %seenids; + my %missingids; + my @list; + foreach my $contactid (@{$args->{ids}}) { + next if $seenids{$contactid}; + $seenids{$contactid} = 1; + my $data = $dbh->selectrow_hashref("SELECT * FROM jcontacts WHERE jcontactid = ?", {}, $contactid); + unless ($data) { + $missingids{$contactid} = 1; + next; + } + + my $item = json_decode($data->{payload}); + + foreach my $key (keys %$item) { + delete $item->{$key} unless _prop_wanted($args, $key); + } + + $item->{id} = $contactid; + + push @list, $item; + } + + return ['contacts', { + list => \@list, + accountId => $accountid, + state => "$user->{jhighestmodseq}", + notFound => (%missingids ? [keys %missingids] : undef), + }]; +} + +sub getContactUpdates { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + return ['error', {type => 'invalidArguments'}] + if not $args->{sinceState}; + return ['error', {type => 'cannotCalculateChanges'}] + if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); + + my $sql = "SELECT jcontactid,active FROM jcontacts WHERE jmodseq > ?"; + + my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); + + if ($args->{maxChanges} and @$data > $args->{maxChanges}) { + return ['error', {type => 'tooManyChanges'}]; + } + + my @changed; + my @removed; + + foreach my $row (@$data) { + if ($row->[1]) { + push @changed, $row->[0]; + } + else { + push @removed, $row->[0]; + } + } + + my @res; + push @res, ['contactUpdates', { + accountId => $accountid, + oldState => "$args->{sinceState}", + newState => "$user->{jhighestmodseq}", + changed => [map { "$_" } @changed], + removed => [map { "$_" } @removed], + }]; + + if ($args->{fetchContacts}) { + push @res, $Self->getContacts({ + accountid => $accountid, + ids => \@changed, + properties => $args->{fetchContactProperties}, + }) if @changed; + } + + return @res; +} + + +1; + + 1; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 36891d3..fdf70d5 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -843,6 +843,91 @@ CREATE TABLE IF NOT EXISTS jfiles ( mtime DATE, active BOOLEAN ); +EOF + + $dbh->do(<do(<do(<do(<do(<do("CREATE INDEX IF NOT EXISTS jcontactmap ON jgroupmap (contactuid)"); + + $dbh->do(< Date: Thu, 9 Jul 2015 05:09:14 -0500 Subject: [PATCH 018/331] Gmail: call it contact --- JMAP/GmailDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index f8986bc..48f0cee 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -802,8 +802,8 @@ CREATE TABLE IF NOT EXISTS iabooks ( EOF $dbh->do(< Date: Thu, 9 Jul 2015 20:21:11 +1000 Subject: [PATCH 019/331] remove double 1 --- JMAP/API.pm | 4 ---- 1 file changed, 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 902a433..c59ff34 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1704,8 +1704,4 @@ sub getContactUpdates { return @res; } - -1; - - 1; From 1e25e6769bf9255619714120b952e8d418cfd531 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 22:55:06 +1000 Subject: [PATCH 020/331] tabs become spaces --- INSTALL | 34 +++---- JMAP/API.pm | 2 +- JMAP/DB.pm | 2 +- JMAP/GmailDB.pm | 2 +- JMAP/ImapDB.pm | 6 +- JMAP/Sync/Common.pm | 4 +- Mail/IMAPTalk.pm | 50 +++++----- bin/server.pl | 4 +- bin/syncserver.pl | 2 +- nginx.conf | 226 ++++++++++++++++++++++---------------------- 10 files changed, 166 insertions(+), 166 deletions(-) diff --git a/INSTALL b/INSTALL index 66283ff..de6a1fa 100644 --- a/INSTALL +++ b/INSTALL @@ -3,23 +3,23 @@ Debian install instructions (port to your own OS as you like) # Might need some software too apt-get install build-essential \ - libanyevent-httpd-perl \ - libdata-uuid-libuuid-perl \ - libdatetime-perl \ - libdbd-sqlite3-perl \ - libdbi-perl \ - libemail-address-perl \ - libemail-mime-perl \ - libhtml-parser-perl \ - libhtml-strip-perl \ - libhttp-tiny-perl \ - libhttp-date-perl \ - libimage-size-perl \ - libio-socket-ssl-perl \ - libjson-perl \ - libjson-xs-perl \ - liblocale-gettext-perl \ - libswitch-perl + libanyevent-httpd-perl \ + libdata-uuid-libuuid-perl \ + libdatetime-perl \ + libdbd-sqlite3-perl \ + libdbi-perl \ + libemail-address-perl \ + libemail-mime-perl \ + libhtml-parser-perl \ + libhtml-strip-perl \ + libhttp-tiny-perl \ + libhttp-date-perl \ + libimage-size-perl \ + libio-socket-ssl-perl \ + libjson-perl \ + libjson-xs-perl \ + liblocale-gettext-perl \ + libswitch-perl cpan AnyEvent::HTTPD::CookiePatch cpan AnyEvent::IMAP diff --git a/JMAP/API.pm b/JMAP/API.pm index c59ff34..796229a 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -753,7 +753,7 @@ sub getMessages { my %wanted; foreach my $prop (@{$args->{properties}}) { next unless $prop =~ m/^headers\.(.*)/; - $item->{headers} ||= {}; # avoid zero matched headers bug + $item->{headers} ||= {}; # avoid zero matched headers bug $wanted{lc $1} = 1; } foreach my $key (keys %{$data->{headers}}) { diff --git a/JMAP/DB.pm b/JMAP/DB.pm index fdf70d5..b43c1fa 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -484,7 +484,7 @@ sub change_message { my $Self = shift; my ($msgid, $data, $newids) = @_; - # doesn't work if only IDs have changed :( + # doesn't work if only IDs have changed :( #return unless $Self->dmaybedirty('jmessages', $data, {msgid => $msgid}); $Self->ddirty('jmessages', $data, {msgid => $msgid}); diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 48f0cee..48c1ea7 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -138,7 +138,7 @@ sub send_email { die "not gmail" unless $token->[0] eq 'gmail'; my $email = Email::Simple->new($rfc822); - sendmail($email, { + sendmail($email, { from => $data->[0], transport => Email::Sender::Transport::GmailSMTP->new({ host => 'smtp.gmail.com', diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 23aa5d5..52983ba 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -431,17 +431,17 @@ sub update_messages { if (exists $action->{isUnread}) { my $act = $action->{isUnread} ? "-flags" : "+flags"; # reverse $Self->log('debug', "STORING $act SEEN for $uid"); - my $res = $imap->store($uid, $act, "(\\Seen)"); + my $res = $imap->store($uid, $act, "(\\Seen)"); } if (exists $action->{isFlagged}) { my $act = $action->{isFlagged} ? "+flags" : "-flags"; $Self->log('debug', "STORING $act FLAGGED for $uid"); - $imap->store($uid, $act, "(\\Flagged)"); + $imap->store($uid, $act, "(\\Flagged)"); } if (exists $action->{isAnswered}) { my $act = $action->{isAnswered} ? "+flags" : "-flags"; $Self->log('debug', "STORING $act ANSWERED for $uid"); - $imap->store($uid, $act, "(\\Answered)"); + $imap->store($uid, $act, "(\\Answered)"); } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1614e22..d8309bf 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -138,14 +138,14 @@ sub imap_move { if ($imap->capability->{move}) { my $res = $imap->move($uids, $newname); unless ($res) { - $res{notMoved} = $uids; + $res{notMoved} = $uids; return \%res; } } else { my $res = $imap->copy($uids, $newname); unless ($res) { - $res{notMoved} = $uids; + $res{notMoved} = $uids; return \%res; } $imap->store($uids, "+flags", "(\\seen \\deleted)"); diff --git a/Mail/IMAPTalk.pm b/Mail/IMAPTalk.pm index 9f34b90..220746d 100755 --- a/Mail/IMAPTalk.pm +++ b/Mail/IMAPTalk.pm @@ -196,7 +196,7 @@ internal methods will always 'die' if they encounter any errors. =item INTERNAL SOCKET FUNCTIONS -These are functions used internally by the C object +These are functions used internally by the C object to read/write data to/from the IMAP connection socket. The class does its own buffering so if you want to read/write to the IMAP socket, you should use these functions. @@ -470,7 +470,7 @@ method for more details. =item B -If supplied, sets the folder name text string separator character. +If supplied, sets the folder name text string separator character. Passed as the second parameter to the C method. =item B @@ -856,7 +856,7 @@ Examples: $IMAP->set_root_folder('inbox', '.', 1, 'user'); # Selects 'Inbox' (because 'Inbox' eq 'inbox' case insensitive) - $IMAP->select('Inbox'); + $IMAP->select('Inbox'); # Selects 'inbox.blah' $IMAP->select('blah'); # Selects 'INBOX.fred' (because 'INBOX' eq 'inbox' case insensitive) @@ -1009,7 +1009,7 @@ sub get_last_error { Returns the last completion response to the tagged command. -This is either the string "ok", "no" or "bad" (always lower case) +This is either the string "ok", "no" or "bad" (always lower case) =cut sub get_last_completion_response { @@ -1217,7 +1217,7 @@ sub unicode_folders { =head1 IMAP FOLDER COMMAND METHODS -B In all cases where a folder name is used, +B In all cases where a folder name is used, the folder name is first manipulated according to the current root folder prefix as described in C. @@ -1227,7 +1227,7 @@ prefix as described in C. =item I Perform the standard IMAP 'select' command to select a folder for -retrieving/moving/adding messages. If $Opts{ReadOnly} is true, the +retrieving/moving/adding messages. If $Opts{ReadOnly} is true, the IMAP EXAMINE verb is used instead of SELECT. Mail::IMAPTalk will cache the currently selected folder, and if you @@ -1673,7 +1673,7 @@ details of a folder/mailbox and possible root quota as well. See RFC 2087 for details of the IMAP quota extension. The result of this command is a little complex. Unfortunately it doesn't map really easily into any structure -since there are several different responses. +since there are several different responses. Basically it's a hash reference. The 'quotaroot' item is the response which lists the root quotas that apply to the given @@ -2401,7 +2401,7 @@ sub xmove { Methods provided by extensions to the cyrus IMAP server -B In all cases where a folder name is used, +B In all cases where a folder name is used, the folder name is first manipulated according to the current root folder prefix as described in C. @@ -2518,13 +2518,13 @@ sub xconvmeta { } elsif (lc($Item) eq 'folderexists') { my %FolderExists = @{$Res->{$Item}}; - $ResHash{folderexists} = { map { + $ResHash{folderexists} = { map { $Self->_unfix_folder_name($_) => int($FolderExists{$_}) } keys %FolderExists }; } elsif (lc($Item) eq 'folderunseen') { my %FolderUnseen = @{$Res->{$Item}}; - $ResHash{folderunseen} = { map { + $ResHash{folderunseen} = { map { $Self->_unfix_folder_name($_) => int($FolderUnseen{$_}) } keys %FolderUnseen }; } @@ -2594,7 +2594,7 @@ messages =cut sub xconvupdates { my ($Self, $Sort, $Window, @Search) = @_; - + my %Results; my %Callbacks = ( @@ -2831,7 +2831,7 @@ Examples: # Parse further to find message components my $MC = $IMAP->find_message($BS); $MC = { 'plain' => ... text body struct ref part ..., - 'html' => ... html body struct ref part (if present) ... + 'html' => ... html body struct ref part (if present) ... 'htmllist' => [ ... html body struct ref parts (if present) ... ] }; # Now get the text part of the message @@ -3166,7 +3166,7 @@ The result in response will look like this: A couple of points to note: -=over +=over =item 1. @@ -3430,7 +3430,7 @@ Would have the result: } } } - + =cut =head1 INTERNAL METHODS @@ -3649,7 +3649,7 @@ sub _send_data { # Handle non-literals if (!$IsLiteral) { $Arg = _quote(ref($Arg) ? $$Arg : $Arg) if $IsQuote; - + # Must be a scalar reference for a non-literal $LineBuffer .= ($AddSpace ? " " : "") . (ref($Arg) ? $$Arg : $Arg); @@ -3958,7 +3958,7 @@ sub _trace { my ($Self, $Line) = @_; $Line =~ s/\015\012/\n/; my $Trace = $Self->{Trace}; - + if (ref($Trace) eq 'GLOB') { print $Trace $Line; } elsif (ref($Trace) eq 'CODE') { @@ -4067,7 +4067,7 @@ sub _next_atom { $$AtomRef = $CurAtom; } } - + # Bracket? elsif ($Line =~ m/\G\(/gc) { # Begin a new sub-array @@ -4365,7 +4365,7 @@ otherwise the function will 'die' with an error if it runs out of data. If $NBytes is not specified (undef), the function will attempt to seek to the end of the file to find the size of the file. - + =cut sub _copy_handle_to_handle { my ($Self, $InHandle, $OutHandle, $NBytes) = @_; @@ -4407,7 +4407,7 @@ have to copy the contents of our buffer first. The number of bytes specified must be available on the IMAP socket, if the function runs out of data it will 'die' with an error. - + =cut sub _copy_imap_socket_to_handle { my ($Self, $OutHandle, $NBytes) = @_; @@ -4436,12 +4436,12 @@ sub _copy_imap_socket_to_handle { # Done return 1; } - + =item I<_quote($String)> Returns an IMAP quoted version of a string. This place "..." around the string, and replaces any internal " with \". - + =cut sub _quote { # Replace " and \ with \" and \\ and surround with "..." @@ -4641,7 +4641,7 @@ from an IMAP fetch (envelope) call into a single RFC 822 email string finally return to the user. This is used to parse an envelope structure returned from a fetch call. - + See the documentation section 'FETCH RESULTS' for more information. =cut @@ -4849,7 +4849,7 @@ sub _parse_bodystructure { =item I<_parse_fetch_annotation($AnnotateItem)> Takes the result from a single IMAP annotation item -into a Perl friendly structure. +into a Perl friendly structure. See the documentation section 'FETCH RESULTS' from more information. @@ -4867,7 +4867,7 @@ sub _parse_fetch_annotation { =item I<_parse_fetch_result($FetchResult)> Takes the result from a single IMAP fetch response line and parses it -into a Perl friendly structure. +into a Perl friendly structure. See the documentation section 'FETCH RESULTS' from more information. @@ -5010,7 +5010,7 @@ sub DESTROY { # If socket exists, and connection is open and authenticated or # selected, do a logout - if ($Self->{Socket} && + if ($Self->{Socket} && ($Self->state() == Authenticated || $Self->state() == Selected) && $Self->is_open()) { $Self->logout(); diff --git a/bin/server.pl b/bin/server.pl index 46b4bed..3ad3cbc 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -193,7 +193,7 @@ sub get_backend { connect => ['127.0.0.1', 5000], on_error => sub { print "CLOSING ON ERROR $accountid\n"; - delete $backend{$accountid}; + delete $backend{$accountid}; }, on_disconnect => sub { print "CLOSING ON DISCONNECT $accountid\n"; @@ -208,7 +208,7 @@ sub get_backend { sub send_backend_request { my $accountid = shift; - my $request = shift; + my $request = shift; my $args = shift; my $cb = shift; my $errcb = shift; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 8212319..0b091bf 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -25,7 +25,7 @@ package SyncServer; sub setup { my $config = shift; - if ($id =~ m/gmail\.com/) { + if ($id =~ m/gmail\.com/) { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; } elsif ($id =~ m/icloud\.com/) { $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; diff --git a/nginx.conf b/nginx.conf index 62c5573..8f7b902 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,118 +1,118 @@ server { - listen 80; - server_name proxy.jmap.io; - location / { - rewrite ^/$ https://proxy.jmap.io/ redirect; - } + listen 80; + server_name proxy.jmap.io; + location / { + rewrite ^/$ https://proxy.jmap.io/ redirect; + } } server { - listen 443; - - ssl on; - ssl_certificate /etc/ssl/certs/proxy.jmap.io.crt; - ssl_certificate_key /etc/ssl/private/proxy.jmap.io.key; - - root /home/jmap/jmap-perl/htdocs/; - index index.html index.htm; - - server_name proxy.jmap.io; - - location / { - # First attempt to serve request as file, then - # as directory, then fall back to displaying a 404. - try_files $uri $uri/ /index.html; - } - - location = / { - if ( $request_method = 'OPTIONS' ) { - add_header 'Access-Control-Allow-Origin' '*'; - # -D GAPING_SECURITY_HOLE - add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; - add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; - add_header 'Access-Control-Max-Age' 600; - add_header 'Content-Type' 'text/plain; charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } - # First attempt to serve request as file, then - # as directory, then fall back to displaying a 404. - proxy_pass http://127.0.0.1:9000/home; - } - - location /events/ { - if ( $request_method = 'OPTIONS' ) { - add_header 'Access-Control-Allow-Origin' '*'; - # -D GAPING_SECURITY_HOLE - add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; - add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; - add_header 'Access-Control-Max-Age' 600; - add_header 'Content-Type' 'text/plain; charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } - # Immediately send backend responses back to client - proxy_buffering off; - # Disable keepalive to browser - keepalive_timeout 0; - # It's a long lived backend connection with potentially a long time between - # push events, make sure proxy doesn't timeout - proxy_read_timeout 7200; - - proxy_pass http://127.0.0.1:9001/events/; - } - - location /files/ { - proxy_pass http://127.0.0.1:9000/files/; - } - - location /jmap/ { - if ( $request_method = 'OPTIONS' ) { - add_header 'Access-Control-Allow-Origin' '*'; - # -D GAPING_SECURITY_HOLE - add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; - add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; - add_header 'Access-Control-Max-Age' 600; - add_header 'Content-Type' 'text/plain; charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } - proxy_pass http://127.0.0.1:9000/jmap/; - } - - location /upload/ { - proxy_pass http://127.0.0.1:9000/upload/; - } - - location /raw/ { - proxy_pass http://127.0.0.1:9000/raw/; - } - - location /A { - proxy_pass http://127.0.0.1:9000/A; - } - - location /J { - proxy_pass http://127.0.0.1:9000/J; - } - - location /U { - proxy_pass http://127.0.0.1:9000/U; - } - - location /register { - proxy_pass http://127.0.0.1:9000/register; - } - - location /signup { - proxy_pass http://127.0.0.1:9000/signup; - } - - location /delete { - proxy_pass http://127.0.0.1:9000/delete; - } - - location /cb { - proxy_pass http://127.0.0.1:9000/cb; - } + listen 443; + + ssl on; + ssl_certificate /etc/ssl/certs/proxy.jmap.io.crt; + ssl_certificate_key /etc/ssl/private/proxy.jmap.io.key; + + root /home/jmap/jmap-perl/htdocs/; + index index.html index.htm; + + server_name proxy.jmap.io; + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ /index.html; + } + + location = / { + if ( $request_method = 'OPTIONS' ) { + add_header 'Access-Control-Allow-Origin' '*'; + # -D GAPING_SECURITY_HOLE + add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; + add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; + add_header 'Access-Control-Max-Age' 600; + add_header 'Content-Type' 'text/plain; charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + proxy_pass http://127.0.0.1:9000/home; + } + + location /events/ { + if ( $request_method = 'OPTIONS' ) { + add_header 'Access-Control-Allow-Origin' '*'; + # -D GAPING_SECURITY_HOLE + add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; + add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; + add_header 'Access-Control-Max-Age' 600; + add_header 'Content-Type' 'text/plain; charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + # Immediately send backend responses back to client + proxy_buffering off; + # Disable keepalive to browser + keepalive_timeout 0; + # It's a long lived backend connection with potentially a long time between + # push events, make sure proxy doesn't timeout + proxy_read_timeout 7200; + + proxy_pass http://127.0.0.1:9001/events/; + } + + location /files/ { + proxy_pass http://127.0.0.1:9000/files/; + } + + location /jmap/ { + if ( $request_method = 'OPTIONS' ) { + add_header 'Access-Control-Allow-Origin' '*'; + # -D GAPING_SECURITY_HOLE + add_header 'Access-Control-Allow-Headers' $http_access_control_request_headers; + add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS'; + add_header 'Access-Control-Max-Age' 600; + add_header 'Content-Type' 'text/plain; charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + proxy_pass http://127.0.0.1:9000/jmap/; + } + + location /upload/ { + proxy_pass http://127.0.0.1:9000/upload/; + } + + location /raw/ { + proxy_pass http://127.0.0.1:9000/raw/; + } + + location /A { + proxy_pass http://127.0.0.1:9000/A; + } + + location /J { + proxy_pass http://127.0.0.1:9000/J; + } + + location /U { + proxy_pass http://127.0.0.1:9000/U; + } + + location /register { + proxy_pass http://127.0.0.1:9000/register; + } + + location /signup { + proxy_pass http://127.0.0.1:9000/signup; + } + + location /delete { + proxy_pass http://127.0.0.1:9000/delete; + } + + location /cb { + proxy_pass http://127.0.0.1:9000/cb; + } } From d5ac6b00b048ac3b034425b873c112c34b8538ad Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 23:08:06 +1000 Subject: [PATCH 021/331] ImapDB: handle intermediate folder lifetime better --- JMAP/ImapDB.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 52983ba..461bfe2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -185,6 +185,9 @@ sub sync_jmailboxes { $byname{$parentid}{$name} = $id; } } + if (@bits) { + $seen{$id} = 1; + } } next unless $name; my %details = ( From 6e85ebc0a5584826482aebd58a9101a2001e03f8 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 23:10:14 +1000 Subject: [PATCH 022/331] ImapDB: create calendar and addressbook entries too --- JMAP/ImapDB.pm | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 461bfe2..05c162a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -683,6 +683,50 @@ CREATE TABLE IF NOT EXISTS ithread ( thrid TEXT ); EOF + + $dbh->do(<do(<do(<do(< Date: Thu, 9 Jul 2015 09:15:50 -0500 Subject: [PATCH 023/331] nearly ready for backend-powered IMAp --- JMAP/ImapDB.pm | 160 ++++++++++++++++++------------------------ JMAP/Sync/Common.pm | 69 +++++++++++++++++- JMAP/Sync/Standard.pm | 2 +- bin/syncserver.pl | 24 ++++++- 4 files changed, 159 insertions(+), 96 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 05c162a..e072143 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -7,7 +7,6 @@ package JMAP::ImapDB; use base qw(JMAP::DB); use DBI; -use Mail::IMAPTalk; use Date::Parse; use JSON::XS qw(encode_json decode_json); use Data::UUID::LibUUID; @@ -16,6 +15,8 @@ use Encode; use Encode::MIME::Header; use Digest::SHA qw(sha1_hex); +our $TAG = 1; + # XXX - specialuse, this is just for iCloud for now my %ROLE_MAP = ( 'inbox' => 'inbox', @@ -28,13 +29,6 @@ my %ROLE_MAP = ( 'trash' => 'trash', ); -sub DESTROY { - my $Self = shift; - if ($Self->{imap}) { - $Self->{imap}->logout(); - } -} - sub setuser { my $Self = shift; my ($hostname, $username, $password) = @_; @@ -70,42 +64,41 @@ sub access_token { return [$hostname, $username, $password]; } -sub connect { +# synchronous backend for now +sub backend_cmd { my $Self = shift; - - if ($Self->{imap}) { - $Self->{lastused} = time(); - return $Self->{imap}; - } - - for (1..3) { - $Self->log('debug', "Looking for server for $Self->{accountid}"); - my $data = $Self->dbh->selectrow_arrayref("SELECT hostname, username, password, lastfoldersync FROM iserver"); - die "UNKNOWN SERVER for $Self->{accountid}" unless ($data and $data->[0]); - my $port = 993; - my $usessl = $port != 143; # we use SSL for anything except default - $Self->log('debug', "getting imaptalk\n"); - $Self->{imap} = Mail::IMAPTalk->new( - Server => $data->[0], - Port => $port, - Username => $data->[1], - Password => $data->[2], - # not configurable right now... - UseSSL => $usessl, - UseBlocking => $usessl, - ); - next unless $Self->{imap}; - $Self->log('debug', "Connected to $data->[0] as $data->[1]"); - eval { $Self->{imap}->enable('condstore') }; - $Self->begin(); - $Self->sync_folders(); - $Self->dmaybeupdate('iserver', {lastfoldersync => time()}, {username => $data->[0]}); - $Self->commit(); - $Self->{lastused} = time(); - return $Self->{imap}; - } - - die "Could not connect to IMAP server: $@"; + my $cmd = shift; + my $args = shift; + unless ($Self->{backend}) { + my $w = AnyEvent->condvar; + my $auth = $Self->access_token(); + tcp_connect('localhost', 5005, sub { + my $fh = shift; + my $handle = AnyEvent::Handle->new(fh => $fh); + $handle->push_write(json => {hostname => $auth->[0], username => $auth->[1], password => $auth->[2]}); + $handle->push_write("\012"); + $handle->push_read(json => sub { + my $hdl = shift; + my $json = shift; + die "Failed to setup " . Dumper($json) unless $json->[0] eq 'setup'; + $w->send($handle); + }); + }); + $Self->{backend} = $w->recv; + } + my $handle = $Self->{backend}; + my $w = AnyEvent->condvar; + my $tag = "T" . $TAG++; + $handle->push_write(json => [$cmd, $args, $tag]); # whatever + $handle->push_write("\012"); + $handle->push_read(json => sub { + my $hdl = shift; + my $json = shift; + die "INVALID RESPONSE" unless $json->[2] eq $tag; + $w->send($json->[1]); + }); + + return $w->recv; } # synchronise list from IMAP server to local folder cache @@ -114,22 +107,22 @@ sub sync_folders { my $Self = shift; my $dbh = $Self->dbh(); - my $imap = $Self->{imap}; - my @folders = $imap->list('', '*'); + my $folders = $Self->backend_cmd('folders', []); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label FROM ifolders"); my %ibylabel = map { $_->[3] => $_ } @$ifolders; my %seen; - foreach my $folder (@folders) { - my $role = $ROLE_MAP{lc $folder->[2]}; - my $label = $role || $folder->[2]; + foreach my $name (sort keys %$folders) { + my $sep = $folders->{$name}[1]; + my $role = $ROLE_MAP{lc $name}; + my $label = $role || $name; my $id = $ibylabel{$label}[0]; if ($id) { - $Self->dmaybeupdate('ifolders', {sep => $folder->[1], imapname => $folder->[2]}, {ifolderid => $id}); + $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); } else { - $id = $Self->dinsert('ifolders', {sep => $folder->[1], imapname => $folder->[2], label => $label}); + $id = $Self->dinsert('ifolders', {sep => $sep, imapname => $name, label => $label}); } $seen{$id} = 1; } @@ -247,7 +240,6 @@ sub labels { sub sync { my $Self = shift; - my $imap = $Self->{imap}; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders"); foreach my $row (@$data) { @@ -267,7 +259,6 @@ sub backfill { sub firstsync { my $Self = shift; - my $imap = $Self->{imap}; my $labels = $Self->labels(); my $ifolderid = $labels->{"inbox"}[0]; @@ -409,7 +400,6 @@ sub update_messages { my $changes = shift; my $dbh = $Self->{dbh}; - my $imap = $Self->{imap}; my %updatemap; foreach my $msgid (keys %$changes) { @@ -417,44 +407,42 @@ sub update_messages { $updatemap{$ifolderid}{$uid} = $changes->{$msgid}; } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); + my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } @$folderdata; + my %jmailmap = map { $_->[4] => $_ } @$folderdata; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; + my $uidvalidity = $foldermap{$ifolderid}[2]; die "NO SUCH FOLDER $ifolderid" unless $imapname; - # we're writing here! - my $r = $imap->select($imapname); - foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { my $action = $updatemap{$ifolderid}{$uid}; if (exists $action->{isUnread}) { - my $act = $action->{isUnread} ? "-flags" : "+flags"; # reverse - $Self->log('debug', "STORING $act SEEN for $uid"); - my $res = $imap->store($uid, $act, "(\\Seen)"); + my $bool = !$action->{isUnread}; + my @flags = ("\\Seen"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isFlagged}) { - my $act = $action->{isFlagged} ? "+flags" : "-flags"; - $Self->log('debug', "STORING $act FLAGGED for $uid"); - $imap->store($uid, $act, "(\\Flagged)"); + my $bool = $action->{isFlagged}; + my @flags = ("\\Flagged"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isAnswered}) { - my $act = $action->{isAnswered} ? "+flags" : "-flags"; - $Self->log('debug', "STORING $act ANSWERED for $uid"); - $imap->store($uid, $act, "(\\Answered)"); + my $bool = $action->{isAnswered}; + my @flags = ("\\Answered"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one my $newfolder = $foldermap{$id}[1]; - $imap->copy($uid, $newfolder); # UIDPLUS? Also the ID changes - $imap->store($uid, '+flags', "(\\Deleted)"); - $imap->uidexpunge($uid); + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } } - $imap->unselect(); } } @@ -463,7 +451,6 @@ sub delete_messages { my $ids = shift; my $dbh = $Self->{dbh}; - my $imap = $Self->{imap}; my %deletemap; foreach my $msgid (@$ids) { @@ -471,25 +458,19 @@ sub delete_messages { $deletemap{$ifolderid}{$uid} = 1; } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); + my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; + my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; foreach my $ifolderid (keys %deletemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; + my $uidvalidity = $foldermap{$ifolderid}[2]; die "NO SUCH FOLDER $ifolderid" unless $imapname; - # we're writing here! - my $r = $imap->select($imapname); - die "SELECT FAILED $r" unless lc($r) eq 'ok'; - my $uids = [sort keys %{$deletemap{$ifolderid}}]; - if (@$uids) { - $imap->store($uids, "+flags", "(\\Deleted)"); - $imap->uidexpunge($uids); - } - $imap->unselect(); + + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder } } @@ -605,19 +586,16 @@ sub fill_messages { $udata{$row->[0]}{$row->[1]} = $row->[2]; } - my $imap = $Self->{imap}; foreach my $ifolderid (sort keys %udata) { - my ($imapname) = $Self->dbh->selectrow_array("SELECT imapname FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); my $uhash = $udata{$ifolderid}; - die "NO folder $ifolderid" unless $imapname; - my $r = $imap->examine($imapname); - - my $messages = $imap->fetch(join(',', sort { $a <=> $b } keys %$uhash), "rfc822"); + my $uids = join(',', sort { $a <=> $b } keys %$uhash); + my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); - foreach my $uid (keys %$messages) { + foreach my $uid (keys %{$res->{data}}) { warn "FETCHED BODY FOR $uid\n"; - my $rfc822 = $messages->{$uid}{rfc822}; + my $rfc822 = $res->{data}{$uid}; my $msgid = $uhash->{$uid}; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index d8309bf..340a806 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -107,6 +107,73 @@ sub imap_status { return $data; } +# no newname == delete +sub imap_update { + my $Self = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uids = shift; + my $isAdd = shift; + my $flags = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->select($imapname); + die "SELECT FAILED $r" unless lc($r) eq 'ok'; + + my $uidvalidity = $imap->get_response_code('uidvalidity'); + + my %res = { + imapname => $imapname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + }; + + if ($olduidvalidity != $uidvalidity) { + return \%res; + } + + $imap->store($uids, $isAdd ? "+flags" : "-flags", "(@$flags)"); + + $res{updated} = $uids; + + return \%res; +} + +sub imap_fill { + my $Self = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uids = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + + my $uidvalidity = $imap->get_response_code('uidvalidity'); + + my %res = { + imapname => $imapname, + newname => $newname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + }; + + if ($olduidvalidity != $uidvalidity) { + return \%res; + } + + my $data = $imap->fetch($uids, "rfc822"); + + my %ids; + foreach my $uid (keys %$data) { + $ids{$uid} = $data->{$uid}{rfc822}; + } + $res{data} = \%ids; + return \%res; +} + # no newname == delete sub imap_move { my $Self = shift; @@ -226,7 +293,7 @@ sub imap_append { my $r = $imap->append($imapname, $flags, $internaldate, ['Literal', $rfc822]); die "APPEND FAILED $r" unless lc($r) eq 'ok'; - my $uid = $Mailbox->get_response_code('appenduid'); + my $uid = $imap->get_response_code('appenduid'); # XXX - fetch the x-gm-msgid or envelope from the server so we know the # the ID that the server gave this message diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 63897f0..238a6f9 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -86,7 +86,7 @@ sub connect_imap { my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; my $name = $folder->[2]; my $label = $role || $folder->[2]; - $Self->{folders}{$name} = $label; + $Self->{folders}{$name} = [$folder->[1], $label]; $Self->{labels}{$label} = $name; } return $Self->{imap}; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 0b091bf..0e97de4 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -25,12 +25,12 @@ package SyncServer; sub setup { my $config = shift; - if ($id =~ m/gmail\.com/) { + if ($config->{hostname} eq 'gmail') { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; - } elsif ($id =~ m/icloud\.com/) { + } elsif ($config->{hostname} eq 'imap.mail.me.com') { $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; } else { - die "UNKNOWN ID $id"; + die "UNKNOWN ID $id ($config->{hostname})"; } warn "$$ Connected $id"; $0 = "[jmap proxy imapsync] $id"; @@ -126,6 +126,24 @@ sub handle_send { return ['sent', $data]; } +sub handle_imap_update { + my $args = shift; + my $data = $backend->imap_update(@$args); + return ['updated', $data]; +} + +sub handle_imap_move { + my $args = shift; + my $data = $backend->imap_move(@$args); + return ['moved', $data]; +} + +sub handle_imap_fill { + my $args = shift; + my $data = $backend->imap_fill(@$args); + return ['filled', $data]; +} + sub mk_handler { my ($db) = @_; From eb16ee3d6fd470b5d453e73380b6653aad1e645b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 09:38:02 -0500 Subject: [PATCH 024/331] massive rewrite of ImapDB to use common sync code --- JMAP/ImapDB.pm | 68 +++++++++++++++++++-------------------------- JMAP/Sync/Common.pm | 30 +++++++++++++++++++- bin/syncserver.pl | 12 ++++++++ 3 files changed, 69 insertions(+), 41 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index e072143..6cb3632 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -299,39 +299,32 @@ sub do_folder { my $imap = $Self->{imap}; my $dbh = $Self->dbh(); - my ($imapname, $olduidfirst, $olduidnext, $olduidvalidity, $oldhighestmodseq) = $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = + $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); die "NO SUCH FOLDER $ifolderid" unless $imapname; - $olduidfirst ||= 0; + $uidfirst ||= 0; - my $r = $imap->examine($imapname); - - my $uidvalidity = $imap->get_response_code('uidvalidity'); - my $uidnext = $imap->get_response_code('uidnext'); - my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; - my $exists = $imap->get_response_code('exists') || 0; - - if ($olduidvalidity and $olduidvalidity != $uidvalidity) { - $oldhighestmodseq = 0; - $olduidfirst = 0; - $olduidnext = 1; - # XXX - delete all the data for this folder and re-sync it - } - elsif ($olduidfirst == 1 and $oldhighestmodseq and $highestmodseq == $oldhighestmodseq) { - $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); - return 0; # yay, nothing to do + my @extra; + if ($uidfirst > 1 and $batchsize) { + my $end = $uidfirst - 1; + $uidfirst -= $batchsize; + $uidfirst = 1 if $uidfirst < 1; + push @extra, backfill => [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; } - $olduidfirst = $uidnext unless $olduidfirst; - $olduidnext = $uidnext unless $olduidnext; + my $res = $Self->backend_cmd('imap_fetch', $imapname, { + uidvalidty => $uidvalidity, + highestmodseq => $highestmodseq, + uidnext => $uidnext, + },{ + @extra, + update => [1, $uidnext - 1, [], $highestmodseq], + new => [$uidnext, '*', [qw(internaldate envelope rfc822.size)]], + }); - my $uidfirst = $olduidfirst; my $didold = 0; - if ($olduidfirst > 1 and $batchsize) { - $uidfirst = $olduidfirst - $batchsize; - $uidfirst = 1 if $uidfirst < 1; - my $to = $olduidfirst - 1; - $Self->log('debug', "FETCHING $imapname: $uidfirst:$to"); - my $new = $imap->fetch("$uidfirst:$to", '(uid flags internaldate envelope rfc822.size)') || {}; + if ($res->{backfill}) { + my $new = $res->{backfill}; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); @@ -341,21 +334,15 @@ sub do_folder { delete $Self->{backfilling}; } - if ($olduidnext > $olduidfirst) { - my $to = $olduidnext - 1; - my @extra; - push @extra, "(changedsince $oldhighestmodseq)" if $oldhighestmodseq; - $Self->log('debug', "UPDATING $imapname: $uidfirst:$to"); - my $changed = $imap->fetch("$uidfirst:$to", "(flags)", @extra) || {}; + if ($res->{update}) { + my $changed = $res->{update}; foreach my $uid (sort { $a <=> $b } keys %$changed) { $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, [$forcelabel]); } } - if ($uidnext > $olduidnext) { - my $to = $uidnext - 1; - $Self->log('debug', "FETCHING $imapname: $olduidnext:$to"); - my $new = $imap->fetch("$olduidnext:$to", '(uid flags internaldate envelope rfc822.size)') || {}; + if ($res->{new}) { + my $new = $res->{new}; foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); @@ -364,10 +351,11 @@ sub do_folder { # need to make changes before counting my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $exists) { + if ($count != $res->{exists}) { my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $uids = $imap->search("UID", "$uidfirst:$to"); + my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + my $uids = $res->{data}; my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); my %exists = map { $_ => 1 } @$uids; foreach my $uid (@$data) { @@ -376,7 +364,7 @@ sub do_folder { } } - $Self->dupdate('ifolders', {highestmodseq => $highestmodseq, uidfirst => $uidfirst, uidnext => $uidnext, uidvalidity => $uidvalidity}, {ifolderid => $ifolderid}); + $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}, uidvalidity => $res->{newstate}{uidvalidity}}, {ifolderid => $ifolderid}); return $didold; } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 340a806..678fe5a 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -155,7 +155,6 @@ sub imap_fill { my %res = { imapname => $imapname, - newname => $newname, olduidvalidity => $olduidvalidity, newuidvalidity => $uidvalidity, }; @@ -174,6 +173,35 @@ sub imap_fill { return \%res; } +sub imap_count { + my $Self = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uids = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + + my $uidvalidity = $imap->get_response_code('uidvalidity'); + + my %res = { + imapname => $imapname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + }; + + if ($olduidvalidity != $uidvalidity) { + return \%res; + } + + my $data = $imap->fetch($uids, "UID"); + + $res{data} = [sort { $a <=> $b } keys %$data]; + return \%res; +} + # no newname == delete sub imap_move { my $Self = shift; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 0e97de4..1d50af8 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -144,6 +144,18 @@ sub handle_imap_fill { return ['filled', $data]; } +sub handle_imap_fetch { + my $args = shift; + my $data = $backend->imap_fetch(@$args); + return ['fetched', $data]; +} + +sub handle_imap_count { + my $args = shift; + my $data = $backend->imap_count(@$args); + return ['counted', $data]; +} + sub mk_handler { my ($db) = @_; From aff882803d96a8773efb8831a795fddd7ac96da2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 9 Jul 2015 22:03:00 -0500 Subject: [PATCH 025/331] WIP lots of stuff --- JMAP/DB.pm | 5 +++-- JMAP/ImapDB.pm | 22 +++++++++++++------- JMAP/Sync/Common.pm | 47 ++++++++++++++++++++++--------------------- JMAP/Sync/Standard.pm | 3 --- bin/apiendpoint.pl | 26 +++++++++--------------- bin/syncserver.pl | 14 ++++++------- 6 files changed, 58 insertions(+), 59 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index b43c1fa..114246a 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -49,7 +49,7 @@ sub log { } else { my ($level, @items) = @_; - return if $level eq 'debug'; + #return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } @@ -678,7 +678,8 @@ sub dupdate { my @keys = sort keys %$values; my @lkeys = sort keys %$limit; - my $sql = "UPDATE $table SET " . join (', ', map { "$_ = ?" } @keys) . " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys); + my $sql = "UPDATE $table SET " . join (', ', map { "$_ = ?" } @keys); + $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; $Self->log('debug', $sql, _dbl(map { $values->{$_} } @keys), _dbl(map { $limit->{$_} } @lkeys)); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 6cb3632..045a061 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -14,6 +14,9 @@ use OAuth2::Tiny; use Encode; use Encode::MIME::Header; use Digest::SHA qw(sha1_hex); +use AnyEvent; +use AnyEvent::Socket; +use Data::Dumper; our $TAG = 1; @@ -34,7 +37,7 @@ sub setuser { my ($hostname, $username, $password) = @_; my $data = $Self->dbh->selectrow_arrayref("SELECT hostname, username, password FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}); + $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}, {hostname => $data->[0]}); } else { $Self->dinsert('iserver', { @@ -45,7 +48,7 @@ sub setuser { } my $user = $Self->dbh->selectrow_arrayref("SELECT email FROM account"); if ($user and $user->[0]) { - $Self->dmaybeupdate('account', {email => $username}); + $Self->dmaybeupdate('account', {email => $username}, {email => $username}); } else { $Self->dinsert('account', { @@ -68,7 +71,7 @@ sub access_token { sub backend_cmd { my $Self = shift; my $cmd = shift; - my $args = shift; + my @args = @_; unless ($Self->{backend}) { my $w = AnyEvent->condvar; my $auth = $Self->access_token(); @@ -89,7 +92,7 @@ sub backend_cmd { my $handle = $Self->{backend}; my $w = AnyEvent->condvar; my $tag = "T" . $TAG++; - $handle->push_write(json => [$cmd, $args, $tag]); # whatever + $handle->push_write(json => [$cmd, \@args, $tag]); # whatever $handle->push_write("\012"); $handle->push_read(json => sub { my $hdl = shift; @@ -97,8 +100,11 @@ sub backend_cmd { die "INVALID RESPONSE" unless $json->[2] eq $tag; $w->send($json->[1]); }); + my $res = $w->recv; - return $w->recv; + warn Dumper ($cmd, \@args, $res); + + return $res; } # synchronise list from IMAP server to local folder cache @@ -259,6 +265,9 @@ sub backfill { sub firstsync { my $Self = shift; + + $Self->sync_folders(); + my $labels = $Self->labels(); my $ifolderid = $labels->{"inbox"}[0]; @@ -302,7 +311,6 @@ sub do_folder { my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); die "NO SUCH FOLDER $ifolderid" unless $imapname; - $uidfirst ||= 0; my @extra; if ($uidfirst > 1 and $batchsize) { @@ -351,7 +359,7 @@ sub do_folder { # need to make changes before counting my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $res->{exists}) { + if ($count != $res->{newstate}{exists}) { my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 678fe5a..1bf3b6a 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -119,15 +119,15 @@ sub imap_update { my $imap = $Self->connect_imap(); my $r = $imap->select($imapname); - die "SELECT FAILED $r" unless lc($r) eq 'ok'; + die "SELECT FAILED $imapname" unless $r; - my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; - my %res = { + my %res = ( imapname => $imapname, olduidvalidity => $olduidvalidity, newuidvalidity => $uidvalidity, - }; + ); if ($olduidvalidity != $uidvalidity) { return \%res; @@ -149,15 +149,15 @@ sub imap_fill { my $imap = $Self->connect_imap(); my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + die "EXAMINE FAILED $imapname" unless $r; - my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; - my %res = { + my %res = ( imapname => $imapname, olduidvalidity => $olduidvalidity, newuidvalidity => $uidvalidity, - }; + ); if ($olduidvalidity != $uidvalidity) { return \%res; @@ -182,15 +182,15 @@ sub imap_count { my $imap = $Self->connect_imap(); my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + die "EXAMINE FAILED $imapname" unless $r; - my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; - my %res = { + my %res = ( imapname => $imapname, olduidvalidity => $olduidvalidity, newuidvalidity => $uidvalidity, - }; + ); if ($olduidvalidity != $uidvalidity) { return \%res; @@ -213,16 +213,16 @@ sub imap_move { my $imap = $Self->connect_imap(); my $r = $imap->select($imapname); - die "SELECT FAILED $r" unless lc($r) eq 'ok'; + die "SELECT FAILED $imapname" unless $r; - my $uidvalidity = $imap->get_response_code('uidvalidity'); + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; - my %res = { + my %res = ( imapname => $imapname, newname => $newname, olduidvalidity => $olduidvalidity, newuidvalidity => $uidvalidity, - }; + ); if ($olduidvalidity != $uidvalidity) { return \%res; @@ -266,10 +266,10 @@ sub imap_fetch { my $imap = $Self->connect_imap(); my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); + die "EXAMINE FAILED $imapname" unless $r; - my $uidvalidity = $imap->get_response_code('uidvalidity'); - my $uidnext = $imap->get_response_code('uidnext'); + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; + my $uidnext = $imap->get_response_code('uidnext') + 0; my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; my $exists = $imap->get_response_code('exists') || 0; @@ -277,10 +277,10 @@ sub imap_fetch { imapname => $imapname, oldstate => $state, newstate => { - uidvalidity => $uidvalidity, - uidnext => $uidnext, - highestmodseq => $highestmodseq, - exists => $exists, + uidvalidity => $uidvalidity + 0, + uidnext => $uidnext + 0, + highestmodseq => $highestmodseq + 0, + exists => $exists + 0, }, ); @@ -297,6 +297,7 @@ sub imap_fetch { my $item = $fetch->{$key}; my $from = $item->[0]; my $to = $item->[1]; + next if ($to eq '*' and $from == $uidnext); my @flags = qw(uid flags); push @flags, @{$item->[2]} if $item->[2]; my @extra; diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 238a6f9..5eae34e 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -61,10 +61,8 @@ sub connect_imap { } for (1..3) { - $Self->log('debug', "Looking for server for $Self->{auth}{username}"); my $port = 993; my $usessl = $port != 143; # we use SSL for anything except default - $Self->log('debug', "getting imaptalk"); $Self->{imap} = Mail::IMAPTalk->new( Server => $Self->{auth}{imapserver}, Port => $port, @@ -75,7 +73,6 @@ sub connect_imap { UseBlocking => $usessl, ); next unless $Self->{imap}; - $Self->log('debug', "Connected as $Self->{auth}{username}"); $Self->{lastused} = time(); my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; my @folders = $Self->{imap}->$list('', '*'); diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 66fa0df..c99e27a 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -20,6 +20,7 @@ package JMAP::Backend; use AnyEvent::Socket; use AnyEvent::Util; use AnyEvent::HTTP; +use EV; use JSON::XS qw(encode_json decode_json); use Net::Server::PreFork; @@ -28,7 +29,6 @@ package JMAP::Backend; # we love globals my $hdl; -my $cv; my $db; my $dbh; my $accountid; @@ -67,7 +67,6 @@ sub getdb { $db->{watcher} = AnyEvent->timer(after => 30, interval => 30, cb => sub { # check if there's more work to do on the account... eval { - $db->connect(); $db->begin(); $db->backfill(); $db->commit(); @@ -78,7 +77,6 @@ sub getdb { sub process_request { my $server = shift; - $cv = AnyEvent->condvar; close STDIN; close STDOUT; @@ -87,12 +85,12 @@ sub process_request { on_error => sub { my ($hdl, $fatal, $msg) = @_; $hdl->destroy; - $cv->send; + EV::unloop; }, on_disconnect => sub { my ($hdl, $fatal, $msg) = @_; $hdl->destroy; - $cv->send; + EV::unloop; }, ); @@ -104,8 +102,7 @@ sub process_request { $handle->push_read(json => mk_handler($accountid)); }); - $cv->recv; - $cv->send; + EV::run; exit 0; } @@ -135,7 +132,6 @@ sub handle_getstate { my $db = shift; my $cmd = shift; - $db->connect(); $db->begin(); my $user = $db->get_user(); $db->commit(); @@ -158,7 +154,7 @@ sub mk_handler { my ($db) = @_; # don't last forever - $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; undef $hdl; $cv->send }); + $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; undef $hdl; EV::unloop }); return sub { my ($hdl, $json) = @_; @@ -212,6 +208,9 @@ sub mk_handler { $res->[2] = $tag; $hdl->push_write(json => $res) if $res->[0]; warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; + if ($res->[0] eq 'error') { + warn Dumper($res); + } } $hdl->push_read(json => mk_handler($db)); }; @@ -219,7 +218,6 @@ sub mk_handler { sub handle_sync { my $db = shift; - $db->connect(); # ensure up-to-date folders $db->begin(); $db->sync(); $db->commit(); @@ -272,7 +270,6 @@ sub handle_cb_google { $db->begin(); $db->setuser($email, $gmaildata->{refresh_token}, $data->{name}, $data->{picture}); $db->commit(); - $db->connect(); # only after user is set $db->begin(); $db->firstsync(); $db->commit(); @@ -310,7 +307,6 @@ sub handle_signup { $db->begin(); $db->setuser(@$detail); $db->commit(); - $db->connect(); # only after user is set $db->begin(); $db->firstsync(); $db->commit(); @@ -322,7 +318,7 @@ sub handle_delete { my $dbh = accountsdb(); $dbh->do("DELETE FROM accounts WHERE accountid = ?", {}, $accountid); $db->delete() if $db; - $hdl->{timer} = AnyEvent->timer(after => 0, cb => sub { undef $hdl; $cv->send }); + $hdl->{timer} = AnyEvent->timer(after => 0, cb => sub { undef $hdl; EV::unloop; }); return ['deleted', $accountid]; } @@ -337,7 +333,6 @@ sub handle_upload { my ($db, $req) = @_; my ($type, $content) = @$req; - $db->connect(); $db->begin(); my $api = JMAP::API->new($db); my ($res) = $api->uploadFile($type, $content); @@ -349,7 +344,6 @@ sub handle_upload { sub handle_download { my ($db, $id) = @_; - $db->connect(); $db->begin(); my $api = JMAP::API->new($db); my ($type, $content) = $api->downloadFile($id); @@ -361,7 +355,6 @@ sub handle_download { sub handle_raw { my ($db, $req) = @_; - $db->connect(); $db->begin(); my $api = JMAP::API->new($db); my ($type, $content, $filename) = $api->getRawMessage($req); @@ -374,7 +367,6 @@ sub handle_jmap { my ($db, $request) = @_; my @res; - $db->connect(); # need to keep the API object around for the entire request for idmap purposes my $api = JMAP::API->new($db); foreach my $item (@$request) { diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 1d50af8..ba50f24 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -12,12 +12,13 @@ package SyncServer; use Net::Server::PreFork; use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; +use EV; +use Data::Dumper; use base qw(Net::Server::PreFork); # we love globals my $hdl; -my $cv; my $id; my $backend; @@ -40,7 +41,6 @@ sub setup { sub process_request { my $server = shift; - $cv = AnyEvent->condvar; close STDIN; close STDOUT; @@ -49,12 +49,12 @@ sub process_request { on_error => sub { my ($hdl, $fatal, $msg) = @_; $hdl->destroy; - $cv->send; + EV::unloop; }, on_disconnect => sub { my ($hdl, $fatal, $msg) = @_; $hdl->destroy; - $cv->send; + EV::unloop; }, ); @@ -67,8 +67,7 @@ sub process_request { $handle->push_read(json => mk_handler()); }); - $cv->recv; - $cv->send; + EV::run; exit 0; } @@ -160,7 +159,7 @@ sub mk_handler { my ($db) = @_; # don't last forever - $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "$$ SHUTTING DOWN $id ON TIMEOUT\n"; undef $hdl; $cv->send }); + $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "$$ SHUTTING DOWN $id ON TIMEOUT\n"; undef $hdl; EV::unloop }); return sub { my ($hdl, $json) = @_; @@ -184,6 +183,7 @@ sub mk_handler { $hdl->push_write("\n"); warn "$$ HANDLED $cmd ($tag) => $res->[0] ($id)\n"; + warn Dumper($json, $res); $hdl->push_read(json => mk_handler($db)); }; } From 63200190c900875d0088a16ed9c6840c9fab92b6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 10 Jul 2015 07:32:26 -0500 Subject: [PATCH 026/331] more status support --- JMAP/Sync/Common.pm | 3 ++- bin/syncserver.pl | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1bf3b6a..d6cb718 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -101,7 +101,8 @@ sub imap_status { my $imap = $Self->connect_imap(); - my $fields = "(uidvalidity uidnext highestmodseq messages)"; + my @fields = qw(uidvalidity uidnext messages); + push @fields, "highestmodseq" if $imap->capability->{condstore}; my $data = $imap->multistatus($fields, @$folders); return $data; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index ba50f24..f6b6504 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -125,6 +125,12 @@ sub handle_send { return ['sent', $data]; } +sub handle_imap_status { + my $args = shift; + my $data = $backend->imap_status(@$args); + return ['status', $data]; +} + sub handle_imap_update { my $args = shift; my $data = $backend->imap_update(@$args); From 5e9a4dda54e6a5b0439f7d196d30675d82f3ab26 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 10 Jul 2015 23:24:57 +1000 Subject: [PATCH 027/331] status code --- JMAP/ImapDB.pm | 53 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 045a061..ac542c2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -115,10 +115,11 @@ sub sync_folders { my $dbh = $Self->dbh(); my $folders = $Self->backend_cmd('folders', []); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label FROM ifolders"); - my %ibylabel = map { $_->[3] => $_ } @$ifolders; + my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label, FROM ifolders"); + my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; + my %getstatus; foreach my $name (sort keys %$folders) { my $sep = $folders->{$name}[1]; my $role = $ROLE_MAP{lc $name}; @@ -131,6 +132,23 @@ sub sync_folders { $id = $Self->dinsert('ifolders', {sep => $sep, imapname => $name, label => $label}); } $seen{$id} = 1; + unless ($ibylabel{$label}[2]) { + # no uidvalidity, we need to get status for this one + $getstatus{$ibylabel{$label}[3]} = $id; + } + } + + if (keys %getstatus) { + my $data = $Self->backend_cmd('status', [keys %getstatus]); + foreach my $name (keys %$data) { + my $status = $data->{$name}; + $Self->dmaybeupdate('ifolders', { + uidvalidity => $status->{uidvalidity}, + uidnext => $status->{uidnext}, + uidfirst => $status->{uidnext}, + highestmodseq => $status->{highestmodseq}, + }, {ifolderid => $id}); + } } foreach my $folder (@$ifolders) { @@ -330,6 +348,11 @@ sub do_folder { new => [$uidnext, '*', [qw(internaldate envelope rfc822.size)]], }); + if ($res->{newstate}{uidvalidity} != $uidvalidity) { + # going to want to nuke everything for the existing folder and create this - but for now, just die + die "UIDVALIDITY CHANGED $imapname: $uidvalidity => $res->{newstate}{uidvalidity}"; + } + my $didold = 0; if ($res->{backfill}) { my $new = $res->{backfill}; @@ -358,21 +381,23 @@ sub do_folder { } # need to make changes before counting - my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $res->{newstate}{exists}) { - my $to = $uidnext - 1; - $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); - my $uids = $res->{data}; - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - my %exists = map { $_ => 1 } @$uids; - foreach my $uid (@$data) { - next if $exists{$uid}; - $Self->deleted_record($ifolderid, $uid); + if ($uidfirst == 1) { + my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + if ($count != $res->{newstate}{exists}) { + my $to = $uidnext - 1; + $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); + my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + my $uids = $res->{data}; + my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + my %exists = map { $_ => 1 } @$uids; + foreach my $uid (@$data) { + next if $exists{$uid}; + $Self->deleted_record($ifolderid, $uid); + } } } - $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}, uidvalidity => $res->{newstate}{uidvalidity}}, {ifolderid => $ifolderid}); + $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); return $didold; } From a4bf7b339c7ef9f33335030b7d103ed489a0deb4 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 10 Jul 2015 09:52:59 -0500 Subject: [PATCH 028/331] getting there --- JMAP/ImapDB.pm | 49 +++++++++++++++++++++++++-------------------- JMAP/Sync/Common.pm | 14 ++++++------- bin/syncserver.pl | 6 ------ 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ac542c2..844330d 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -30,6 +30,12 @@ my %ROLE_MAP = ( 'sent messages' => 'sent', 'sent items' => 'sent', 'trash' => 'trash', + '\\inbox' => 'inbox', + '\\trash' => 'trash', + '\\sent' => 'sent', + '\\junk' => 'junk', + '\\archive' => 'archive', + '\\drafts' => 'drafts', ); sub setuser { @@ -115,15 +121,15 @@ sub sync_folders { my $dbh = $Self->dbh(); my $folders = $Self->backend_cmd('folders', []); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label, FROM ifolders"); + my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; my %getstatus; foreach my $name (sort keys %$folders) { - my $sep = $folders->{$name}[1]; - my $role = $ROLE_MAP{lc $name}; - my $label = $role || $name; + my $sep = $folders->{$name}[0]; + my $role = $ROLE_MAP{lc $folders->{$name}[1]}; + my $label = $role || $folders->{$name}[1]; my $id = $ibylabel{$label}[0]; if ($id) { $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); @@ -134,12 +140,12 @@ sub sync_folders { $seen{$id} = 1; unless ($ibylabel{$label}[2]) { # no uidvalidity, we need to get status for this one - $getstatus{$ibylabel{$label}[3]} = $id; + $getstatus{$name} = $id; } } if (keys %getstatus) { - my $data = $Self->backend_cmd('status', [keys %getstatus]); + my $data = $Self->backend_cmd('imap_status', [keys %getstatus]); foreach my $name (keys %$data) { my $status = $data->{$name}; $Self->dmaybeupdate('ifolders', { @@ -147,7 +153,7 @@ sub sync_folders { uidnext => $status->{uidnext}, uidfirst => $status->{uidnext}, highestmodseq => $status->{highestmodseq}, - }, {ifolderid => $id}); + }, {ifolderid => $getstatus{$name}}); } } @@ -180,6 +186,7 @@ sub sync_jmailboxes { my %seen; foreach my $folder (@$ifolders) { my $fname = $folder->[2]; + warn " MAPPING $fname ($folder->[1])"; $fname =~ s/^INBOX\.//; # check for roles first my @bits = split "[$folder->[1]]", $fname; @@ -191,6 +198,7 @@ sub sync_jmailboxes { $precedence = 2 if $role; $precedence = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { + $seen{$id} = 1 if $id; $name = $item; $parentid = $id; $id = $byname{$parentid}{$name}; @@ -202,9 +210,6 @@ sub sync_jmailboxes { $byname{$parentid}{$name} = $id; } } - if (@bits) { - $seen{$id} = 1; - } } next unless $name; my %details = ( @@ -330,23 +335,22 @@ sub do_folder { $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); die "NO SUCH FOLDER $ifolderid" unless $imapname; - my @extra; + my %fetches; + $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size)]]; + $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; + if ($uidfirst > 1 and $batchsize) { my $end = $uidfirst - 1; $uidfirst -= $batchsize; $uidfirst = 1 if $uidfirst < 1; - push @extra, backfill => [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; + $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; } my $res = $Self->backend_cmd('imap_fetch', $imapname, { - uidvalidty => $uidvalidity, + uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, uidnext => $uidnext, - },{ - @extra, - update => [1, $uidnext - 1, [], $highestmodseq], - new => [$uidnext, '*', [qw(internaldate envelope rfc822.size)]], - }); + },\%fetches); if ($res->{newstate}{uidvalidity} != $uidvalidity) { # going to want to nuke everything for the existing folder and create this - but for now, just die @@ -355,7 +359,7 @@ sub do_folder { my $didold = 0; if ($res->{backfill}) { - my $new = $res->{backfill}; + my $new = $res->{backfill}[1]; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); @@ -366,14 +370,14 @@ sub do_folder { } if ($res->{update}) { - my $changed = $res->{update}; + my $changed = $res->{update}[1]; foreach my $uid (sort { $a <=> $b } keys %$changed) { $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, [$forcelabel]); } } if ($res->{new}) { - my $new = $res->{new}; + my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); @@ -574,7 +578,8 @@ sub apply_data { } sub _envelopedata { - my $envelope = decode_json(shift); + my $data = shift; + my $envelope = decode_json($data); my $encsub = decode('MIME-Header', $envelope->{Subject}); return ( msgsubject => $encsub, diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index d6cb718..4357835 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -103,7 +103,7 @@ sub imap_status { my @fields = qw(uidvalidity uidnext messages); push @fields, "highestmodseq" if $imap->capability->{condstore}; - my $data = $imap->multistatus($fields, @$folders); + my $data = $imap->multistatus("(@fields)", @$folders); return $data; } @@ -286,11 +286,8 @@ sub imap_fetch { ); if (($state->{uidvalidity} || 0) != $uidvalidity) { - return \%res; - } - - if ($highestmodseq and $highestmodseq == ($state->{highestmodseq} || 0)) { - $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); + warn "UIDVALID $state->{uidvalidity} $uidvalidity\n"; + $res{uidfail} = 1; return \%res; } @@ -298,12 +295,13 @@ sub imap_fetch { my $item = $fetch->{$key}; my $from = $item->[0]; my $to = $item->[1]; - next if ($to eq '*' and $from == $uidnext); + $to = $uidnext - 1 if $to eq '*'; + next if $from > $to; my @flags = qw(uid flags); push @flags, @{$item->[2]} if $item->[2]; + next if ($highestmodseq and $item->[3] and $item->[3] == $highestmodseq); my @extra; push @extra, "(changedsince $item->[3])" if $item->[3]; - $Self->log('debug', "FETCHING $imapname: $from:$to @flags @extra"); my $data = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; $res{$key} = [$item, $data]; } diff --git a/bin/syncserver.pl b/bin/syncserver.pl index f6b6504..965b2ec 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -77,12 +77,6 @@ sub handle_ping { return ['pong', $id]; } -sub handle_status { - my $args = shift; - my $data = $backend->fetch_status(@$args); - return ['status', $data]; -} - sub handle_folder { my $args = shift; my $data = $backend->fetch_folder(@$args); From 9f7d6819e500fb3ca07caee474b3e61fab927258 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 10 Jul 2015 10:16:41 -0500 Subject: [PATCH 029/331] WIP WIP Calendar sync (gotta sleep now) --- JMAP/ImapDB.pm | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 844330d..c450d6f 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -258,6 +258,65 @@ sub sync_jmailboxes { } } +# synchronise list from CalDAV server to local folder cache +# call in transaction +sub sync_calendars { + my $Self = shift; + + my $dbh = $Self->dbh(); + + my $calendars = $Self->backend_cmd('calendars', []); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour FROM icalendars"); + my %byhref = map { $_->[1] => $_ } @$icalendars; + + foreach my $calendar (@$calendars) { + my $id = $byhref{$calendar->{href}}[0]; + my $data = {isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, + colour => $calendar->{colour}, name => $calendar->{name}}; + if ($id) { + $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); + } + else { + $id = $Self->dinsert('icalendars', $data); + } + $seen{$id} = 1; + } + + foreach my $calendar (@$icalendars) { + my $id = $calendar->[0]; + next if $seen{$id}; + $dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); + } + + $Self->sync_jcalendars(); +} + +# synchronise from the imap folder cache to the jmap mailbox listing +# call in transaction +sub sync_jmailboxes { + my $Self = shift; + my $dbh = $Self->dbh(); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); + my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); + + my %jbyid; + my %roletoid; + my %byname; + foreach my $calendar (@$jcalendars) { + $jbyid{$calendar->[0]} = $calendar; + } + + foreach my $calendar (@$icalendars) { + + } + + foreach my $calendar (@$jcalendars) { + my $id = $calendar->[0]; + next if $seen{$id}; + $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); + } +} + sub labels { my $Self = shift; unless ($Self->{t}{labels}) { @@ -290,6 +349,8 @@ sub firstsync { my $Self = shift; $Self->sync_folders(); + $Self->sync_calendars(); + $Self->sync_addressbooks(); my $labels = $Self->labels(); @@ -696,6 +757,7 @@ CREATE TABLE IF NOT EXISTS icalendars ( isReadOnly INTEGER, colour TEXT, syncToken TEXT, + jcalendarid INTEGER, mtime DATE NOT NULL ); EOF @@ -717,6 +779,7 @@ CREATE TABLE IF NOT EXISTS iabooks ( name TEXT, isReadOnly INTEGER, syncToken TEXT, + jabookid INTEGER, mtime DATE NOT NULL ); EOF From 57aee95a31e9088315f53689e14a249c6cc1d251 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 12:09:55 +1000 Subject: [PATCH 030/331] more calendar handling --- JMAP/ImapDB.pm | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c450d6f..9965924 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -293,21 +293,27 @@ sub sync_calendars { # synchronise from the imap folder cache to the jmap mailbox listing # call in transaction -sub sync_jmailboxes { +sub sync_jcalendars { my $Self = shift; my $dbh = $Self->dbh(); my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); my %jbyid; - my %roletoid; - my %byname; foreach my $calendar (@$jcalendars) { $jbyid{$calendar->[0]} = $calendar; } + my %seen; foreach my $calendar (@$icalendars) { - + if ($jbyid{$calendar->[3]}) { + $Self->dmaybeupdate('jcalendars', {name => $calendar->[1], colour => $calendar->[2]}, {jcalendarid => $calendar->[3]}); + $seen{$calendar->[3]} = 1; + } + else { + my $id = $Self->dinsert('jcalendars', {name => $calendar->[1], colour => $calendar->[2]}); + $seen{$id} = 1; + } } foreach my $calendar (@$jcalendars) { From ef6a2dde7071695b38bb2d32c07830a6cf194a0b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 12:42:10 +1000 Subject: [PATCH 031/331] some more steps --- JMAP/ImapDB.pm | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9965924..ce5692c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -323,6 +323,30 @@ sub sync_jcalendars { } } +sub do_calendar { + my $Self = shift; + my $calendarid = shift; + + my $dbh = $Self->dbh(); + + my ($href) = $dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource FROM ievents WHERE icalendarid = ?", {}, $calendarid); + my %res = map { $_->[1] => $_->[0] } @$exists; + + my $events = $Self->backend_cmd('events', {href => $href}); + + foreach my $resource (keys %$events) { + my $id = delete $res{$resource}; + if ($id) { + $Self->dmaybeupdate('ievents', {content => $events->{$resource}}, {ieventid => id}); + } + else { + $Self->dinsert('ievents', {content => $events->{$resource}, resource => $resource}); + } + # XXX - parse UID and calculate which JMAP record to update + } +} + sub labels { my $Self = shift; unless ($Self->{t}{labels}) { From 9129f5952ede42e3877cc54ab1ac273266b8d074 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 12:42:54 +1000 Subject: [PATCH 032/331] sync all the calendars on startup (for initial testing only) --- JMAP/ImapDB.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ce5692c..ba1c272 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -289,6 +289,10 @@ sub sync_calendars { } $Self->sync_jcalendars(); + + foreach my $id (keys %seen) { + $Self->do_calendar($id); + } } # synchronise from the imap folder cache to the jmap mailbox listing From a22b320419f9e1c0e27a328d2f361ec5c3b1534a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 15:41:06 +1000 Subject: [PATCH 033/331] API: fix event fetching --- JMAP/API.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 796229a..66be170 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1288,7 +1288,7 @@ sub getCalendarEventList { my $start = $args->{position} || 0; return ['error', {type => 'invalidArguments'}] if $start < 0; - my $data = $dbh->selectall_arrayref("SELECT jeventid,jcalendarid FROM jevents WHERE active = 1 ORDER BY jeventid"); + my $data = $dbh->selectall_arrayref("SELECT eventuid,jcalendarid FROM jevents WHERE active = 1 ORDER BY eventuid"); $data = $Self->_event_filter($data, $args->{filter}, {}) if $args->{filter}; @@ -1337,7 +1337,7 @@ sub getCalendarEvents { foreach my $eventid (@{$args->{ids}}) { next if $seenids{$eventid}; $seenids{$eventid} = 1; - my $data = $dbh->selectrow_hashref("SELECT * FROM jevents WHERE jeventid = ?", {}, $eventid); + my $data = $dbh->selectrow_hashref("SELECT * FROM jevents WHERE eventuid = ?", {}, $eventid); unless ($data) { $missingids{$eventid} = 1; next; @@ -1378,7 +1378,7 @@ sub getCalendarEventUpdates { return ['error', {type => 'cannotCalculateChanges'}] if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); - my $sql = "SELECT jeventid,active FROM jevents WHERE jmodseq > ?"; + my $sql = "SELECT eventuid,active FROM jevents WHERE jmodseq > ?"; my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); From f6940eb23cf057c39bbcceabaad69f2f1f1bb024 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 10 Jul 2015 23:30:56 -0500 Subject: [PATCH 034/331] make things work again --- JMAP/ImapDB.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ba1c272..71ad2be 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -269,6 +269,7 @@ sub sync_calendars { my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; + my %seen; foreach my $calendar (@$calendars) { my $id = $byhref{$calendar->{href}}[0]; my $data = {isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, @@ -342,7 +343,7 @@ sub do_calendar { foreach my $resource (keys %$events) { my $id = delete $res{$resource}; if ($id) { - $Self->dmaybeupdate('ievents', {content => $events->{$resource}}, {ieventid => id}); + $Self->dmaybeupdate('ievents', {content => $events->{$resource}}, {ieventid => $id}); } else { $Self->dinsert('ievents', {content => $events->{$resource}, resource => $resource}); @@ -384,7 +385,7 @@ sub firstsync { $Self->sync_folders(); $Self->sync_calendars(); - $Self->sync_addressbooks(); + #$Self->sync_addressbooks(); my $labels = $Self->labels(); From 52cf8587e02388ccae6bc8148c4663135411f23c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 00:46:02 -0500 Subject: [PATCH 035/331] events --- JMAP/DB.pm | 28 ++++++++++++++++++++++++++++ JMAP/ImapDB.pm | 30 +++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 114246a..bcdaff8 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -20,6 +20,7 @@ use Encode; use Encode::MIME::Header; use DateTime; use Date::Parse; +use Net::CalDAVTalk; sub new { my $class = shift; @@ -607,6 +608,33 @@ sub report_messages { return ($msgids, []); } +sub parse_event { + my $Self = shift; + my $raw = shift; + my $CalDAV = Net::CalDAVTalk->new(url => 'http://localhost/'); # empty caldav + my ($event) = $CalDAV->vcalendarToEvents($raw); + return $event; +} + +sub set_event { + my $Self = shift; + my $jcalendarid = shift; + my $event = shift; + my $eventuid = delete $event->{uid}; + $Self->dmake('jevents', { + eventuid => $eventuid, + jcalendarid => $jcalendarid, + payload => json_encode($event), + }); +} + +sub delete_event { + my $Self = shift; + my $jcalendarid = shift; # doesn't matter + my $eventuid = shift; + return $Self->dupdate('jevents', {active => 0}, {eventuid => $eventuid}); +} + sub create_file { my $Self = shift; my $type = shift; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 71ad2be..1f7825a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -20,7 +20,7 @@ use Data::Dumper; our $TAG = 1; -# XXX - specialuse, this is just for iCloud for now +# special use or name magic my %ROLE_MAP = ( 'inbox' => 'inbox', 'drafts' => 'drafts', @@ -334,21 +334,33 @@ sub do_calendar { my $dbh = $Self->dbh(); - my ($href) = $dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendarid); - my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource FROM ievents WHERE icalendarid = ?", {}, $calendarid); - my %res = map { $_->[1] => $_->[0] } @$exists; + my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); + my %res = map { $_->[1] => $_ } @$exists; my $events = $Self->backend_cmd('events', {href => $href}); foreach my $resource (keys %$events) { - my $id = delete $res{$resource}; - if ($id) { - $Self->dmaybeupdate('ievents', {content => $events->{$resource}}, {ieventid => $id}); + my $data = delete $res{$resource}; + my $raw = $events->{$resource}; + if ($data) { + my $id = $data->[0]; + next if $raw eq $data->[2]; + $Self->dmaybeupdate('ievents', {content => $raw, resource => $resource}, {ieventid => $id}); } else { - $Self->dinsert('ievents', {content => $events->{$resource}, resource => $resource}); + $Self->dinsert('ievents', {content => $raw, resource => $resource}); } - # XXX - parse UID and calculate which JMAP record to update + my $event = $Self->parse_event($raw); + $Self->set_event($jcalendarid, $event); + } + + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $id = $data->[0]; + $Self->ddelete('ievents', {ieventid => $id}); + my $event = $Self->parse_event($data->[2]); + $Self->delete_event($jcalendarid, $event->{uid}); } } From 45d1ab6c766ac7ab61064b5dcbb064b9b6b1af75 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 00:57:36 -0500 Subject: [PATCH 036/331] calendars: make API read-only but present --- JMAP/API.pm | 4 ++-- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 66be170..715d0c8 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1343,7 +1343,7 @@ sub getCalendarEvents { next; } - my $item = json_decode($data->{payload}); + my $item = decode_json($data->{payload}); foreach my $key (keys %$item) { delete $item->{$key} unless _prop_wanted($args, $key); @@ -1629,7 +1629,7 @@ sub getContacts { next; } - my $item = json_decode($data->{payload}); + my $item = decode_json($data->{payload}); foreach my $key (keys %$item) { delete $item->{$key} unless _prop_wanted($args, $key); diff --git a/JMAP/DB.pm b/JMAP/DB.pm index bcdaff8..b6e428b 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -624,7 +624,7 @@ sub set_event { $Self->dmake('jevents', { eventuid => $eventuid, jcalendarid => $jcalendarid, - payload => json_encode($event), + payload => encode_json($event), }); } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 1f7825a..5c78a03 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -311,12 +311,24 @@ sub sync_jcalendars { my %seen; foreach my $calendar (@$icalendars) { + my $data = { + name => $calendar->[1], + colour => $calendar->[2], + isVisible => 1, + mayReadFreeBusy => 1, + mayReadItems => 1, + mayAddItems => 0, + mayModifyItems => 0, + mayRemoveItems => 0, + mayDelete => 0, + mayRename => 0, + }; if ($jbyid{$calendar->[3]}) { - $Self->dmaybeupdate('jcalendars', {name => $calendar->[1], colour => $calendar->[2]}, {jcalendarid => $calendar->[3]}); + $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); $seen{$calendar->[3]} = 1; } else { - my $id = $Self->dinsert('jcalendars', {name => $calendar->[1], colour => $calendar->[2]}); + my $id = $Self->dmake('jcalendars', $data); $seen{$id} = 1; } } From b2371ad5bd58368d28f38197c2d6fc7db8b9502b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 01:15:57 -0500 Subject: [PATCH 037/331] calendarId --- JMAP/API.pm | 1 + JMAP/ImapDB.pm | 1 + 2 files changed, 2 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 715d0c8..bb16552 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1350,6 +1350,7 @@ sub getCalendarEvents { } $item->{id} = $eventid; + $item->{calendarId} = $data->{jcalendarid} if _prop_wanted($args, "calendarId"); push @list, $item; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 5c78a03..b1af9cc 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -329,6 +329,7 @@ sub sync_jcalendars { } else { my $id = $Self->dmake('jcalendars', $data); + $Self->dupdate('icalendars', {jcalendarid => $id}, {icalendarid => $calendar->[0]}); $seen{$id} = 1; } } From 4f75d56af371928059e07004601e558cb7a5dc11 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 03:30:22 -0500 Subject: [PATCH 038/331] quote the calendarId --- JMAP/API.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index bb16552..dd44067 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1350,7 +1350,7 @@ sub getCalendarEvents { } $item->{id} = $eventid; - $item->{calendarId} = $data->{jcalendarid} if _prop_wanted($args, "calendarId"); + $item->{calendarId} = "$data->{jcalendarid}" if _prop_wanted($args, "calendarId"); push @list, $item; } From 3b6c70cbe1b4d6193a700f54259aa1e138f2a4ac Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 11 Jul 2015 09:29:14 -0500 Subject: [PATCH 039/331] non-blocking backend cmd --- JMAP/ImapDB.pm | 57 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b1af9cc..56f415f 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -77,9 +77,37 @@ sub access_token { sub backend_cmd { my $Self = shift; my $cmd = shift; + my $cb; + if (ref($cmd)) { + $cb = $cmd; + $cmd = shift; + } my @args = @_; - unless ($Self->{backend}) { - my $w = AnyEvent->condvar; + + my $w = AnyEvent->condvar; + + my $action = sub { + my $handle = shift; + my $tag = "T" . $TAG++; + $handle->push_write(json => [$cmd, \@args, $tag]); # whatever + $handle->push_write("\012"); + $handle->push_read(json => sub { + my $hdl = shift; + my $json = shift; + die "INVALID RESPONSE" unless $json->[2] eq $tag; + if ($cb) { + $cb->($json); + } + else { + $w->send($json); + } + }); + }; + if ($Self->{backend}) { + $action->($Self->{backend}); + } + else { + my $h = AnyEvent->condvar; my $auth = $Self->access_token(); tcp_connect('localhost', 5005, sub { my $fh = shift; @@ -90,26 +118,21 @@ sub backend_cmd { my $hdl = shift; my $json = shift; die "Failed to setup " . Dumper($json) unless $json->[0] eq 'setup'; - $w->send($handle); + $action->($handle); + $h->send($handle); }); + # XXX - handle destroy correctly + # handle backend going away, etc }); - $Self->{backend} = $w->recv; + + # synchronous startup to avoid race condition on setting up channel + $Self->{backend} = $h->recv; } - my $handle = $Self->{backend}; - my $w = AnyEvent->condvar; - my $tag = "T" . $TAG++; - $handle->push_write(json => [$cmd, \@args, $tag]); # whatever - $handle->push_write("\012"); - $handle->push_read(json => sub { - my $hdl = shift; - my $json = shift; - die "INVALID RESPONSE" unless $json->[2] eq $tag; - $w->send($json->[1]); - }); - my $res = $w->recv; - warn Dumper ($cmd, \@args, $res); + return if $cb; # async usage + my $res = $w->recv; + warn Dumper ($cmd, \@args, $res); return $res; } From b797aa7d7e4d0adfe1ced800ab8a0ae55024e0f5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 00:46:16 -0500 Subject: [PATCH 040/331] addressbooks - read and pass through --- JMAP/ImapDB.pm | 142 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 9 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 56f415f..5c8ff10 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -295,8 +295,12 @@ sub sync_calendars { my %seen; foreach my $calendar (@$calendars) { my $id = $byhref{$calendar->{href}}[0]; - my $data = {isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, - colour => $calendar->{colour}, name => $calendar->{name}}; + my $data = { + isReadOnly => $calendar->{isReadOnly}, + href => $calendar->{href}, + colour => $calendar->{colour}, + name => $calendar->{name}, + }; if ($id) { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); } @@ -400,6 +404,126 @@ sub do_calendar { } } +# synchronise list from CardDAV server to local folder cache +# call in transaction +sub sync_addressbooks { + my $Self = shift; + + my $dbh = $Self->dbh(); + + my $addressbooks = $Self->backend_cmd('addressbooks', []); + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly FROM iaddressbooks"); + my %byhref = map { $_->[1] => $_ } @$iaddressbooks; + + my %seen; + foreach my $addressbook (@$addressbooks) { + my $id = $byhref{$addressbook->{href}}[0]; + my $data = { + isReadOnly => $addressbook->{isReadOnly}, + href => $addressbook->{href}, + name => $addressbook->{name}, + }; + if ($id) { + $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + } + else { + $id = $Self->dinsert('iaddressbooks', $data); + } + $seen{$id} = 1; + } + + foreach my $addressbook (@$iaddressbooks) { + my $id = $addressbook->[0]; + next if $seen{$id}; + $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + } + + $Self->sync_jaddressbooks(); + + foreach my $id (keys %seen) { + $Self->do_addressbook($id); + } +} + +# synchronise from the imap folder cache to the jmap mailbox listing +# call in transaction +sub sync_jaddressbooks { + my $Self = shift; + my $dbh = $Self->dbh(); + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks"); + my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks"); + + my %jbyid; + foreach my $addressbook (@$jaddressbooks) { + $jbyid{$addressbook->[0]} = $addressbook; + } + + my %seen; + foreach my $addressbook (@$iaddressbooks) { + my $data = { + name => $addressbook->[1], + isVisible => 1, + mayReadItems => 1, + mayAddItems => 0, + mayModifyItems => 0, + mayRemoveItems => 0, + mayDelete => 0, + mayRename => 0, + }; + if ($jbyid{$addressbook->[2]}) { + $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $addressbook->[2]}); + $seen{$addressbook->[2]} = 1; + } + else { + my $id = $Self->dmake('jaddressbooks', $data); + $Self->dupdate('iaddressbooks', {jaddressbookid => $id}, {iaddressbookid => $addressbook->[0]}); + $seen{$id} = 1; + } + } + + foreach my $addressbook (@$jaddressbooks) { + my $id = $addressbook->[0]; + next if $seen{$id}; + $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $id}); + } +} + +sub do_addressbook { + my $Self = shift; + my $addressbookid = shift; + + my $dbh = $Self->dbh(); + + my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); + my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); + my %res = map { $_->[1] => $_ } @$exists; + + my $cards = $Self->backend_cmd('cards', {href => $href}); + + foreach my $resource (keys %$events) { + my $data = delete $res{$resource}; + my $raw = $events->{$resource}; + if ($data) { + my $id = $data->[0]; + next if $raw eq $data->[2]; + $Self->dmaybeupdate('icards', {content => $raw, resource => $resource}, {icardid => $id}); + } + else { + $Self->dinsert('icards', {content => $raw, resource => $resource}); + } + my $card = $Self->parse_card($raw); + $Self->set_card($jaddressbookid, $card); + } + + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $id = $data->[0]; + $Self->ddelete('icards', {icardid => $id}); + my $card = $Self->parse_card($data->[2]); + $Self->delete_card($jaddressbookid, $card->{uid}); + } +} + sub labels { my $Self = shift; unless ($Self->{t}{labels}) { @@ -433,7 +557,7 @@ sub firstsync { $Self->sync_folders(); $Self->sync_calendars(); - #$Self->sync_addressbooks(); + $Self->sync_addressbooks(); my $labels = $Self->labels(); @@ -856,21 +980,21 @@ CREATE TABLE IF NOT EXISTS ievents ( EOF $dbh->do(<do(< Date: Sun, 12 Jul 2015 01:40:01 -0500 Subject: [PATCH 041/331] card management code --- JMAP/DB.pm | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index b6e428b..fea4eee 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -21,6 +21,7 @@ use Encode::MIME::Header; use DateTime; use Date::Parse; use Net::CalDAVTalk; +use Net::CardDAVTalk::VCard; sub new { my $class = shift; @@ -635,6 +636,78 @@ sub delete_event { return $Self->dupdate('jevents', {active => 0}, {eventuid => $eventuid}); } +sub parse_card { + my $Self = shift; + my $raw = shift; + my ($card) = Net::CalDAVTalk::VCard->new_fromstring($raw); + + my %hash; + + $hash{uid} = $card->uid(); + $hash{kind} = $card->VKind(); + + if ($hash{kind} eq 'contact') { + $hash{lastName} = $card->VLastName(); + $hash{firstName} = $card->VFirstName(); + $hash{prefix} = $card->VTitle(); + + $hash{company} = $card->VCompany(); + $hash{department} = $card->VDepartment(); + + $hash{emails} = [$card->VEmails()]; + $hash{addresses} = [$card->VAddresses()]; + $hash{phones} = [$card->VPhones()]; + $hash{online} = [$card->VOnline()]; + + $hash{nickname} = $card->VNickname(); + $hash{birthday} = $card->VBirthday(); + $hash{notes} = $card->VNotes(); + } + else { + $hash{name} = $card->VFN(); + $hash{members} = [$card->VGroupContactUIDs()]; + } + + return \%hash; +} + +sub set_card { + my $Self = shift; + my $jcalendarid = shift; + my $card = shift; + my $carduid = delete $card->{uid}; + my $kind = delete $card->{kind}; + if ($kind eq 'contact') { + $Self->dmake('jcontacts', { + carduid => $carduid, + jaddressbookid => $jaddressbookid, + payload => encode_json($card), + }); + } + else { + $Self->ddelete('jcontactgroups', {carduid => $carduid, jaddressbookid => $jaddressbookid}); + my $gid = $Self->dmake('jcontactgroups', { + carduid => $carduid, + jaddressbookid => $jaddressbookid, + name => $card->{name}, + }); + foreach my $item (@{$card->{members}}) { + $Self->dmake('jcontactgroupmap', { + jcontactgroupid => $gid, + contactuid => $item, + }); + } + } +} + +sub delete_card { + my $Self = shift; + my $jcalendarid = shift; # doesn't matter + my $carduid = shift; + $Self->dupdate('jcontactgroups', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dupdate('jcontactgroupmap', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); +} + sub create_file { my $Self = shift; my $type = shift; From d190f3034f86538a104c1a52145f303a8b16a430 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 01:41:17 -0500 Subject: [PATCH 042/331] Addressbook support --- JMAP/DB.pm | 9 +++++++-- JMAP/ImapDB.pm | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index fea4eee..d1bc9ae 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -704,8 +704,13 @@ sub delete_card { my $Self = shift; my $jcalendarid = shift; # doesn't matter my $carduid = shift; - $Self->dupdate('jcontactgroups', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); - $Self->dupdate('jcontactgroupmap', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + my $kind = shift; + if ($kind eq 'contact') { + $Self->dupdate('jcontactgroups', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + } + else { + $Self->dupdate('jcontactgroupmap', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + } } sub create_file { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 5c8ff10..c9b236a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -520,7 +520,7 @@ sub do_addressbook { my $id = $data->[0]; $Self->ddelete('icards', {icardid => $id}); my $card = $Self->parse_card($data->[2]); - $Self->delete_card($jaddressbookid, $card->{uid}); + $Self->delete_card($jaddressbookid, $card->{uid}, $card->{kind}); } } From 74b99398ccd64ca7ee3c8e4f06038eb90278e32f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 01:44:41 -0500 Subject: [PATCH 043/331] fixup compile --- JMAP/DB.pm | 4 ++-- JMAP/ImapDB.pm | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index d1bc9ae..539315d 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -673,7 +673,7 @@ sub parse_card { sub set_card { my $Self = shift; - my $jcalendarid = shift; + my $jaddressbookid = shift; my $card = shift; my $carduid = delete $card->{uid}; my $kind = delete $card->{kind}; @@ -702,7 +702,7 @@ sub set_card { sub delete_card { my $Self = shift; - my $jcalendarid = shift; # doesn't matter + my $jaddressbookid = shift; # doesn't matter my $carduid = shift; my $kind = shift; if ($kind eq 'contact') { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c9b236a..e1c5b35 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -500,9 +500,9 @@ sub do_addressbook { my $cards = $Self->backend_cmd('cards', {href => $href}); - foreach my $resource (keys %$events) { + foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; - my $raw = $events->{$resource}; + my $raw = $cards->{$resource}; if ($data) { my $id = $data->[0]; next if $raw eq $data->[2]; From 7be8eb5b7b6d3893e6835adc3369a3eb1d69c662 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 01:56:24 -0500 Subject: [PATCH 044/331] cards --- JMAP/ImapDB.pm | 4 ++-- JMAP/Sync/Common.pm | 4 ++-- bin/syncserver.pl | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index e1c5b35..bfbcbf0 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -96,10 +96,10 @@ sub backend_cmd { my $json = shift; die "INVALID RESPONSE" unless $json->[2] eq $tag; if ($cb) { - $cb->($json); + $cb->($json->[1]); } else { - $w->send($json); + $w->send($json->[1]); } }); }; diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 4357835..fa71295 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -52,7 +52,7 @@ sub get_events { return \%res; } -sub get_abooks { +sub get_addressbooks { my $Self = shift; my $talk = $Self->connect_contacts(); @@ -61,7 +61,7 @@ sub get_abooks { return $data; } -sub get_contacts { +sub get_cards { my $Self = shift; my $Args = shift; my $talk = $Self->connect_contacts(); diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 965b2ec..be6068b 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -101,16 +101,16 @@ sub handle_events { return ['events', $data]; } -sub handle_abooks { +sub handle_addressbooks { my $args = shift; - my $data = $backend->get_abooks(@$args); - return ['abooks', $data]; + my $data = $backend->get_addressbooks(@$args); + return ['addressbooks', $data]; } -sub handle_contacts { +sub handle_cards { my $args = shift; - my $data = $backend->get_contacts(@$args); - return ['contacts', $data]; + my $data = $backend->get_cards(@$args); + return ['cards', $data]; } sub handle_send { From 2e3e98674bedf22c7e072d9b22e4767cdb4a76fb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 02:07:20 -0500 Subject: [PATCH 045/331] fix URL --- JMAP/Sync/Standard.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 5eae34e..2b34cd9 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -45,7 +45,7 @@ sub connect_contacts { $Self->{contacts} = Net::CardDAVTalk->new( user => $Self->{auth}{username}, password => $Self->{auth}{password}, - url => $Self->{auth}{abookurl}, + url => $Self->{auth}{addressbookurl}, expandurl => 1, ); From 6cc2568a53f695445354ec7fb67cb3ba1972da23 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 02:08:37 -0500 Subject: [PATCH 046/331] more URL --- JMAP/Sync/ICloud.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm index c54172f..218ce44 100644 --- a/JMAP/Sync/ICloud.pm +++ b/JMAP/Sync/ICloud.pm @@ -13,7 +13,7 @@ sub new { imapserver => 'imap.mail.me.com', smtpserver => 'smtp.mail.me.com', calurl => 'https://caldav.icloud.com/', - aburl => 'https://contacts.icloud.com/', + addressbookurl => 'https://contacts.icloud.com/', %$auth, ); return JMAP::Sync::Standard->new(\%a); From afcf98d8d932d86d1ca0afc46a54ebbba4f0e666 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 02:13:24 -0500 Subject: [PATCH 047/331] should be href --- JMAP/Sync/Common.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index fa71295..4e31b2f 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -66,7 +66,7 @@ sub get_cards { my $Args = shift; my $talk = $Self->connect_contacts(); - my $data = $talk->GetContacts($Args->{path}); + my $data = $talk->GetContacts($Args->{href}); my %res; foreach my $item (@$data) { From dcca6f887c9a912140454d895d3a05c4f4e03c81 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 02:48:29 -0500 Subject: [PATCH 048/331] getting there - contact group name normalisation --- JMAP/DB.pm | 28 +++++++++++++--------------- JMAP/ImapDB.pm | 5 +++-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 539315d..6cb9100 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -639,7 +639,7 @@ sub delete_event { sub parse_card { my $Self = shift; my $raw = shift; - my ($card) = Net::CalDAVTalk::VCard->new_fromstring($raw); + my ($card) = Net::CardDAVTalk::VCard->new_fromstring($raw); my %hash; @@ -679,21 +679,21 @@ sub set_card { my $kind = delete $card->{kind}; if ($kind eq 'contact') { $Self->dmake('jcontacts', { - carduid => $carduid, + contactuid => $carduid, jaddressbookid => $jaddressbookid, payload => encode_json($card), }); } else { - $Self->ddelete('jcontactgroups', {carduid => $carduid, jaddressbookid => $jaddressbookid}); - my $gid = $Self->dmake('jcontactgroups', { - carduid => $carduid, + $Self->dmake('jcontactgroups', { + groupuid => $carduid, jaddressbookid => $jaddressbookid, name => $card->{name}, }); + $Self->ddelete('jcontactgroups', {groupuid => $carduid}); foreach my $item (@{$card->{members}}) { - $Self->dmake('jcontactgroupmap', { - jcontactgroupid => $gid, + $Self->dinsert('jcontactgroupmap', { + groupuid => $carduid, contactuid => $item, }); } @@ -1003,7 +1003,7 @@ EOF $dbh->do(<do(<do("CREATE INDEX IF NOT EXISTS jcontactmap ON jgroupmap (contactuid)"); + $dbh->do("CREATE INDEX IF NOT EXISTS jcontactmap ON jcontactgroupmap (contactuid)"); $dbh->do(<sync_folders(); - $Self->sync_calendars(); + #$Self->sync_folders(); + #$Self->sync_calendars(); $Self->sync_addressbooks(); + return 0; my $labels = $Self->labels(); From 6888c8cb7eaf02bc3f2ac391b5b4f5999bb030d6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 12 Jul 2015 03:23:38 -0500 Subject: [PATCH 049/331] sync everything --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 6cb9100..5571179 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -1025,7 +1025,7 @@ EOF $dbh->do(<sync_folders(); - #$Self->sync_calendars(); + $Self->sync_folders(); + $Self->sync_calendars(); $Self->sync_addressbooks(); - return 0; my $labels = $Self->labels(); From 2f79ff628f637ea29bf7358682a88984cce0e375 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 14 Jul 2015 02:27:50 -0500 Subject: [PATCH 050/331] less noisy --- bin/apiendpoint.pl | 2 +- bin/syncserver.pl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index c99e27a..7280c73 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -153,7 +153,7 @@ sub handle_getstate { sub mk_handler { my ($db) = @_; - # don't last forever + # don't last forever - XXX send a "bye" packet? $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; undef $hdl; EV::unloop }); return sub { diff --git a/bin/syncserver.pl b/bin/syncserver.pl index be6068b..55a7069 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -183,7 +183,6 @@ sub mk_handler { $hdl->push_write("\n"); warn "$$ HANDLED $cmd ($tag) => $res->[0] ($id)\n"; - warn Dumper($json, $res); $hdl->push_read(json => mk_handler($db)); }; } From 2bba2f70b497d18e94e71b7e43ce3e5a6de72166 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 14 Jul 2015 14:38:57 -0500 Subject: [PATCH 051/331] API: Records and fix GetContacts --- JMAP/API.pm | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index dd44067..86393d6 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -280,7 +280,7 @@ sub getMailboxUpdates { onlyCountsChanged => $onlyCounts ? JSON::true : JSON::false, }]); - if (@changed and $args->{fetchMailboxes}) { + if (@changed and $args->{fetchRecords}) { my %items = ( accountid => $accountid, ids => \@changed, @@ -288,8 +288,8 @@ sub getMailboxUpdates { if ($onlyCounts) { $items{properties} = [qw(totalMessages unreadMessages totalThreads unreadThreads)]; } - elsif ($args->{fetchMailboxProperties}) { - $items{properties} = $args->{fetchMailboxProperties}; + elsif ($args->{fetchRecordProperties}) { + $items{properties} = $args->{fetchRecordProperties}; } push @res, $Self->getMailboxes(\%items); } @@ -861,11 +861,11 @@ sub getMessageUpdates { removed => [map { "$_" } @removed], }]; - if ($args->{fetchMessages}) { + if ($args->{fetchRecords}) { push @res, $Self->getMessages({ accountid => $accountid, ids => \@changed, - properties => $args->{fetchMessageProperties}, + properties => $args->{fetchRecordProperties}, }) if @changed; } @@ -1111,7 +1111,7 @@ sub getThreadUpdates { removed => \@removed, }]; - if ($args->{fetchThreads}) { + if ($args->{fetchRecords}) { push @res, $Self->getThreads({ accountid => $accountid, ids => \@changed, @@ -1615,28 +1615,30 @@ sub getContacts { return ['error', {type => 'accountNotFound'}] if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] unless $args->{ids}; #properties: String[] A list of properties to fetch for each message. - my %seenids; - my %missingids; + my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts", 'jcontactid', {Slice => {}}); + + my %want; + if ($args->{ids}) { + %want = map { $_ => 1 } @{$args->{ids}}; + } + else { + %want = keys %$data; + } + my @list; - foreach my $contactid (@{$args->{ids}}) { - next if $seenids{$contactid}; - $seenids{$contactid} = 1; - my $data = $dbh->selectrow_hashref("SELECT * FROM jcontacts WHERE jcontactid = ?", {}, $contactid); - unless ($data) { - $missingids{$contactid} = 1; - next; - } + foreach my $id (%want) { + next unless $data->{$id}; + delete $want{$id}; - my $item = decode_json($data->{payload}); + my $item = decode_json($data->{$id}{payload}); foreach my $key (keys %$item) { delete $item->{$key} unless _prop_wanted($args, $key); } - $item->{id} = $contactid; + $item->{id} = $id; push @list, $item; } @@ -1645,7 +1647,7 @@ sub getContacts { list => \@list, accountId => $accountid, state => "$user->{jhighestmodseq}", - notFound => (%missingids ? [keys %missingids] : undef), + notFound => (%want ? [keys %want] : undef), }]; } From 3d4a40d06d94def198279872273fb432cab89959 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 14 Jul 2015 15:05:05 -0500 Subject: [PATCH 052/331] API and lots of changes - sorry about these bogus messages --- JMAP/API.pm | 121 +++++++++++++++++++++++++++++++++++++++++--- JMAP/ImapDB.pm | 22 ++++++-- JMAP/Sync/Common.pm | 2 + bin/apiendpoint.pl | 4 +- bin/syncserver.pl | 3 +- 5 files changed, 139 insertions(+), 13 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 86393d6..dede8b3 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1575,7 +1575,7 @@ sub getContactList { my $start = $args->{position} || 0; return ['error', {type => 'invalidArguments'}] if $start < 0; - my $data = $dbh->selectall_arrayref("SELECT jcontactid,jaddressbookid FROM jcontacts WHERE active = 1 ORDER BY jcontactid"); + my $data = $dbh->selectall_arrayref("SELECT contactuid,jaddressbookid FROM jcontacts WHERE active = 1 ORDER BY contactuid"); $data = $Self->_event_filter($data, $args->{filter}, {}) if $args->{filter}; @@ -1617,18 +1617,18 @@ sub getContacts { #properties: String[] A list of properties to fetch for each message. - my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts", 'jcontactid', {Slice => {}}); + my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts", 'contactuid', {Slice => {}}); my %want; if ($args->{ids}) { %want = map { $_ => 1 } @{$args->{ids}}; } else { - %want = keys %$data; + %want = %$data; } my @list; - foreach my $id (%want) { + foreach my $id (keys %want) { next unless $data->{$id}; delete $want{$id}; @@ -1667,7 +1667,7 @@ sub getContactUpdates { return ['error', {type => 'cannotCalculateChanges'}] if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); - my $sql = "SELECT jcontactid,active FROM jcontacts WHERE jmodseq > ?"; + my $sql = "SELECT contactuid,active FROM jcontacts WHERE jmodseq > ?"; my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); @@ -1696,11 +1696,118 @@ sub getContactUpdates { removed => [map { "$_" } @removed], }]; - if ($args->{fetchContacts}) { + if ($args->{fetchRecords}) { push @res, $Self->getContacts({ accountid => $accountid, ids => \@changed, - properties => $args->{fetchContactProperties}, + properties => $args->{fetchRecordProperties}, + }) if @changed; + } + + return @res; +} + +sub getContactGroups { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + #properties: String[] A list of properties to fetch for each message. + + my $data = $dbh->selectall_hashref("SELECT * FROM jcontactgroups", 'groupuid', {Slice => {}}); + + my %want; + if ($args->{ids}) { + %want = map { $_ => 1 } @{$args->{ids}}; + } + else { + %want = %$data; + } + + my @list; + foreach my $id (keys %want) { + next unless $data->{$id}; + delete $want{$id}; + + my $item = {}; + $item->{id} = $id; + + if (_prop_wanted('name', $args)) { + $item->{name} = $data->{name}; + } + + if (_prop_wanted('contactIds', $args)) { + my $ids = $dbh->selectcol_arrayref("SELECT contactuid FROM jgroupmap WHERE groupuid = ?", {}, $id); + $item->{contactIds} = $ids; + } + + push @list, $item; + } + + return ['contactGroups', { + list => \@list, + accountId => $accountid, + state => "$user->{jhighestmodseq}", + notFound => (%want ? [keys %want] : undef), + }]; +} + +sub getContactGroupUpdates { + my $Self = shift; + my $args = shift; + + my $dbh = $Self->{db}->dbh(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return ['error', {type => 'accountNotFound'}] + if ($args->{accountId} and $args->{accountId} ne $accountid); + + return ['error', {type => 'invalidArguments'}] + if not $args->{sinceState}; + return ['error', {type => 'cannotCalculateChanges'}] + if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); + + my $sql = "SELECT groupuid,active FROM jcontactgroups WHERE jmodseq > ?"; + + my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); + + if ($args->{maxChanges} and @$data > $args->{maxChanges}) { + return ['error', {type => 'tooManyChanges'}]; + } + + my @changed; + my @removed; + + foreach my $row (@$data) { + if ($row->[1]) { + push @changed, $row->[0]; + } + else { + push @removed, $row->[0]; + } + } + + my @res; + push @res, ['contactGroupUpdates', { + accountId => $accountid, + oldState => "$args->{sinceState}", + newState => "$user->{jhighestmodseq}", + changed => [map { "$_" } @changed], + removed => [map { "$_" } @removed], + }]; + + if ($args->{fetchRecords}) { + push @res, $Self->getContactGroups({ + accountid => $accountid, + ids => \@changed, + properties => $args->{fetchRecordProperties}, }) if @changed; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index bfbcbf0..cb36ebd 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -111,7 +111,17 @@ sub backend_cmd { my $auth = $Self->access_token(); tcp_connect('localhost', 5005, sub { my $fh = shift; - my $handle = AnyEvent::Handle->new(fh => $fh); + my $handle; + $handle = AnyEvent::Handle->new(fh => $fh, + on_disconnect => sub { + delete $Self->{backend}; + undef $h; + }, + on_disconnect => sub { + delete $Self->{backend}; + undef $h; + }, + ); $handle->push_write(json => {hostname => $auth->[0], username => $auth->[1], password => $auth->[2]}); $handle->push_write("\012"); $handle->push_read(json => sub { @@ -132,7 +142,7 @@ sub backend_cmd { return if $cb; # async usage my $res = $w->recv; - warn Dumper ($cmd, \@args, $res); + #warn Dumper ($cmd, \@args, $res); return $res; } @@ -535,10 +545,14 @@ sub labels { sub sync { my $Self = shift; - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders"); + my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); + my @imapnames = map { $_->[1] } @$data; + my $status = $Self->backend_cmd('imap_status', \@imapnames); foreach my $row (@$data) { - $Self->do_folder(@$row); + # XXX - better handling of UIDvalidity change? + next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); + $Self->do_folder($row->[0], $row->[4]); } } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 4e31b2f..ead2e31 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -101,6 +101,8 @@ sub imap_status { my $imap = $Self->connect_imap(); + $imap->unselect(); + my @fields = qw(uidvalidity uidnext messages); push @fields, "highestmodseq" if $imap->capability->{condstore}; my $data = $imap->multistatus("(@fields)", @$folders); diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 7280c73..255f3e8 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -116,7 +116,9 @@ sub change_cb { clientId => undef, accountStates => { $db->accountid() => { - mailState => "$state", + messages => "$state", + threads => "$state", + mailboxes => "$state", }, }, }; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 55a7069..b0578ff 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -4,7 +4,7 @@ use lib '/home/jmap/jmap-perl'; package SyncServer; -#use Mail::IMAPTalk qw(:trace); +use Mail::IMAPTalk qw(:trace); use AnyEvent; use AnyEvent::Handle; @@ -179,6 +179,7 @@ sub mk_handler { $res = ['error', "$@"] } $res->[2] = $tag; + #warn Dumper($json, $res); $hdl->push_write(json => $res); $hdl->push_write("\n"); From 3f74bcc42e22e060850b33abecb8e52a0045b53f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 14 Jul 2015 17:05:57 -0500 Subject: [PATCH 053/331] server: bye support - stuff from backends --- bin/server.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/server.pl b/bin/server.pl index 3ad3cbc..0855485 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -164,10 +164,15 @@ sub mk_json { if ($res->[0] eq 'push') { PushEvent($accountid, event => "state", data => $res->[1]); } + elsif ($res->[0] eq 'bye') { + print "SERVER CLOSING $accountid\n"; + delete $backend{$accountid}; + } elsif ($waiting{$accountid}{$res->[2]}) { if ($res->[0] eq 'error') { $waiting{$accountid}{$res->[2]}[1]->($res->[1]); # start again... + print "ERROR $res->[1] on $accountid (dropping backend)\n"; delete $backend{$accountid}; } else { From 96735d1675a382bffd723f762907a3930e644e83 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 14 Jul 2015 17:38:44 -0500 Subject: [PATCH 054/331] add token check --- JMAP/ImapDB.pm | 15 ++++++++++++--- bin/apiendpoint.pl | 11 ++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index cb36ebd..a343ef9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -422,19 +422,25 @@ sub sync_addressbooks { my $dbh = $Self->dbh(); my $addressbooks = $Self->backend_cmd('addressbooks', []); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly FROM iaddressbooks"); + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); my %byhref = map { $_->[1] => $_ } @$iaddressbooks; my %seen; + my @todo; foreach my $addressbook (@$addressbooks) { my $id = $byhref{$addressbook->{href}}[0]; my $data = { isReadOnly => $addressbook->{isReadOnly}, href => $addressbook->{href}, name => $addressbook->{name}, + syncToken => $addressbook->{syncToken}, }; if ($id) { - $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + my $token = $byhref{$addressbook->{href}}[4]; + if ($token ne $addressbook->{syncToken}) { + push @todo, $id; + $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + } } else { $id = $Self->dinsert('iaddressbooks', $data); @@ -450,7 +456,7 @@ sub sync_addressbooks { $Self->sync_jaddressbooks(); - foreach my $id (keys %seen) { + foreach my $id (@todo) { $Self->do_addressbook($id); } } @@ -554,6 +560,9 @@ sub sync { next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); $Self->do_folder($row->[0], $row->[4]); } + + $Self->sync_calendars(); + $Self->sync_addressbooks(); } sub backfill { diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 255f3e8..843e048 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -123,7 +123,7 @@ sub change_cb { }, }; - $hdl->push_write(json => ['push', $data]); + $hdl->push_write(json => ['push', $data]) if $hdl; } sub handle_ping { @@ -155,8 +155,13 @@ sub handle_getstate { sub mk_handler { my ($db) = @_; - # don't last forever - XXX send a "bye" packet? - $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; undef $hdl; EV::unloop }); + $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { + warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; + $hdl->push_write(json => ['bye']); + $hdl->push_shutdown(); + undef $hdl; + EV::unloop; + }); return sub { my ($hdl, $json) = @_; From f530133154a18814e4f407a14f0900964d7dc7ff Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 02:18:44 +1000 Subject: [PATCH 055/331] separate IMAP sync, add SyncToken support for everything --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 16 ++++++++++------ JMAP/Sync/Common.pm | 4 ++-- bin/apiendpoint.pl | 11 ++++++++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index dede8b3..28fce0b 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -890,7 +890,7 @@ sub setMessages { my ($updated, $notUpdated) = $Self->{db}->update_messages($update); my ($deleted, $notDeleted) = $Self->{db}->delete_messages($delete); - $Self->{db}->sync(); + $Self->{db}->sync_imap(); foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index a343ef9..857400b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -299,10 +299,11 @@ sub sync_calendars { my $dbh = $Self->dbh(); my $calendars = $Self->backend_cmd('calendars', []); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour FROM icalendars"); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; my %seen; + my @todo; foreach my $calendar (@$calendars) { my $id = $byhref{$calendar->{href}}[0]; my $data = { @@ -310,9 +311,15 @@ sub sync_calendars { href => $calendar->{href}, colour => $calendar->{colour}, name => $calendar->{name}, + syncToken => $calendar->{syncToken}, }; if ($id) { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); + my $token = $byhref{$addressbook->{href}}[5]; + if ($token ne $addressbook->{syncToken}) { + push @todo, $id; + $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + } } else { $id = $Self->dinsert('icalendars', $data); @@ -328,7 +335,7 @@ sub sync_calendars { $Self->sync_jcalendars(); - foreach my $id (keys %seen) { + foreach my $id (@todo) { $Self->do_calendar($id); } } @@ -549,7 +556,7 @@ sub labels { return $Self->{t}{labels}; } -sub sync { +sub sync_imap { my $Self = shift; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); my @imapnames = map { $_->[1] } @$data; @@ -560,9 +567,6 @@ sub sync { next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); $Self->do_folder($row->[0], $row->[4]); } - - $Self->sync_calendars(); - $Self->sync_addressbooks(); } sub backfill { diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index ead2e31..2ffa457 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -32,7 +32,7 @@ sub get_calendars { my $Self = shift; my $talk = $Self->connect_calendars(); - my $data = $talk->GetCalendars(); + my $data = $talk->GetCalendars(Sync => 1); return $data; } @@ -56,7 +56,7 @@ sub get_addressbooks { my $Self = shift; my $talk = $Self->connect_contacts(); - my $data = $talk->GetAddressBooks(); + my $data = $talk->GetAddressBooks(Sync => 1); return $data; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 843e048..dfa1337 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -226,7 +226,16 @@ sub mk_handler { sub handle_sync { my $db = shift; $db->begin(); - $db->sync(); + $db->sync_imap(); + $db->commit(); + return ['sync', $JSON::true]; +} + +sub handle_davsync { + my $db = shift; + $db->begin(); + $db->sync_calendars(); + $db->sync_addressbooks(); $db->commit(); return ['sync', $JSON::true]; } From 852db5a7a4c947bcb2dab5fb3721293f47d9a024 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 02:52:12 +1000 Subject: [PATCH 056/331] backend: stay alive rather than dropping off the API layer will close if it needs to --- JMAP/Sync/Common.pm | 9 +++++++++ bin/syncserver.pl | 10 ++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 2ffa457..9cfccc8 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -95,6 +95,15 @@ sub labels { return $Self->{labels}; } +sub imap_noop { + my $Self = shift; + my $folders = shift; + + my $imap = $Self->connect_imap(); + + $imap->noop(); +} + sub imap_status { my $Self = shift; my $folders = shift; diff --git a/bin/syncserver.pl b/bin/syncserver.pl index b0578ff..cf852ab 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -155,11 +155,17 @@ sub handle_imap_count { return ['counted', $data]; } +sub reset_keepalive { + $hdl->{keepalive} = AnyEvent->timer(after => 600, cb => sub { + $backend->imap_noop(); + reset_keepalive(); + }); +} + sub mk_handler { my ($db) = @_; - # don't last forever - $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "$$ SHUTTING DOWN $id ON TIMEOUT\n"; undef $hdl; EV::unloop }); + reset_keepalive(); return sub { my ($hdl, $json) = @_; From dc535e510b3ab257090d1dcfdbf7d2f921032d49 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 03:11:52 +1000 Subject: [PATCH 057/331] respond with update details and errors on update --- JMAP/ImapDB.pm | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 857400b..16671fc 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -724,21 +724,27 @@ sub update_messages { my %updatemap; foreach my $msgid (keys %$changes) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $updatemap{$ifolderid}{$uid} = $changes->{$msgid}; + $updatemap{$ifolderid}{$uid} = $msgid; } my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } @$folderdata; + my %notchanged; + my @changed; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; my $uidvalidity = $foldermap{$ifolderid}[2]; - die "NO SUCH FOLDER $ifolderid" unless $imapname; foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { - my $action = $updatemap{$ifolderid}{$uid}; + my $msgid = $updatemap{$ifolderid}{$uid}; + my $action = $changes->{$msgid}; + unless ($imapname and $uidvaldity) { + $notchanged{$msgid} = "No folder found"; + next; + } if (exists $action->{isUnread}) { my $bool = !$action->{isUnread}; my @flags = ("\\Seen"); @@ -762,8 +768,12 @@ sub update_messages { my $newfolder = $foldermap{$id}[1]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } + # XXX - handle errors from backend commands + push @changed, $msgid; } } + + return (\@changed, \%notchanged); } sub delete_messages { @@ -775,23 +785,26 @@ sub delete_messages { my %deletemap; foreach my $msgid (@$ids) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $deletemap{$ifolderid}{$uid} = 1; + $deletemap{$ifolderid}{$uid} = $msgid; } my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; + my @deleted, %notdeleted; foreach my $ifolderid (keys %deletemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; my $uidvalidity = $foldermap{$ifolderid}[2]; - die "NO SUCH FOLDER $ifolderid" unless $imapname; - + unless ($imapname) { + $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; + } my $uids = [sort keys %{$deletemap{$ifolderid}}]; - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder + push @deleted, values %{$deletemap{$ifolderid}}; } + return (\@deleted, \%notdeleted); } sub deleted_record { From 5fb3a5dd722edb7e1b94403ce8ac1ff3f6c2b00e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 11:59:10 -0500 Subject: [PATCH 058/331] copypast --- JMAP/ImapDB.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 16671fc..b18a81e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -315,10 +315,10 @@ sub sync_calendars { }; if ($id) { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); - my $token = $byhref{$addressbook->{href}}[5]; - if ($token ne $addressbook->{syncToken}) { + my $token = $byhref{$calendar->{href}}[5]; + if ($token ne $calendar->{syncToken}) { push @todo, $id; - $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); } } else { From 8ff0b314b39a8aaa7834bc73ebb23b0871a213f6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 13:34:28 -0500 Subject: [PATCH 059/331] fix nits --- JMAP/ImapDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b18a81e..b9dfd20 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -741,7 +741,7 @@ sub update_messages { foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { my $msgid = $updatemap{$ifolderid}{$uid}; my $action = $changes->{$msgid}; - unless ($imapname and $uidvaldity) { + unless ($imapname and $uidvalidity) { $notchanged{$msgid} = "No folder found"; next; } @@ -792,7 +792,7 @@ sub delete_messages { my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; - my @deleted, %notdeleted; + my (@deleted, %notdeleted); foreach my $ifolderid (keys %deletemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; From ee60f46def4bef9e9985fbbd4e0b33e49e070c42 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 10:50:45 +1000 Subject: [PATCH 060/331] apiendpoint - needs mail --- bin/apiendpoint.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index dfa1337..5dc389c 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -10,6 +10,7 @@ package JMAP::Backend; use warnings; use AnyEvent; use AnyEvent::Gmail; +use Mail::IMAPTalk; use Data::Dumper; use AnyEvent::HTTPD; use JMAP::GmailDB; From 75ea5a9ed54d82769bb4eb4ab0d393e7efd33b41 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 10:55:34 +1000 Subject: [PATCH 061/331] check for transaction in DB calls --- JMAP/DB.pm | 5 +++++ bin/apiendpoint.pl | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 5571179..24e428d 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -62,6 +62,11 @@ sub dbh { return $Self->{dbh}; } +sub in_transaction { + my $Self = shift; + return $Self->{t} ? 1 : 0; +} + sub begin { my $Self = shift; confess("ALREADY IN TRANSACTION") if $Self->{t}; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 5dc389c..3f80c23 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -66,6 +66,7 @@ sub getdb { } $db->{change_cb} = \&change_cb; $db->{watcher} = AnyEvent->timer(after => 30, interval => 30, cb => sub { + return if $db->in_transaction(); # check if there's more work to do on the account... eval { $db->begin(); @@ -73,6 +74,16 @@ sub getdb { $db->commit(); }; }); + $db->{calsync} = AnyEvent->timer(after => 10, interval => 300, cb => sub { + return if $db->in_transaction(); + # check if there's more work to do on the account... + eval { + $db->begin(); + $db->backfill(); + $db->commit(); + }; + }); + return $db; } @@ -217,7 +228,7 @@ sub mk_handler { $hdl->push_write(json => $res) if $res->[0]; warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; if ($res->[0] eq 'error') { - warn Dumper($res); + warn Dumper($res); } } $hdl->push_read(json => mk_handler($db)); From 76d2e26203b91b4081b60aac2063d2024ad6aefb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 17:26:31 -0500 Subject: [PATCH 062/331] sync calendars and addressbooks if present --- bin/apiendpoint.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 3f80c23..7f6c66e 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -74,16 +74,16 @@ sub getdb { $db->commit(); }; }); - $db->{calsync} = AnyEvent->timer(after => 10, interval => 300, cb => sub { + $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { return if $db->in_transaction(); # check if there's more work to do on the account... eval { $db->begin(); - $db->backfill(); + $db->sync_calendars(); + $db->sync_addressbooks(); $db->commit(); }; }); - return $db; } From 6da7277266ed860ab2cedd958ef67ef59910c5de Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 17:28:29 -0500 Subject: [PATCH 063/331] sync: handle no calendars/contacts backends --- JMAP/Sync/Common.pm | 4 ++++ JMAP/Sync/Standard.pm | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 9cfccc8..8c8ee1f 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -31,6 +31,7 @@ sub DESTROY { sub get_calendars { my $Self = shift; my $talk = $Self->connect_calendars(); + return unless $talk; my $data = $talk->GetCalendars(Sync => 1); @@ -41,6 +42,7 @@ sub get_events { my $Self = shift; my $Args = shift; my $talk = $Self->connect_calendars(); + return unless $talk; my $data = $talk->GetEvents($Args->{href}, Full => 1); @@ -55,6 +57,7 @@ sub get_events { sub get_addressbooks { my $Self = shift; my $talk = $Self->connect_contacts(); + return unless $talk; my $data = $talk->GetAddressBooks(Sync => 1); @@ -65,6 +68,7 @@ sub get_cards { my $Self = shift; my $Args = shift; my $talk = $Self->connect_contacts(); + return unless $talk; my $data = $talk->GetContacts($Args->{href}); diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 2b34cd9..39573d0 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -19,6 +19,8 @@ my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSel sub connect_calendars { my $Self = shift; + return unless $Self->{auth}{calurl}; + if ($Self->{calendars}) { $Self->{lastused} = time(); return $Self->{calendars}; @@ -37,6 +39,8 @@ sub connect_calendars { sub connect_contacts { my $Self = shift; + return unless $Self->{auth}{addressbookurl}; + if ($Self->{contacts}) { $Self->{lastused} = time(); return $Self->{contacts}; From 70919eb4ef994a59e7292d37b033af92d3c6704d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 17:29:33 -0500 Subject: [PATCH 064/331] more protect against no calendar/contact backend --- JMAP/ImapDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b9dfd20..55e9032 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -299,6 +299,7 @@ sub sync_calendars { my $dbh = $Self->dbh(); my $calendars = $Self->backend_cmd('calendars', []); + return unless $calendars; my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; @@ -429,6 +430,7 @@ sub sync_addressbooks { my $dbh = $Self->dbh(); my $addressbooks = $Self->backend_cmd('addressbooks', []); + return unless $addressbooks; my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); my %byhref = map { $_->[1] => $_ } @$iaddressbooks; From 6007ea1e2aa1bbcd1cb45cf12cc80fd73b6241b1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 18:08:54 -0500 Subject: [PATCH 065/331] GmailDB: port to new --- JMAP/GmailDB.pm | 833 +++++++++++++++++++++++++++++++----------------- 1 file changed, 536 insertions(+), 297 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 48c1ea7..7c24fef 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -7,45 +7,48 @@ package JMAP::GmailDB; use base qw(JMAP::DB); use DBI; -use Mail::GmailTalk; use Date::Parse; use JSON::XS qw(encode_json decode_json); use Data::UUID::LibUUID; use OAuth2::Tiny; use Encode; use Encode::MIME::Header; -use Date::Format; -use Email::Simple; -use Email::Sender::Simple qw(sendmail); -use Email::Sender::Transport::GmailSMTP; -use IO::All; +use Digest::SHA qw(sha1_hex); +use AnyEvent; +use AnyEvent::Socket; +use Data::Dumper; -my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); +our $TAG = 1; + +# special use or name magic my %ROLE_MAP = ( - '\\Inbox' => 'inbox', - '\\Drafts' => 'drafts', - '\\Spam' => 'spam', - '\\Trash' => 'trash', - '\\AllMail' => 'archive', - '\\Sent' => 'sent', + 'inbox' => 'inbox', + 'drafts' => 'drafts', + 'junk' => 'spam', + 'deleted messages' => 'trash', + 'archive' => 'archive', + 'sent messages' => 'sent', + 'sent items' => 'sent', + 'trash' => 'trash', + '\\inbox' => 'inbox', + '\\trash' => 'trash', + '\\sent' => 'sent', + '\\junk' => 'junk', + '\\archive' => 'archive', + '\\drafts' => 'drafts', + '\\allmail' => 'allmail', ); -sub DESTROY { - my $Self = shift; - if ($Self->{imap}) { - $Self->{imap}->logout(); - } -} - sub setuser { my $Self = shift; my ($username, $refresh_token, $displayname, $picture) = @_; my $data = $Self->dbh->selectrow_arrayref("SELECT username, refresh_token FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {username => $username, refresh_token => $refresh_token}); + $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}, {hostname => $data->[0]}); } else { $Self->dinsert('iserver', { + hostname => $hostname, username => $username, refresh_token => $refresh_token, }); @@ -65,89 +68,85 @@ sub setuser { } } -my $O; -sub O { - unless ($O) { - my $data = io->file("/home/jmap/jmap-perl/config.json")->slurp; - my $config = decode_json($data); - $O = OAuth2::Tiny->new(%$config); - } - return $O; -} - sub access_token { my $Self = shift; - my $username = shift; - my $refresh_token = shift; - - unless ($refresh_token) { - ($username, $refresh_token) = $Self->dbh->selectrow_array("SELECT username, refresh_token FROM iserver"); - } - my $O = $Self->O(); - my $data = $O->refresh($refresh_token); + my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT hostname, username, password FROM iserver"); - return ['gmail', $username, $data->{access_token}]; + return [$hostname, $username, $password]; } -sub connect { +# synchronous backend for now +sub backend_cmd { my $Self = shift; + my $cmd = shift; + my $cb; + if (ref($cmd)) { + $cb = $cmd; + $cmd = shift; + } + my @args = @_; + + my $w = AnyEvent->condvar; + + my $action = sub { + my $handle = shift; + my $tag = "T" . $TAG++; + $handle->push_write(json => [$cmd, \@args, $tag]); # whatever + $handle->push_write("\012"); + $handle->push_read(json => sub { + my $hdl = shift; + my $json = shift; + die "INVALID RESPONSE" unless $json->[2] eq $tag; + if ($cb) { + $cb->($json->[1]); + } + else { + $w->send($json->[1]); + } + }); + }; + if ($Self->{backend}) { + $action->($Self->{backend}); + } + else { + my $h = AnyEvent->condvar; + my $auth = $Self->access_token(); + tcp_connect('localhost', 5005, sub { + my $fh = shift; + my $handle; + $handle = AnyEvent::Handle->new(fh => $fh, + on_disconnect => sub { + delete $Self->{backend}; + undef $h; + }, + on_disconnect => sub { + delete $Self->{backend}; + undef $h; + }, + ); + $handle->push_write(json => {hostname => $auth->[0], username => $auth->[1], password => $auth->[2]}); + $handle->push_write("\012"); + $handle->push_read(json => sub { + my $hdl = shift; + my $json = shift; + die "Failed to setup " . Dumper($json) unless $json->[0] eq 'setup'; + $action->($handle); + $h->send($handle); + }); + # XXX - handle destroy correctly + # handle backend going away, etc + }); - if ($Self->{imap}) { - $Self->{lastused} = time(); - return $Self->{imap}; - } - - for (1..3) { - $Self->log('debug', "Looking for server for $Self->{accountid}"); - my $data = $Self->dbh->selectrow_arrayref("SELECT username, refresh_token, lastfoldersync FROM iserver"); - die "UNKNOWN SERVER for $Self->{accountid}" unless ($data and $data->[0]); - $Self->log('debug', "getting access token for $data->[0]"); - my $token = $Self->access_token($data->[0], $data->[1]); - my $port = 993; - my $usessl = $port != 143; # we use SSL for anything except default - $Self->log('debug', "getting imaptalk"); - $Self->{imap} = Mail::GmailTalk->new( - Server => 'imap.gmail.com', - Port => $port, - Username => $data->[0], - Password => $token->[2], # bogus, but here we go... - # not configurable right now... - UseSSL => $usessl, - UseBlocking => $usessl, - ); - next unless $Self->{imap}; - $Self->log('debug', "Connected as $data->[0]"); - $Self->begin(); - $Self->sync_folders(); - $Self->dmaybeupdate('iserver', {lastfoldersync => time()}, {username => $data->[0]}); - $Self->commit(); - $Self->{lastused} = time(); - return $Self->{imap}; + # synchronous startup to avoid race condition on setting up channel + $Self->{backend} = $h->recv; } - die "Could not connect to IMAP server: $@"; -} + return if $cb; # async usage -sub send_email { - my $Self = shift; - my $rfc822 = shift; - my $data = $Self->dbh->selectrow_arrayref("SELECT username, refresh_token FROM iserver"); - die "UNKNOWN SERVER for $Self->{accountid}" unless ($data and $data->[0]); - my $token = $Self->access_token($data->[0], $data->[1]); - die "not gmail" unless $token->[0] eq 'gmail'; - - my $email = Email::Simple->new($rfc822); - sendmail($email, { - from => $data->[0], - transport => Email::Sender::Transport::GmailSMTP->new({ - host => 'smtp.gmail.com', - port => 465, - ssl => 1, - sasl_username => $token->[1], - access_token => $token->[2], - }) - }); + my $res = $w->recv; + #warn Dumper ($cmd, \@args, $res); + return $res; } # synchronise list from IMAP server to local folder cache @@ -156,24 +155,43 @@ sub sync_folders { my $Self = shift; my $dbh = $Self->dbh(); - my $imap = $Self->{imap}; - my @folders = $imap->xlist('', '*'); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label FROM ifolders"); - my %ibylabel = map { $_->[3] => $_ } @$ifolders; + my $folders = $Self->backend_cmd('folders', []); + my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); + my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; - foreach my $folder (@folders) { - my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; - my $label = $role || $folder->[2]; + my %getstatus; + foreach my $name (sort keys %$folders) { + my $sep = $folders->{$name}[0]; + my $role = $ROLE_MAP{lc $folders->{$name}[1]}; + my $label = $role || $folders->{$name}[1]; my $id = $ibylabel{$label}[0]; if ($id) { - $Self->dmaybeupdate('ifolders', {sep => $folder->[1], imapname => $folder->[2]}, {ifolderid => $id}); + $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); } else { - $id = $Self->dinsert('ifolders', {sep => $folder->[1], imapname => $folder->[2], label => $label}); + $id = $Self->dinsert('ifolders', {sep => $sep, imapname => $name, label => $label}); } $seen{$id} = 1; + unless ($ibylabel{$label}[2]) { + # no uidvalidity, we need to get status for this one + next unless ($label eq 'allmail' or $label eq 'trash'); + $getstatus{$name} = $id; + } + } + + if (keys %getstatus) { + my $data = $Self->backend_cmd('imap_status', [keys %getstatus]); + foreach my $name (keys %$data) { + my $status = $data->{$name}; + $Self->dmaybeupdate('ifolders', { + uidvalidity => $status->{uidvalidity}, + uidnext => $status->{uidnext}, + uidfirst => $status->{uidnext}, + highestmodseq => $status->{highestmodseq}, + }, {ifolderid => $getstatus{$name}}); + } } foreach my $folder (@$ifolders) { @@ -203,31 +221,33 @@ sub sync_jmailboxes { foreach my $mailbox (@$jmailboxes) { $jbyid{$mailbox->[0]} = $mailbox; $roletoid{$mailbox->[3]} = $mailbox->[0] if $mailbox->[3]; - $byname{$mailbox->[2]||'0'}{$mailbox->[1]} = $mailbox->[0]; + $byname{$mailbox->[2]}{$mailbox->[1]} = $mailbox->[0]; } my %seen; foreach my $folder (@$ifolders) { + my $fname = $folder->[2]; + warn " MAPPING $fname ($folder->[1])"; + $fname =~ s/^INBOX\.//; # check for roles first - my $role = $ROLE_MAP{$folder->[3]}; - my @bits = split $folder->[1], $folder->[2]; + my @bits = split "[$folder->[1]]", $fname; + my $role = $ROLE_MAP{lc $folder->[3]} || $ROLE_MAP{lc $fname}; my $id = 0; my $parentid = 0; my $name; my $precedence = 3; - $precedence = 1 if ($role||'' eq 'inbox'); + $precedence = 2 if $role; + $precedence = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { - if ($item eq '[Gmail]') { - $precedence = 2 if $role; - next; - } + $seen{$id} = 1 if $id; $name = $item; $parentid = $id; $id = $byname{$parentid}{$name}; unless ($id) { if (@bits) { # need to create intermediate folder ... - $id = $Self->dmake('jmailboxes', {name => $name, parentid => $parentid}); + # XXX - label noselect? + $id = $Self->dmake('jmailboxes', {name => $name, precedence => 4, parentid => $parentid}); $byname{$parentid}{$name} = $id; } } @@ -272,31 +292,268 @@ sub sync_jmailboxes { $Self->dmaybeupdate('ifolders', {jmailboxid => $id}, {ifolderid => $folder->[0]}); } - my $haveoutbox = 0; foreach my $mailbox (@$jmailboxes) { my $id = $mailbox->[0]; - if (($mailbox->[3]||'') eq 'outbox') { - $haveoutbox = 1; - next; - } next if $seen{$id}; $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); } +} + +# synchronise list from CalDAV server to local folder cache +# call in transaction +sub sync_calendars { + my $Self = shift; + + my $dbh = $Self->dbh(); + + my $calendars = $Self->backend_cmd('calendars', []); + return unless $calendars; + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); + my %byhref = map { $_->[1] => $_ } @$icalendars; + + my %seen; + my @todo; + foreach my $calendar (@$calendars) { + my $id = $byhref{$calendar->{href}}[0]; + my $data = { + isReadOnly => $calendar->{isReadOnly}, + href => $calendar->{href}, + colour => $calendar->{colour}, + name => $calendar->{name}, + syncToken => $calendar->{syncToken}, + }; + if ($id) { + $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); + my $token = $byhref{$calendar->{href}}[5]; + if ($token ne $calendar->{syncToken}) { + push @todo, $id; + $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); + } + } + else { + $id = $Self->dinsert('icalendars', $data); + } + $seen{$id} = 1; + } + + foreach my $calendar (@$icalendars) { + my $id = $calendar->[0]; + next if $seen{$id}; + $dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); + } + + $Self->sync_jcalendars(); + + foreach my $id (@todo) { + $Self->do_calendar($id); + } +} + +# synchronise from the imap folder cache to the jmap mailbox listing +# call in transaction +sub sync_jcalendars { + my $Self = shift; + my $dbh = $Self->dbh(); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); + my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); + + my %jbyid; + foreach my $calendar (@$jcalendars) { + $jbyid{$calendar->[0]} = $calendar; + } - unless ($haveoutbox) { - $Self->dmake('jmailboxes', { - role => "outbox", - name => "Outbox", - parentid => 0, - precedence => 1, - mustBeOnly => 0, + my %seen; + foreach my $calendar (@$icalendars) { + my $data = { + name => $calendar->[1], + colour => $calendar->[2], + isVisible => 1, + mayReadFreeBusy => 1, + mayReadItems => 1, + mayAddItems => 0, + mayModifyItems => 0, + mayRemoveItems => 0, mayDelete => 0, mayRename => 0, - mayAdd => 1, - mayRemove => 1, - mayChild => 0, # don't go fiddling around - mayRead => 1, - }); + }; + if ($jbyid{$calendar->[3]}) { + $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); + $seen{$calendar->[3]} = 1; + } + else { + my $id = $Self->dmake('jcalendars', $data); + $Self->dupdate('icalendars', {jcalendarid => $id}, {icalendarid => $calendar->[0]}); + $seen{$id} = 1; + } + } + + foreach my $calendar (@$jcalendars) { + my $id = $calendar->[0]; + next if $seen{$id}; + $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); + } +} + +sub do_calendar { + my $Self = shift; + my $calendarid = shift; + + my $dbh = $Self->dbh(); + + my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); + my %res = map { $_->[1] => $_ } @$exists; + + my $events = $Self->backend_cmd('events', {href => $href}); + + foreach my $resource (keys %$events) { + my $data = delete $res{$resource}; + my $raw = $events->{$resource}; + if ($data) { + my $id = $data->[0]; + next if $raw eq $data->[2]; + $Self->dmaybeupdate('ievents', {content => $raw, resource => $resource}, {ieventid => $id}); + } + else { + $Self->dinsert('ievents', {content => $raw, resource => $resource}); + } + my $event = $Self->parse_event($raw); + $Self->set_event($jcalendarid, $event); + } + + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $id = $data->[0]; + $Self->ddelete('ievents', {ieventid => $id}); + my $event = $Self->parse_event($data->[2]); + $Self->delete_event($jcalendarid, $event->{uid}); + } +} + +# synchronise list from CardDAV server to local folder cache +# call in transaction +sub sync_addressbooks { + my $Self = shift; + + my $dbh = $Self->dbh(); + + my $addressbooks = $Self->backend_cmd('addressbooks', []); + return unless $addressbooks; + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); + my %byhref = map { $_->[1] => $_ } @$iaddressbooks; + + my %seen; + my @todo; + foreach my $addressbook (@$addressbooks) { + my $id = $byhref{$addressbook->{href}}[0]; + my $data = { + isReadOnly => $addressbook->{isReadOnly}, + href => $addressbook->{href}, + name => $addressbook->{name}, + syncToken => $addressbook->{syncToken}, + }; + if ($id) { + my $token = $byhref{$addressbook->{href}}[4]; + if ($token ne $addressbook->{syncToken}) { + push @todo, $id; + $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + } + } + else { + $id = $Self->dinsert('iaddressbooks', $data); + } + $seen{$id} = 1; + } + + foreach my $addressbook (@$iaddressbooks) { + my $id = $addressbook->[0]; + next if $seen{$id}; + $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + } + + $Self->sync_jaddressbooks(); + + foreach my $id (@todo) { + $Self->do_addressbook($id); + } +} + +# synchronise from the imap folder cache to the jmap mailbox listing +# call in transaction +sub sync_jaddressbooks { + my $Self = shift; + my $dbh = $Self->dbh(); + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks"); + my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks"); + + my %jbyid; + foreach my $addressbook (@$jaddressbooks) { + $jbyid{$addressbook->[0]} = $addressbook; + } + + my %seen; + foreach my $addressbook (@$iaddressbooks) { + my $data = { + name => $addressbook->[1], + isVisible => 1, + mayReadItems => 1, + mayAddItems => 0, + mayModifyItems => 0, + mayRemoveItems => 0, + mayDelete => 0, + mayRename => 0, + }; + if ($jbyid{$addressbook->[2]}) { + $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $addressbook->[2]}); + $seen{$addressbook->[2]} = 1; + } + else { + my $id = $Self->dmake('jaddressbooks', $data); + $Self->dupdate('iaddressbooks', {jaddressbookid => $id}, {iaddressbookid => $addressbook->[0]}); + $seen{$id} = 1; + } + } + + foreach my $addressbook (@$jaddressbooks) { + my $id = $addressbook->[0]; + next if $seen{$id}; + $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $id}); + } +} + +sub do_addressbook { + my $Self = shift; + my $addressbookid = shift; + + my $dbh = $Self->dbh(); + + my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); + my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); + my %res = map { $_->[1] => $_ } @$exists; + + my $cards = $Self->backend_cmd('cards', {href => $href}); + + foreach my $resource (keys %$cards) { + my $data = delete $res{$resource}; + my $raw = $cards->{$resource}; + if ($data) { + my $id = $data->[0]; + next if $raw eq $data->[2]; + $Self->dmaybeupdate('icards', {content => $raw, resource => $resource}, {icardid => $id}); + } + else { + $Self->dinsert('icards', {content => $raw, resource => $resource}); + } + my $card = $Self->parse_card($raw); + $Self->set_card($jaddressbookid, $card); + } + + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $id = $data->[0]; + $Self->ddelete('icards', {icardid => $id}); + my $card = $Self->parse_card($data->[2]); + $Self->delete_card($jaddressbookid, $card->{uid}, $card->{kind}); } } @@ -309,33 +566,39 @@ sub labels { return $Self->{t}{labels}; } -sub sync { +sub sync_imap { my $Self = shift; - my $imap = $Self->{imap}; - my $labels = $Self->labels(); - - # there's some special casing to care about here... we force the \\Trash label on Trash UIDs - $Self->do_folder($labels->{"\\allmail"}[0]); - if ($labels->{"\\trash"}[0]) { - $Self->do_folder($labels->{"\\trash"}[0], "\\Trash"); + my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); + my @imapnames = map { $_->[1] } grep { $_->[2] } @$data; + my $status = $Self->backend_cmd('imap_status', \@imapnames); + + foreach my $row (@$data) { + # XXX - better handling of UIDvalidity change? + next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); + $Self->do_folder($row->[0], $row->[4]); } } sub backfill { my $Self = shift; - my $old = $Self->dbh->selectcol_arrayref("SELECT ifolderid FROM ifolders WHERE uidnext > 1 AND uidfirst > 1"); - foreach my $ifolderid (@$old) { - warn "SYNCING OLD FOLDER $ifolderid\n"; - $Self->do_folder($ifolderid); + my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime"); + my $rest = 500; + foreach my $row (@$data) { + $rest -= $Self->do_folder(@$row, $rest); + last if $rest < 10; } } sub firstsync { my $Self = shift; - my $imap = $Self->{imap}; + + $Self->sync_folders(); + $Self->sync_calendars(); + $Self->sync_addressbooks(); + my $labels = $Self->labels(); - my $ifolderid = $labels->{"\\allmail"}[0]; + my $ifolderid = $labels->{"allmail"}[0]; $Self->do_folder($ifolderid, undef, 50); my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); @@ -344,93 +607,108 @@ sub firstsync { $Self->fill_messages(@$msgids); } +sub calcmsgid { + my $Self = shift; + my $envelope = shift; + my $json = JSON::XS->new->allow_nonref->canonical; + my $coded = $json->encode($envelope); + my $msgid = sha1_hex($coded); + + my $replyto = lc($envelope->{'In-Reply-To'} || ''); + my $messageid = lc($envelope->{'Message-ID'} || ''); + my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?)", {}, $replyto, $messageid); + $thrid ||= $msgid; + foreach my $id ($replyto, $messageid) { + next if $id eq ''; + $Self->dbh->do("INSERT OR IGNORE INTO ithread (messageid, thrid) VALUES (?, ?)", {}, $id, $thrid); + } + + return ($msgid, $thrid); +} + sub do_folder { my $Self = shift; my $ifolderid = shift; my $forcelabel = shift; - my $batchsize = shift || 500; + my $batchsize = shift; Carp::confess("NO FOLDERID") unless $ifolderid; my $imap = $Self->{imap}; my $dbh = $Self->dbh(); - my ($imapname, $olduidfirst, $olduidnext, $olduidvalidity, $oldhighestmodseq) = $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = + $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); die "NO SUCH FOLDER $ifolderid" unless $imapname; - $olduidfirst ||= 0; - - my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'read-only'); - my $uidvalidity = $imap->get_response_code('uidvalidity'); - my $uidnext = $imap->get_response_code('uidnext'); - my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; - my $exists = $imap->get_response_code('exists') || 0; + my %fetches; + $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size x-gm-msgid x-gm-thrid)]]; + $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; - if ($olduidvalidity and $olduidvalidity != $uidvalidity) { - $oldhighestmodseq = 0; - $olduidfirst = 0; - $olduidnext = 1; - # XXX - delete all the data for this folder and re-sync it - } - elsif ($olduidfirst == 1 and $oldhighestmodseq and $highestmodseq == $oldhighestmodseq) { - $Self->log('debug', "Nothing to do for $imapname at $highestmodseq"); - return 0; # yay, nothing to do + if ($uidfirst > 1 and $batchsize) { + my $end = $uidfirst - 1; + $uidfirst -= $batchsize; + $uidfirst = 1 if $uidfirst < 1; + $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size x-gm-msgid x-gm-thrid)]]; } - $olduidfirst = $uidnext unless $olduidfirst; - $olduidnext = $uidnext unless $olduidnext; + my $res = $Self->backend_cmd('imap_fetch', $imapname, { + uidvalidity => $uidvalidity, + highestmodseq => $highestmodseq, + uidnext => $uidnext, + },\%fetches); - my $uidfirst = $olduidfirst; - if ($olduidfirst > 1) { - $uidfirst = $olduidfirst - $batchsize; - $uidfirst = 1 if $uidfirst < 1; - my $to = $olduidfirst - 1; - $Self->log('debug', "FETCHING $imapname: $uidfirst:$to"); - my $new = $imap->fetch("$uidfirst:$to", '(uid flags internaldate envelope rfc822.size x-gm-msgid x-gm-thrid x-gm-labels)') || {}; + if ($res->{newstate}{uidvalidity} != $uidvalidity) { + # going to want to nuke everything for the existing folder and create this - but for now, just die + die "UIDVALIDITY CHANGED $imapname: $uidvalidity => $res->{newstate}{uidvalidity}"; + } + + my $didold = 0; + if ($res->{backfill}) { + my $new = $res->{backfill}[1]; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, $forcelabel ? [$forcelabel] : $new->{$uid}{'x-gm-labels'}, $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}, $new->{$uid}{'rfc822.size'}); + my ($msgid, $thrid) = ($new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}); + $didold++; + $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } delete $Self->{backfilling}; } - if ($olduidnext > $olduidfirst) { - my $to = $olduidnext - 1; - my @extra; - push @extra, "(changedsince $oldhighestmodseq)" if $oldhighestmodseq; - $Self->log('debug', "UPDATING $imapname: $uidfirst:$to"); - my $changed = $imap->fetch("$uidfirst:$to", "(flags x-gm-labels)", @extra) || {}; + if ($res->{update}) { + my $changed = $res->{update}[1]; foreach my $uid (sort { $a <=> $b } keys %$changed) { - $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, $forcelabel ? [$forcelabel] : $changed->{$uid}{'x-gm-labels'}); + $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, [$forcelabel]); } } - if ($uidnext > $olduidnext) { - my $to = $uidnext - 1; - $Self->log('debug', "FETCHING $imapname: $olduidnext:$to"); - my $new = $imap->fetch("$olduidnext:$to", '(uid flags internaldate envelope rfc822.size x-gm-msgid x-gm-thrid x-gm-labels)') || {}; + if ($res->{new}) { + my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, $forcelabel ? [$forcelabel] : $new->{$uid}{'x-gm-labels'}, $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}, $new->{$uid}{'rfc822.size'}); + my ($msgid, $thrid) = ($new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}); + $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } } # need to make changes before counting - my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $exists) { - my $to = $uidnext - 1; - $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $uids = $imap->search("UID", "$uidfirst:$to"); - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - my %exists = map { $_ => 1 } @$uids; - foreach my $uid (@$data) { - next if $exists{$uid}; - $Self->deleted_record($ifolderid, $uid); + if ($uidfirst == 1) { + my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + if ($count != $res->{newstate}{exists}) { + my $to = $uidnext - 1; + $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); + my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + my $uids = $res->{data}; + my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + my %exists = map { $_ => 1 } @$uids; + foreach my $uid (@$data) { + next if $exists{$uid}; + $Self->deleted_record($ifolderid, $uid); + } } } - $Self->dupdate('ifolders', {highestmodseq => $highestmodseq, uidfirst => $uidfirst, uidnext => $uidnext, uidvalidity => $uidvalidity}, {ifolderid => $ifolderid}); + $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); - return $uidfirst; + return $didold; } sub changed_record { @@ -447,100 +725,64 @@ sub changed_record { $Self->apply_data($msgid, $flaglist, $labellist); } -sub import_message { - my $Self = shift; - my $message = shift; - my $mailboxIds = shift; - my %flags = @_; - - my $dbh = $Self->{dbh}; - my $imap = $Self->{imap}; - - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); - my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; - - # store to the first named folder - we can use labels on gmail to add to other folders later. - my $foldername = $jmailmap{$mailboxIds->[0]}[1]; - $imap->select($foldername); - - my @flags; - push @flags, "\\Seen" unless $flags{isUnread}; - push @flags, "\\Answered" if $flags{isAnswered}; - push @flags, "\\Flagged" if $flags{isFlagged}; - - my $internaldate = time(); # XXX - allow setting? - my $date = Date::Format::time2str('%e-%b-%Y %T %z', $internaldate); - $imap->append($foldername, "(@flags)", $date, { Literal => $message }); - my $uid = $imap->get_response_code('appenduid'); - - if (@$mailboxIds > 1) { - my $labels = join(" ", grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @$mailboxIds); - $imap->store($uid->[1], "X-GM-LABELS", "($labels)"); - } - - my $new = $imap->fetch($uid->[1], '(x-gm-msgid x-gm-thrid)'); - my $msgid = $new->{$uid->[1]}{'x-gm-msgid'}; - my $thrid = $new->{$uid->[1]}{'x-gm-thrid'}; - - return ($msgid, $thrid); -} - sub update_messages { my $Self = shift; my $changes = shift; - my @updated; - my %notUpdated; - my $dbh = $Self->{dbh}; - my $imap = $Self->{imap}; my %updatemap; foreach my $msgid (keys %$changes) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $updatemap{$ifolderid}{$uid} = [$changes->{$msgid}, $msgid]; + $updatemap{$ifolderid}{$uid} = $msgid; } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); + my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; + my %jmailmap = map { $_->[4] => $_ } @$folderdata; + my %notchanged; + my @changed; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; - die "NO SUCH FOLDER $ifolderid" unless $imapname; + my $uidvalidity = $foldermap{$ifolderid}[2]; - # we're writing here! - my $r = $imap->select($imapname); - die "SELECT FAILED $r" unless lc($r) eq 'ok'; - - # XXX - error handling foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { - my $action = $updatemap{$ifolderid}{$uid}[0]; - my $msgid = $updatemap{$ifolderid}{$uid}[1]; + my $msgid = $updatemap{$ifolderid}{$uid}; + my $action = $changes->{$msgid}; + unless ($imapname and $uidvalidity) { + $notchanged{$msgid} = "No folder found"; + next; + } if (exists $action->{isUnread}) { - my $act = $action->{isUnread} ? "-flags" : "+flags"; # reverse - my $res = $imap->store($uid, $act, "(\\Seen)"); + my $bool = !$action->{isUnread}; + my @flags = ("\\Seen"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isFlagged}) { - my $act = $action->{isFlagged} ? "+flags" : "-flags"; - $imap->store($uid, $act, "(\\Flagged)"); + my $bool = $action->{isFlagged}; + my @flags = ("\\Flagged"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isAnswered}) { - my $act = $action->{isAnswered} ? "+flags" : "-flags"; - $imap->store($uid, $act, "(\\Answered)"); + my $bool = $action->{isAnswered}; + my @flags = ("\\Answered"); + $Self->log('debug', "STORING $bool @flags for $uid"); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { - my $labels = join(" ", grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @{$action->{mailboxIds}}); - $imap->store($uid, "X-GM-LABELS", "($labels)"); + my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @{$action->{mailboxIds}}; + $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); } - push @updated, $msgid; + # XXX - handle errors from backend commands + push @changed, $msgid; } - $imap->unselect(); } - return (\@updated, \%notUpdated); + return (\@changed, \%notchanged); } sub delete_messages { @@ -548,39 +790,30 @@ sub delete_messages { my $ids = shift; my $dbh = $Self->{dbh}; - my $imap = $Self->{imap}; - - my @deleted; - my %notDeleted; my %deletemap; foreach my $msgid (@$ids) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $deletemap{$ifolderid}{$uid} = 1; + $deletemap{$ifolderid}{$uid} = $msgid; } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); + my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; + my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; + my (@deleted, %notdeleted); foreach my $ifolderid (keys %deletemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; - die "NO SUCH FOLDER $ifolderid" unless $imapname; - - # we're writing here! - my $r = $imap->select($imapname); - die "SELECT FAILED $r" unless lc($r) eq 'ok'; - - my $uids = [sort keys %{$deletemap{$ifolderid}}]; - if (@$uids) { - $imap->store($uids, "+flags", "(\\Deleted)"); - $imap->uidexpunge($uids); + my $uidvalidity = $foldermap{$ifolderid}[2]; + unless ($imapname) { + $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; } - $imap->unselect(); + my $uids = [sort keys %{$deletemap{$ifolderid}}]; + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder + push @deleted, values %{$deletemap{$ifolderid}}; } - - return (\@deleted, \%notDeleted); + return (\@deleted, \%notdeleted); } sub deleted_record { @@ -622,7 +855,6 @@ sub new_record { sub apply_data { my $Self = shift; my ($msgid, $flaglist, $labellist) = @_; - @$labellist = ('\\allmail') unless @$labellist; my %flagdata = ( isUnread => 1, @@ -663,9 +895,11 @@ sub apply_data { } sub _envelopedata { - my $envelope = decode_json(shift); + my $data = shift; + my $envelope = decode_json($data); + my $encsub = decode('MIME-Header', $envelope->{Subject}); return ( - msgsubject => decode('MIME-Header', $envelope->{Subject}), + msgsubject => $encsub, msgfrom => $envelope->{From}, msgto => $envelope->{To}, msgcc => $envelope->{Cc}, @@ -695,20 +929,16 @@ sub fill_messages { $udata{$row->[0]}{$row->[1]} = $row->[2]; } - my $imap = $Self->{imap}; foreach my $ifolderid (sort keys %udata) { - my ($imapname) = $Self->dbh->selectrow_array("SELECT imapname FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); my $uhash = $udata{$ifolderid}; - die "NO folder $ifolderid" unless $imapname; - my $r = $imap->examine($imapname); - die "EXAMINE FAILED $r" unless lc($r) eq 'ok'; - - my $messages = $imap->fetch(join(',', sort { $a <=> $b } keys %$uhash), "rfc822"); + my $uids = join(',', sort { $a <=> $b } keys %$uhash); + my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); - foreach my $uid (keys %$messages) { + foreach my $uid (keys %{$res->{data}}) { warn "FETCHED BODY FOR $uid\n"; - my $rfc822 = $messages->{$uid}{rfc822}; + my $rfc822 = $res->{data}{$uid}; my $msgid = $uhash->{$uid}; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } @@ -730,6 +960,7 @@ sub _initdb { CREATE TABLE IF NOT EXISTS iserver ( username TEXT PRIMARY KEY, refresh_token TEXT, + hostname TEXT, lastfoldersync DATE, mtime DATE NOT NULL ); @@ -766,6 +997,12 @@ CREATE TABLE IF NOT EXISTS imessages ( size INTEGER, mtime DATE NOT NULL ); +EOF + $dbh->do(<do(<do(<do(< Date: Wed, 15 Jul 2015 19:45:53 -0500 Subject: [PATCH 066/331] make Gmail work --- JMAP/GmailDB.pm | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 7c24fef..3e155d7 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -17,6 +17,7 @@ use Digest::SHA qw(sha1_hex); use AnyEvent; use AnyEvent::Socket; use Data::Dumper; +use IO::All; our $TAG = 1; @@ -39,16 +40,41 @@ my %ROLE_MAP = ( '\\allmail' => 'allmail', ); +my $O; +sub O { + unless ($O) { + my $data = io->file("/home/jmap/jmap-perl/config.json")->slurp; + my $config = decode_json($data); + $O = OAuth2::Tiny->new(%$config); + } + return $O; +} + +sub access_token { + my $Self = shift; + my $username = shift; + my $refresh_token = shift; + + unless ($refresh_token) { + ($username, $refresh_token) = $Self->dbh->selectrow_array("SELECT username, refresh_token FROM iserver"); + } + + my $O = $Self->O(); + my $data = $O->refresh($refresh_token); + + return ['gmail', $username, $data->{access_token}]; +} + + sub setuser { my $Self = shift; my ($username, $refresh_token, $displayname, $picture) = @_; my $data = $Self->dbh->selectrow_arrayref("SELECT username, refresh_token FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}, {hostname => $data->[0]}); + $Self->dmaybeupdate('iserver', {username => $username, refresh_token => $refresh_token}, {username => $data->[0]}); } else { $Self->dinsert('iserver', { - hostname => $hostname, username => $username, refresh_token => $refresh_token, }); @@ -68,14 +94,6 @@ sub setuser { } } -sub access_token { - my $Self = shift; - - my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT hostname, username, password FROM iserver"); - - return [$hostname, $username, $password]; -} - # synchronous backend for now sub backend_cmd { my $Self = shift; From 7b1a20e2f7c4a2665348f91c24055d909e4d2fa5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 23:31:25 -0500 Subject: [PATCH 067/331] fix group map, more debug --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 3 +++ bin/apiendpoint.pl | 6 ++++++ bin/syncserver.pl | 6 ++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 24e428d..d601824 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -695,7 +695,7 @@ sub set_card { jaddressbookid => $jaddressbookid, name => $card->{name}, }); - $Self->ddelete('jcontactgroups', {groupuid => $carduid}); + $Self->ddelete('jcontactgroupmap', {groupuid => $carduid}); foreach my $item (@{$card->{members}}) { $Self->dinsert('jcontactgroupmap', { groupuid => $carduid, diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 55e9032..2e7d2a7 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -321,6 +321,9 @@ sub sync_calendars { push @todo, $id; $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); } + else { + warn "match $token $calendar->{syncToken}\n"; + } } else { $id = $Self->dinsert('icalendars', $data); diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 7f6c66e..1e31d4d 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -96,12 +96,16 @@ sub process_request { fh => $server->{server}{client}, on_error => sub { my ($hdl, $fatal, $msg) = @_; + warn "SHUTDOWN ON ERROR $accountid"; $hdl->destroy; + undef $hdl; EV::unloop; }, on_disconnect => sub { my ($hdl, $fatal, $msg) = @_; + warn "SHUTDOWN ON DISCONNECT $accountid"; $hdl->destroy; + undef $hdl; EV::unloop; }, ); @@ -114,7 +118,9 @@ sub process_request { $handle->push_read(json => mk_handler($accountid)); }); + warn "STARTING LOOP"; EV::run; + warn "ENDING LOOP"; exit 0; } diff --git a/bin/syncserver.pl b/bin/syncserver.pl index cf852ab..abb5cb9 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -48,12 +48,16 @@ sub process_request { fh => $server->{server}{client}, on_error => sub { my ($hdl, $fatal, $msg) = @_; + warn "CLOSING ON ERROR $id"; $hdl->destroy; + undef $hdl; EV::unloop; }, on_disconnect => sub { my ($hdl, $fatal, $msg) = @_; + warn "CLOSING ON DISCONNECT $id"; $hdl->destroy; + undef $hdl; EV::unloop; }, ); @@ -67,7 +71,9 @@ sub process_request { $handle->push_read(json => mk_handler()); }); + warn "STARTING UP"; EV::run; + warn "CLOSING DOWN"; exit 0; } From fcefe91f03828d0b0f798bd58d10fbebcf919b6c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 23:32:28 -0500 Subject: [PATCH 068/331] sync on start --- JMAP/ImapDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2e7d2a7..3f45854 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -326,6 +326,7 @@ sub sync_calendars { } } else { + push @todo, $id; $id = $Self->dinsert('icalendars', $data); } $seen{$id} = 1; @@ -455,6 +456,7 @@ sub sync_addressbooks { } } else { + push @todo, $id; $id = $Self->dinsert('iaddressbooks', $data); } $seen{$id} = 1; From 08a4537bdd68ff36b0a623c8e932b768a04e5fcd Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 15 Jul 2015 23:36:29 -0500 Subject: [PATCH 069/331] create first --- JMAP/ImapDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 3f45854..7e86a3f 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -326,8 +326,8 @@ sub sync_calendars { } } else { - push @todo, $id; $id = $Self->dinsert('icalendars', $data); + push @todo, $id; } $seen{$id} = 1; } @@ -456,8 +456,8 @@ sub sync_addressbooks { } } else { - push @todo, $id; $id = $Self->dinsert('iaddressbooks', $data); + push @todo, $id; } $seen{$id} = 1; } From 2ccfa4105fb4e30859465b4d9c1021ba2076c736 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 01:42:45 -0500 Subject: [PATCH 070/331] FastMail module --- JMAP/Sync/Fastmail.pm | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 JMAP/Sync/Fastmail.pm diff --git a/JMAP/Sync/Fastmail.pm b/JMAP/Sync/Fastmail.pm new file mode 100644 index 0000000..9d39079 --- /dev/null +++ b/JMAP/Sync/Fastmail.pm @@ -0,0 +1,22 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Sync::Fastmail; +use base qw(JMAP::Sync::Standard); + +sub new { + my $class = shift; + my $auth = shift; + my %a = ( + imapserver => 'mail.messagingengine.com', + smtpserver => 'mail.messagingengine.com', + calurl => 'https://caldav.messagingengine.com/', + addressbookurl => 'https://carddav.messagingengine.com/', + %$auth, + ); + return JMAP::Sync::Standard->new(\%a); +} + +1; From d96abf911fce05bfd732c59e72f864e6b8b91ed1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 01:42:56 -0500 Subject: [PATCH 071/331] better error handling --- JMAP/API.pm | 8 ++++---- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 17 +++++++++++------ bin/apiendpoint.pl | 1 + bin/syncserver.pl | 12 ++++++++++-- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 28fce0b..909e58e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1738,12 +1738,12 @@ sub getContactGroups { my $item = {}; $item->{id} = $id; - if (_prop_wanted('name', $args)) { - $item->{name} = $data->{name}; + if (_prop_wanted($args, 'name')) { + $item->{name} = $data->{$id}{name}; } - if (_prop_wanted('contactIds', $args)) { - my $ids = $dbh->selectcol_arrayref("SELECT contactuid FROM jgroupmap WHERE groupuid = ?", {}, $id); + if (_prop_wanted($args, 'contactIds')) { + my $ids = $dbh->selectcol_arrayref("SELECT contactuid FROM jcontactgroupmap WHERE groupuid = ?", {}, $id); $item->{contactIds} = $ids; } diff --git a/JMAP/DB.pm b/JMAP/DB.pm index d601824..5b6fcb5 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -51,7 +51,7 @@ sub log { } else { my ($level, @items) = @_; - #return if $level eq 'debug'; + return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7e86a3f..3c01fdb 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -127,9 +127,14 @@ sub backend_cmd { $handle->push_read(json => sub { my $hdl = shift; my $json = shift; - die "Failed to setup " . Dumper($json) unless $json->[0] eq 'setup'; - $action->($handle); - $h->send($handle); + if ($json->[0] eq 'setup') { + $action->($handle); + $h->send($handle); + } + else { + warn "FAILED $json->[1]"; + $h->send(undef); + } }); # XXX - handle destroy correctly # handle backend going away, etc @@ -139,6 +144,8 @@ sub backend_cmd { $Self->{backend} = $h->recv; } + die "Failed to get a backend" unless $Self->{backend}; + return if $cb; # async usage my $res = $w->recv; @@ -321,9 +328,6 @@ sub sync_calendars { push @todo, $id; $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); } - else { - warn "match $token $calendar->{syncToken}\n"; - } } else { $id = $Self->dinsert('icalendars', $data); @@ -646,6 +650,7 @@ sub do_folder { $uidfirst -= $batchsize; $uidfirst = 1 if $uidfirst < 1; $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; + warn "BACKFILLING $imapname: $uidfirst:$end ($uidnext)\n"; } my $res = $Self->backend_cmd('imap_fetch', $imapname, { diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 1e31d4d..ff87823 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -77,6 +77,7 @@ sub getdb { $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { return if $db->in_transaction(); # check if there's more work to do on the account... + warn "DAV SYNC running $accountid"; eval { $db->begin(); $db->sync_calendars(); diff --git a/bin/syncserver.pl b/bin/syncserver.pl index abb5cb9..5e9852c 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -37,6 +37,7 @@ sub setup { $0 = "[jmap proxy imapsync] $id"; $hdl->push_write(json => [ 'setup', $id ]); $hdl->push_write("\n"); + return 1; } sub process_request { @@ -67,8 +68,15 @@ sub process_request { my $handle = shift; my $json = shift; $id = $json->{username}; - setup($json); - $handle->push_read(json => mk_handler()); + if (eval { setup($json) }) { + $handle->push_read(json => mk_handler()); + } + else { + $handle->push_write(json => ['error', "$@"]); + $handle->push_shutdown(); + undef $hdl; + EV::unloop; + } }); warn "STARTING UP"; From 31e6d4c98d45d6e0933b77268d63d1b40b23c721 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 01:44:24 -0500 Subject: [PATCH 072/331] support FM --- bin/syncserver.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 5e9852c..2ac313e 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -12,6 +12,7 @@ package SyncServer; use Net::Server::PreFork; use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; +use JMAP::Sync::Fastmail; use EV; use Data::Dumper; @@ -30,6 +31,8 @@ sub setup { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; } elsif ($config->{hostname} eq 'imap.mail.me.com') { $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; + } elsif ($config->{hostname} eq 'mail.messagingengine.com') { + $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $id"; } else { die "UNKNOWN ID $id ($config->{hostname})"; } From caf14daa4673564f412011132087d786bceae9c7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 02:41:39 -0500 Subject: [PATCH 073/331] Fix gmail and non-gmail contacts, NAMESPACE prefixes, release transactions --- JMAP/ImapDB.pm | 31 +++++++++++++++++++++---------- JMAP/Sync/Common.pm | 2 ++ JMAP/Sync/Gmail.pm | 1 + JMAP/Sync/Standard.pm | 9 ++++++++- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 3c01fdb..5629a31 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -84,6 +84,9 @@ sub backend_cmd { } my @args = @_; + # release the transaction for the duration + $Self->commit(); + my $w = AnyEvent->condvar; my $action = sub { @@ -96,10 +99,10 @@ sub backend_cmd { my $json = shift; die "INVALID RESPONSE" unless $json->[2] eq $tag; if ($cb) { - $cb->($json->[1]); + $cb->($json); } else { - $w->send($json->[1]); + $w->send($json); } }); }; @@ -149,8 +152,12 @@ sub backend_cmd { return if $cb; # async usage my $res = $w->recv; - #warn Dumper ($cmd, \@args, $res); - return $res; + $Self->begin(); + if ($res->[0] eq 'error') { + warn Dumper ($cmd, \@args, $res); + die "$res->[1]"; + } + return $res->[1]; } # synchronise list from IMAP server to local folder cache @@ -170,6 +177,7 @@ sub sync_folders { my $sep = $folders->{$name}[0]; my $role = $ROLE_MAP{lc $folders->{$name}[1]}; my $label = $role || $folders->{$name}[1]; + warn "FOUND $name => $label"; my $id = $ibylabel{$label}[0]; if ($id) { $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); @@ -313,7 +321,7 @@ sub sync_calendars { my %seen; my @todo; foreach my $calendar (@$calendars) { - my $id = $byhref{$calendar->{href}}[0]; + my $id = $calendar->{href} ? $byhref{$calendar->{href}}[0] : 0; my $data = { isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, @@ -376,7 +384,7 @@ sub sync_jcalendars { mayDelete => 0, mayRename => 0, }; - if ($jbyid{$calendar->[3]}) { + if ($calendar->[3] && $jbyid{$calendar->[3]}) { $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); $seen{$calendar->[3]} = 1; } @@ -504,7 +512,7 @@ sub sync_jaddressbooks { mayDelete => 0, mayRename => 0, }; - if ($jbyid{$addressbook->[2]}) { + if ($addressbook->[2] && $jbyid{$addressbook->[2]}) { $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $addressbook->[2]}); $seen{$addressbook->[2]} = 1; } @@ -933,16 +941,19 @@ sub fill_messages { } foreach my $ifolderid (sort keys %udata) { - my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); my $uhash = $udata{$ifolderid}; + my $uids = join(',', sort { $a <=> $b } grep { not $result{$uhash->{$_}} } keys %$uhash); + next unless $uids; + + my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + next unless $imapname; - my $uids = join(',', sort { $a <=> $b } keys %$uhash); my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); foreach my $uid (keys %{$res->{data}}) { - warn "FETCHED BODY FOR $uid\n"; my $rfc822 = $res->{data}{$uid}; my $msgid = $uhash->{$uid}; + next unless $rfc822; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 8c8ee1f..1cb66ac 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -44,6 +44,7 @@ sub get_events { my $talk = $Self->connect_calendars(); return unless $talk; + $Args->{href} =~ s{/$}{}; my $data = $talk->GetEvents($Args->{href}, Full => 1); my %res; @@ -70,6 +71,7 @@ sub get_cards { my $talk = $Self->connect_contacts(); return unless $talk; + $Args->{href} =~ s{/$}{}; my $data = $talk->GetContacts($Args->{href}); my %res; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 689c5c3..8504db7 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -28,6 +28,7 @@ sub connect_calendars { user => $Self->{auth}{username}, access_token => $Self->{auth}{access_token}, url => "https://apidata.googleusercontent.com/caldav/v2", + is_google => 1, expandurl => 1, ); diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 39573d0..4d2b2f5 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -78,6 +78,8 @@ sub connect_imap { ); next unless $Self->{imap}; $Self->{lastused} = time(); + my $namespace = $Self->{imap}->namespace(); + my $prefix = $namespace->[0][0][0]; my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; my @folders = $Self->{imap}->$list('', '*'); @@ -86,7 +88,12 @@ sub connect_imap { foreach my $folder (@folders) { my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; my $name = $folder->[2]; - my $label = $role || $folder->[2]; + my $label = $role; + unless ($label) { + $label = $folder->[2]; + $label =~ s{^$prefix}{}; + $label =~ s{^[$folder->[1]]}{}; # just in case prefix was missing sep + } $Self->{folders}{$name} = [$folder->[1], $label]; $Self->{labels}{$label} = $name; } From 8f12bb8a29164fdb64b3ba995677e6aea353dead Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 03:24:29 -0500 Subject: [PATCH 074/331] simplify msgid for ImapDB, more debug --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 5b6fcb5..86eeedf 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -26,7 +26,7 @@ use Net::CardDAVTalk::VCard; sub new { my $class = shift; my $accountid = shift || die; - my $dbh = DBI->connect("dbi:SQLite:dbname=/home/jmap/data/$accountid.sqlite3"); + my $dbh = DBI->connect("dbi:SQLite:dbname=/home/jmap/data/$accountid.sqlite3", undef, undef, { RaiseError => 1 }); my $Self = bless { accountid => $accountid, dbh => $dbh, start => time() }, ref($class) || $class; $Self->_initdb($dbh); return $Self; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 5629a31..061be1a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -77,11 +77,6 @@ sub access_token { sub backend_cmd { my $Self = shift; my $cmd = shift; - my $cb; - if (ref($cmd)) { - $cb = $cmd; - $cmd = shift; - } my @args = @_; # release the transaction for the duration @@ -98,14 +93,10 @@ sub backend_cmd { my $hdl = shift; my $json = shift; die "INVALID RESPONSE" unless $json->[2] eq $tag; - if ($cb) { - $cb->($json); - } - else { - $w->send($json); - } + $w->send($json); }); }; + if ($Self->{backend}) { $action->($Self->{backend}); } @@ -149,8 +140,6 @@ sub backend_cmd { die "Failed to get a backend" unless $Self->{backend}; - return if $cb; # async usage - my $res = $w->recv; $Self->begin(); if ($res->[0] eq 'error') { @@ -621,7 +610,7 @@ sub calcmsgid { my $envelope = shift; my $json = JSON::XS->new->allow_nonref->canonical; my $coded = $json->encode($envelope); - my $msgid = sha1_hex($coded); + my $msgid = 's' . substr(sha1_hex($coded), 0, 11); my $replyto = lc($envelope->{'In-Reply-To'} || ''); my $messageid = lc($envelope->{'Message-ID'} || ''); @@ -665,7 +654,7 @@ sub do_folder { uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, uidnext => $uidnext, - },\%fetches); + }, \%fetches); if ($res->{newstate}{uidvalidity} != $uidvalidity) { # going to want to nuke everything for the existing folder and create this - but for now, just die @@ -954,6 +943,7 @@ sub fill_messages { my $rfc822 = $res->{data}{$uid}; my $msgid = $uhash->{$uid}; next unless $rfc822; + warn "ADDING RAW MESSAGE $imapname: $uid => $msgid\n"; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } } From 05bbac551f2850d2f5d08fb0054ae4146ac7508c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 03:26:44 -0500 Subject: [PATCH 075/331] should never be in a transaction when called! --- bin/apiendpoint.pl | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index ff87823..d7a1085 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -66,7 +66,6 @@ sub getdb { } $db->{change_cb} = \&change_cb; $db->{watcher} = AnyEvent->timer(after => 30, interval => 30, cb => sub { - return if $db->in_transaction(); # check if there's more work to do on the account... eval { $db->begin(); @@ -75,7 +74,6 @@ sub getdb { }; }); $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { - return if $db->in_transaction(); # check if there's more work to do on the account... warn "DAV SYNC running $accountid"; eval { From c067c520176e4416cca4ba87eca4280a3a74533b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 04:42:52 -0500 Subject: [PATCH 076/331] don't crash on bogus messages --- JMAP/DB.pm | 26 +++++++++++++++++--------- JMAP/ImapDB.pm | 13 +++++++------ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 86eeedf..5af0275 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -237,17 +237,18 @@ sub add_raw_message { my $eml = Email::MIME->new($rfc822); my $message = $Self->parse_message($msgid, $eml); + my $parsed = encode_json($message); # fiddle the top-level fields my $data = { msgid => $msgid, rfc822 => $rfc822, - parsed => encode_json($message), + parsed => $parsed, }; - $Self->dinsert('jrawmessage', $data); + my $jid = $Self->dinsert('jrawmessage', $data); - return $message; + return decode_json($parsed); } sub parse_date { @@ -294,8 +295,8 @@ sub parse_message { bcc => [$Self->parse_emails($eml->header('Bcc'))], from => [$Self->parse_emails($eml->header('From'))]->[0], replyTo => [$Self->parse_emails($eml->header('Reply-To'))]->[0], - subject => decode('MIME-Header', $eml->header('Subject')), - date => $Self->isodate($Self->parse_date($eml->header('Date'))), + subject => scalar(decode('MIME-Header', $eml->header('Subject'))), + date => scalar($Self->isodate($Self->parse_date($eml->header('Date')))), preview => $preview, textBody => $textpart, htmlBody => $htmlpart, @@ -409,11 +410,18 @@ sub _clean { return $text; } +sub _body_str { + my $eml = shift; + my $str = eval { $eml->body_str() }; + return $str if $str; + return Encode::decode('us-ascii', $eml->body_raw()); +} + sub textpart { my $eml = shift; my $type = $eml->content_type() || 'text/plain'; if ($type =~ m{^text/plain}i) { - return _clean($type, $eml->body_str()); + return _clean($type, _body_str($eml)); } foreach my $sub ($eml->subparts()) { my $res = textpart($sub); @@ -426,7 +434,7 @@ sub htmlpart { my $eml = shift; my $type = $eml->content_type() || 'text/plain'; if ($type =~ m{^text/html}i) { - return _clean($type, $eml->body_str()); + return _clean($type, _body_str($eml)); } foreach my $sub ($eml->subparts()) { my $res = htmlpart($sub); @@ -447,11 +455,11 @@ sub preview { my $eml = shift; my $type = $eml->content_type() || 'text/plain'; if ($type =~ m{text/plain}i) { - my $text = _clean($type, $eml->body_str()); + my $text = _clean($type, _body_str($eml)); return make_preview($text); } if ($type =~ m{text/html}i) { - my $text = _clean($type, $eml->body_str()); + my $text = _clean($type, _body_str($eml)); return make_preview(htmltotext($text)); } foreach my $sub ($eml->subparts()) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 061be1a..b231849 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -223,7 +223,6 @@ sub sync_jmailboxes { my %seen; foreach my $folder (@$ifolders) { my $fname = $folder->[2]; - warn " MAPPING $fname ($folder->[1])"; $fname =~ s/^INBOX\.//; # check for roles first my @bits = split "[$folder->[1]]", $fname; @@ -608,8 +607,9 @@ sub firstsync { sub calcmsgid { my $Self = shift; my $envelope = shift; + my $size = shift; my $json = JSON::XS->new->allow_nonref->canonical; - my $coded = $json->encode($envelope); + my $coded = $json->encode([$envelope, $size]); my $msgid = 's' . substr(sha1_hex($coded), 0, 11); my $replyto = lc($envelope->{'In-Reply-To'} || ''); @@ -666,7 +666,7 @@ sub do_folder { my $new = $res->{backfill}[1]; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); + my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}, $new->{$uid}{'rfc822.size'}); $didold++; $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } @@ -683,7 +683,7 @@ sub do_folder { if ($res->{new}) { my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}); + my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}, $new->{$uid}{'rfc822.size'}); $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } } @@ -939,9 +939,10 @@ sub fill_messages { my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); - foreach my $uid (keys %{$res->{data}}) { - my $rfc822 = $res->{data}{$uid}; + foreach my $uid (sort { $a <=> $b } keys %{$res->{data}}) { my $msgid = $uhash->{$uid}; + next if $result{$msgid}; + my $rfc822 = $res->{data}{$uid}; next unless $rfc822; warn "ADDING RAW MESSAGE $imapname: $uid => $msgid\n"; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); From aa5cbf2ebe3dce5870f60a1e4cb615faf07ccda5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 04:56:03 -0500 Subject: [PATCH 077/331] multi-backend support --- JMAP/API.pm | 4 ++-- JMAP/ImapDB.pm | 52 +++++++++++++++++++++++++--------------------- bin/apiendpoint.pl | 2 +- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 909e58e..32936aa 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -724,7 +724,7 @@ sub getMessages { # need to load messages from the server if ($need_content) { - my $content = $Self->{db}->fill_messages(map { $_->{id} } @list); + my $content = $Self->{db}->fill_messages('interactive', map { $_->{id} } @list); foreach my $item (@list) { my $data = $content->{$item->{id}}; foreach my $prop (qw(preview textBody htmlBody)) { @@ -890,7 +890,7 @@ sub setMessages { my ($updated, $notUpdated) = $Self->{db}->update_messages($update); my ($deleted, $notDeleted) = $Self->{db}->delete_messages($delete); - $Self->{db}->sync_imap(); + $Self->{db}->sync_imap('interactive'); foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b231849..df59a38 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -74,8 +74,9 @@ sub access_token { } # synchronous backend for now -sub backend_cmd { +sub async_cmd { my $Self = shift; + my $backend = shift; my $cmd = shift; my @args = @_; @@ -97,8 +98,8 @@ sub backend_cmd { }); }; - if ($Self->{backend}) { - $action->($Self->{backend}); + if ($Self->{$backend}) { + $action->($Self->{$backend}); } else { my $h = AnyEvent->condvar; @@ -135,10 +136,10 @@ sub backend_cmd { }); # synchronous startup to avoid race condition on setting up channel - $Self->{backend} = $h->recv; + $Self->{$backend} = $h->recv; } - die "Failed to get a backend" unless $Self->{backend}; + die "Failed to get a $backend" unless $Self->{$backend}; my $res = $w->recv; $Self->begin(); @@ -156,7 +157,7 @@ sub sync_folders { my $dbh = $Self->dbh(); - my $folders = $Self->backend_cmd('folders', []); + my $folders = $Self->async_cmd('sync', 'folders', []); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; @@ -182,7 +183,7 @@ sub sync_folders { } if (keys %getstatus) { - my $data = $Self->backend_cmd('imap_status', [keys %getstatus]); + my $data = $Self->async_cmd('sync', 'imap_status', [keys %getstatus]); foreach my $name (keys %$data) { my $status = $data->{$name}; $Self->dmaybeupdate('ifolders', { @@ -301,7 +302,7 @@ sub sync_calendars { my $dbh = $Self->dbh(); - my $calendars = $Self->backend_cmd('calendars', []); + my $calendars = $Self->async_cmd('dav', 'calendars', []); return unless $calendars; my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; @@ -400,7 +401,7 @@ sub do_calendar { my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); my %res = map { $_->[1] => $_ } @$exists; - my $events = $Self->backend_cmd('events', {href => $href}); + my $events = $Self->async_cmd('dav', 'events', {href => $href}); foreach my $resource (keys %$events) { my $data = delete $res{$resource}; @@ -433,7 +434,7 @@ sub sync_addressbooks { my $dbh = $Self->dbh(); - my $addressbooks = $Self->backend_cmd('addressbooks', []); + my $addressbooks = $Self->async_cmd('dav', 'addressbooks', []); return unless $addressbooks; my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); my %byhref = map { $_->[1] => $_ } @$iaddressbooks; @@ -528,7 +529,7 @@ sub do_addressbook { my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); my %res = map { $_->[1] => $_ } @$exists; - my $cards = $Self->backend_cmd('cards', {href => $href}); + my $cards = $Self->async_cmd('dav', 'cards', {href => $href}); foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; @@ -565,14 +566,15 @@ sub labels { sub sync_imap { my $Self = shift; + my $syncname = shift || 'sync'; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); my @imapnames = map { $_->[1] } @$data; - my $status = $Self->backend_cmd('imap_status', \@imapnames); + my $status = $Self->async_cmd($syncname, 'imap_status', \@imapnames); foreach my $row (@$data) { # XXX - better handling of UIDvalidity change? next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); - $Self->do_folder($row->[0], $row->[4]); + $Self->do_folder($syncname, $row->[0], $row->[4]); } } @@ -581,7 +583,7 @@ sub backfill { my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime"); my $rest = 500; foreach my $row (@$data) { - $rest -= $Self->do_folder(@$row, $rest); + $rest -= $Self->do_folder('backfill', @$row, $rest); last if $rest < 10; } } @@ -596,12 +598,12 @@ sub firstsync { my $labels = $Self->labels(); my $ifolderid = $labels->{"inbox"}[0]; - $Self->do_folder($ifolderid, "inbox", 50); + $Self->do_folder('sync', $ifolderid, "inbox", 50); my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); # pre-load the INBOX! - $Self->fill_messages(@$msgids); + $Self->fill_messages('sync', @$msgids); } sub calcmsgid { @@ -626,6 +628,7 @@ sub calcmsgid { sub do_folder { my $Self = shift; + my $syncname = shift; my $ifolderid = shift; my $forcelabel = shift; my $batchsize = shift; @@ -650,7 +653,7 @@ sub do_folder { warn "BACKFILLING $imapname: $uidfirst:$end ($uidnext)\n"; } - my $res = $Self->backend_cmd('imap_fetch', $imapname, { + my $res = $Self->async_cmd($syncname, 'imap_fetch', $imapname, { uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, uidnext => $uidnext, @@ -694,7 +697,7 @@ sub do_folder { if ($count != $res->{newstate}{exists}) { my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + my $res = $Self->async_cmd($syncname, 'imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); my $uids = $res->{data}; my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); my %exists = map { $_ => 1 } @$uids; @@ -758,24 +761,24 @@ sub update_messages { my $bool = !$action->{isUnread}; my @flags = ("\\Seen"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isFlagged}) { my $bool = $action->{isFlagged}; my @flags = ("\\Flagged"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isAnswered}) { my $bool = $action->{isAnswered}; my @flags = ("\\Answered"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one my $newfolder = $foldermap{$id}[1]; - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); + $Self->async_cmd('interactive', 'imap_move', $imapname, $uidvalidity, $uid, $newfolder); } # XXX - handle errors from backend commands push @changed, $msgid; @@ -810,7 +813,7 @@ sub delete_messages { $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; } my $uids = [sort keys %{$deletemap{$ifolderid}}]; - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder + $Self->async_cmd('interactive', 'imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder push @deleted, values %{$deletemap{$ifolderid}}; } return (\@deleted, \%notdeleted); @@ -912,6 +915,7 @@ sub _envelopedata { sub fill_messages { my $Self = shift; + my $synctype = shift; my @ids = @_; my $data = $Self->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage WHERE msgid IN (" . join(', ', map { "?" } @ids) . ")", {}, @ids); @@ -937,7 +941,7 @@ sub fill_messages { my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); next unless $imapname; - my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); + my $res = $Self->async_cmd($synctype, 'imap_fill', $imapname, $uidvalidity, $uids); foreach my $uid (sort { $a <=> $b } keys %{$res->{data}}) { my $msgid = $uhash->{$uid}; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index d7a1085..dcbe7d0 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -243,7 +243,7 @@ sub mk_handler { sub handle_sync { my $db = shift; $db->begin(); - $db->sync_imap(); + $db->sync_imap('sync'); $db->commit(); return ['sync', $JSON::true]; } From 5d375ad2ace3014ea1ec3d382345eb09442ae76f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 05:30:28 -0500 Subject: [PATCH 078/331] explicit transactional lifetimes! --- JMAP/API.pm | 95 ++++++++++++++++++++++++++++++++++++++++++---- JMAP/ImapDB.pm | 71 ++++++++++++++++++++++++---------- bin/apiendpoint.pl | 35 +---------------- 3 files changed, 139 insertions(+), 62 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 32936aa..cc75da3 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -28,9 +28,9 @@ sub getAccounts { my $Self = shift; my $args = shift; - my $dbh = $Self->{db}->dbh(); - + $Self->begin(); my $user = $Self->{db}->get_user(); + $Self->commit(); my @list; push @list, { @@ -53,9 +53,9 @@ sub getPreferences { my $Self = shift; my $args = shift; - my $dbh = $Self->{db}->dbh(); - + $Self->begin(); my $user = $Self->{db}->get_user(); + $Self->commit(); my @list; @@ -68,9 +68,9 @@ sub getSavedSearches { my $Self = shift; my $args = shift; - my $dbh = $Self->{db}->dbh(); - + $Self->begin(); my $user = $Self->{db}->get_user(); + $Self->commit(); my @list; @@ -84,9 +84,9 @@ sub getPersonalities { my $Self = shift; my $args = shift; - my $dbh = $Self->{db}->dbh(); - + $Self->begin(); my $user = $Self->{db}->get_user(); + $Self->commit(); my @list; push @list, { @@ -125,6 +125,8 @@ sub getMailboxes { my $Self = shift; my $args = shift; + # XXX - ideally this is transacted inside the DB + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -202,6 +204,8 @@ sub getMailboxes { } my %missingids = %ids; + $Self->commit(); + return ['mailboxes', { list => \@list, accountId => $accountid, @@ -214,8 +218,10 @@ sub getIdentities { my $Self = shift; my $args = shift; + $Self->begin(); my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); + $Self->commit(); return ['identities', { accountId => $accountid, @@ -235,6 +241,7 @@ sub getMailboxUpdates { my $args = shift; my $dbh = $Self->{db}->dbh(); + $Self->begin(); my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); return ['error', {type => 'accountNotFound'}] @@ -271,6 +278,8 @@ sub getMailboxUpdates { } } + $Self->commit(); + my @res = (['mailboxUpdates', { accountId => $accountid, oldState => "$sinceState", @@ -407,6 +416,7 @@ sub getMessageList { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -444,6 +454,9 @@ sub getMessageList { } gotit: + + $Self->commit(); + my $end = $args->{limit} ? $start + $args->{limit} - 1 : $#$data; $end = $#$data if $end > $#$data; @@ -485,6 +498,7 @@ sub getMessageListUpdates { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -614,6 +628,8 @@ sub getMessageListUpdates { } } + $Self->commit(); + my @res; push @res, ['messageListUpdates', { accountId => $accountid, @@ -634,6 +650,7 @@ sub getMessages { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -722,6 +739,8 @@ sub getMessages { push @list, $item; } + $Self->commit(); + # need to load messages from the server if ($need_content) { my $content = $Self->{db}->fill_messages('interactive', map { $_->{id} } @list); @@ -791,6 +810,7 @@ sub getRawMessage { $filename = $2; } + # skipping transactions here my $dbh = $Self->{db}->dbh(); my ($content) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); return unless $content; @@ -820,6 +840,8 @@ sub getMessageUpdates { my $Self = shift; my $args = shift; + $Self->begin(); + my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -852,6 +874,8 @@ sub getMessageUpdates { } } + $Self->commit(); + my @res; push @res, ['messageUpdates', { accountId => $accountid, @@ -876,11 +900,15 @@ sub setMessages { my $Self = shift; my $args = shift; + $Self->begin(); + my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); return ['error', {type => 'accountNotFound'}] if ($args->{accountId} and $args->{accountId} ne $accountid); + $Self->commit(); + my $create = $args->{create} || {}; my $update = $args->{update} || {}; my $delete = $args->{delete} || []; @@ -919,6 +947,8 @@ sub importMessage { my $Self = shift; my $args = shift; + $Self->begin(); + my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); return ['error', {type => 'accountNotFound'}] @@ -933,6 +963,8 @@ sub importMessage { return ['error', {type => 'notFound'}] if (not $type or $type ne 'message/rfc822'); + $Self->commit(); + # import to a normal mailbox (or boxes) my ($msgid, $thrid) = $Self->import_message($message, $args->{mailboxIds}, isUnread => $args->{isUnread}, @@ -959,6 +991,8 @@ sub reportMessages { my $Self = shift; my $args = shift; + $Self->begin(); + my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); return ['error', {type => 'accountNotFound'}] @@ -970,6 +1004,8 @@ sub reportMessages { return ['error', {type => 'invalidArguments'}] if not exists $args->{asSpam}; + $Self->commit(); + my ($reported, $notfound) = $Self->report_messages($args->{messageIds}, $args->{asSpam}); my @res; @@ -987,6 +1023,7 @@ sub getThreads { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1037,6 +1074,8 @@ sub getThreads { push @allmsgs, @msgs; } + $Self->commit(); + my @res; push @res, ['threads', { list => \@list, @@ -1060,6 +1099,8 @@ sub getThreadUpdates { my $Self = shift; my $args = shift; + $Self->begin(); + my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1102,6 +1143,8 @@ sub getThreadUpdates { my @changed = keys %threads; + $Self->commit(); + my @res; push @res, ['threadUpdates', { accountId => $accountid, @@ -1134,6 +1177,7 @@ sub getCalendars { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1178,6 +1222,8 @@ sub getCalendars { } my %missingids = %ids; + $Self->commit(); + return ['calendars', { list => \@list, accountId => $accountid, @@ -1189,6 +1235,9 @@ sub getCalendars { sub getCalendarUpdates { my $Self = shift; my $args = shift; + + $Self->begin(); + my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1227,6 +1276,8 @@ sub getCalendarUpdates { } } + $Self->commit(); + my @res = (['calendarUpdates', { accountId => $accountid, oldState => "$sinceState", @@ -1278,6 +1329,7 @@ sub getCalendarEventList { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1297,6 +1349,8 @@ sub getCalendarEventList { my @result = map { $data->[$_][0] } $start..$end; + $Self->commit(); + my @res; push @res, ['calendarEventList', { accountId => $accountid, @@ -1321,6 +1375,7 @@ sub getCalendarEvents { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1355,6 +1410,8 @@ sub getCalendarEvents { push @list, $item; } + $Self->commit(); + return ['calendarEvents', { list => \@list, accountId => $accountid, @@ -1367,6 +1424,8 @@ sub getCalendarEventUpdates { my $Self = shift; my $args = shift; + $Self->begin(); + my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1387,6 +1446,8 @@ sub getCalendarEventUpdates { return ['error', {type => 'tooManyChanges'}]; } + $Self->commit(); + my @changed; my @removed; @@ -1423,6 +1484,7 @@ sub getAddressbooks { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1464,6 +1526,8 @@ sub getAddressbooks { push @list, \%rec; } my %missingids = %ids; + + $Self->commit(); return ['addressbooks', { list => \@list, @@ -1476,6 +1540,8 @@ sub getAddressbooks { sub getAddressbookUpdates { my $Self = shift; my $args = shift; + + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1514,6 +1580,8 @@ sub getAddressbookUpdates { } } + $Self->commit(); + my @res = (['addressbookUpdates', { accountId => $accountid, oldState => "$sinceState", @@ -1565,6 +1633,7 @@ sub getContactList { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1584,6 +1653,8 @@ sub getContactList { my @result = map { $data->[$_][0] } $start..$end; + $Self->commit(); + my @res; push @res, ['contactList', { accountId => $accountid, @@ -1608,6 +1679,7 @@ sub getContacts { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1642,6 +1714,7 @@ sub getContacts { push @list, $item; } + $Self->commit(); return ['contacts', { list => \@list, @@ -1655,6 +1728,7 @@ sub getContactUpdates { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1674,6 +1748,7 @@ sub getContactUpdates { if ($args->{maxChanges} and @$data > $args->{maxChanges}) { return ['error', {type => 'tooManyChanges'}]; } + $Self->commit(); my @changed; my @removed; @@ -1711,6 +1786,7 @@ sub getContactGroups { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1749,6 +1825,7 @@ sub getContactGroups { push @list, $item; } + $Self->commit(); return ['contactGroups', { list => \@list, @@ -1762,6 +1839,7 @@ sub getContactGroupUpdates { my $Self = shift; my $args = shift; + $Self->begin(); my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1793,6 +1871,7 @@ sub getContactGroupUpdates { push @removed, $row->[0]; } } + $Self->commit(); my @res; push @res, ['contactGroupUpdates', { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index df59a38..4eab5d5 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -80,9 +80,6 @@ sub async_cmd { my $cmd = shift; my @args = @_; - # release the transaction for the duration - $Self->commit(); - my $w = AnyEvent->condvar; my $action = sub { @@ -142,7 +139,6 @@ sub async_cmd { die "Failed to get a $backend" unless $Self->{$backend}; my $res = $w->recv; - $Self->begin(); if ($res->[0] eq 'error') { warn Dumper ($cmd, \@args, $res); die "$res->[1]"; @@ -155,9 +151,11 @@ sub async_cmd { sub sync_folders { my $Self = shift; + my $folders = $Self->async_cmd('sync', 'folders', []); + + $Self->begin(); my $dbh = $Self->dbh(); - my $folders = $Self->async_cmd('sync', 'folders', []); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; @@ -182,8 +180,17 @@ sub sync_folders { } } + foreach my $folder (@$ifolders) { + my $id = $folder->[0]; + next if $seen{$id}; + $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); + } + + $Self->commit(); + if (keys %getstatus) { my $data = $Self->async_cmd('sync', 'imap_status', [keys %getstatus]); + $Self->begin(); foreach my $name (keys %$data) { my $status = $data->{$name}; $Self->dmaybeupdate('ifolders', { @@ -193,12 +200,7 @@ sub sync_folders { highestmodseq => $status->{highestmodseq}, }, {ifolderid => $getstatus{$name}}); } - } - - foreach my $folder (@$ifolders) { - my $id = $folder->[0]; - next if $seen{$id}; - $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); + $Self->commit(); } $Self->sync_jmailboxes(); @@ -208,6 +210,7 @@ sub sync_folders { # call in transaction sub sync_jmailboxes { my $Self = shift; + $Self->begin(); my $dbh = $Self->dbh(); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentid, role, active FROM jmailboxes"); @@ -293,16 +296,18 @@ sub sync_jmailboxes { next if $seen{$id}; $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); } + $Self->commit(); } # synchronise list from CalDAV server to local folder cache -# call in transaction sub sync_calendars { my $Self = shift; + my $calendars = $Self->async_cmd('dav', 'calendars', []); + + $Self->begin(); my $dbh = $Self->dbh(); - my $calendars = $Self->async_cmd('dav', 'calendars', []); return unless $calendars; my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; @@ -339,6 +344,8 @@ sub sync_calendars { $dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); } + $Self->commit(); + $Self->sync_jcalendars(); foreach my $id (@todo) { @@ -350,6 +357,7 @@ sub sync_calendars { # call in transaction sub sync_jcalendars { my $Self = shift; + $Self->begin(); my $dbh = $Self->dbh(); my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); @@ -389,6 +397,7 @@ sub sync_jcalendars { next if $seen{$id}; $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); } + $Self->commit(); } sub do_calendar { @@ -398,11 +407,12 @@ sub do_calendar { my $dbh = $Self->dbh(); my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my $events = $Self->async_cmd('dav', 'events', {href => $href}); + + $Self->begin(); my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); my %res = map { $_->[1] => $_ } @$exists; - my $events = $Self->async_cmd('dav', 'events', {href => $href}); - foreach my $resource (keys %$events) { my $data = delete $res{$resource}; my $raw = $events->{$resource}; @@ -425,6 +435,8 @@ sub do_calendar { my $event = $Self->parse_event($data->[2]); $Self->delete_event($jcalendarid, $event->{uid}); } + + $Self->commit(); } # synchronise list from CardDAV server to local folder cache @@ -432,10 +444,12 @@ sub do_calendar { sub sync_addressbooks { my $Self = shift; - my $dbh = $Self->dbh(); - my $addressbooks = $Self->async_cmd('dav', 'addressbooks', []); return unless $addressbooks; + + $Self->begin(); + my $dbh = $Self->dbh(); + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); my %byhref = map { $_->[1] => $_ } @$iaddressbooks; @@ -469,6 +483,8 @@ sub sync_addressbooks { $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); } + $Self->commit(); + $Self->sync_jaddressbooks(); foreach my $id (@todo) { @@ -480,6 +496,7 @@ sub sync_addressbooks { # call in transaction sub sync_jaddressbooks { my $Self = shift; + $Self->begin(); my $dbh = $Self->dbh(); my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks"); my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks"); @@ -517,6 +534,7 @@ sub sync_jaddressbooks { next if $seen{$id}; $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $id}); } + $Self->commit(); } sub do_addressbook { @@ -526,11 +544,13 @@ sub do_addressbook { my $dbh = $Self->dbh(); my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); + my $cards = $Self->async_cmd('dav', 'cards', {href => $href}); + + $Self->begin(); + my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); my %res = map { $_->[1] => $_ } @$exists; - my $cards = $Self->async_cmd('dav', 'cards', {href => $href}); - foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; my $raw = $cards->{$resource}; @@ -553,6 +573,7 @@ sub do_addressbook { my $card = $Self->parse_card($data->[2]); $Self->delete_card($jaddressbookid, $card->{uid}, $card->{kind}); } + $Self->commit(); } sub labels { @@ -634,7 +655,7 @@ sub do_folder { my $batchsize = shift; Carp::confess("NO FOLDERID") unless $ifolderid; - my $imap = $Self->{imap}; + $Self->begin(); my $dbh = $Self->dbh(); my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = @@ -653,6 +674,8 @@ sub do_folder { warn "BACKFILLING $imapname: $uidfirst:$end ($uidnext)\n"; } + $Self->commit(); + my $res = $Self->async_cmd($syncname, 'imap_fetch', $imapname, { uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, @@ -664,6 +687,8 @@ sub do_folder { die "UIDVALIDITY CHANGED $imapname: $uidvalidity => $res->{newstate}{uidvalidity}"; } + $Self->begin(); + my $didold = 0; if ($res->{backfill}) { my $new = $res->{backfill}[1]; @@ -710,6 +735,8 @@ sub do_folder { $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); + $Self->commit(); + return $didold; } @@ -943,6 +970,8 @@ sub fill_messages { my $res = $Self->async_cmd($synctype, 'imap_fill', $imapname, $uidvalidity, $uids); + $Self->begin(); + foreach my $uid (sort { $a <=> $b } keys %{$res->{data}}) { my $msgid = $uhash->{$uid}; next if $result{$msgid}; @@ -951,6 +980,8 @@ sub fill_messages { warn "ADDING RAW MESSAGE $imapname: $uid => $msgid\n"; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } + + $Self->commit(); } my @stillneed = grep { not $result{$_} } @ids; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index dcbe7d0..da3d1c4 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -68,19 +68,15 @@ sub getdb { $db->{watcher} = AnyEvent->timer(after => 30, interval => 30, cb => sub { # check if there's more work to do on the account... eval { - $db->begin(); $db->backfill(); - $db->commit(); }; }); $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { # check if there's more work to do on the account... warn "DAV SYNC running $accountid"; eval { - $db->begin(); $db->sync_calendars(); $db->sync_addressbooks(); - $db->commit(); }; }); return $db; @@ -151,9 +147,7 @@ sub handle_getstate { my $db = shift; my $cmd = shift; - $db->begin(); my $user = $db->get_user(); - $db->commit(); die "Failed to get user" unless $user; my $state = "$user->{jhighestmodseq}"; @@ -242,18 +236,14 @@ sub mk_handler { sub handle_sync { my $db = shift; - $db->begin(); $db->sync_imap('sync'); - $db->commit(); return ['sync', $JSON::true]; } sub handle_davsync { my $db = shift; - $db->begin(); $db->sync_calendars(); $db->sync_addressbooks(); - $db->commit(); return ['sync', $JSON::true]; } @@ -300,12 +290,8 @@ sub handle_cb_google { } getdb(); - $db->begin(); $db->setuser($email, $gmaildata->{refresh_token}, $data->{name}, $data->{picture}); - $db->commit(); - $db->begin(); $db->firstsync(); - $db->commit(); return ['registered', [$accountid, $email]]; } @@ -337,12 +323,8 @@ sub handle_signup { $dbh->do("INSERT INTO accounts (email, accountid, type) VALUES (?, ?, ?)", {}, $detail->[1], $accountid, 'imap'); } getdb(); - $db->begin(); $db->setuser(@$detail); - $db->commit(); - $db->begin(); $db->firstsync(); - $db->commit(); return ['signedup', [$accountid, $detail->[1]]]; } @@ -366,10 +348,8 @@ sub handle_upload { my ($db, $req) = @_; my ($type, $content) = @$req; - $db->begin(); my $api = JMAP::API->new($db); my ($res) = $api->uploadFile($type, $content); - $db->commit(); return ['upload', $res]; } @@ -377,10 +357,8 @@ sub handle_upload { sub handle_download { my ($db, $id) = @_; - $db->begin(); my $api = JMAP::API->new($db); my ($type, $content) = $api->downloadFile($id); - $db->commit(); return ['download', [$type, $content]]; } @@ -388,10 +366,8 @@ sub handle_download { sub handle_raw { my ($db, $req) = @_; - $db->begin(); my $api = JMAP::API->new($db); my ($type, $content, $filename) = $api->getRawMessage($req); - $db->commit(); return ['raw', [$type, $content, $filename]]; } @@ -407,16 +383,7 @@ sub handle_jmap { my @items; my $FuncRef = $api->can($command); if ($FuncRef) { - $db->begin(); - my $res = eval { @items = $api->$command($args, $tag); return 1 }; - if ($res) { - $db->commit(); - } - else { - my $error = $@; - @items = ['error', { type => 'serverError', description => $error }]; - $db->rollback(); - } + @items = $api->$command($args, $tag); } else { @items = ['error', { type => 'unknownMethod' }]; From c86caf7ee34f98384621b1128188ff0d5e7d1bff Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 05:53:28 -0500 Subject: [PATCH 079/331] much nicer error handling --- JMAP/API.pm | 10 ++++++++++ JMAP/ImapDB.pm | 23 +++++++++++++++-------- bin/apiendpoint.pl | 30 +++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index cc75da3..136654a 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -121,6 +121,16 @@ sub getPersonalities { }]; } +sub begin { + my $Self = shift; + $Self->{db}->begin(); +} + +sub commit { + my $Self = shift; + $Self->{db}->commit(); +} + sub getMailboxes { my $Self = shift; my $args = shift; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 4eab5d5..35b62c6 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -41,6 +41,7 @@ my %ROLE_MAP = ( sub setuser { my $Self = shift; my ($hostname, $username, $password) = @_; + $Self->begin(); my $data = $Self->dbh->selectrow_arrayref("SELECT hostname, username, password FROM iserver"); if ($data and $data->[0]) { $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}, {hostname => $data->[0]}); @@ -63,6 +64,7 @@ sub setuser { jhighestmodseq => 1, }); } + $Self->commit(); } sub access_token { @@ -82,6 +84,9 @@ sub async_cmd { my $w = AnyEvent->condvar; + use Carp; + Carp::confess("in transaction") if $Self->in_transaction(); + my $action = sub { my $handle = shift; my $tag = "T" . $TAG++; @@ -304,11 +309,11 @@ sub sync_calendars { my $Self = shift; my $calendars = $Self->async_cmd('dav', 'calendars', []); + return unless $calendars; $Self->begin(); my $dbh = $Self->dbh(); - return unless $calendars; my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; @@ -578,11 +583,11 @@ sub do_addressbook { sub labels { my $Self = shift; - unless ($Self->{t}{labels}) { + unless ($Self->{labels}) { my $data = $Self->dbh->selectall_arrayref("SELECT label, ifolderid, jmailboxid, imapname FROM ifolders"); - $Self->{t}{labels} = { map { lc $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; + $Self->{labels} = { map { lc $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; } - return $Self->{t}{labels}; + return $Self->{labels}; } sub sync_imap { @@ -716,6 +721,10 @@ sub do_folder { } } + $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); + + $Self->commit(); + # need to make changes before counting if ($uidfirst == 1) { my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); @@ -723,6 +732,7 @@ sub do_folder { my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); my $res = $Self->async_cmd($syncname, 'imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + $Self->begin(); my $uids = $res->{data}; my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); my %exists = map { $_ => 1 } @$uids; @@ -730,13 +740,10 @@ sub do_folder { next if $exists{$uid}; $Self->deleted_record($ifolderid, $uid); } + $Self->commit(); } } - $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); - - $Self->commit(); - return $didold; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index da3d1c4..a32e2e9 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -103,6 +103,10 @@ sub process_request { undef $hdl; EV::unloop; }, + on_shutdown => sub { + undef $hdl; + EV::unloop; + } ); # send some request line @@ -147,9 +151,11 @@ sub handle_getstate { my $db = shift; my $cmd = shift; + $db->begin(); my $user = $db->get_user(); die "Failed to get user" unless $user; my $state = "$user->{jhighestmodseq}"; + $db->commit(); my $data = { clientId => undef, @@ -222,15 +228,17 @@ sub mk_handler { unless ($res) { $res = ['error', "$@"] } - if ($res->[0]) { - $res->[2] = $tag; - $hdl->push_write(json => $res) if $res->[0]; - warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; - if ($res->[0] eq 'error') { - warn Dumper($res); - } + $res->[2] = $tag; + $hdl->push_write(json => $res) if $res->[0]; + warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; + if ($res->[0] eq 'error') { + warn Dumper($res); + # this process won't handle any more connections + $hdl->push_shutdown(); + } + else { + $hdl->push_read(json => mk_handler($db)); } - $hdl->push_read(json => mk_handler($db)); }; } @@ -383,7 +391,11 @@ sub handle_jmap { my @items; my $FuncRef = $api->can($command); if ($FuncRef) { - @items = $api->$command($args, $tag); + @items = eval { $api->$command($args, $tag) }; + if ($@) { + @items = ['error', { type => "serverError", message => "$@" }]; + eval { $api->rollback() }; + } } else { @items = ['error', { type => 'unknownMethod' }]; From 3905ab73c22676b5329412b9483616ef1f49fca1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 06:04:22 -0500 Subject: [PATCH 080/331] Imap: we just have to re-id every time :( --- JMAP/ImapDB.pm | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 35b62c6..d15c4f2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -634,10 +634,11 @@ sub firstsync { sub calcmsgid { my $Self = shift; - my $envelope = shift; - my $size = shift; + my $imapname = shift; + my $uid = shift; + my $data = shift; my $json = JSON::XS->new->allow_nonref->canonical; - my $coded = $json->encode([$envelope, $size]); + my $coded = $json->encode([$imapname, $uid, $data]); my $msgid = 's' . substr(sha1_hex($coded), 0, 11); my $replyto = lc($envelope->{'In-Reply-To'} || ''); @@ -699,7 +700,7 @@ sub do_folder { my $new = $res->{backfill}[1]; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}, $new->{$uid}{'rfc822.size'}); + my ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); $didold++; $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } @@ -716,7 +717,7 @@ sub do_folder { if ($res->{new}) { my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($new->{$uid}{envelope}, $new->{$uid}{'rfc822.size'}); + my ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } } From 6c8fb8de020c075e92b5ed546b69e0ba6374dcf1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 06:13:44 -0500 Subject: [PATCH 081/331] better missing message detection --- JMAP/ImapDB.pm | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d15c4f2..8debdde 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -637,6 +637,7 @@ sub calcmsgid { my $imapname = shift; my $uid = shift; my $data = shift; + my $envelope = $data->{envelope}; my $json = JSON::XS->new->allow_nonref->canonical; my $coded = $json->encode([$imapname, $uid, $data]); my $msgid = 's' . substr(sha1_hex($coded), 0, 11); @@ -769,16 +770,21 @@ sub update_messages { my $dbh = $Self->{dbh}; my %updatemap; + my %notchanged; foreach my $msgid (keys %$changes) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $updatemap{$ifolderid}{$uid} = $msgid; + if ($ifolderid and $uid) { + $updatemap{$ifolderid}{$uid} = $msgid; + } + else { + $notchanged{$msgid} = "No such message on server"; + } } my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } @$folderdata; - my %notchanged; my @changed; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? @@ -830,16 +836,21 @@ sub delete_messages { my $dbh = $Self->{dbh}; my %deletemap; + my %notdeleted; foreach my $msgid (@$ids) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $deletemap{$ifolderid}{$uid} = $msgid; + if ($ifolderid and $uid) { + $deletemap{$ifolderid}{$uid} = $msgid; + else { + $notdeleted{$msgid} = "No such message on server"; + } } my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; - my (@deleted, %notdeleted); + my @deleted; foreach my $ifolderid (keys %deletemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; From b1f30c293712eee66d7945696027fb7373a4c339 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 12:58:56 -0500 Subject: [PATCH 082/331] little fixes - don't hold connections forever --- JMAP/API.pm | 4 ++-- JMAP/ImapDB.pm | 1 + JMAP/Sync/Common.pm | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 136654a..5fe0646 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -447,7 +447,7 @@ sub getMessageList { return ['error', {type => 'invalidArguments'}] if $start < 0; my $sort = $Self->_build_sort($args->{sort}); - my $data = $dbh->selectall_arrayref("SELECT msgid,thrid FROM jmessages WHERE active = 1 ORDER BY $sort"); + my $data = $dbh->selectall_arrayref("SELECT DISTINCT msgid,thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmessages.active = 1 AND jmessagemap.active = 1 ORDER BY $sort"); $data = $Self->_filter($data, $args->{filter}, {}) if $args->{filter}; $data = $Self->_collapse($data) if $args->{collapseThreads}; @@ -1144,7 +1144,7 @@ sub getThreadUpdates { my @removed; foreach my $key (keys %delcheck) { - my ($exists) = $dbh->selectrow_array("SELECT COUNT(*) FROM jmessages WHERE thrid = ? AND active = 1", {}, $key); + my ($exists) = $dbh->selectrow_array("SELECT COUNT(DISTINCT msgid) FROM jmessages JOIN jmessagemap WHERE thrid = ? AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $key); unless ($exists) { delete $threads{$key}; push @removed, $key; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 8debdde..ce15c1d 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -841,6 +841,7 @@ sub delete_messages { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); if ($ifolderid and $uid) { $deletemap{$ifolderid}{$uid} = $msgid; + } else { $notdeleted{$msgid} = "No such message on server"; } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1cb66ac..f9e056d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -25,6 +25,15 @@ sub DESTROY { my $Self = shift; if ($Self->{imap}) { $Self->{imap}->logout(); + delete $Self->{imap}; + } +} + +sub disconnect { + my $Self = shift; + if ($Self->{imap}) { + $Self->{imap}->logout(); + delete $Self->{imap}; } } From 55d9e28e845c41856dd6baaeabde654f1c854a5d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 13:48:10 -0500 Subject: [PATCH 083/331] better handling: --- bin/apiendpoint.pl | 6 +++--- bin/syncserver.pl | 24 +++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index a32e2e9..837b0d5 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -24,9 +24,9 @@ package JMAP::Backend; use EV; use JSON::XS qw(encode_json decode_json); -use Net::Server::PreFork; +use Net::Server::Fork; -use base qw(Net::Server::PreFork); +use base qw(Net::Server::Fork); # we love globals my $hdl; @@ -53,7 +53,7 @@ sub getdb { die "no accountid" unless $accountid; $dbh ||= accountsdb(); my ($email, $type) = $dbh->selectrow_array("SELECT email,type FROM accounts WHERE accountid = ?", {}, $accountid); - die "no type" unless $type; + die "no type for $accountid" unless $type; warn "CONNECTING: $email $type\n"; if ($type eq 'gmail') { $db = JMAP::GmailDB->new($accountid); diff --git a/bin/syncserver.pl b/bin/syncserver.pl index 2ac313e..2763d69 100644 --- a/bin/syncserver.pl +++ b/bin/syncserver.pl @@ -9,21 +9,21 @@ package SyncServer; use AnyEvent; use AnyEvent::Handle; use JSON::XS qw(encode_json decode_json); -use Net::Server::PreFork; +use Net::Server::Fork; use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; use JMAP::Sync::Fastmail; use EV; use Data::Dumper; -use base qw(Net::Server::PreFork); +use base qw(Net::Server::Fork); # we love globals my $hdl; my $id; my $backend; -$0 = '[jmap proxy imapsync]'; +$0 = '[jmap proxy syncserver]'; sub setup { my $config = shift; @@ -37,7 +37,7 @@ sub setup { die "UNKNOWN ID $id ($config->{hostname})"; } warn "$$ Connected $id"; - $0 = "[jmap proxy imapsync] $id"; + $0 = "[jmap proxy syncserver] $id"; $hdl->push_write(json => [ 'setup', $id ]); $hdl->push_write("\n"); return 1; @@ -53,16 +53,19 @@ sub process_request { on_error => sub { my ($hdl, $fatal, $msg) = @_; warn "CLOSING ON ERROR $id"; - $hdl->destroy; - undef $hdl; - EV::unloop; + eval { $backend->disconnect() }; + exit 0; }, on_disconnect => sub { my ($hdl, $fatal, $msg) = @_; warn "CLOSING ON DISCONNECT $id"; - $hdl->destroy; - undef $hdl; - EV::unloop; + eval { $backend->disconnect() }; + exit 0; + }, + on_shutdown => sub { + warn "CLOSING DOWN ON SHUTDOWN"; + eval { $backend->disconnect() }; + exit 0; }, ); @@ -84,7 +87,6 @@ sub process_request { warn "STARTING UP"; EV::run; - warn "CLOSING DOWN"; exit 0; } From 3a33509b820e22d895b170e2ca8edddf65ef1bd7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 04:57:51 +1000 Subject: [PATCH 084/331] API: rollback transaction on errors --- JMAP/API.pm | 143 ++++++++++++++++++++++++++++------------------------ 1 file changed, 78 insertions(+), 65 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 5fe0646..b21a388 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -131,6 +131,14 @@ sub commit { $Self->{db}->commit(); } +sub _transError { + my $Self = shift; + if ($Self->{db}->in_transaction()) { + $Self->{db}->rollback(); + } + return @_; +} + sub getMailboxes { my $Self = shift; my $args = shift; @@ -141,7 +149,7 @@ sub getMailboxes { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, precedence, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); @@ -254,13 +262,13 @@ sub getMailboxUpdates { $Self->begin(); my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $sinceState = $args->{sinceState}; - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT jmailboxid, jmodseq, jcountsmodseq, active FROM jmailboxes ORDER BY jmailboxid"); @@ -431,20 +439,20 @@ sub getMessageList { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if (exists $args->{position} and exists $args->{anchor}); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if (not exists $args->{position} and not exists $args->{anchor}); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if (exists $args->{anchor} and not exists $args->{anchorOffset}); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if (not exists $args->{anchor} and exists $args->{anchorOffset}); my $start = $args->{position} || 0; - return ['error', {type => 'invalidArguments'}] if $start < 0; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if $start < 0; my $sort = $Self->_build_sort($args->{sort}); my $data = $dbh->selectall_arrayref("SELECT DISTINCT msgid,thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmessages.active = 1 AND jmessagemap.active = 1 ORDER BY $sort"); @@ -460,7 +468,7 @@ sub getMessageList { $start = 0 if $start < 0; goto gotit; } - return ['error', {type => 'anchorNotFound'}]; + return $Self->_transError(['error', {type => 'anchorNotFound'}]); } gotit: @@ -513,16 +521,17 @@ sub getMessageListUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $start = $args->{position} || 0; - return ['error', {type => 'invalidArguments'}] if $start < 0; + return $Self->_transError(['error', {type => 'invalidArguments'}]) + if $start < 0; my $sort = $Self->_build_sort($args->{sort}); my $data = $dbh->selectall_arrayref("SELECT msgid,thrid,jmodseq,active FROM jmessages ORDER BY $sort"); @@ -591,7 +600,7 @@ sub getMessageListUpdates { } if ($args->{maxChanges} and $changes > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } if ($args->{upToMessageId} and $args->{upToMessageId} eq $msgid) { @@ -628,7 +637,7 @@ sub getMessageListUpdates { } if ($args->{maxChanges} and $changes > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } if ($args->{upToMessageId} and $args->{upToMessageId} eq $msgid) { @@ -665,10 +674,11 @@ sub getMessages { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] unless $args->{ids}; + return $Self->_transError(['error', {type => 'invalidArguments'}]) + unless $args->{ids}; #properties: String[] A list of properties to fetch for each message. # XXX - lots to do about properties here @@ -856,12 +866,12 @@ sub getMessageUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT msgid,active FROM jmessages WHERE jmodseq > ?"; @@ -869,7 +879,7 @@ sub getMessageUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } my @changed; @@ -914,7 +924,7 @@ sub setMessages { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); $Self->commit(); @@ -961,16 +971,16 @@ sub importMessage { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{file}; - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{mailboxIds}; my ($type, $message) = $Self->get_file($args->{file}); - return ['error', {type => 'notFound'}] + return $Self->_transError(['error', {type => 'notFound'}]) if (not $type or $type ne 'message/rfc822'); $Self->commit(); @@ -994,7 +1004,7 @@ sub importMessage { sub copyMessages { my $Self = shift; - return ['error', {type => 'notImplemented'}]; + return $Self->_transError(['error', {type => 'notImplemented'}]); } sub reportMessages { @@ -1005,13 +1015,13 @@ sub reportMessages { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{messageIds}; - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not exists $args->{asSpam}; $Self->commit(); @@ -1038,7 +1048,7 @@ sub getThreads { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); # XXX - error if no IDs @@ -1115,12 +1125,12 @@ sub getThreadUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT thrid,active FROM jmessages WHERE jmodseq > ?"; @@ -1132,7 +1142,7 @@ sub getThreadUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } my %threads; @@ -1192,7 +1202,7 @@ sub getCalendars { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $data = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, isVisible, mayReadFreeBusy, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jcalendars WHERE active = 1"); @@ -1252,13 +1262,13 @@ sub getCalendarUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $sinceState = $args->{sinceState}; - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT jcalendarid, jmodseq, active FROM jcalendars ORDER BY jcalendarid"); @@ -1344,11 +1354,12 @@ sub getCalendarEventList { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $start = $args->{position} || 0; - return ['error', {type => 'invalidArguments'}] if $start < 0; + return $Self->_transError(['error', {type => 'invalidArguments'}]) + if $start < 0; my $data = $dbh->selectall_arrayref("SELECT eventuid,jcalendarid FROM jevents WHERE active = 1 ORDER BY eventuid"); @@ -1390,10 +1401,11 @@ sub getCalendarEvents { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] unless $args->{ids}; + return $Self->_transError(['error', {type => 'invalidArguments'}]) + unless $args->{ids}; #properties: String[] A list of properties to fetch for each message. my %seenids; @@ -1440,12 +1452,12 @@ sub getCalendarEventUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT eventuid,active FROM jevents WHERE jmodseq > ?"; @@ -1453,7 +1465,7 @@ sub getCalendarEventUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } $Self->commit(); @@ -1499,7 +1511,7 @@ sub getAddressbooks { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, name, isVisible, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jaddressbooks WHERE active = 1"); @@ -1556,13 +1568,13 @@ sub getAddressbookUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $sinceState = $args->{sinceState}; - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, jmodseq, active FROM jaddressbooks ORDER BY jaddressbookid"); @@ -1648,11 +1660,12 @@ sub getContactList { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); my $start = $args->{position} || 0; - return ['error', {type => 'invalidArguments'}] if $start < 0; + return $Self->_transError(['error', {type => 'invalidArguments'}]) + if $start < 0; my $data = $dbh->selectall_arrayref("SELECT contactuid,jaddressbookid FROM jcontacts WHERE active = 1 ORDER BY contactuid"); @@ -1694,7 +1707,7 @@ sub getContacts { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); #properties: String[] A list of properties to fetch for each message. @@ -1743,12 +1756,12 @@ sub getContactUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT contactuid,active FROM jcontacts WHERE jmodseq > ?"; @@ -1756,7 +1769,7 @@ sub getContactUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } $Self->commit(); @@ -1801,7 +1814,7 @@ sub getContactGroups { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); #properties: String[] A list of properties to fetch for each message. @@ -1854,12 +1867,12 @@ sub getContactGroupUpdates { my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); - return ['error', {type => 'accountNotFound'}] + return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - return ['error', {type => 'invalidArguments'}] + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return ['error', {type => 'cannotCalculateChanges'}] + return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT groupuid,active FROM jcontactgroups WHERE jmodseq > ?"; @@ -1867,7 +1880,7 @@ sub getContactGroupUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return ['error', {type => 'tooManyChanges'}]; + return $Self->_transError(['error', {type => 'tooManyChanges'}]); } my @changed; From b6fda6288091af73f9efd367243f56f41716dd96 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 15:09:41 -0500 Subject: [PATCH 085/331] remove warnings --- JMAP/ImapDB.pm | 29 +++++++++++++++++------------ bin/apiendpoint.pl | 26 ++++++++++++-------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ce15c1d..89ff268 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -170,7 +170,6 @@ sub sync_folders { my $sep = $folders->{$name}[0]; my $role = $ROLE_MAP{lc $folders->{$name}[1]}; my $label = $role || $folders->{$name}[1]; - warn "FOUND $name => $label"; my $id = $ibylabel{$label}[0]; if ($id) { $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); @@ -607,11 +606,13 @@ sub sync_imap { sub backfill { my $Self = shift; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime"); - my $rest = 500; + return unless @$data; + my $rest = 50; foreach my $row (@$data) { $rest -= $Self->do_folder('backfill', @$row, $rest); last if $rest < 10; } + return 1; } sub firstsync { @@ -670,19 +671,24 @@ sub do_folder { die "NO SUCH FOLDER $ifolderid" unless $imapname; my %fetches; - $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size)]]; - $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; - - if ($uidfirst > 1 and $batchsize) { - my $end = $uidfirst - 1; - $uidfirst -= $batchsize; - $uidfirst = 1 if $uidfirst < 1; - $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; - warn "BACKFILLING $imapname: $uidfirst:$end ($uidnext)\n"; + + if ($batchsize) { + if ($uidfirst > 1) { + my $end = $uidfirst - 1; + $uidfirst -= $batchsize; + $uidfirst = 1 if $uidfirst < 1; + $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; + } + } + else { + $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size)]]; + $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; } $Self->commit(); + return unless keys %fetches; + my $res = $Self->async_cmd($syncname, 'imap_fetch', $imapname, { uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, @@ -997,7 +1003,6 @@ sub fill_messages { next if $result{$msgid}; my $rfc822 = $res->{data}{$uid}; next unless $rfc822; - warn "ADDING RAW MESSAGE $imapname: $uid => $msgid\n"; $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 837b0d5..998290d 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -48,6 +48,12 @@ sub handle_getinfo { return ['info', [$email, $type]]; } +sub do_backfill { + # check if there's more work to do on the account... + my $did = $db->in_transaction ? 1 : eval { $db->backfill() }; + $db->{backfiller} = $did ? AnyEvent->timer(after => 2, cb => sub { do_backfill() }) : undef; +} + sub getdb { return $db if $db; die "no accountid" unless $accountid; @@ -65,15 +71,10 @@ sub getdb { die "Weird type $type"; } $db->{change_cb} = \&change_cb; - $db->{watcher} = AnyEvent->timer(after => 30, interval => 30, cb => sub { - # check if there's more work to do on the account... - eval { - $db->backfill(); - }; - }); + $db->{backfiller} = AnyEvent->timer(after => 2, cb => sub { do_backfill() }); $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { # check if there's more work to do on the account... - warn "DAV SYNC running $accountid"; + return if $db->in_transaction(); eval { $db->sync_calendars(); $db->sync_addressbooks(); @@ -114,12 +115,10 @@ sub process_request { my $handle = shift; set_accountid(shift); warn "Connected $accountid\n"; - $handle->push_read(json => mk_handler($accountid)); + $handle->push_read(json => mk_handler($accountid, 1)); }); - warn "STARTING LOOP"; EV::run; - warn "ENDING LOOP"; exit 0; } @@ -170,7 +169,7 @@ sub handle_getstate { } sub mk_handler { - my ($db) = @_; + my ($db, $n) = @_; $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; @@ -182,6 +181,7 @@ sub mk_handler { return sub { my ($hdl, $json) = @_; + $hdl->push_read(json => mk_handler($db, $n+1)); # make sure we have a connection @@ -233,12 +233,10 @@ sub mk_handler { warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; if ($res->[0] eq 'error') { warn Dumper($res); + warn "DIED AFTER COMMAND $n"; # this process won't handle any more connections $hdl->push_shutdown(); } - else { - $hdl->push_read(json => mk_handler($db)); - } }; } From b3528214dc8bcf8723e2d9ff5ac0b6ae944ab148 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 15:44:03 -0500 Subject: [PATCH 086/331] WIP make delete right --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 5af0275..50fd8df 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -51,7 +51,7 @@ sub log { } else { my ($level, @items) = @_; - return if $level eq 'debug'; + #return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 89ff268..c4544ed 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -89,7 +89,7 @@ sub async_cmd { my $action = sub { my $handle = shift; - my $tag = "T" . $TAG++; + my $tag = "$backend" . $TAG++; $handle->push_write(json => [$cmd, \@args, $tag]); # whatever $handle->push_write("\012"); $handle->push_read(json => sub { @@ -640,7 +640,7 @@ sub calcmsgid { my $data = shift; my $envelope = $data->{envelope}; my $json = JSON::XS->new->allow_nonref->canonical; - my $coded = $json->encode([$imapname, $uid, $data]); + my $coded = $json->encode([$envelope, $data->{'rfc822.size'}]); my $msgid = 's' . substr(sha1_hex($coded), 0, 11); my $replyto = lc($envelope->{'In-Reply-To'} || ''); @@ -880,7 +880,10 @@ sub deleted_record { $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - $Self->apply_data($msgid, [], []); + # NOT FOR GMAIL + my ($labels) = $Self->{dbh}->selectcol_arrayref("SELECT label FROM jmessagemap JOIN ifolders USING (jmailboxid) WHERE msgid = ? AND ifolderid != ? AND active = 1", {}, $msgid, $folder); + + $Self->apply_data($msgid, [], $labels); } sub new_record { @@ -912,6 +915,9 @@ sub apply_data { my $Self = shift; my ($msgid, $flaglist, $labellist) = @_; + # spurious temporary old message during move + return if grep { lc $_ eq '\\deleted' } @$flaglist; + my %flagdata = ( isUnread => 1, isFlagged => 0, From 938df2fae4c036d663ca7eb1e2743026855c735c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 07:45:31 +1000 Subject: [PATCH 087/331] ImapDB: switch to using the backend in-proc --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 113 ++++++++++++++----------------------------------- 2 files changed, 33 insertions(+), 82 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index b21a388..3e88f0e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -938,7 +938,7 @@ sub setMessages { my ($updated, $notUpdated) = $Self->{db}->update_messages($update); my ($deleted, $notDeleted) = $Self->{db}->delete_messages($delete); - $Self->{db}->sync_imap('interactive'); + $Self->{db}->sync_imap(); foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c4544ed..146fb1f 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -76,79 +76,32 @@ sub access_token { } # synchronous backend for now -sub async_cmd { +sub backend_cmd { my $Self = shift; - my $backend = shift; my $cmd = shift; my @args = @_; - my $w = AnyEvent->condvar; - use Carp; Carp::confess("in transaction") if $Self->in_transaction(); - my $action = sub { - my $handle = shift; - my $tag = "$backend" . $TAG++; - $handle->push_write(json => [$cmd, \@args, $tag]); # whatever - $handle->push_write("\012"); - $handle->push_read(json => sub { - my $hdl = shift; - my $json = shift; - die "INVALID RESPONSE" unless $json->[2] eq $tag; - $w->send($json); - }); - }; - - if ($Self->{$backend}) { - $action->($Self->{$backend}); - } - else { - my $h = AnyEvent->condvar; + unless ($Self->{backend}) { my $auth = $Self->access_token(); - tcp_connect('localhost', 5005, sub { - my $fh = shift; - my $handle; - $handle = AnyEvent::Handle->new(fh => $fh, - on_disconnect => sub { - delete $Self->{backend}; - undef $h; - }, - on_disconnect => sub { - delete $Self->{backend}; - undef $h; - }, - ); - $handle->push_write(json => {hostname => $auth->[0], username => $auth->[1], password => $auth->[2]}); - $handle->push_write("\012"); - $handle->push_read(json => sub { - my $hdl = shift; - my $json = shift; - if ($json->[0] eq 'setup') { - $action->($handle); - $h->send($handle); - } - else { - warn "FAILED $json->[1]"; - $h->send(undef); - } - }); - # XXX - handle destroy correctly - # handle backend going away, etc - }); - - # synchronous startup to avoid race condition on setting up channel - $Self->{$backend} = $h->recv; + my $config = { hostname => $auth->[0], username => $auth->[1], password => $auth->[2] }; + my $backend; + if ($config->{hostname} eq 'gmail') { + $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; + } elsif ($config->{hostname} eq 'imap.mail.me.com') { + $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; + } elsif ($config->{hostname} eq 'mail.messagingengine.com') { + $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $id"; + } else { + die "UNKNOWN ID $id ($config->{hostname})"; + } + $Self->{backend} = $backend; } - die "Failed to get a $backend" unless $Self->{$backend}; - - my $res = $w->recv; - if ($res->[0] eq 'error') { - warn Dumper ($cmd, \@args, $res); - die "$res->[1]"; - } - return $res->[1]; + die "No such command $cmd" unless $backend->can($cmd); + return $backend->$cmd(@args); } # synchronise list from IMAP server to local folder cache @@ -156,7 +109,7 @@ sub async_cmd { sub sync_folders { my $Self = shift; - my $folders = $Self->async_cmd('sync', 'folders', []); + my $folders = $Self->backend_cmd('folders', []); $Self->begin(); my $dbh = $Self->dbh(); @@ -193,7 +146,7 @@ sub sync_folders { $Self->commit(); if (keys %getstatus) { - my $data = $Self->async_cmd('sync', 'imap_status', [keys %getstatus]); + my $data = $Self->backend_cmd('imap_status', [keys %getstatus]); $Self->begin(); foreach my $name (keys %$data) { my $status = $data->{$name}; @@ -307,7 +260,7 @@ sub sync_jmailboxes { sub sync_calendars { my $Self = shift; - my $calendars = $Self->async_cmd('dav', 'calendars', []); + my $calendars = $Self->backend_cmd('get_calendars', []); return unless $calendars; $Self->begin(); @@ -411,7 +364,7 @@ sub do_calendar { my $dbh = $Self->dbh(); my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); - my $events = $Self->async_cmd('dav', 'events', {href => $href}); + my $events = $Self->backend_cmd('get_events', {href => $href}); $Self->begin(); my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); @@ -448,7 +401,7 @@ sub do_calendar { sub sync_addressbooks { my $Self = shift; - my $addressbooks = $Self->async_cmd('dav', 'addressbooks', []); + my $addressbooks = $Self->backend_cmd('get_addressbooks', []); return unless $addressbooks; $Self->begin(); @@ -548,7 +501,7 @@ sub do_addressbook { my $dbh = $Self->dbh(); my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); - my $cards = $Self->async_cmd('dav', 'cards', {href => $href}); + my $cards = $Self->backend_cmd('get_cards', {href => $href}); $Self->begin(); @@ -591,10 +544,9 @@ sub labels { sub sync_imap { my $Self = shift; - my $syncname = shift || 'sync'; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); my @imapnames = map { $_->[1] } @$data; - my $status = $Self->async_cmd($syncname, 'imap_status', \@imapnames); + my $status = $Self->backend_cmd('imap_status', \@imapnames); foreach my $row (@$data) { # XXX - better handling of UIDvalidity change? @@ -630,7 +582,7 @@ sub firstsync { my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); # pre-load the INBOX! - $Self->fill_messages('sync', @$msgids); + $Self->fill_messages(@$msgids); } sub calcmsgid { @@ -689,7 +641,7 @@ sub do_folder { return unless keys %fetches; - my $res = $Self->async_cmd($syncname, 'imap_fetch', $imapname, { + my $res = $Self->backend_cmd('imap_fetch', $imapname, { uidvalidity => $uidvalidity, highestmodseq => $highestmodseq, uidnext => $uidnext, @@ -739,7 +691,7 @@ sub do_folder { if ($count != $res->{newstate}{exists}) { my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $res = $Self->async_cmd($syncname, 'imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); $Self->begin(); my $uids = $res->{data}; my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); @@ -808,24 +760,24 @@ sub update_messages { my $bool = !$action->{isUnread}; my @flags = ("\\Seen"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isFlagged}) { my $bool = $action->{isFlagged}; my @flags = ("\\Flagged"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{isAnswered}) { my $bool = $action->{isAnswered}; my @flags = ("\\Answered"); $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->async_cmd('interactive', 'imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one my $newfolder = $foldermap{$id}[1]; - $Self->async_cmd('interactive', 'imap_move', $imapname, $uidvalidity, $uid, $newfolder); + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } # XXX - handle errors from backend commands push @changed, $msgid; @@ -866,7 +818,7 @@ sub delete_messages { $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; } my $uids = [sort keys %{$deletemap{$ifolderid}}]; - $Self->async_cmd('interactive', 'imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder push @deleted, values %{$deletemap{$ifolderid}}; } return (\@deleted, \%notdeleted); @@ -974,7 +926,6 @@ sub _envelopedata { sub fill_messages { my $Self = shift; - my $synctype = shift; my @ids = @_; my $data = $Self->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage WHERE msgid IN (" . join(', ', map { "?" } @ids) . ")", {}, @ids); @@ -1000,7 +951,7 @@ sub fill_messages { my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); next unless $imapname; - my $res = $Self->async_cmd($synctype, 'imap_fill', $imapname, $uidvalidity, $uids); + my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); $Self->begin(); From 9c31efac3aa0f88e8015f542c52ab1dccb0cae04 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 08:18:43 +1000 Subject: [PATCH 088/331] Mail::IMAPTalk that supports move --- Mail/IMAPTalk.pm | 359 ++++++++++++++++++++++++----------------------- 1 file changed, 186 insertions(+), 173 deletions(-) diff --git a/Mail/IMAPTalk.pm b/Mail/IMAPTalk.pm index 220746d..bbb0be3 100755 --- a/Mail/IMAPTalk.pm +++ b/Mail/IMAPTalk.pm @@ -196,7 +196,7 @@ internal methods will always 'die' if they encounter any errors. =item INTERNAL SOCKET FUNCTIONS -These are functions used internally by the C object +These are functions used internally by the C object to read/write data to/from the IMAP connection socket. The class does its own buffering so if you want to read/write to the IMAP socket, you should use these functions. @@ -428,6 +428,13 @@ If supplied and true, and a socket is supplied via the C option, checks that a greeting line is supplied by the server and reads the greeting line. +=item B + +For historical reasons, the special name "INBOX" is rewritten as +Inbox because it looks nicer on the way out, and back on the way +in. If you want to preserve the name INBOX on the outside, set +this flag to true. + =back =item B @@ -470,14 +477,9 @@ method for more details. =item B -If supplied, sets the folder name text string separator character. +If supplied, sets the folder name text string separator character. Passed as the second parameter to the C method. -=item B - -If supplied, passed along with RootFolder to the C -method. - =item B If supplied, passed along with RootFolder to the C @@ -488,20 +490,19 @@ method. Examples: $imap = Mail::IMAPTalk->new( - Server => 'foo.com', - Port => 143, - Username => 'joebloggs', - Password => 'mypassword', - Separator => '.', - RootFolder => 'inbox', - CaseInsensitive => 1) - || die "Connection to foo.com failed. Reason: $@"; + Server => 'foo.com', + Port => 143, + Username => 'joebloggs', + Password => 'mypassword', + Separator => '.', + RootFolder => 'INBOX', + ) || die "Connection to foo.com failed. Reason: $@"; $imap = Mail::IMAPTalk->new( - Socket => $SSLSocket, - State => Mail::IMAPTalk::Authenticated, - Uid => 0) - || die "Could not query on existing socket. Reason: $@"; + Socket => $SSLSocket, + State => Mail::IMAPTalk::Authenticated, + Uid => 0 + ) || die "Could not query on existing socket. Reason: $@"; =cut sub new { @@ -572,6 +573,7 @@ sub new { $Self->{LocalFD} = fileno($Socket); $Self->{UseBlocking} = $Args{UseBlocking}; $Self->{Pedantic} = $Args{Pedantic}; + $Self->{PreserveINBOX} = $Args{PreserveINBOX}; # Do this now, so we trace greeting line as well $Self->set_tracing($AlwaysTrace); @@ -600,7 +602,7 @@ sub new { # Set root folder and separator (if supplied) $Self->set_root_folder( - $Args{RootFolder}, $Args{Separator}, $Args{CaseInsensitive}, $Args{AltRootRegexp}); + $Args{RootFolder}, $Args{Separator}, $Args{AltRootRegexp}); return $Self; } @@ -829,20 +831,17 @@ sub is_open { } -=item I +=item I Change the root folder prefix. Some IMAP servers require that all user folders/mailboxes live under a root folder prefix (current versions of B for example use 'INBOX' for personal folders and 'user' for other users folders). If no value is specified, it sets it to ''. You might want to use the B method to find out what roots are -available. The $CaseInsensitive argument is a flag that determines -whether the root folder should be matched in a case sensitive or -insensitive way. See below. +available. Setting this affects all commands that take a folder argument. Basically -if the foldername begins with root folder prefix (case sensitive or -insensitive based on the second argument), it's left as is, +if the foldername begins with root folder prefix, it's left as is, otherwise the root folder prefix and separator char are prefixed to the folder name. @@ -853,43 +852,40 @@ other namespaces in your IMAP server. Examples: # This is what cyrus uses - $IMAP->set_root_folder('inbox', '.', 1, 'user'); + $IMAP->set_root_folder('INBOX', '.', qr/^user/); # Selects 'Inbox' (because 'Inbox' eq 'inbox' case insensitive) $IMAP->select('Inbox'); - # Selects 'inbox.blah' + # Selects 'INBOX.blah' $IMAP->select('blah'); - # Selects 'INBOX.fred' (because 'INBOX' eq 'inbox' case insensitive) - #IMAP->select('INBOX.fred'); # Selects 'INBOX.fred' + # Selects 'INBOX.Inbox.fred' + #IMAP->select('Inbox.fred'); # Selects 'user.john' (because 'user' is alt root) #IMAP->select('user.john'); # Selects 'user.john' =cut sub set_root_folder { - my ($Self, $RootFolder, $Separator, $CaseInsensitive, $AltRootRegexp) = @_; + my ($Self, $RootFolder, $Separator, $AltRootRegexp) = @_; $RootFolder = '' if !defined($RootFolder); - $Separator = '' if !defined($Separator); - $AltRootRegexp = '' if !defined($AltRootRegexp); + $Separator = '.' if !defined($Separator); - # Strip of the Separator, if the IMAP-Server already appended it + # Strip off the Separator, if the IMAP-Server already appended it $RootFolder =~ s/\Q$Separator\E$//; $Self->{RootFolder} = $RootFolder; $Self->{AltRootRegexp} = $AltRootRegexp; $Self->{Separator} = $Separator; - $Self->{CaseInsensitive} = $CaseInsensitive; - # A little tricky. We want to promote INBOX.blah -> blah, but - # we have to be careful not to loose things like INBOX.inbox - # which we leave alone + # We map canonical IMAP INBOX to nicer looking Inbox, + # but have to be careful if the root is INBOX as well - # INBOX -> INBOX + # INBOX -> Inbox (done in _fix_folder_name) # INBOX.blah -> blah # INBOX.inbox -> INBOX.inbox - # INBOX.INBOX -> INBOX.INBOX + # INBOX.Inbox -> INBOX.Inbox # INBOX.inbox.inbox -> INBOX.inbox.inbox - # INBOX.inbox.blah -> INBOX.blah + # INBOX.Inbox.blah -> Inbox.blah # user.xyz -> user.xyz # RootFolderMatch @@ -900,26 +896,21 @@ sub set_root_folder { # If folder returned matches this, strip $RootFolder . $Separator # eg strip inbox. if folder /^inbox\.(?!inbox(\.inbox)*)/ - my ($RootFolderMatch, $UnrootFolderMatch, $RootFolderNormalise); + my ($RootFolderMatch, $UnrootFolderMatch); if ($RootFolder) { - if ($CaseInsensitive) { - $RootFolderMatch = qr/\Q${RootFolder}\E(?i:\Q${Separator}${RootFolder}\E)*/i; - $UnrootFolderMatch = qr/^\Q${RootFolder}${Separator}\E(?!${RootFolderMatch}$)/i; - $RootFolderNormalise = qr/^\Q${RootFolder}\E(\Q${Separator}\E|$)/i; - } else { - $RootFolderMatch = qr/\Q${RootFolder}\E(?:\Q${Separator}${RootFolder}\E)*/; - $UnrootFolderMatch = qr/^\Q${RootFolder}${Separator}\E(?!${RootFolderMatch}$)/; - $RootFolderNormalise = qr/^\Q${RootFolder}(?:\Q${Separator}\E|$)/; - } + # Note the /i on the end to make this case-insensitive + $RootFolderMatch = qr/\Q${RootFolder}\E(?:\Q${Separator}${RootFolder}\E)*/i; + $UnrootFolderMatch = qr/^\Q${RootFolder}${Separator}\E(?!${RootFolderMatch}$)/; - $RootFolderMatch = qr/^${RootFolderMatch}$/; if ($AltRootRegexp) { - $RootFolderMatch = qr/$RootFolderMatch|^(?:${AltRootRegexp})$|^(?:${AltRootRegexp})\Q${Separator}\E/; + $RootFolderMatch = qr/^${RootFolderMatch}$|^(?:${AltRootRegexp}(?:\Q${Separator}\E|$))/; + } else { + $RootFolderMatch = qr/^${RootFolderMatch}$/; } - } - @$Self{qw(RootFolderMatch UnrootFolderMatch RootFolderNormalise)} - = ($RootFolderMatch, $UnrootFolderMatch, $RootFolderNormalise); + + @$Self{qw(RootFolderMatch UnrootFolderMatch)} + = ($RootFolderMatch, $UnrootFolderMatch); return 1; } @@ -932,12 +923,11 @@ Separator. =cut sub _set_separator { - my ($Self,$Separator) = @_; + my ($Self, $Separator) = @_; #Nothing to do, if we have the same Separator as before return 1 if (defined($Separator) && ($Self->{Separator} eq $Separator)); - return $Self->set_root_folder($Self->{RootFolder}, $Separator, - $Self->{CaseInsensitive}, $Self->{AltRootRegexp}); + return $Self->set_root_folder($Self->{RootFolder}, $Separator, $Self->{AltRootRegexp}); } =item I @@ -1009,7 +999,7 @@ sub get_last_error { Returns the last completion response to the tagged command. -This is either the string "ok", "no" or "bad" (always lower case) +This is either the string "ok", "no" or "bad" (always lower case) =cut sub get_last_completion_response { @@ -1217,7 +1207,7 @@ sub unicode_folders { =head1 IMAP FOLDER COMMAND METHODS -B In all cases where a folder name is used, +B In all cases where a folder name is used, the folder name is first manipulated according to the current root folder prefix as described in C. @@ -1227,7 +1217,7 @@ prefix as described in C. =item I Perform the standard IMAP 'select' command to select a folder for -retrieving/moving/adding messages. If $Opts{ReadOnly} is true, the +retrieving/moving/adding messages. If $Opts{ReadOnly} is true, the IMAP EXAMINE verb is used instead of SELECT. Mail::IMAPTalk will cache the currently selected folder, and if you @@ -1673,7 +1663,7 @@ details of a folder/mailbox and possible root quota as well. See RFC 2087 for details of the IMAP quota extension. The result of this command is a little complex. Unfortunately it doesn't map really easily into any structure -since there are several different responses. +since there are several different responses. Basically it's a hash reference. The 'quotaroot' item is the response which lists the root quotas that apply to the given @@ -1813,7 +1803,6 @@ sub getannotation { return $Self->_imap_cmd("getannotation", 0, "annotation", $Self->_fix_folder_name(+shift, 1), { Quote => $_[0] }, { Quote => $_[1] }); } - =item I Perform the IMAP 'getmetadata' command to get the metadata items @@ -1854,6 +1843,43 @@ sub getmetadata { return $Self->_imap_cmd("getmetadata", 0, "metadata", @Args, { Quote => [ @_ ] }); } +=item I + +Performs many IMAP 'getmetadata' commands on a list of folders. Sends +all the commands at once and wait for responses. This speeds up latency +issues. + +Returns a hash ref of folder name => metadata results. + +If an error occurs, the annotation result is a scalar ref to the completion +response string (eg 'bad', 'no', etc) + +=cut +sub multigetmetadata { + my ($Self, $Entries, @FolderList) = @_; + $Self->_require_capability('metadata') || return undef; + + # Send all commands at once + my $FirstId = $Self->{CmdId}; + for (@FolderList) { + $Self->_send_cmd("getmetadata", $Self->_fix_folder_name($_, 1), { Quote => ref($Entries) ? $Entries : [ $Entries ] }); + $Self->{CmdId}++; + } + + # Parse responses + my %Resp; + $Self->{CmdId} = $FirstId; + for (@FolderList) { + my ($CompletionResp, $DataResp) = $Self->_parse_response("metadata"); + $Resp{$_} = ref($DataResp) ? + (ref($Entries) ? $DataResp->{$_} : $DataResp->{$_}->{$Entries}) : + \$CompletionResp; + $Self->{CmdId}++; + } + + return \%Resp; +} + =item I Perform the IMAP 'setannotation' command to get the annotation(s) @@ -1887,40 +1913,6 @@ sub setmetadata { return $Self->_imap_cmd("setmetadata", 0, "metadata", $Self->_fix_folder_name(+shift, 1), { Quote => [ @_ ] }); } -=item I - -Performs many IMAP 'getannotation' commands on a list of folders. Sends -all the commands at once and wait for responses. This speeds up latency -issues. - -Returns a hash ref of folder name => annotation results. - -If an error occurs, the annotation result is a scalar ref to the completion -response string (eg 'bad', 'no', etc) - -=cut -sub multigetannotation { - my ($Self, $Entry, $Attribute, @FolderList) = @_; - - # Send all commands at once - my $FirstId = $Self->{CmdId}; - for (@FolderList) { - $Self->_send_cmd("getannotation", $Self->_fix_folder_name($_, 1), { Quote => $Entry }, { Quote => $Attribute }); - $Self->{CmdId}++; - } - - # Parse responses - my %Resp; - $Self->{CmdId} = $FirstId; - for (@FolderList) { - my ($CompletionResp, $DataResp) = $Self->_parse_response("annotation"); - $Resp{$_} = ref($DataResp) ? $DataResp->{$_}->{$Entry}->{$Attribute} : \$CompletionResp; - $Self->{CmdId}++; - } - - return \%Resp; -} - =item I Perform the standard IMAP 'close' command to expunge deleted messages @@ -2386,6 +2378,14 @@ sub fetch_meta { return \%FetchRes; } +sub move { + my $Self = shift; + my $Uids = _fix_message_ids(+shift); + my $FolderName = $Self->_fix_folder_name(+shift); + $Self->cb_folder_changed($FolderName); + return $Self->_imap_cmd("move", 1, "", $Uids, $FolderName, @_); +} + sub xmove { my $Self = shift; my $Uids = _fix_message_ids(+shift); @@ -2401,7 +2401,7 @@ sub xmove { Methods provided by extensions to the cyrus IMAP server -B In all cases where a folder name is used, +B In all cases where a folder name is used, the folder name is first manipulated according to the current root folder prefix as described in C. @@ -2518,13 +2518,13 @@ sub xconvmeta { } elsif (lc($Item) eq 'folderexists') { my %FolderExists = @{$Res->{$Item}}; - $ResHash{folderexists} = { map { + $ResHash{folderexists} = { map { $Self->_unfix_folder_name($_) => int($FolderExists{$_}) } keys %FolderExists }; } elsif (lc($Item) eq 'folderunseen') { my %FolderUnseen = @{$Res->{$Item}}; - $ResHash{folderunseen} = { map { + $ResHash{folderunseen} = { map { $Self->_unfix_folder_name($_) => int($FolderUnseen{$_}) } keys %FolderUnseen }; } @@ -2594,7 +2594,7 @@ messages =cut sub xconvupdates { my ($Self, $Sort, $Window, @Search) = @_; - + my %Results; my %Callbacks = ( @@ -2831,7 +2831,7 @@ Examples: # Parse further to find message components my $MC = $IMAP->find_message($BS); $MC = { 'plain' => ... text body struct ref part ..., - 'html' => ... html body struct ref part (if present) ... + 'html' => ... html body struct ref part (if present) ... 'htmllist' => [ ... html body struct ref parts (if present) ... ] }; # Now get the text part of the message @@ -2841,7 +2841,11 @@ Examples: sub find_message { my (%MsgComponents); - my %KnownTextParts = map { $_ => 1 } qw(plain html text enriched); + my %KnownTextParts = ( + plain => 'text', text => 'text', enriched => 'text', + html => 'html', + 'application/octet-stream' => 'text' + ); my @PartList = ([ undef, $_[0], 0, '', \(my $Tmp = '') ]); @@ -2850,6 +2854,7 @@ sub find_message { my ($Parent, $BS, $Pos, $InMultiList, $MultiTypeRef) = @$Part; my $InsideAlt = $InMultiList =~ /\balternative\b/ ? 1 : 0; + my $InsideEnc = $InMultiList =~ /\bencrypted\b/ ? 1 : 0; # Pull out common MIME fields we'll look at my ($MTT, $MT, $ST, $SP) = @$BS{qw(MIME-TxtType MIME-Type MIME-Subtype MIME-Subparts)}; @@ -2858,57 +2863,62 @@ sub find_message { # with "$DT ne 'attachment'", rather than "$DT eq 'inline'" my ($DT, $CD) = @$BS{qw(Disposition-Type Content-Disposition)}; - # Yay, found text component that ins't an attachment or has a filename - if ($MT eq 'text' && ($DT ne 'attachment' && !$CD->{filename} && !$CD->{'filename*'})) { - - # See if it's a sub-type we understand/want - if ($KnownTextParts{$ST}) { - - # Map plain, text, enriched -> text - my $UT = $ST; - $UT = 'text' if $ST eq 'plain' || $ST eq 'enriched'; - - # Found it if not already found one of this type - if ( !exists $MsgComponents{$UT} ) { - - # Don't treat html parts in a multipart/mixed as an - # alternative representation unless the first part - if ( $ST eq 'html' - && $Parent - && $Parent->{'MIME-Subtype'} eq 'mixed' - && $Pos > 0 ) - { - } - else { - $MsgComponents{$UT} ||= $BS; - } - - } - - # Override existing part if old part is <= 10 bytes (eg 5 blank - # lines), and new part is > 10 bytes. Or if old part has - # 0 lines and new part has some lines - elsif ( ( $MsgComponents{$UT}->{'Size'} <= 10 && $BS->{'Size'} > 10 ) - || ( $MsgComponents{$UT}->{Lines} < 1 && $BS->{Lines} > 0 ) ) + # Parts we want to treat as "text" + my $IsInline = 0; + # Text component of type we understand that isn't an attachment or has a filename + $IsInline = 1 if $MT eq 'text' && + $KnownTextParts{$ST} && + $DT ne 'attachment' && + !$CD->{filename} && + !$CD->{'filename*'}; + # Bah, PGP has application/octet-stream inside an application/pgp-encrypted part + $IsInline = 1 if $MTT eq 'application/octet-stream' && + $InsideEnc && + $CD->{filename} =~ /encrypted/; + + if ($IsInline) { + # Map to just text or html type + my $UT = $KnownTextParts{$MTT} // $KnownTextParts{$ST}; + + # Found it if not already found one of this type + if ( !exists $MsgComponents{$UT} ) { + + # Don't treat html parts in a multipart/mixed as an + # alternative representation unless the first part + if ( $ST eq 'html' + && $Parent + && $Parent->{'MIME-Subtype'} eq 'mixed' + && $Pos > 0 ) { - $MsgComponents{$UT} = $BS; - } - - # Add to textlist/htmllist if not in alternative part - # or best part type if we are - if ($UT eq 'text' || !$InsideAlt) { - push @{$MsgComponents{'textlist'}}, $BS; - $$MultiTypeRef ||= $UT; } - if ($UT eq 'html' || !$InsideAlt) { - push @{$MsgComponents{'htmllist'}}, $BS; - $$MultiTypeRef ||= $UT; + else { + $MsgComponents{$UT} ||= $BS; } - # Ok got a known part, move to next - next; } - # Wasn't a known text type, will add as an attachment + + # Override existing part if old part is <= 10 bytes (eg 5 blank + # lines), and new part is > 10 bytes. Or if old part has + # 0 lines and new part has some lines + elsif ( ( $MsgComponents{$UT}->{'Size'} <= 10 && $BS->{'Size'} > 10 ) + || ( $MsgComponents{$UT}->{Lines} < 1 && $BS->{Lines} > 0 ) ) + { + $MsgComponents{$UT} = $BS; + } + + # Add to textlist/htmllist if not in alternative part + # or best part type if we are + if ($UT eq 'text' || !$InsideAlt) { + push @{$MsgComponents{'textlist'}}, $BS; + $$MultiTypeRef ||= $UT; + } + if ($UT eq 'html' || !$InsideAlt) { + push @{$MsgComponents{'htmllist'}}, $BS; + $$MultiTypeRef ||= $UT; + } + + # Ok got a known part, move to next + next; } elsif ($MT eq 'image') { @@ -2961,7 +2971,11 @@ sub find_message { }; } - delete $MsgComponents{htmllist} if !$MsgComponents{html}; + # If there's no simple text or html bit, remove any + # corresponding list versions as well + for (qw(text html)) { + $MsgComponents{$_} || delete $MsgComponents{$_ . "list"}; + } return \%MsgComponents; } @@ -3166,7 +3180,7 @@ The result in response will look like this: A couple of points to note: -=over +=over =item 1. @@ -3430,7 +3444,7 @@ Would have the result: } } } - + =cut =head1 INTERNAL METHODS @@ -3649,7 +3663,7 @@ sub _send_data { # Handle non-literals if (!$IsLiteral) { $Arg = _quote(ref($Arg) ? $$Arg : $Arg) if $IsQuote; - + # Must be a scalar reference for a non-literal $LineBuffer .= ($AddSpace ? " " : "") . (ref($Arg) ? $$Arg : $Arg); @@ -3903,7 +3917,8 @@ sub _parse_response { } elsif ($Res1 eq 'metadata') { my ($Name, $Bits) = @{$Self->_remaining_atoms()}; $Name = ($UnfixCache{$Name} ||= $Self->_unfix_folder_name($Name)); - $DataResp{metadata}->{$Name}->{$Bits->[0]} = $Bits->[1]; + my %Hash = @$Bits; + $DataResp{metadata}->{$Name}->{$_} = $Hash{$_} for keys %Hash; } elsif (($Res1 eq 'bye') && ($Self->{LastCmd} ne 'logout')) { $Self->{Cache}->{bye} = $Self->_remaining_line(); @@ -3958,7 +3973,7 @@ sub _trace { my ($Self, $Line) = @_; $Line =~ s/\015\012/\n/; my $Trace = $Self->{Trace}; - + if (ref($Trace) eq 'GLOB') { print $Trace $Line; } elsif (ref($Trace) eq 'CODE') { @@ -4067,7 +4082,7 @@ sub _next_atom { $$AtomRef = $CurAtom; } } - + # Bracket? elsif ($Line =~ m/\G\(/gc) { # Begin a new sub-array @@ -4365,7 +4380,7 @@ otherwise the function will 'die' with an error if it runs out of data. If $NBytes is not specified (undef), the function will attempt to seek to the end of the file to find the size of the file. - + =cut sub _copy_handle_to_handle { my ($Self, $InHandle, $OutHandle, $NBytes) = @_; @@ -4407,7 +4422,7 @@ have to copy the contents of our buffer first. The number of bytes specified must be available on the IMAP socket, if the function runs out of data it will 'die' with an error. - + =cut sub _copy_imap_socket_to_handle { my ($Self, $OutHandle, $NBytes) = @_; @@ -4436,12 +4451,12 @@ sub _copy_imap_socket_to_handle { # Done return 1; } - + =item I<_quote($String)> Returns an IMAP quoted version of a string. This place "..." around the string, and replaces any internal " with \". - + =cut sub _quote { # Replace " and \ with \" and \\ and surround with "..." @@ -4508,14 +4523,13 @@ sub _fix_folder_name { return '' if $FolderName eq ''; + # Map nicer looking Inbox to canonical INBOX + return 'INBOX' if ($FolderName eq 'Inbox' and not $Self->{PreserveINBOX}); + $FolderName = $Self->_fix_folder_encoding($FolderName); return $FolderName if $WildCard && $FolderName =~ /[\*\%]/; - # XXX - make more general/configurable - return $FolderName if $FolderName =~ m{^DELETED\.user\.}; - return $FolderName if $FolderName =~ m{^RESTORED\.}; - my $RootFolderMatch = $Self->{RootFolderMatch} || return $FolderName; @@ -4552,13 +4566,12 @@ with the C call. sub _unfix_folder_name { my ($Self, $FolderName) = @_; - # Normalise root folder part - my $RFN = $Self->{RootFolderNormalise}; - $FolderName =~ s/^$RFN/$Self->{RootFolder}$1/ if $RFN; - my $UFM = $Self->{UnrootFolderMatch}; $FolderName =~ s/^$UFM// if $UFM; + # Map canonical INBOX to nicer looking Inbox + $FolderName = "Inbox" if ($FolderName eq "INBOX" && not $Self->{PreserveINBOX}); + my $UnicodeFolders = $Self->unicode_folders(); if ( $UnicodeFolders && ( $FolderName =~ m{&} ) ) { @@ -4641,7 +4654,7 @@ from an IMAP fetch (envelope) call into a single RFC 822 email string finally return to the user. This is used to parse an envelope structure returned from a fetch call. - + See the documentation section 'FETCH RESULTS' for more information. =cut @@ -4849,7 +4862,7 @@ sub _parse_bodystructure { =item I<_parse_fetch_annotation($AnnotateItem)> Takes the result from a single IMAP annotation item -into a Perl friendly structure. +into a Perl friendly structure. See the documentation section 'FETCH RESULTS' from more information. @@ -4867,7 +4880,7 @@ sub _parse_fetch_annotation { =item I<_parse_fetch_result($FetchResult)> Takes the result from a single IMAP fetch response line and parses it -into a Perl friendly structure. +into a Perl friendly structure. See the documentation section 'FETCH RESULTS' from more information. @@ -5010,7 +5023,7 @@ sub DESTROY { # If socket exists, and connection is open and authenticated or # selected, do a logout - if ($Self->{Socket} && + if ($Self->{Socket} && ($Self->state() == Authenticated || $Self->state() == Selected) && $Self->is_open()) { $Self->logout(); From ff435028a7c6871a3fe56adcd6d3481ae57e4df2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 08:26:33 +1000 Subject: [PATCH 089/331] Don't ever use move --- JMAP/Sync/Common.pm | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index f9e056d..b32935d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -257,14 +257,14 @@ sub imap_move { if ($newname) { # move - if ($imap->capability->{move}) { - my $res = $imap->move($uids, $newname); - unless ($res) { - $res{notMoved} = $uids; - return \%res; - } - } - else { + #if ($imap->capability->{move}) { + #my $res = $imap->move($uids, $newname); + #unless ($res) { + #$res{notMoved} = $uids; + #return \%res; + #} + #} + #else { my $res = $imap->copy($uids, $newname); unless ($res) { $res{notMoved} = $uids; @@ -272,7 +272,7 @@ sub imap_move { } $imap->store($uids, "+flags", "(\\seen \\deleted)"); $imap->uidexpunge($uids); - } + #} } else { $imap->store($uids, "+flags", "(\\seen \\deleted)"); From 571f01ace9a80be5fa891e22cd30e2ccbbea0aea Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 16:50:42 -0500 Subject: [PATCH 090/331] no syncserver --- bin/syncserver.pl | 214 ---------------------------------------------- 1 file changed, 214 deletions(-) delete mode 100644 bin/syncserver.pl diff --git a/bin/syncserver.pl b/bin/syncserver.pl deleted file mode 100644 index 2763d69..0000000 --- a/bin/syncserver.pl +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/perl -w - -use strict; -use lib '/home/jmap/jmap-perl'; -package SyncServer; - -use Mail::IMAPTalk qw(:trace); - -use AnyEvent; -use AnyEvent::Handle; -use JSON::XS qw(encode_json decode_json); -use Net::Server::Fork; -use JMAP::Sync::Gmail; -use JMAP::Sync::ICloud; -use JMAP::Sync::Fastmail; -use EV; -use Data::Dumper; - -use base qw(Net::Server::Fork); - -# we love globals -my $hdl; -my $id; -my $backend; - -$0 = '[jmap proxy syncserver]'; - -sub setup { - my $config = shift; - if ($config->{hostname} eq 'gmail') { - $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; - } elsif ($config->{hostname} eq 'imap.mail.me.com') { - $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; - } elsif ($config->{hostname} eq 'mail.messagingengine.com') { - $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $id"; - } else { - die "UNKNOWN ID $id ($config->{hostname})"; - } - warn "$$ Connected $id"; - $0 = "[jmap proxy syncserver] $id"; - $hdl->push_write(json => [ 'setup', $id ]); - $hdl->push_write("\n"); - return 1; -} - -sub process_request { - my $server = shift; - - close STDIN; - close STDOUT; - $hdl = AnyEvent::Handle->new( - fh => $server->{server}{client}, - on_error => sub { - my ($hdl, $fatal, $msg) = @_; - warn "CLOSING ON ERROR $id"; - eval { $backend->disconnect() }; - exit 0; - }, - on_disconnect => sub { - my ($hdl, $fatal, $msg) = @_; - warn "CLOSING ON DISCONNECT $id"; - eval { $backend->disconnect() }; - exit 0; - }, - on_shutdown => sub { - warn "CLOSING DOWN ON SHUTDOWN"; - eval { $backend->disconnect() }; - exit 0; - }, - ); - - # first item is always an authentication - $hdl->push_read(json => sub { - my $handle = shift; - my $json = shift; - $id = $json->{username}; - if (eval { setup($json) }) { - $handle->push_read(json => mk_handler()); - } - else { - $handle->push_write(json => ['error', "$@"]); - $handle->push_shutdown(); - undef $hdl; - EV::unloop; - } - }); - - warn "STARTING UP"; - EV::run; - exit 0; -} - -SyncServer->run(host => '127.0.0.1', port => 5005); - -sub handle_ping { - return ['pong', $id]; -} - -sub handle_folder { - my $args = shift; - my $data = $backend->fetch_folder(@$args); - return ['folder', $data]; -} - -sub handle_folders { - my $args = shift; - my $data = $backend->folders(@$args); - return ['folders', $data]; -} - -sub handle_calendars { - my $args = shift; - my $data = $backend->get_calendars(@$args); - return ['calendars', $data]; -} - -sub handle_events { - my $args = shift; - my $data = $backend->get_events(@$args); - return ['events', $data]; -} - -sub handle_addressbooks { - my $args = shift; - my $data = $backend->get_addressbooks(@$args); - return ['addressbooks', $data]; -} - -sub handle_cards { - my $args = shift; - my $data = $backend->get_cards(@$args); - return ['cards', $data]; -} - -sub handle_send { - my $args = shift; - my $data = $backend->send_email(@$args); - return ['sent', $data]; -} - -sub handle_imap_status { - my $args = shift; - my $data = $backend->imap_status(@$args); - return ['status', $data]; -} - -sub handle_imap_update { - my $args = shift; - my $data = $backend->imap_update(@$args); - return ['updated', $data]; -} - -sub handle_imap_move { - my $args = shift; - my $data = $backend->imap_move(@$args); - return ['moved', $data]; -} - -sub handle_imap_fill { - my $args = shift; - my $data = $backend->imap_fill(@$args); - return ['filled', $data]; -} - -sub handle_imap_fetch { - my $args = shift; - my $data = $backend->imap_fetch(@$args); - return ['fetched', $data]; -} - -sub handle_imap_count { - my $args = shift; - my $data = $backend->imap_count(@$args); - return ['counted', $data]; -} - -sub reset_keepalive { - $hdl->{keepalive} = AnyEvent->timer(after => 600, cb => sub { - $backend->imap_noop(); - reset_keepalive(); - }); -} - -sub mk_handler { - my ($db) = @_; - - reset_keepalive(); - - return sub { - my ($hdl, $json) = @_; - - # make sure we have a connection - - my ($cmd, $args, $tag) = @$json; - my $res = eval { - my $fn = "handle_$cmd"; - if (SyncServer->can($fn)) { - no strict 'refs'; - return $fn->($args); - } - die "Unknown command $cmd"; - }; - unless ($res) { - $res = ['error', "$@"] - } - $res->[2] = $tag; - #warn Dumper($json, $res); - $hdl->push_write(json => $res); - $hdl->push_write("\n"); - - warn "$$ HANDLED $cmd ($tag) => $res->[0] ($id)\n"; - $hdl->push_read(json => mk_handler($db)); - }; -} From 56a92adf7a275a4aafbbff0c0559a55c946f1aec Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:01:00 -0500 Subject: [PATCH 091/331] fix up simplification layers --- JMAP/ImapDB.pm | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 146fb1f..65b4132 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -17,6 +17,9 @@ use Digest::SHA qw(sha1_hex); use AnyEvent; use AnyEvent::Socket; use Data::Dumper; +use JMAP::Sync::Gmail; +use JMAP::Sync::ICloud; +use JMAP::Sync::Fastmail; our $TAG = 1; @@ -89,19 +92,19 @@ sub backend_cmd { my $config = { hostname => $auth->[0], username => $auth->[1], password => $auth->[2] }; my $backend; if ($config->{hostname} eq 'gmail') { - $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $id"; + $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $auth->[1]"; } elsif ($config->{hostname} eq 'imap.mail.me.com') { - $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $id"; + $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $auth->[1]"; } elsif ($config->{hostname} eq 'mail.messagingengine.com') { - $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $id"; + $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $auth->[1]"; } else { - die "UNKNOWN ID $id ($config->{hostname})"; + die "UNKNOWN ID $config->{username} ($config->{hostname})"; } $Self->{backend} = $backend; } - die "No such command $cmd" unless $backend->can($cmd); - return $backend->$cmd(@args); + die "No such command $cmd" unless $Self->{backend}->can($cmd); + return $Self->{backend}->$cmd(@args); } # synchronise list from IMAP server to local folder cache @@ -551,7 +554,7 @@ sub sync_imap { foreach my $row (@$data) { # XXX - better handling of UIDvalidity change? next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); - $Self->do_folder($syncname, $row->[0], $row->[4]); + $Self->do_folder($row->[0], $row->[4]); } } @@ -561,7 +564,7 @@ sub backfill { return unless @$data; my $rest = 50; foreach my $row (@$data) { - $rest -= $Self->do_folder('backfill', @$row, $rest); + $rest -= $Self->do_folder(@$row, $rest); last if $rest < 10; } return 1; @@ -577,7 +580,7 @@ sub firstsync { my $labels = $Self->labels(); my $ifolderid = $labels->{"inbox"}[0]; - $Self->do_folder('sync', $ifolderid, "inbox", 50); + $Self->do_folder($ifolderid, "inbox", 50); my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); @@ -609,7 +612,6 @@ sub calcmsgid { sub do_folder { my $Self = shift; - my $syncname = shift; my $ifolderid = shift; my $forcelabel = shift; my $batchsize = shift; From ef63fbc7d8afc9907d845e7c3c8440a750cd347a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:23:49 -0500 Subject: [PATCH 092/331] WIP slow down backfill --- bin/apiendpoint.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 998290d..54b4230 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -51,7 +51,7 @@ sub handle_getinfo { sub do_backfill { # check if there's more work to do on the account... my $did = $db->in_transaction ? 1 : eval { $db->backfill() }; - $db->{backfiller} = $did ? AnyEvent->timer(after => 2, cb => sub { do_backfill() }) : undef; + $db->{backfiller} = $did ? AnyEvent->timer(after => 20, cb => sub { do_backfill() }) : undef; } sub getdb { From 4006ce1e430bc08940e735f7d519c5726e7fc87e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:29:05 -0500 Subject: [PATCH 093/331] WIP more - no backfill at all --- bin/apiendpoint.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 54b4230..1e06f1f 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -50,8 +50,9 @@ sub handle_getinfo { sub do_backfill { # check if there's more work to do on the account... + return 0; my $did = $db->in_transaction ? 1 : eval { $db->backfill() }; - $db->{backfiller} = $did ? AnyEvent->timer(after => 20, cb => sub { do_backfill() }) : undef; + $db->{backfiller} = $did ? AnyEvent->timer(after => 5, cb => sub { do_backfill() }) : undef; } sub getdb { From a65cbaac85f411f93a06fe129e73e3adce01afb0 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:35:13 -0500 Subject: [PATCH 094/331] Revert "Don't ever use move" This reverts commit 81af0c64df5e73a245f3130065e18fdaedf16e84. --- JMAP/Sync/Common.pm | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index b32935d..f9e056d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -257,14 +257,14 @@ sub imap_move { if ($newname) { # move - #if ($imap->capability->{move}) { - #my $res = $imap->move($uids, $newname); - #unless ($res) { - #$res{notMoved} = $uids; - #return \%res; - #} - #} - #else { + if ($imap->capability->{move}) { + my $res = $imap->move($uids, $newname); + unless ($res) { + $res{notMoved} = $uids; + return \%res; + } + } + else { my $res = $imap->copy($uids, $newname); unless ($res) { $res{notMoved} = $uids; @@ -272,7 +272,7 @@ sub imap_move { } $imap->store($uids, "+flags", "(\\seen \\deleted)"); $imap->uidexpunge($uids); - #} + } } else { $imap->store($uids, "+flags", "(\\seen \\deleted)"); From 4d010e95c807228f4899b180ab747dec464141d1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:44:03 -0500 Subject: [PATCH 095/331] always look for missing messages while backfilling --- JMAP/ImapDB.pm | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 65b4132..1d17b4e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -688,22 +688,21 @@ sub do_folder { $Self->commit(); # need to make changes before counting - if ($uidfirst == 1) { - my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $res->{newstate}{exists}) { - my $to = $uidnext - 1; - $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); - $Self->begin(); - my $uids = $res->{data}; - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - my %exists = map { $_ => 1 } @$uids; - foreach my $uid (@$data) { - next if $exists{$uid}; - $Self->deleted_record($ifolderid, $uid); - } - $Self->commit(); + my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + # if we don't know everything, we have to ALWAYS check or moves break + if ($uidfirst != 1 or $count != $res->{newstate}{exists}) { + my $to = $uidnext - 1; + $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); + my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); + $Self->begin(); + my $uids = $res->{data}; + my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + my %exists = map { $_ => 1 } @$uids; + foreach my $uid (@$data) { + next if $exists{$uid}; + $Self->deleted_record($ifolderid, $uid); } + $Self->commit(); } return $didold; From ffc50032391723ccf595047f45bfce4e1e35c1a9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 17:46:01 -0500 Subject: [PATCH 096/331] count - don't get race-conditioned into deleting messages --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 1d17b4e..ede77b4 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -696,7 +696,7 @@ sub do_folder { my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); $Self->begin(); my $uids = $res->{data}; - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ? AND uid >= ?", {}, $ifolderid, $uidfirst); my %exists = map { $_ => 1 } @$uids; foreach my $uid (@$data) { next if $exists{$uid}; From bb78ccb95ea694619ed5137c90edde3d47f17e0d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 18:05:58 -0500 Subject: [PATCH 097/331] handle both not deleting from the future and better deletion --- JMAP/ImapDB.pm | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ede77b4..0064b5c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -691,6 +691,8 @@ sub do_folder { my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); # if we don't know everything, we have to ALWAYS check or moves break if ($uidfirst != 1 or $count != $res->{newstate}{exists}) { + # welcome to the future + $uidnext = $res->{newstate}{uidnext}; my $to = $uidnext - 1; $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); @@ -829,14 +831,12 @@ sub deleted_record { my $Self = shift; my ($folder, $uid) = @_; - my ($msgid) = $Self->{dbh}->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); + my ($msgid, $jmailboxid) = $Self->{dbh}->selectrow_array("SELECT msgid, jmailboxid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); + return unless $msgid; $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - # NOT FOR GMAIL - my ($labels) = $Self->{dbh}->selectcol_arrayref("SELECT label FROM jmessagemap JOIN ifolders USING (jmailboxid) WHERE msgid = ? AND ifolderid != ? AND active = 1", {}, $msgid, $folder); - - $Self->apply_data($msgid, [], $labels); + $Self->delete_message_from_mailbox($msgid, $jmailboxid); } sub new_record { From 8e6341aae5f174df11eea519527a0a4559884e38 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 18:26:12 -0500 Subject: [PATCH 098/331] JMAP: fix SQL --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 0064b5c..14610ab 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -831,7 +831,7 @@ sub deleted_record { my $Self = shift; my ($folder, $uid) = @_; - my ($msgid, $jmailboxid) = $Self->{dbh}->selectrow_array("SELECT msgid, jmailboxid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); + my ($msgid, $jmailboxid) = $Self->{dbh}->selectrow_array("SELECT msgid, jmailboxid FROM imessages JOIN ifolders USING (ifolderid) WHERE imessages.ifolderid = ? AND uid = ?", {}, $folder, $uid); return unless $msgid; $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); From d09995728ebda158c16bf0938e9a489a1c0b1964 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 16 Jul 2015 18:56:55 -0500 Subject: [PATCH 099/331] server: do the backfilling --- JMAP/ImapDB.pm | 2 -- bin/apiendpoint.pl | 33 +++++++++++++++------------------ bin/server.pl | 24 +++++++++++++++++++++--- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 14610ab..25919fb 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -574,8 +574,6 @@ sub firstsync { my $Self = shift; $Self->sync_folders(); - $Self->sync_calendars(); - $Self->sync_addressbooks(); my $labels = $Self->labels(); diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 1e06f1f..687514f 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -39,6 +39,7 @@ package JMAP::Backend; sub set_accountid { $accountid = shift; $0 = "[jmap proxy] $accountid"; + $accountid =~ s/:.*//; # strip section splitter } sub handle_getinfo { @@ -48,13 +49,6 @@ sub handle_getinfo { return ['info', [$email, $type]]; } -sub do_backfill { - # check if there's more work to do on the account... - return 0; - my $did = $db->in_transaction ? 1 : eval { $db->backfill() }; - $db->{backfiller} = $did ? AnyEvent->timer(after => 5, cb => sub { do_backfill() }) : undef; -} - sub getdb { return $db if $db; die "no accountid" unless $accountid; @@ -72,15 +66,6 @@ sub getdb { die "Weird type $type"; } $db->{change_cb} = \&change_cb; - $db->{backfiller} = AnyEvent->timer(after => 2, cb => sub { do_backfill() }); - $db->{calsync} = AnyEvent->timer(after => 10, interval => 100, cb => sub { - # check if there's more work to do on the account... - return if $db->in_transaction(); - eval { - $db->sync_calendars(); - $db->sync_addressbooks(); - }; - }); return $db; } @@ -221,6 +206,12 @@ sub mk_handler { if ($cmd eq 'sync') { return handle_sync(getdb(), $args, $tag); } + if ($cmd eq 'davsync') { + return handle_davsync(getdb(), $args, $tag); + } + if ($cmd eq 'backfill') { + return handle_backfill(getdb(), $args, $tag); + } if ($cmd eq 'getinfo') { return handle_getinfo(); } @@ -243,15 +234,21 @@ sub mk_handler { sub handle_sync { my $db = shift; - $db->sync_imap('sync'); + $db->sync_imap(); return ['sync', $JSON::true]; } +sub handle_backfill { + my $db = shift; + my $res = $db->backfill(); + return ['sync', $res ? $JSON::true : $JSON::false]; +} + sub handle_davsync { my $db = shift; $db->sync_calendars(); $db->sync_addressbooks(); - return ['sync', $JSON::true]; + return ['davsync', $JSON::true]; } sub accountsdb { diff --git a/bin/server.pl b/bin/server.pl index 0855485..10d6aa0 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -42,7 +42,7 @@ sub idler { my $accountid = shift; my $edgecb = shift; - send_backend_request($accountid, 'gettoken', $accountid, sub { + send_backend_request("$accountid:sync", 'gettoken', $accountid, sub { my ($data) = @_; if ($data) { my $imap = $data->[0] eq 'gmail' ? AnyEvent::Gmail->new( @@ -78,6 +78,10 @@ sub idler { $imap->connect(); $idler{$accountid}{idler} = $imap; + + $idler{$accountid}{dav} = AnyEvent->timer(after => 1, interval => 300, sub { + send_backend_request("$accountid:dav", 'davsync', $accountid); + }); } else { # clean up so next attempt will try again @@ -581,17 +585,29 @@ sub HandleEventSource { }); } +sub prod_backfill { + my $force = shift; + return if $idler{$accountid}{backfilling}; + $idler{$accountid}{backfilling} = 1; + + send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { + prod_backfill(@_); + }); +} + sub prod_idler { my $accountid = shift; unless ($idler{$accountid}) { idler($accountid, sub { - send_backend_request($accountid, 'sync', $accountid); + send_backend_request("$accountid:sync", 'sync', $accountid); }, ); } + prod_backfill(); + $idler{$accountid}{lastused} = time(); } @@ -629,7 +645,7 @@ sub ShutdownPushChannel { sub ShutdownHandle { my ($Handle, $Msg) = @_; - $Handle->push_write($Msg); + $Handle->push_write($Msg) if $Msg; $Handle->on_drain(sub { $Handle->destroy(); }); } @@ -644,8 +660,10 @@ sub HandleKeepAlive { next if $PushMap{$accountid}; # nothing to do if ($idler{$accountid}{lastused} < $Now - KEEPIDLE_TIME) { my $old = $idler{$accountid}{idler}; + my $sync = delete $backends{"$accountid:sync"}; delete $idler{$accountid}; eval { $old->disconnect() }; + eval { ShutdownHandle($sync) }; } } } From bf853040af69e4329924e5c6164c6c4f56b92209 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 01:38:56 -0500 Subject: [PATCH 100/331] lock around transaction --- JMAP/DB.pm | 5 ++++- bin/server.pl | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 50fd8df..10cc908 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -9,6 +9,7 @@ use Data::Dumper; use DBI; use Carp qw(confess); +use IO::LockedFile; use JSON::XS qw(encode_json decode_json); use Email::MIME; # seriously, it's parsable, get over it @@ -70,8 +71,10 @@ sub in_transaction { sub begin { my $Self = shift; confess("ALREADY IN TRANSACTION") if $Self->{t}; - $Self->dbh->begin_work(); + my $accountid = $Self->accountid(); $Self->{t} = {}; + $Self->{t}{lock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); + $Self->dbh->begin_work(); } sub commit { diff --git a/bin/server.pl b/bin/server.pl index 10d6aa0..cecf982 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -79,8 +79,8 @@ sub idler { $idler{$accountid}{idler} = $imap; - $idler{$accountid}{dav} = AnyEvent->timer(after => 1, interval => 300, sub { - send_backend_request("$accountid:dav", 'davsync', $accountid); + $idler{$accountid}{dav} = AnyEvent->timer(after => 1, interval => 300, cb => sub { + send_backend_request("$accountid:dav", 'davsync', $accountid, sub { }); }); } else { @@ -223,6 +223,7 @@ sub send_backend_request { my $errcb = shift; my $backend = get_backend($accountid); my $cmd = "#" . $backend->[1]++; + warn "SENDING $accountid $request\n"; $waiting{$accountid}{$cmd} = [$cb || sub {return 1}, $errcb || sub {return 1}]; $backend->[0]->push_write(json => [$request, $args, $cmd]); } @@ -586,12 +587,13 @@ sub HandleEventSource { } sub prod_backfill { + my $accountid = shift; my $force = shift; - return if $idler{$accountid}{backfilling}; + return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { - prod_backfill(@_); + prod_backfill($accountid, @_); }); } @@ -606,7 +608,7 @@ sub prod_idler { ); } - prod_backfill(); + prod_backfill($accountid); $idler{$accountid}{lastused} = time(); } @@ -660,10 +662,14 @@ sub HandleKeepAlive { next if $PushMap{$accountid}; # nothing to do if ($idler{$accountid}{lastused} < $Now - KEEPIDLE_TIME) { my $old = $idler{$accountid}{idler}; - my $sync = delete $backends{"$accountid:sync"}; + my $sync = delete $backend{"$accountid:sync"}; + my $dav = delete $backend{"$accountid:dav"}; + my $backfill = delete $backend{"$accountid:backfill"}; delete $idler{$accountid}; eval { $old->disconnect() }; eval { ShutdownHandle($sync) }; + eval { ShutdownHandle($dav) }; + eval { ShutdownHandle($backfill) }; } } } From 69417aea42d1fd13db4743b12fa7e2c806d1df60 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 16:41:06 +1000 Subject: [PATCH 101/331] some renamings from the new JMAP spec --- JMAP/API.pm | 8 ++++---- JMAP/DB.pm | 4 ++-- JMAP/GmailDB.pm | 22 +++++++++++----------- JMAP/ImapDB.pm | 22 +++++++++++----------- bin/apiendpoint.pl | 8 ++------ 5 files changed, 30 insertions(+), 34 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 3e88f0e..dde6ca4 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -152,7 +152,7 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, precedence, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, order, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); my %ids; if ($args->{ids}) { @@ -176,7 +176,7 @@ sub getMailboxes { parentId => ($item->[1] ? "$item->[1]" : undef), name => $item->[2], role => $item->[3], - precedence => $item->[4], + order => $item->[4], mustBeOnlyMailbox => $item->[5] ? $JSON::true : $JSON::false, mayDeleteMailbox => $item->[6] ? $JSON::true : $JSON::false, mayRenameMailbox => $item->[7] ? $JSON::true : $JSON::false, @@ -1205,7 +1205,7 @@ sub getCalendars { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, isVisible, mayReadFreeBusy, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jcalendars WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, isVisible, mayReadFreeBusy, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jcalendars WHERE active = 1"); my %ids; if ($args->{ids}) { @@ -1223,7 +1223,7 @@ sub getCalendars { my %rec = ( id => "$item->[0]", name => $item->[1], - colour => $item->[2], + color => $item->[2], isVisible => $item->[3] ? $JSON::true : $JSON::false, mayReadFreeBusy => $item->[4] ? $JSON::true : $JSON::false, mayReadItems => $item->[5] ? $JSON::true : $JSON::false, diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 10cc908..4045b7e 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -908,7 +908,7 @@ CREATE TABLE IF NOT EXISTS jmailboxes ( parentid INTEGER, role TEXT, name TEXT, - precedence INTEGER, + order INTEGER, mustBeOnly BOOLEAN, mayDelete BOOLEAN, mayRename BOOLEAN, @@ -972,7 +972,7 @@ EOF CREATE TABLE IF NOT EXISTS jcalendars ( jcalendarid INTEGER PRIMARY KEY, name TEXT, - colour TEXT, + color TEXT, isVisible BOOLEAN, mayReadFreeBusy BOOLEAN, mayReadItems BOOLEAN, diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 3e155d7..8312ea1 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -253,9 +253,9 @@ sub sync_jmailboxes { my $id = 0; my $parentid = 0; my $name; - my $precedence = 3; - $precedence = 2 if $role; - $precedence = 1 if ($role||'') eq 'inbox'; + my $order = 3; + $order = 2 if $role; + $order = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; @@ -265,7 +265,7 @@ sub sync_jmailboxes { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, precedence => 4, parentid => $parentid}); + $id = $Self->dmake('jmailboxes', {name => $name, order => 4, parentid => $parentid}); $byname{$parentid}{$name} = $id; } } @@ -274,7 +274,7 @@ sub sync_jmailboxes { my %details = ( name => $name, parentid => $parentid, - precedence => $precedence, + order => $order, mustBeOnly => $ONLY_MAILBOXES{$role||''}, mayDelete => (not $PROTECTED_MAILBOXES{$role||''}), mayRename => (not $NO_RENAME{$role||''}), @@ -326,7 +326,7 @@ sub sync_calendars { my $calendars = $Self->backend_cmd('calendars', []); return unless $calendars; - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; my %seen; @@ -336,7 +336,7 @@ sub sync_calendars { my $data = { isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, - colour => $calendar->{colour}, + color => $calendar->{color}, name => $calendar->{name}, syncToken => $calendar->{syncToken}, }; @@ -372,8 +372,8 @@ sub sync_calendars { sub sync_jcalendars { my $Self = shift; my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); - my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); + my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); my %jbyid; foreach my $calendar (@$jcalendars) { @@ -384,7 +384,7 @@ sub sync_jcalendars { foreach my $calendar (@$icalendars) { my $data = { name => $calendar->[1], - colour => $calendar->[2], + color => $calendar->[2], isVisible => 1, mayReadFreeBusy => 1, mayReadItems => 1, @@ -1029,7 +1029,7 @@ CREATE TABLE IF NOT EXISTS icalendars ( href TEXT, name TEXT, isReadOnly INTEGER, - colour TEXT, + color TEXT, syncToken TEXT, jcalendarid INTEGER, mtime DATE NOT NULL diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 25919fb..c779879 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -194,9 +194,9 @@ sub sync_jmailboxes { my $id = 0; my $parentid = 0; my $name; - my $precedence = 3; - $precedence = 2 if $role; - $precedence = 1 if ($role||'') eq 'inbox'; + my $order = 3; + $order = 2 if $role; + $order = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; @@ -206,7 +206,7 @@ sub sync_jmailboxes { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, precedence => 4, parentid => $parentid}); + $id = $Self->dmake('jmailboxes', {name => $name, order => 4, parentid => $parentid}); $byname{$parentid}{$name} = $id; } } @@ -215,7 +215,7 @@ sub sync_jmailboxes { my %details = ( name => $name, parentid => $parentid, - precedence => $precedence, + order => $order, mustBeOnly => 1, mayDelete => 0, mayRename => 0, @@ -269,7 +269,7 @@ sub sync_calendars { $Self->begin(); my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, colour, syncToken FROM icalendars"); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; my %seen; @@ -279,7 +279,7 @@ sub sync_calendars { my $data = { isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, - colour => $calendar->{colour}, + color => $calendar->{color}, name => $calendar->{name}, syncToken => $calendar->{syncToken}, }; @@ -319,8 +319,8 @@ sub sync_jcalendars { my $Self = shift; $Self->begin(); my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, colour, jcalendarid FROM icalendars"); - my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, colour, active FROM jcalendars"); + my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); + my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); my %jbyid; foreach my $calendar (@$jcalendars) { @@ -331,7 +331,7 @@ sub sync_jcalendars { foreach my $calendar (@$icalendars) { my $data = { name => $calendar->[1], - colour => $calendar->[2], + color => $calendar->[2], isVisible => 1, mayReadFreeBusy => 1, mayReadItems => 1, @@ -1032,7 +1032,7 @@ CREATE TABLE IF NOT EXISTS icalendars ( href TEXT, name TEXT, isReadOnly INTEGER, - colour TEXT, + color TEXT, syncToken TEXT, jcalendarid INTEGER, mtime DATE NOT NULL diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 687514f..7a1584f 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -112,16 +112,12 @@ sub process_request { sub change_cb { my $db = shift; - my $state = shift; + my $states = shift; my $data = { clientId => undef, accountStates => { - $db->accountid() => { - messages => "$state", - threads => "$state", - mailboxes => "$state", - }, + $db->accountid() => $states, }, }; From a30e17fa1eb9a3f85eedf919c409a175faff4c3c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 17:29:36 +1000 Subject: [PATCH 102/331] return blobId rather than url --- JMAP/API.pm | 6 +++--- JMAP/DB.pm | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index dde6ca4..1ea6ba6 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -752,8 +752,8 @@ sub getMessages { $item->{size} = $data->{msgsize}; } - if (_prop_wanted($args, 'rawUrl')) { - $item->{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; + if (_prop_wanted($args, 'blobId')) { + $item->{blobId} = "$msgid"; } push @list, $item; @@ -942,7 +942,7 @@ sub setMessages { foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; - $created->{$cid}{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; + $created->{$cid}{blobId} = "$msgid"; } my @res; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 4045b7e..6b226da 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -395,7 +395,7 @@ sub attachments { push @res, { id => $id, type => $type, - url => "https://proxy.jmap.io/raw/$accountid/$messageid/$id/$filename", + blobId => "$message/$id"; name => $filename, size => length($body), isInline => $isInline, From 7ab43d19a5a5ec0c87543b183e6095461f956049 Mon Sep 17 00:00:00 2001 From: Neil Jenkins Date: Fri, 17 Jul 2015 10:39:21 -0700 Subject: [PATCH 103/331] Update proxy description --- htdocs/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/htdocs/index.html b/htdocs/index.html index 7457483..3a39f0a 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -15,11 +15,11 @@

JMAP Proxy

The JMAP proxy is a work in a progress. It is currently stable enough to test out and get a feel for JMAP in action. Not all methods are implemented yet; see below for what is and isn't available.

-

The eventual goal is to have a proxy you can stick in front of any IMAP server and get a usable JMAP out the other end. For now, the proxy only supports Gmail as it makes use of some of their custom additions to IMAP to provide thread ids and immutable message ids. Source code will be available soon on GitHub and will be MIT licensed.

+

The eventual goal is to have a proxy you can stick in front of any IMAP server and get a usable JMAP out the other end. The proxy supports Gmail and modern IMAP implementations (it MUST implement CONDSTORE). Source code is available on GitHub and is MIT licensed.

-

The web client hooked up for this demo is the FastMail web interface. For now, this is not open source, however we anticipate open sourcing at least the data-model library for other projects to use. The code is unminified, so takes much longer to initially load than the normal FastMail UI, but can be viewed with standard web-dev tools. These can also show the data exchanges being made to load the data and get delta updates.

+

The web client hooked up for this demo is a simple example (no compose, contacts, calendars). It will be open-sourced very soon. The code is unminified, so takes much longer to initially load than it would in a production setting, but can be viewed with standard web-dev tools. These can also show the data exchanges being made to load the data and get delta updates.

-

HTML email is not currently being sanitised in any way, and the service is not running with all of the security measures employed on a production site such as FastMail, so DO NOT USE THIS PROXY FOR ACCOUNTS WITH SENSITIVE DATA.

+

The service is not running with all of the security measures employed on a production site such as FastMail, so DO NOT USE THIS PROXY FOR ACCOUNTS WITH SENSITIVE DATA.

When you create your account, the most recent 50 emails will be downloaded in their entirety, so the first page should be snappy immediately. After that, you are redirected to the landing page. A background task will continue to pull in batches of messages and add them to your account, so you will see older messages appear while you are using the interface.

From 13baa58657e30da4a3632a7119a440ed11a6289a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 02:50:40 -0500 Subject: [PATCH 104/331] server: slow down the backfill a bit --- bin/server.pl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/server.pl b/bin/server.pl index cecf982..299f027 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -586,14 +586,19 @@ sub HandleEventSource { }); } + sub prod_backfill { my $accountid = shift; my $force = shift; return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; - send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { - prod_backfill($accountid, @_); + my $timer; + $timer = AnyEvent->timer(after => 5, cb => sub { + send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { + $timer = undef; + prod_backfill($accountid, @_); + }); }); } From 783beaa2dcb8fd8bded7f897cf49f7922619984d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 03:02:42 -0500 Subject: [PATCH 105/331] bind_part: fix subpart iteration --- JMAP/DB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 6b226da..24f7ae5 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -335,7 +335,7 @@ sub find_part { $type =~ s/;.*//; return ($type, $sub->body()) if ($id eq $target); if ($type =~ m{^multipart/}) { - my @res = find_part($sub, $id); + my @res = find_part($sub, $target, $id); return @res if @res; } } From eb1c6bf6fe443ba13573e94a3484751df7933d46 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 03:06:32 -0500 Subject: [PATCH 106/331] support both types of URL --- JMAP/API.pm | 5 +++++ JMAP/DB.pm | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 1ea6ba6..a2b3138 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -752,6 +752,10 @@ sub getMessages { $item->{size} = $data->{msgsize}; } + if (_prop_wanted($args, 'rawUrl')) { + $item->{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; + } + if (_prop_wanted($args, 'blobId')) { $item->{blobId} = "$msgid"; } @@ -942,6 +946,7 @@ sub setMessages { foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; + $created->{$cid}{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; $created->{$cid}{blobId} = "$msgid"; } diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 24f7ae5..d07e994 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -395,7 +395,8 @@ sub attachments { push @res, { id => $id, type => $type, - blobId => "$message/$id"; + url => "https://proxy.jmap.io/raw/$accountid/$messageid/$id/$filename", # XXX dep + blobId => "$messageid/$id", name => $filename, size => length($body), isInline => $isInline, From b6fecbec9480f3af1d8b46719f4913717e82a383 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 03:24:14 -0500 Subject: [PATCH 107/331] order becomes sortOrder --- JMAP/API.pm | 4 ++-- JMAP/DB.pm | 2 +- JMAP/GmailDB.pm | 10 +++++----- JMAP/ImapDB.pm | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index a2b3138..c7b4834 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -152,7 +152,7 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, order, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, sortOrder, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); my %ids; if ($args->{ids}) { @@ -176,7 +176,7 @@ sub getMailboxes { parentId => ($item->[1] ? "$item->[1]" : undef), name => $item->[2], role => $item->[3], - order => $item->[4], + sortOrder => $item->[4], mustBeOnlyMailbox => $item->[5] ? $JSON::true : $JSON::false, mayDeleteMailbox => $item->[6] ? $JSON::true : $JSON::false, mayRenameMailbox => $item->[7] ? $JSON::true : $JSON::false, diff --git a/JMAP/DB.pm b/JMAP/DB.pm index d07e994..6473e34 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -909,7 +909,7 @@ CREATE TABLE IF NOT EXISTS jmailboxes ( parentid INTEGER, role TEXT, name TEXT, - order INTEGER, + sortOrder INTEGER, mustBeOnly BOOLEAN, mayDelete BOOLEAN, mayRename BOOLEAN, diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 8312ea1..7e41df8 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -253,9 +253,9 @@ sub sync_jmailboxes { my $id = 0; my $parentid = 0; my $name; - my $order = 3; - $order = 2 if $role; - $order = 1 if ($role||'') eq 'inbox'; + my $sortOrder = 3; + $sortOrder = 2 if $role; + $sortOrder = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; @@ -265,7 +265,7 @@ sub sync_jmailboxes { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, order => 4, parentid => $parentid}); + $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentid => $parentid}); $byname{$parentid}{$name} = $id; } } @@ -274,7 +274,7 @@ sub sync_jmailboxes { my %details = ( name => $name, parentid => $parentid, - order => $order, + sortOrder => $sortOrder, mustBeOnly => $ONLY_MAILBOXES{$role||''}, mayDelete => (not $PROTECTED_MAILBOXES{$role||''}), mayRename => (not $NO_RENAME{$role||''}), diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c779879..19e7dd3 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -194,9 +194,9 @@ sub sync_jmailboxes { my $id = 0; my $parentid = 0; my $name; - my $order = 3; - $order = 2 if $role; - $order = 1 if ($role||'') eq 'inbox'; + my $sortOrder = 3; + $sortOrder = 2 if $role; + $sortOrder = 1 if ($role||'') eq 'inbox'; while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; @@ -206,7 +206,7 @@ sub sync_jmailboxes { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, order => 4, parentid => $parentid}); + $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentid => $parentid}); $byname{$parentid}{$name} = $id; } } @@ -215,7 +215,7 @@ sub sync_jmailboxes { my %details = ( name => $name, parentid => $parentid, - order => $order, + sortOrder => $sortOrder, mustBeOnly => 1, mayDelete => 0, mayRename => 0, From 30658509a6207b1a34c7fdb3548a2537117de273 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:14:16 -0500 Subject: [PATCH 108/331] fix Literal for save --- JMAP/Sync/Common.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index f9e056d..44890a0 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -344,15 +344,15 @@ sub imap_append { my $imap = $Self->connect_imap(); - my $r = $imap->append($imapname, $flags, $internaldate, ['Literal', $rfc822]); - die "APPEND FAILED $r" unless lc($r) eq 'ok'; + my $r = $imap->append($imapname, $flags, $internaldate, {'Literal' => $rfc822}); + die "APPEND FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'appenduid'); # what's with that?? my $uid = $imap->get_response_code('appenduid'); # XXX - fetch the x-gm-msgid or envelope from the server so we know the # the ID that the server gave this message - return ['append', $imapname, $uid]; + return ['append', $imapname, @$uid]; } 1; From b85838d3beabc8ba93c8d009376e722af3f0514b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:15:07 -0500 Subject: [PATCH 109/331] JMAP: fix destroy --- JMAP/API.pm | 8 +++---- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 62 +++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index c7b4834..eddd9c3 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -935,12 +935,12 @@ sub setMessages { my $create = $args->{create} || {}; my $update = $args->{update} || {}; - my $delete = $args->{delete} || []; + my $destroy = $args->{destroy} || []; # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_messages($create); my ($updated, $notUpdated) = $Self->{db}->update_messages($update); - my ($deleted, $notDeleted) = $Self->{db}->delete_messages($delete); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_messages($destroy); $Self->{db}->sync_imap(); @@ -961,8 +961,8 @@ sub setMessages { notCreated => $notCreated, updated => $updated, notUpdated => $notUpdated, - deleted => $deleted, - notDeleted => $notDeleted, + destroyed => $destroyed, + notDestroyed => $notDestroyed, }]; return @res; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 6473e34..21d92d5 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -603,7 +603,7 @@ sub update_messages { die "Virtual method"; } -sub delete_messages { +sub destroy_messages { my $Self = shift; die "Virtual method"; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 19e7dd3..0525bf6 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -16,6 +16,7 @@ use Encode::MIME::Header; use Digest::SHA qw(sha1_hex); use AnyEvent; use AnyEvent::Socket; +use Date::Format; use Data::Dumper; use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; @@ -712,7 +713,7 @@ sub changed_record { my $Self = shift; my ($folder, $uid, $flaglist, $labellist) = @_; - my $flags = encode_json([sort @$flaglist]); + my $flags = encode_json([grep { lc $_ ne '\\recent' } sort @$flaglist]); my $labels = encode_json([sort @$labellist]); my ($msgid) = $Self->{dbh}->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); @@ -722,6 +723,41 @@ sub changed_record { $Self->apply_data($msgid, $flaglist, $labellist); } +sub import_message { + my $Self = shift; + my $message = shift; + my $mailboxIds = shift; + my %flags = @_; + + my $dbh = $Self->dbh(); + my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); + my %foldermap = map { $_->[0] => $_ } @$folderdata; + my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; + + # store to the first named folder - we can use labels on gmail to add to other folders later. + my $imapname = $jmailmap{$mailboxIds->[0]}[1]; + + my @flags; + push @flags, "\\Seen" unless $flags{isUnread}; + push @flags, "\\Answered" if $flags{isAnswered}; + push @flags, "\\Flagged" if $flags{isFlagged}; + + my $internaldate = time(); # XXX - allow setting? + my $date = Date::Format::time2str('%e-%b-%Y %T %z', $internaldate); + + my $data = $Self->backend_cmd('imap_append', $imapname, "(@flags)", $date, $message); + warn Dumper($data); + # XXX - compare $data->[2] with uidvalidity + my $uid = $data->[3]; + + # make sure we're up to date: XXX - imap only + $Self->do_folder($jmailmap{$mailboxIds->[0]}[0], $mailboxIds->[0]); + + my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$mailboxIds->[0]}[0], $uid); + + return ($msgid, $thrid); +} + sub update_messages { my $Self = shift; my $changes = shift; @@ -788,21 +824,21 @@ sub update_messages { return (\@changed, \%notchanged); } -sub delete_messages { +sub destroy_messages { my $Self = shift; my $ids = shift; my $dbh = $Self->{dbh}; - my %deletemap; - my %notdeleted; + my %destroymap; + my %notdestroyed; foreach my $msgid (@$ids) { my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); if ($ifolderid and $uid) { - $deletemap{$ifolderid}{$uid} = $msgid; + $destroymap{$ifolderid}{$uid} = $msgid; } else { - $notdeleted{$msgid} = "No such message on server"; + $notdestroyed{$msgid} = "No such message on server"; } } @@ -810,19 +846,19 @@ sub delete_messages { my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; - my @deleted; - foreach my $ifolderid (keys %deletemap) { + my @destroyed; + foreach my $ifolderid (keys %destroymap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; my $uidvalidity = $foldermap{$ifolderid}[2]; unless ($imapname) { - $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; + $notdestroyed{$_} = "No folder" for values %{$destroymap{$ifolderid}}; } - my $uids = [sort keys %{$deletemap{$ifolderid}}]; + my $uids = [sort keys %{$destroymap{$ifolderid}}]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder - push @deleted, values %{$deletemap{$ifolderid}}; + push @destroyed, values %{$destroymap{$ifolderid}}; } - return (\@deleted, \%notdeleted); + return (\@destroyed, \%notdestroyed); } sub deleted_record { @@ -841,7 +877,7 @@ sub new_record { my $Self = shift; my ($ifolderid, $uid, $flaglist, $labellist, $envelope, $internaldate, $msgid, $thrid, $size) = @_; - my $flags = encode_json([sort @$flaglist]); + my $flags = encode_json([grep { lc $_ ne '\\recent' } sort @$flaglist]); my $labels = encode_json([sort @$labellist]); my $data = { From 100e00b1e1c14154ff7a442687ef3866c0fa2a31 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:29:31 -0500 Subject: [PATCH 110/331] import: correct folder --- JMAP/ImapDB.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 0525bf6..52a374b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -741,6 +741,7 @@ sub import_message { push @flags, "\\Seen" unless $flags{isUnread}; push @flags, "\\Answered" if $flags{isAnswered}; push @flags, "\\Flagged" if $flags{isFlagged}; + push @flags, "\\Draft" if $flags{isDraft}; my $internaldate = time(); # XXX - allow setting? my $date = Date::Format::time2str('%e-%b-%Y %T %z', $internaldate); @@ -751,7 +752,7 @@ sub import_message { my $uid = $data->[3]; # make sure we're up to date: XXX - imap only - $Self->do_folder($jmailmap{$mailboxIds->[0]}[0], $mailboxIds->[0]); + $Self->do_folder($jmailmap{$mailboxIds->[0]}[0], $jmailmap{$mailboxIds->[0]}[2]); my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$mailboxIds->[0]}[0], $uid); From b5878234b5ec7e1763a7883a226bcde986c723d9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:45:45 -0500 Subject: [PATCH 111/331] don't require condstore --- bin/apiendpoint.pl | 2 -- bin/server.pl | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 7a1584f..db3c643 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -312,8 +312,6 @@ sub handle_signup { my $capa = $imap->capability(); $imap->logout(); - die "THIS PROXY REQUIRES A SERVER THAT SUPPORTS CONDSTORE: $detail->[0]" unless $capa->{condstore}; - my $dbh = accountsdb(); my ($existing, $type) = $dbh->selectrow_array("SELECT accountid, type FROM accounts WHERE email = ?", {}, $detail->[1]); if ($existing) { diff --git a/bin/server.pl b/bin/server.pl index 299f027..9e8c86d 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -94,12 +94,10 @@ sub setup_examine { my $edgecb = shift; my $imap = shift; - $imap->send_cmd('ENABLE CONDSTORE', sub { - $imap->send_cmd('SELECT "INBOX"', sub { - $edgecb->("initial"); + $imap->send_cmd('SELECT "INBOX"', sub { + $edgecb->("initial"); - setup_idle($edgecb, $imap); - }); + setup_idle($edgecb, $imap); }); } From 42ad57b794066ac562e41f45ff214eb1541afa64 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:49:46 -0500 Subject: [PATCH 112/331] Yahoo endpoint --- JMAP/Sync/Yahoo.pm | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 JMAP/Sync/Yahoo.pm diff --git a/JMAP/Sync/Yahoo.pm b/JMAP/Sync/Yahoo.pm new file mode 100644 index 0000000..3d232e8 --- /dev/null +++ b/JMAP/Sync/Yahoo.pm @@ -0,0 +1,22 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Sync::Yahoo; +use base qw(JMAP::Sync::Standard); + +sub new { + my $class = shift; + my $auth = shift; + my %a = ( + imapserver => 'imap.mail.yahoo.com', + smtpserver => 'smtp.mail.yahoo.com', + calurl => 'https://caldav.calendar.yahoo.com', + addressbookurl => 'https://carddav.address.yahoo.com', + %$auth, + ); + return JMAP::Sync::Standard->new(\%a); +} + +1; From adf784501a1d7201350c7d1ca09a4e5d358bddfb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 12:50:39 -0500 Subject: [PATCH 113/331] ImapDB: use Yahoo --- JMAP/ImapDB.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 52a374b..33cb8fd 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -21,6 +21,7 @@ use Data::Dumper; use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; use JMAP::Sync::Fastmail; +use JMAP::Sync::Yahoo; our $TAG = 1; @@ -98,6 +99,8 @@ sub backend_cmd { $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $auth->[1]"; } elsif ($config->{hostname} eq 'mail.messagingengine.com') { $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $auth->[1]"; + } elsif ($config->{hostname} eq 'imap.mail.yahoo.com') { + $backend = JMAP::Sync::Yahoo->new($config) || die "failed to setup $auth->[1]"; } else { die "UNKNOWN ID $config->{username} ($config->{hostname})"; } From dbeb7df88a11fd8bf5f85e5d52f5a64a2d99cab2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 04:01:05 +1000 Subject: [PATCH 114/331] fix up the parentId and other fields --- JMAP/API.pm | 2 +- JMAP/DB.pm | 22 +++++++++++----------- JMAP/GmailDB.pm | 16 ++++++++-------- JMAP/ImapDB.pm | 28 +++++++++++++++------------- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index eddd9c3..28230d1 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -152,7 +152,7 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentid, name, role, sortOrder, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); my %ids; if ($args->{ids}) { diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 21d92d5..c555288 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -138,7 +138,7 @@ sub get_mailboxes { my $Self = shift; confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{mailboxes}) { - $Self->{t}{mailboxes} = $Self->dbh->selectall_hashref("SELECT jmailboxid, jmodseq, label, name, parentid, nummessages, numumessages, numthreads, numuthreads, active FROM jmailboxes", 'jmailboxid', {Slice => {}}); + $Self->{t}{mailboxes} = $Self->dbh->selectall_hashref("SELECT jmailboxid, jmodseq, label, name, parentId, nummessages, numumessages, numthreads, numuthreads, active FROM jmailboxes", 'jmailboxid', {Slice => {}}); } return $Self->{t}{mailboxes}; } @@ -151,16 +151,16 @@ sub get_mailbox { sub add_mailbox { my $Self = shift; - my ($name, $label, $parentid) = @_; + my ($name, $label, $parentId) = @_; my $mailboxes = $Self->get_mailboxes(); - confess("ALREADY EXISTS $name in $parentid") if grep { $_->{name} eq $name and $_->{parentid} == $parentid } values %$mailboxes; + confess("ALREADY EXISTS $name in $parentId") if grep { $_->{name} eq $name and $_->{parentId} == $parentId } values %$mailboxes; my $data = { name => $name, label => $label, - parentid => $parentid, + parentId => $parentId, nummessages => 0, numthreads => 0, numumessages => 0, @@ -906,17 +906,17 @@ EOF $dbh->do(<dbh(); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); - my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentid, role, active FROM jmailboxes"); + my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); my %jbyid; my %roletoid; @@ -251,7 +251,7 @@ sub sync_jmailboxes { my @bits = split "[$folder->[1]]", $fname; my $role = $ROLE_MAP{lc $folder->[3]} || $ROLE_MAP{lc $fname}; my $id = 0; - my $parentid = 0; + my $parentId = 0; my $name; my $sortOrder = 3; $sortOrder = 2 if $role; @@ -259,21 +259,21 @@ sub sync_jmailboxes { while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; - $parentid = $id; - $id = $byname{$parentid}{$name}; + $parentId = $id; + $id = $byname{$parentId}{$name}; unless ($id) { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentid => $parentid}); - $byname{$parentid}{$name} = $id; + $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentId => $parentid}); + $byname{$parentId}{$name} = $id; } } } next unless $name; my %details = ( name => $name, - parentid => $parentid, + parentId => $parentid, sortOrder => $sortOrder, mustBeOnly => $ONLY_MAILBOXES{$role||''}, mayDelete => (not $PROTECTED_MAILBOXES{$role||''}), @@ -302,7 +302,7 @@ sub sync_jmailboxes { } else { $id = $Self->dmake('jmailboxes', {role => $role, %details}); - $byname{$parentid}{$name} = $id; + $byname{$parentId}{$name} = $id; $roletoid{$role} = $id if $role; } } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 33cb8fd..65fac7a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -177,7 +177,7 @@ sub sync_jmailboxes { $Self->begin(); my $dbh = $Self->dbh(); my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); - my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentid, role, active FROM jmailboxes"); + my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); my %jbyid; my %roletoid; @@ -196,7 +196,7 @@ sub sync_jmailboxes { my @bits = split "[$folder->[1]]", $fname; my $role = $ROLE_MAP{lc $fname}; my $id = 0; - my $parentid = 0; + my $parentId = 0; my $name; my $sortOrder = 3; $sortOrder = 2 if $role; @@ -204,29 +204,31 @@ sub sync_jmailboxes { while (my $item = shift @bits) { $seen{$id} = 1 if $id; $name = $item; - $parentid = $id; - $id = $byname{$parentid}{$name}; + $parentId = $id; + $id = $byname{$parentId}{$name}; unless ($id) { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentid => $parentid}); - $byname{$parentid}{$name} = $id; + $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentId => $parentId}); + $byname{$parentId}{$name} = $id; } } } next unless $name; my %details = ( name => $name, - parentid => $parentid, + parentId => $parentId, sortOrder => $sortOrder, - mustBeOnly => 1, + mustBeOnlyMailbox => 1, mayDelete => 0, + mayReadItems => 1, + mayAddItems => 1, + mayRemoveItems => 1, + mayCreateChild => 0, +>>>>>>> WIP mayRename => 0, - mayAdd => 1, - mayRemove => 1, - mayChild => 0, - mayRead => 1, + mayDelete => 0, ); if ($id) { if ($role and $roletoid{$role} and $roletoid{$role} != $id) { @@ -247,7 +249,7 @@ sub sync_jmailboxes { } else { $id = $Self->dmake('jmailboxes', {role => $role, %details}); - $byname{$parentid}{$name} = $id; + $byname{$parentId}{$name} = $id; $roletoid{$role} = $id if $role; } } From 4b3156618c736023903b6cb40064df51d97e2fe4 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:06:36 -0500 Subject: [PATCH 115/331] debug imap --- bin/apiendpoint.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index db3c643..06517e6 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -3,7 +3,7 @@ use lib '/home/jmap/jmap-perl'; package JMAP::Backend; -#use Mail::IMAPTalk qw(:trace); +use Mail::IMAPTalk qw(:trace); use Carp qw(verbose); use strict; From d63978fc67b0a0bb7e7ca3c8a1bcedb4d54e9e5d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:07:19 -0500 Subject: [PATCH 116/331] missed one --- JMAP/GmailDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 7a8ef62..7cabaab 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -265,7 +265,7 @@ sub sync_jmailboxes { if (@bits) { # need to create intermediate folder ... # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentId => $parentid}); + $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentId => $parentId}); $byname{$parentId}{$name} = $id; } } @@ -273,7 +273,7 @@ sub sync_jmailboxes { next unless $name; my %details = ( name => $name, - parentId => $parentid, + parentId => $parentId, sortOrder => $sortOrder, mustBeOnly => $ONLY_MAILBOXES{$role||''}, mayDelete => (not $PROTECTED_MAILBOXES{$role||''}), From 179eb57b8f495f58684cbce0a350bdafc1b48dda Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:07:44 -0500 Subject: [PATCH 117/331] another one --- JMAP/ImapDB.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 65fac7a..848688a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -226,7 +226,6 @@ sub sync_jmailboxes { mayAddItems => 1, mayRemoveItems => 1, mayCreateChild => 0, ->>>>>>> WIP mayRename => 0, mayDelete => 0, ); From 327d8b07f4961552084f7a0025e586edbeb93326 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:13:58 -0500 Subject: [PATCH 118/331] pretend to be google --- JMAP/API.pm | 14 +++++++------- JMAP/Sync/Yahoo.pm | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 28230d1..532ffc4 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -152,7 +152,7 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnly, mayDelete, mayRename, mayAdd, mayRemove, mayChild, mayRead FROM jmailboxes WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnlyMailbox, mayReadItems, mayAddItems, mayRemoveItems, mayCreateChild, mayRename, mayDelete FROM jmailboxes WHERE active = 1"); my %ids; if ($args->{ids}) { @@ -178,12 +178,12 @@ sub getMailboxes { role => $item->[3], sortOrder => $item->[4], mustBeOnlyMailbox => $item->[5] ? $JSON::true : $JSON::false, - mayDeleteMailbox => $item->[6] ? $JSON::true : $JSON::false, - mayRenameMailbox => $item->[7] ? $JSON::true : $JSON::false, - mayAddMessages => $item->[8] ? $JSON::true : $JSON::false, - mayRemoveMessages => $item->[9] ? $JSON::true : $JSON::false, - mayCreateChild => $item->[10] ? $JSON::true : $JSON::false, - mayReadMessageList => $item->[11] ? $JSON::true : $JSON::false, + mayReadItems => $item->[6] ? $JSON::true : $JSON::false, + mayAddItems => $item->[7] ? $JSON::true : $JSON::false, + mayRemoveItems => $item->[8] ? $JSON::true : $JSON::false, + mayCreateChild => $item->[9] ? $JSON::true : $JSON::false, + mayRename => $item->[10] ? $JSON::true : $JSON::false, + mayDelete => $item->[11] ? $JSON::true : $JSON::false, ); foreach my $key (keys %rec) { diff --git a/JMAP/Sync/Yahoo.pm b/JMAP/Sync/Yahoo.pm index 3d232e8..8ad0303 100644 --- a/JMAP/Sync/Yahoo.pm +++ b/JMAP/Sync/Yahoo.pm @@ -6,6 +6,28 @@ use warnings; package JMAP::Sync::Yahoo; use base qw(JMAP::Sync::Standard); +sub connect_contacts { + my $Self = shift; + + return unless $Self->{auth}{addressbookurl}; + + if ($Self->{contacts}) { + $Self->{lastused} = time(); + return $Self->{contacts}; + } + + $Self->{contacts} = Net::CardDAVTalk->new( + user => $Self->{auth}{username}, + password => $Self->{auth}{password}, + url => $Self->{auth}{addressbookurl}, + expandurl => 1, + is_google => 1, + ); + + return $Self->{contacts}; +} + + sub new { my $class = shift; my $auth = shift; From c24642eb1d7b3239b48a45618c347a9a6313fef1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:40:09 -0500 Subject: [PATCH 119/331] handle capabilities for yahoo --- JMAP/Sync/Common.pm | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 44890a0..ab22de7 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -125,10 +125,15 @@ sub imap_status { my $imap = $Self->connect_imap(); - $imap->unselect(); + if ($imap->capability->{unselect}) { + $imap->unselect(); + } + else { + $imap->close(); + } my @fields = qw(uidvalidity uidnext messages); - push @fields, "highestmodseq" if $imap->capability->{condstore}; + push @fields, "highestmodseq" if ($imap->capability->{condstore} or $imap->capability->{xmyhighestmodseq}); my $data = $imap->multistatus("(@fields)", @$folders); return $data; @@ -327,7 +332,7 @@ sub imap_fetch { push @flags, @{$item->[2]} if $item->[2]; next if ($highestmodseq and $item->[3] and $item->[3] == $highestmodseq); my @extra; - push @extra, "(changedsince $item->[3])" if $item->[3]; + push @extra, "(changedsince $item->[3])" if ($item->[3] and $imap->capability->{condstore}); my $data = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; $res{$key} = [$item, $data]; } From 5d415c6fa0968774abf9d1d3f33604fd5b643732 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 13:41:02 -0500 Subject: [PATCH 120/331] not my, ym --- JMAP/Sync/Common.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index ab22de7..1a8e9b3 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -133,7 +133,7 @@ sub imap_status { } my @fields = qw(uidvalidity uidnext messages); - push @fields, "highestmodseq" if ($imap->capability->{condstore} or $imap->capability->{xmyhighestmodseq}); + push @fields, "highestmodseq" if ($imap->capability->{condstore} or $imap->capability->{xymhighestmodseq}); my $data = $imap->multistatus("(@fields)", @$folders); return $data; From ec2592c339c663933e63a049a4a9f1969980fd63 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 15:48:24 -0500 Subject: [PATCH 121/331] MIME for names --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index c555288..8094f01 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -274,7 +274,7 @@ sub parse_emails { my $emails = shift; my @addrs = eval { Email::Address->parse($emails) }; - return map { { name => Encode::decode_utf8($_->name()), email => $_->address() } } @addrs; + return map { { name => decode('MIME-Header', $_->name()), email => $_->address() } } @addrs; } sub parse_message { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 848688a..8eda264 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -11,8 +11,6 @@ use Date::Parse; use JSON::XS qw(encode_json decode_json); use Data::UUID::LibUUID; use OAuth2::Tiny; -use Encode; -use Encode::MIME::Header; use Digest::SHA qw(sha1_hex); use AnyEvent; use AnyEvent::Socket; From 5eca8c2bff7a4a2eef50ddd06ae52f4381628fb5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 17 Jul 2015 15:57:49 -0500 Subject: [PATCH 122/331] fix naming --- JMAP/DB.pm | 3 ++- JMAP/ImapDB.pm | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 8094f01..6c2fbae 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -73,6 +73,7 @@ sub begin { confess("ALREADY IN TRANSACTION") if $Self->{t}; my $accountid = $Self->accountid(); $Self->{t} = {}; + # we need this because sqlite locking isn't as robust as you might hope $Self->{t}{lock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); $Self->dbh->begin_work(); } @@ -274,7 +275,7 @@ sub parse_emails { my $emails = shift; my @addrs = eval { Email::Address->parse($emails) }; - return map { { name => decode('MIME-Header', $_->name()), email => $_->address() } } @addrs; + return map { { name => Encode::decode('MIME-Header', $_->name()), email => $_->address() } } @addrs; } sub parse_message { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 8eda264..46ff761 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -12,6 +12,8 @@ use JSON::XS qw(encode_json decode_json); use Data::UUID::LibUUID; use OAuth2::Tiny; use Digest::SHA qw(sha1_hex); +use Encode; +use Encode::MIME::Header; use AnyEvent; use AnyEvent::Socket; use Date::Format; @@ -26,13 +28,27 @@ our $TAG = 1; # special use or name magic my %ROLE_MAP = ( 'inbox' => 'inbox', + 'drafts' => 'drafts', + 'draft' => 'drafts', + 'draft messages' => 'drafts', + + 'bulk' => 'spam', + 'bulk mail' => 'spam', 'junk' => 'spam', - 'deleted messages' => 'trash', + 'junk mail' => 'spam', + 'spam' => 'spam', + 'spam mail' => 'spam', + 'spam messages' => 'spam', + 'archive' => 'archive', - 'sent messages' => 'sent', + 'sent' => 'sent', 'sent items' => 'sent', + 'sent messages' => 'sent', + + 'deleted messages' => 'trash', 'trash' => 'trash', + '\\inbox' => 'inbox', '\\trash' => 'trash', '\\sent' => 'sent', @@ -949,7 +965,7 @@ sub apply_data { sub _envelopedata { my $data = shift; my $envelope = decode_json($data); - my $encsub = decode('MIME-Header', $envelope->{Subject}); + my $encsub = Encode::decode('MIME-Header', $envelope->{Subject}); return ( msgsubject => $encsub, msgfrom => $envelope->{From}, From 01315e9cbd828fd0711280f5250aa17944263475 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 00:35:46 -0400 Subject: [PATCH 123/331] fix some push stuff and the set APIs --- JMAP/API.pm | 164 ++++++++++++++++++++++++++++++++++++++++++++- JMAP/DB.pm | 6 +- JMAP/ImapDB.pm | 3 +- JMAP/Sync/Yahoo.pm | 2 +- bin/apiendpoint.pl | 10 +-- 5 files changed, 174 insertions(+), 11 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 532ffc4..d52fc07 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -954,9 +954,7 @@ sub setMessages { push @res, ['messagesSet', { accountId => $accountid, oldState => undef, # proxy can't guarantee the old state - # this is actually the state BEFORE the changes, so the client will get a spurious duplicate of the change, but there's - # no nice way to avoid that... - newState => "$user->{jhighestmodseq}", + newState => undef, # or give a new state created => $created, notCreated => $notCreated, updated => $updated, @@ -1921,4 +1919,164 @@ sub getContactGroupUpdates { return @res; } +sub setContactGroups { + my $Self = shift; + my $args = shift; + + $Self->begin(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return $Self->_transError(['error', {type => 'accountNotFound'}]) + if ($args->{accountId} and $args->{accountId} ne $accountid); + + $Self->commit(); + + my $create = $args->{create} || {}; + my $update = $args->{update} || {}; + my $destroy = $args->{destroy} || []; + + # XXX - idmap support + my ($created, $notCreated) = $Self->{db}->create_contact_groups($create); + my ($updated, $notUpdated) = $Self->{db}->update_contact_groups($update); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contact_groups($destroy); + + $Self->{db}->sync_addressbooks(); + + my @res; + push @res, ['contactGroupsSet', { + accountId => $accountid, + oldState => undef, # proxy can't guarantee the old state + newState => undef, # or give a new state + created => $created, + notCreated => $notCreated, + updated => $updated, + notUpdated => $notUpdated, + destroyed => $destroyed, + notDestroyed => $notDestroyed, + }]; + + return @res; +} + +sub setContacts { + my $Self = shift; + my $args = shift; + + $Self->begin(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return $Self->_transError(['error', {type => 'accountNotFound'}]) + if ($args->{accountId} and $args->{accountId} ne $accountid); + + $Self->commit(); + + my $create = $args->{create} || {}; + my $update = $args->{update} || {}; + my $destroy = $args->{destroy} || []; + + # XXX - idmap support + my ($created, $notCreated) = $Self->{db}->create_contacts($create); + my ($updated, $notUpdated) = $Self->{db}->update_contacts($update); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contacts($destroy); + + $Self->{db}->sync_addressbooks(); + + my @res; + push @res, ['contactsSet', { + accountId => $accountid, + oldState => undef, # proxy can't guarantee the old state + newState => undef, # or give a new state + created => $created, + notCreated => $notCreated, + updated => $updated, + notUpdated => $notUpdated, + destroyed => $destroyed, + notDestroyed => $notDestroyed, + }]; + + return @res; +} + +sub setCalendarEvents { + my $Self = shift; + my $args = shift; + + $Self->begin(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return $Self->_transError(['error', {type => 'accountNotFound'}]) + if ($args->{accountId} and $args->{accountId} ne $accountid); + + $Self->commit(); + + my $create = $args->{create} || {}; + my $update = $args->{update} || {}; + my $destroy = $args->{destroy} || []; + + # XXX - idmap support + my ($created, $notCreated) = $Self->{db}->create_calendar_events($create); + my ($updated, $notUpdated) = $Self->{db}->update_calendar_events($update); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendar_events($destroy); + + $Self->{db}->sync_calendars(); + + my @res; + push @res, ['calendarEventsSet', { + accountId => $accountid, + oldState => undef, # proxy can't guarantee the old state + newState => undef, # or give a new state + created => $created, + notCreated => $notCreated, + updated => $updated, + notUpdated => $notUpdated, + destroyed => $destroyed, + notDestroyed => $notDestroyed, + }]; + + return @res; +} + +sub setCalendars { + my $Self = shift; + my $args = shift; + + $Self->begin(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return $Self->_transError(['error', {type => 'accountNotFound'}]) + if ($args->{accountId} and $args->{accountId} ne $accountid); + + $Self->commit(); + + my $create = $args->{create} || {}; + my $update = $args->{update} || {}; + my $destroy = $args->{destroy} || []; + + # XXX - idmap support + my ($created, $notCreated) = $Self->{db}->create_calendars($create); + my ($updated, $notUpdated) = $Self->{db}->update_calendars($update); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendars($destroy); + + $Self->{db}->sync_calendars(); + + my @res; + push @res, ['calendarsSet', { + accountId => $accountid, + oldState => undef, # proxy can't guarantee the old state + newState => undef, # or give a new state + created => $created, + notCreated => $notCreated, + updated => $updated, + notUpdated => $notUpdated, + destroyed => $destroyed, + notDestroyed => $notDestroyed, + }]; + + return @res; +} + 1; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 6c2fbae..4116509 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -86,7 +86,11 @@ sub commit { # push an update if anything to tell.. if ($t->{modseq} and $Self->{change_cb}) { - $Self->{change_cb}->($Self, "$t->{modseq}"); # aka stateString + $Self->{change_cb}->($Self, { + Mailbox => "$t->{modseq}", + Thread => "$t->{modseq}", + Message => "$t->{modseq}", + }); } } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 46ff761..45748b4 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -22,6 +22,7 @@ use JMAP::Sync::Gmail; use JMAP::Sync::ICloud; use JMAP::Sync::Fastmail; use JMAP::Sync::Yahoo; +use JMAP::Sync::Standard; our $TAG = 1; @@ -116,7 +117,7 @@ sub backend_cmd { } elsif ($config->{hostname} eq 'imap.mail.yahoo.com') { $backend = JMAP::Sync::Yahoo->new($config) || die "failed to setup $auth->[1]"; } else { - die "UNKNOWN ID $config->{username} ($config->{hostname})"; + $backend = JMAP::Sync::Standard->new($config) || die "failed to setup $auth->[1]"; } $Self->{backend} = $backend; } diff --git a/JMAP/Sync/Yahoo.pm b/JMAP/Sync/Yahoo.pm index 8ad0303..fd00fce 100644 --- a/JMAP/Sync/Yahoo.pm +++ b/JMAP/Sync/Yahoo.pm @@ -38,7 +38,7 @@ sub new { addressbookurl => 'https://carddav.address.yahoo.com', %$auth, ); - return JMAP::Sync::Standard->new(\%a); + return $class->SUPER::new(\%a); } 1; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 06517e6..6c065ff 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -115,8 +115,7 @@ sub change_cb { my $states = shift; my $data = { - clientId => undef, - accountStates => { + changed => { $db->accountid() => $states, }, }; @@ -139,10 +138,11 @@ sub handle_getstate { $db->commit(); my $data = { - clientId => undef, - accountStates => { + changed => { $db->accountid() => { - mailState => "$state", + Mailbox => "$state", + Thread => "$state", + Message => "$state", }, }, }; From edfe3d7a4d4742327de39bea6ef95b2e39eeafd5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 15:38:47 +1000 Subject: [PATCH 124/331] massive contacts/calendars writing experiment --- JMAP/ImapDB.pm | 335 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 321 insertions(+), 14 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 45748b4..04c8926 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -355,11 +355,11 @@ sub sync_jcalendars { isVisible => 1, mayReadFreeBusy => 1, mayReadItems => 1, - mayAddItems => 0, - mayModifyItems => 0, - mayRemoveItems => 0, - mayDelete => 0, - mayRename => 0, + mayAddItems => 1, + mayModifyItems => 1, + mayRemoveItems => 1, + mayDelete => 1, + mayRename => 1, }; if ($calendar->[3] && $jbyid{$calendar->[3]}) { $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); @@ -396,15 +396,16 @@ sub do_calendar { foreach my $resource (keys %$events) { my $data = delete $res{$resource}; my $raw = $events->{$resource}; + my $event = $Self->parse_event($raw); + my $uid = $event->{uid}; if ($data) { my $id = $data->[0]; next if $raw eq $data->[2]; - $Self->dmaybeupdate('ievents', {content => $raw, resource => $resource}, {ieventid => $id}); + $Self->dmaybeupdate('ievents', {uid => $uid, content => $raw, resource => $resource}, {ieventid => $id}); } else { - $Self->dinsert('ievents', {content => $raw, resource => $resource}); + $Self->dinsert('ievents', {uid => $uid, content => $raw, resource => $resource}); } - my $event = $Self->parse_event($raw); $Self->set_event($jcalendarid, $event); } @@ -492,9 +493,9 @@ sub sync_jaddressbooks { name => $addressbook->[1], isVisible => 1, mayReadItems => 1, - mayAddItems => 0, - mayModifyItems => 0, - mayRemoveItems => 0, + mayAddItems => 1, + mayModifyItems => 1, + mayRemoveItems => 1, mayDelete => 0, mayRename => 0, }; @@ -534,15 +535,16 @@ sub do_addressbook { foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; my $raw = $cards->{$resource}; + my $card = $Self->parse_card($raw); + my $uid = $card->{uid}; if ($data) { my $id = $data->[0]; next if $raw eq $data->[2]; - $Self->dmaybeupdate('icards', {content => $raw, resource => $resource}, {icardid => $id}); + $Self->dmaybeupdate('icards', {uid => $uid, content => $raw, resource => $resource}, {icardid => $id}); } else { - $Self->dinsert('icards', {content => $raw, resource => $resource}); + $Self->dinsert('icards', {uid => $uid, content => $raw, resource => $resource}); } - my $card = $Self->parse_card($raw); $Self->set_card($jaddressbookid, $card); } @@ -1026,6 +1028,305 @@ sub fill_messages { return \%result; } +sub create_calendar_events { + my $Self = shift; + my $new = shift; + + my $dbh = $Self->{dbh}; + + my %createmap; + my %notcreated; + foreach my $cid (keys %$new) { + my $calendar = $new->{$cid}; + my ($href) = $dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendar->{calendarId}); + unless ($href) { + $notcreated{$msgid} = "No such calendar on server"; + next; + } + my $uid = new_uuid_string(); + + $Self->backend_cmd('new_event', $href, {%$calendar, uid => $uid}); + $createmap{$cid} = { id => $uid }; + } + + return (\%createmap, \%notcreated); +} + +sub update_calendar_events { + my $Self = shift; + my $update = shift; + + my $dbh = $Self->{dbh}; + + my @updated; + my %notupdated; + foreach my $uid (keys %$update) { + my $calendar = $updated->{$uid}; + my ($href) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uuid); + unless ($href) { + $notcreated{$uid} = "No such calendar on server"; + next; + } + + $Self->backend_cmd('update_event', $href, $resource, $calendar); + push @updated, $uid; + } + + return (\@updated, \%notupdated); +} + +sub destroy_calendar_events { + my $Self = shift; + my $destroy = shift; + + my $dbh = $Self->{dbh}; + + my @destroyed; + my %notdestroyed; + foreach my $uid (@$destroy); + my ($href) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uuid); + unless ($href) { + $notdestroyed{$uid} = "No such calendar on server"; + next; + } + + $Self->backend_cmd('delete_event', $href, $resource); + push @destroyed, $uid; + } + + return (\@destroyed, \%notdestroyed); +} + +sub update_contact_groups { + my $Self = shift; + my $changes = shift; + + my $dbh = $Self->{dbh}; + + my @updated; + my %notchanged; + foreach my $carduid (keys %$changes) { + my $contact = $changes->{$carduid}; + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notchanged{$msgid} = "No such card on server"; + next; + } + my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); + $card->VKind('group'); + $card->VName($contact->{name}) if exists $contact->{name}; + $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; + + $Self->backend_cmd('set_card', $href, $resource, $card); + push @updated, $carduid; + } + + return (\@updated, \%notchanged); +} + +sub destroy_contact_groups { + my $Self = shift; + my $destroy = shift; + + my $dbh = $Self->{dbh}; + + my @destroyed; + my %notdestroyed; + foreach my $carduid (@$destroy) { + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notdestroyed{$msgid} = "No such card on server"; + next; + } + $Self->backend_cmd('set_card', $href, $resource, undef); + push @destroyed, $carduid; + } + + return (\@destroyed, \%notdestroyed); +} + +sub create_contact_groups { + my $Self = shift; + my $new = shift; + + my $dbh = $Self->{dbh}; + + my %createmap; + my %notcreated; + foreach my $cid (keys %$new) { + my $contact = $new->{$cid}; + my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + unless ($href) { + $notcreated{$msgid} = "No such addressbook on server"; + next; + } + my ($card) = Net::CardDAVTalk::VCard->new(); + my $uid = new_uuid_string(); + $card->uid($uid); + $card->VKind('group'); + $card->VName($contact->{name}) if exists $contact->{name}; + $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; + + $Self->backend_cmd('set_card', $href, $resource, $card); + $createmap{$cid} = { id => $uid }; + } + + return (\%createmap, \%notcreated); +} + +sub update_contact_groups { + my $Self = shift; + my $changes = shift; + + my $dbh = $Self->{dbh}; + + my @updated; + my %notchanged; + foreach my $carduid (keys %$changes) { + my $contact = $changes->{$carduid}; + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notchanged{$msgid} = "No such card on server"; + next; + } + my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); + $card->VKind('group'); + $card->VName($contact->{name}) if exists $contact->{name}; + $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; + + $Self->backend_cmd('set_card', $href, $resource, $card); + push @updated, $carduid; + } + + return (\@updated, \%notchanged); +} + +sub destroy_contact_groups { + my $Self = shift; + my $destroy = shift; + + my $dbh = $Self->{dbh}; + + my @destroyed; + my %notdestroyed; + foreach my $carduid (@$destroy) { + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notdestroyed{$msgid} = "No such card on server"; + next; + } + $Self->backend_cmd('set_card', $href, $resource, undef); + push @destroyed, $carduid; + } + + return (\@destroyed, \%notdestroyed); +} + +sub create_contacts { + my $Self = shift; + my $new = shift; + + my $dbh = $Self->{dbh}; + + my %createmap; + my %notcreated; + foreach my $cid (keys %$new) { + my $contact = $new->{$cid}; + my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + unless ($href) { + $notcreated{$msgid} = "No such addressbook on server"; + next; + } + my ($card) = Net::CardDAVTalk::VCard->new(); + my $uid = new_uuid_string(); + $card->uid($uid); + $card->VLastName($contact->{lastName}) if exists $contact->{lastName}; + $card->VFirstName($contact->{firstName}) if exists $contact->{firstName}; + $card->VTitle($contact->{prefix}) if exists $contact->{prefix}; + + $card->VCompany($contact->{company}) if exists $contact->{company}; + $card->VDepartment($contact->{department}) if exists $contact->{department}; + + $card->VEmails($contact->{emails}) if exists $contact->{emails}; + $card->VAddresses($contact->{addresses}) if exists $contact->{addresses}; + $card->VPhones($contact->{phones}) if exists $contact->{phones}; + $card->VOnline($contact->{online}) if exists $contact->{online}; + + $card->VOnline($contact->{nickname}) if exists $contact->{nickname}; + $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; + $card->VNotes($contact->{notes}) if exists $contact->{notes}; + + $Self->backend_cmd('set_card', $href, $resource, $card); + $createmap{$cid} = { id => $uid }; + } + + return (\%createmap, \%notcreated); +} + +sub update_contacts { + my $Self = shift; + my $changes = shift; + + my $dbh = $Self->{dbh}; + + my @updated; + my %notchanged; + foreach my $carduid (keys %$changes) { + my $contact = $changes->{$carduid}; + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notchanged{$msgid} = "No such card on server"; + next; + } + my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); + $card->VLastName($contact->{lastName}) if exists $contact->{lastName}; + $card->VFirstName($contact->{firstName}) if exists $contact->{firstName}; + $card->VTitle($contact->{prefix}) if exists $contact->{prefix}; + + $card->VCompany($contact->{company}) if exists $contact->{company}; + $card->VDepartment($contact->{department}) if exists $contact->{department}; + + $card->VEmails($contact->{emails}) if exists $contact->{emails}; + $card->VAddresses($contact->{addresses}) if exists $contact->{addresses}; + $card->VPhones($contact->{phones}) if exists $contact->{phones}; + $card->VOnline($contact->{online}) if exists $contact->{online}; + + $card->VOnline($contact->{nickname}) if exists $contact->{nickname}; + $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; + $card->VNotes($contact->{notes}) if exists $contact->{notes}; + + $Self->backend_cmd('set_card', $href, $resource, $card); + push @updated, $carduid; + } + + return (\@updated, \%notchanged); +} + +sub destroy_contacts { + my $Self = shift; + my $destroy = shift; + + my $dbh = $Self->{dbh}; + + my @destroyed; + my %notdestroyed; + foreach my $carduid (@$destroy) { + my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); + unless ($href and $resource) { + $notdestroyed{$msgid} = "No such card on server"; + next; + } + $Self->backend_cmd('set_card', $href, $resource, undef); + push @destroyed, $carduid; + } + + return (\@destroyed, \%notdestroyed); +} + +sub _initdb { + my $Self = shift; + my $dbh = shift; + sub _initdb { my $Self = shift; my $dbh = shift; @@ -1100,11 +1401,14 @@ CREATE TABLE IF NOT EXISTS ievents ( ieventid INTEGER PRIMARY KEY NOT NULL, icalendarid INTEGER, resource TEXT, + uid TEXT, content TEXT, mtime DATE NOT NULL ); EOF + $dbh->do("CREATE INDEX ieventuid ON ievents (uid)"); + $dbh->do(<do("CREATE INDEX icarduid ON icards (uid)"); + } 1; From e0fb5d4ab677e0f87637ff297ff1a67ac3b23f29 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 01:53:45 -0400 Subject: [PATCH 125/331] fix ImapDB and implkement backend --- JMAP/ImapDB.pm | 84 +++++++++------------------------------------ JMAP/Sync/Common.pm | 74 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 67 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 04c8926..2e26e5a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1040,7 +1040,7 @@ sub create_calendar_events { my $calendar = $new->{$cid}; my ($href) = $dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendar->{calendarId}); unless ($href) { - $notcreated{$msgid} = "No such calendar on server"; + $notcreated{$cid} = "No such calendar on server"; next; } my $uid = new_uuid_string(); @@ -1061,10 +1061,10 @@ sub update_calendar_events { my @updated; my %notupdated; foreach my $uid (keys %$update) { - my $calendar = $updated->{$uid}; - my ($href) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uuid); - unless ($href) { - $notcreated{$uid} = "No such calendar on server"; + my $calendar = $update->{$uid}; + my ($href, $resource) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uid); + unless ($href and $resource) { + $notupdated{$uid} = "No such event on server"; next; } @@ -1083,10 +1083,10 @@ sub destroy_calendar_events { my @destroyed; my %notdestroyed; - foreach my $uid (@$destroy); - my ($href) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uuid); - unless ($href) { - $notdestroyed{$uid} = "No such calendar on server"; + foreach my $uid (@$destroy) { + my ($href, $resource) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uid); + unless ($href and $resource) { + $notdestroyed{$uid} = "No such event on server"; next; } @@ -1097,54 +1097,6 @@ sub destroy_calendar_events { return (\@destroyed, \%notdestroyed); } -sub update_contact_groups { - my $Self = shift; - my $changes = shift; - - my $dbh = $Self->{dbh}; - - my @updated; - my %notchanged; - foreach my $carduid (keys %$changes) { - my $contact = $changes->{$carduid}; - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { - $notchanged{$msgid} = "No such card on server"; - next; - } - my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); - $card->VKind('group'); - $card->VName($contact->{name}) if exists $contact->{name}; - $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; - - $Self->backend_cmd('set_card', $href, $resource, $card); - push @updated, $carduid; - } - - return (\@updated, \%notchanged); -} - -sub destroy_contact_groups { - my $Self = shift; - my $destroy = shift; - - my $dbh = $Self->{dbh}; - - my @destroyed; - my %notdestroyed; - foreach my $carduid (@$destroy) { - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { - $notdestroyed{$msgid} = "No such card on server"; - next; - } - $Self->backend_cmd('set_card', $href, $resource, undef); - push @destroyed, $carduid; - } - - return (\@destroyed, \%notdestroyed); -} - sub create_contact_groups { my $Self = shift; my $new = shift; @@ -1157,11 +1109,12 @@ sub create_contact_groups { my $contact = $new->{$cid}; my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); unless ($href) { - $notcreated{$msgid} = "No such addressbook on server"; + $notcreated{$cid} = "No such addressbook on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new(); my $uid = new_uuid_string(); + my $resource = "$uid.vcf"; $card->uid($uid); $card->VKind('group'); $card->VName($contact->{name}) if exists $contact->{name}; @@ -1186,7 +1139,7 @@ sub update_contact_groups { my $contact = $changes->{$carduid}; my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); unless ($href and $resource) { - $notchanged{$msgid} = "No such card on server"; + $notchanged{$carduid} = "No such card on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); @@ -1212,7 +1165,7 @@ sub destroy_contact_groups { foreach my $carduid (@$destroy) { my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); unless ($href and $resource) { - $notdestroyed{$msgid} = "No such card on server"; + $notdestroyed{$carduid} = "No such card on server"; next; } $Self->backend_cmd('set_card', $href, $resource, undef); @@ -1234,11 +1187,12 @@ sub create_contacts { my $contact = $new->{$cid}; my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); unless ($href) { - $notcreated{$msgid} = "No such addressbook on server"; + $notcreated{$cid} = "No such addressbook on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new(); my $uid = new_uuid_string(); + my $resource = "$uid.vcf"; $card->uid($uid); $card->VLastName($contact->{lastName}) if exists $contact->{lastName}; $card->VFirstName($contact->{firstName}) if exists $contact->{firstName}; @@ -1275,7 +1229,7 @@ sub update_contacts { my $contact = $changes->{$carduid}; my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); unless ($href and $resource) { - $notchanged{$msgid} = "No such card on server"; + $notchanged{$carduid} = "No such card on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); @@ -1313,7 +1267,7 @@ sub destroy_contacts { foreach my $carduid (@$destroy) { my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); unless ($href and $resource) { - $notdestroyed{$msgid} = "No such card on server"; + $notdestroyed{$carduid} = "No such card on server"; next; } $Self->backend_cmd('set_card', $href, $resource, undef); @@ -1323,10 +1277,6 @@ sub destroy_contacts { return (\@destroyed, \%notdestroyed); } -sub _initdb { - my $Self = shift; - my $dbh = shift; - sub _initdb { my $Self = shift; my $dbh = shift; diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1a8e9b3..12412ad 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -64,6 +64,43 @@ sub get_events { return \%res; } +sub new_event { + my $Self = shift; + my $href = shift; + my $event = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_calendars(); + return unless $talk; + + $talk->NewEvent($href, $event); +} + +sub update_event { + my $Self = shift; + my $href = shift; + my $resource = shift; + my $event = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_calendars(); + return unless $talk; + + $talk->NewEvent("$href/$resource", $event); +} + +sub delete_event { + my $Self = shift; + my $href = shift; + my $resource = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_calendars(); + return unless $talk; + + $talk->DeleteEvent({href => "$href/$resource"}); +} + sub get_addressbooks { my $Self = shift; my $talk = $Self->connect_contacts(); @@ -91,6 +128,43 @@ sub get_cards { return \%res; } +sub new_card { + my $Self = shift; + my $href = shift; + my $card = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_contacts(); + return unless $talk; + + $talk->NewContact($href, $card); +} + +sub update_card { + my $Self = shift; + my $href = shift; + my $resource = shift; + my $card = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_contacts(); + return unless $talk; + + $talk->UpdateContact("$href/$resource", $card); +} + +sub delete_card { + my $Self = shift; + my $href = shift; + my $resource = shift; + $href =~ s{/$}{}; + + my $talk = $Self->connect_contacts(); + return unless $talk; + + $talk->DeleteContact("$href/$resource"); +} + # read folder list from the server sub folders { my $Self = shift; From 3dea06b584be9b6c5866bc2c3f8e5b364985cce7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 02:03:03 -0400 Subject: [PATCH 126/331] bogus prefs for now --- JMAP/API.pm | 23 +++++++++++++++++++++++ JMAP/ImapDB.pm | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index d52fc07..c9459db 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1196,6 +1196,29 @@ sub _prop_wanted { return 0; } +sub getCalendarPreferences { + return ['calendarPreferences', { + autoAddCalendarId => '', + autoAddInvitations => JSON::false, + autoAddGroupId => JSON::null, + autoRSVPGroupId => JSON::null, + autoRSVP => JSON::false, + autoUpdate => JSON::false, + birthdaysAreVisible => JSON::false, + defaultAlerts => [], + defaultAllDayAlerts => [], + defaultCalendarId => '', + firstDayOfWeek => 1, + markReadAndFileAutoAdd => JSON::false, + markReadAndFileAutoUpdate => JSON::false, + onlyAutoAddIfInGroup => JSON::false, + onlyAutoRSVPIfInGroup => JSON::false, + showWeekNumbers => JSON::false, + timeZone => JSON::null, + useTimeZones => JSON::false, + }]; +} + sub getCalendars { my $Self = shift; my $args = shift; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2e26e5a..eb63a9a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1357,7 +1357,7 @@ CREATE TABLE IF NOT EXISTS ievents ( ); EOF - $dbh->do("CREATE INDEX ieventuid ON ievents (uid)"); + $dbh->do("CREATE INDEX IF NOT EXISTS ieventuid ON ievents (uid)"); $dbh->do(<do("CREATE INDEX icarduid ON icards (uid)"); + $dbh->do("CREATE INDEX IF NOT EXISTS icarduid ON icards (uid)"); } From 021f190f2f90e7cb83c573d975d773f66005a375 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 03:08:45 -0400 Subject: [PATCH 127/331] lots more fixes --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 75 ++++++++++++++++++++++----------------------- JMAP/Sync/Common.pm | 16 +++------- 3 files changed, 42 insertions(+), 51 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 4116509..0e46da2 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -728,7 +728,7 @@ sub delete_card { my $carduid = shift; my $kind = shift; if ($kind eq 'contact') { - $Self->dupdate('jcontactgroups', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dupdate('jcontacts', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); } else { $Self->dupdate('jcontactgroupmap', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index eb63a9a..3b4ccd1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -401,10 +401,10 @@ sub do_calendar { if ($data) { my $id = $data->[0]; next if $raw eq $data->[2]; - $Self->dmaybeupdate('ievents', {uid => $uid, content => $raw, resource => $resource}, {ieventid => $id}); + $Self->dmaybeupdate('ievents', {icalendarid => $calendarid, uid => $uid, content => $raw, resource => $resource}, {ieventid => $id}); } else { - $Self->dinsert('ievents', {uid => $uid, content => $raw, resource => $resource}); + $Self->dinsert('ievents', {icalendarid => $calendarid, uid => $uid, content => $raw, resource => $resource}); } $Self->set_event($jcalendarid, $event); } @@ -540,10 +540,10 @@ sub do_addressbook { if ($data) { my $id = $data->[0]; next if $raw eq $data->[2]; - $Self->dmaybeupdate('icards', {uid => $uid, content => $raw, resource => $resource}, {icardid => $id}); + $Self->dmaybeupdate('icards', {iaddressbookid => $addressbookid, uid => $uid, content => $raw, resource => $resource}, {icardid => $id}); } else { - $Self->dinsert('icards', {uid => $uid, content => $raw, resource => $resource}); + $Self->dinsert('icards', {iaddressbookid => $addressbookid, uid => $uid, content => $raw, resource => $resource}); } $Self->set_card($jaddressbookid, $card); } @@ -1062,13 +1062,13 @@ sub update_calendar_events { my %notupdated; foreach my $uid (keys %$update) { my $calendar = $update->{$uid}; - my ($href, $resource) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uid); - unless ($href and $resource) { + my ($resource) = $dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); + unless ($resource) { $notupdated{$uid} = "No such event on server"; next; } - $Self->backend_cmd('update_event', $href, $resource, $calendar); + $Self->backend_cmd('update_event', $resource, $calendar); push @updated, $uid; } @@ -1084,13 +1084,13 @@ sub destroy_calendar_events { my @destroyed; my %notdestroyed; foreach my $uid (@$destroy) { - my ($href, $resource) = $dbh->selectrow_array("SELECT href, resource FROM ievents JOIN icalendars USING (icalendarid) WHERE uid = ?", {}, $uid); - unless ($href and $resource) { + my ($resource) = $dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); + unless ($resource) { $notdestroyed{$uid} = "No such event on server"; next; } - $Self->backend_cmd('delete_event', $href, $resource); + $Self->backend_cmd('delete_event', $resource); push @destroyed, $uid; } @@ -1107,20 +1107,20 @@ sub create_contact_groups { my %notcreated; foreach my $cid (keys %$new) { my $contact = $new->{$cid}; - my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + #my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { $notcreated{$cid} = "No such addressbook on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new(); my $uid = new_uuid_string(); - my $resource = "$uid.vcf"; $card->uid($uid); $card->VKind('group'); $card->VName($contact->{name}) if exists $contact->{name}; $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; - $Self->backend_cmd('set_card', $href, $resource, $card); + $Self->backend_cmd('new_card', $href, $card); $createmap{$cid} = { id => $uid }; } @@ -1137,8 +1137,8 @@ sub update_contact_groups { my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { + my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + unless ($resource) { $notchanged{$carduid} = "No such card on server"; next; } @@ -1147,7 +1147,7 @@ sub update_contact_groups { $card->VName($contact->{name}) if exists $contact->{name}; $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; - $Self->backend_cmd('set_card', $href, $resource, $card); + $Self->backend_cmd('update_card', $resource, $card); push @updated, $carduid; } @@ -1163,12 +1163,12 @@ sub destroy_contact_groups { my @destroyed; my %notdestroyed; foreach my $carduid (@$destroy) { - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { + my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + unless ($resource) { $notdestroyed{$carduid} = "No such card on server"; next; } - $Self->backend_cmd('set_card', $href, $resource, undef); + $Self->backend_cmd('delete_card', $resource); push @destroyed, $carduid; } @@ -1185,14 +1185,13 @@ sub create_contacts { my %notcreated; foreach my $cid (keys %$new) { my $contact = $new->{$cid}; - my ($href) = $dbh->selectrow_array("SELECT href FROM iaddresses WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { $notcreated{$cid} = "No such addressbook on server"; next; } my ($card) = Net::CardDAVTalk::VCard->new(); my $uid = new_uuid_string(); - my $resource = "$uid.vcf"; $card->uid($uid); $card->VLastName($contact->{lastName}) if exists $contact->{lastName}; $card->VFirstName($contact->{firstName}) if exists $contact->{firstName}; @@ -1201,16 +1200,16 @@ sub create_contacts { $card->VCompany($contact->{company}) if exists $contact->{company}; $card->VDepartment($contact->{department}) if exists $contact->{department}; - $card->VEmails($contact->{emails}) if exists $contact->{emails}; - $card->VAddresses($contact->{addresses}) if exists $contact->{addresses}; - $card->VPhones($contact->{phones}) if exists $contact->{phones}; - $card->VOnline($contact->{online}) if exists $contact->{online}; + $card->VEmails(@{$contact->{emails}}) if exists $contact->{emails}; + $card->VAddresses(@{$contact->{addresses}}) if exists $contact->{addresses}; + $card->VPhones(@{$contact->{phones}}) if exists $contact->{phones}; + $card->VOnline(@{$contact->{online}}) if exists $contact->{online}; - $card->VOnline($contact->{nickname}) if exists $contact->{nickname}; + $card->VNickname($contact->{nickname}) if exists $contact->{nickname}; $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; $card->VNotes($contact->{notes}) if exists $contact->{notes}; - $Self->backend_cmd('set_card', $href, $resource, $card); + $Self->backend_cmd('new_card', $href, $card); $createmap{$cid} = { id => $uid }; } @@ -1227,8 +1226,8 @@ sub update_contacts { my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { + my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + unless ($resource) { $notchanged{$carduid} = "No such card on server"; next; } @@ -1240,16 +1239,16 @@ sub update_contacts { $card->VCompany($contact->{company}) if exists $contact->{company}; $card->VDepartment($contact->{department}) if exists $contact->{department}; - $card->VEmails($contact->{emails}) if exists $contact->{emails}; - $card->VAddresses($contact->{addresses}) if exists $contact->{addresses}; - $card->VPhones($contact->{phones}) if exists $contact->{phones}; - $card->VOnline($contact->{online}) if exists $contact->{online}; + $card->VEmails(@{$contact->{emails}}) if exists $contact->{emails}; + $card->VAddresses(@{$contact->{addresses}}) if exists $contact->{addresses}; + $card->VPhones(@{$contact->{phones}}) if exists $contact->{phones}; + $card->VOnline(@{$contact->{online}}) if exists $contact->{online}; - $card->VOnline($contact->{nickname}) if exists $contact->{nickname}; + $card->VNickname($contact->{nickname}) if exists $contact->{nickname}; $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; $card->VNotes($contact->{notes}) if exists $contact->{notes}; - $Self->backend_cmd('set_card', $href, $resource, $card); + $Self->backend_cmd('update_card', $resource, $card); push @updated, $carduid; } @@ -1265,12 +1264,12 @@ sub destroy_contacts { my @destroyed; my %notdestroyed; foreach my $carduid (@$destroy) { - my ($href, $resource, $content) = $dbh->selectrow_array("SELECT href, resource, content FROM icards JOIN iaddresses USING (iaddressbookid) WHERE uid = ?", {}, $carduid); - unless ($href and $resource) { + my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + unless ($resource) { $notdestroyed{$carduid} = "No such card on server"; next; } - $Self->backend_cmd('set_card', $href, $resource, undef); + $Self->backend_cmd('delete_card', $resource); push @destroyed, $carduid; } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 12412ad..1e95b5d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -78,27 +78,23 @@ sub new_event { sub update_event { my $Self = shift; - my $href = shift; my $resource = shift; my $event = shift; - $href =~ s{/$}{}; my $talk = $Self->connect_calendars(); return unless $talk; - $talk->NewEvent("$href/$resource", $event); + $talk->UpdateEvent($resource, $event); } sub delete_event { my $Self = shift; - my $href = shift; my $resource = shift; - $href =~ s{/$}{}; my $talk = $Self->connect_calendars(); return unless $talk; - $talk->DeleteEvent({href => "$href/$resource"}); + $talk->DeleteEvent({href => $resource}); } sub get_addressbooks { @@ -142,27 +138,23 @@ sub new_card { sub update_card { my $Self = shift; - my $href = shift; my $resource = shift; my $card = shift; - $href =~ s{/$}{}; my $talk = $Self->connect_contacts(); return unless $talk; - $talk->UpdateContact("$href/$resource", $card); + $talk->UpdateContact($resource, $card); } sub delete_card { my $Self = shift; - my $href = shift; my $resource = shift; - $href =~ s{/$}{}; my $talk = $Self->connect_contacts(); return unless $talk; - $talk->DeleteContact("$href/$resource"); + $talk->DeleteContact($resource); } # read folder list from the server From d71ddea6790b914000d50b5676371d734d54f57c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 03:11:27 -0400 Subject: [PATCH 128/331] carduid --- JMAP/DB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 0e46da2..eb19a3b 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -728,10 +728,10 @@ sub delete_card { my $carduid = shift; my $kind = shift; if ($kind eq 'contact') { - $Self->dupdate('jcontacts', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dupdate('jcontacts', {active => 0}, {contactuid => $carduid, jaddressbookid => $jaddressbookid}); } else { - $Self->dupdate('jcontactgroupmap', {active => 0}, {carduid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dupdate('jcontactgroupmap', {active => 0}, {groupuid => $carduid, jaddressbookid => $jaddressbookid}); } } From 567051068abf981b59400926611af632254e157c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 03:53:13 -0400 Subject: [PATCH 129/331] fix threads, draft ordering, perm delete --- JMAP/API.pm | 5 +++-- JMAP/ImapDB.pm | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index c9459db..2abe7e8 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1063,8 +1063,8 @@ sub getThreads { foreach my $thrid (@{$args->{ids}}) { next if $seenids{$thrid}; $seenids{$thrid} = 1; - my $data = $dbh->selectall_arrayref("SELECT msgid,isDraft,msgmessageid,msginreplyto FROM jmessages WHERE thrid = ? ORDER BY internaldate", {}, $thrid); - unless ($data) { + my $data = $dbh->selectall_arrayref("SELECT msgid,isDraft,msgmessageid,msginreplyto FROM jmessages WHERE thrid = ? AND active = 1 ORDER BY internaldate", {}, $thrid); + unless (@$data) { $missingids{$thrid} = 1; next; } @@ -1073,6 +1073,7 @@ sub getThreads { my %seenmsgs; foreach my $item (@$data) { next unless $item->[1]; + next unless $item->[3]; # push the rest of the drafts to the end push @{$drafts{$item->[3]}}, $item->[0]; } foreach my $item (@$data) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 3b4ccd1..2f02af9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -892,6 +892,7 @@ sub deleted_record { $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); + $Self->ddirty('jmessages', {}, {msgid => $msgid}); # dump modeseq $Self->delete_message_from_mailbox($msgid, $jmailboxid); } From 894a21f0c0e9e6830f91da54a794c56a53abb225 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 15:28:47 -0400 Subject: [PATCH 130/331] some stuff for drafts and compose --- JMAP/DB.pm | 99 ++++++++++++++++++++++++++++++++++++++++++++------ JMAP/ImapDB.pm | 6 ++- bin/server.pl | 2 + 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index eb19a3b..cb4e381 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -23,6 +23,7 @@ use DateTime; use Date::Parse; use Net::CalDAVTalk; use Net::CardDAVTalk::VCard; +use MIME::Base64 qw(encode_base64); sub new { my $class = shift; @@ -549,26 +550,100 @@ sub _mkemail { sub _makemsg { my $Self = shift; my $args = shift; + my $isDraft = shift; my %replyHeaders; if ($args->{inReplyToMessageId}) { # XXX - get replyheaders } - my $MIME = Email::Simple->create( - header => [ - From => _mkone($args->{from}), - To => _mkemail($args->{to}), - Cc => _mkemail($args->{cc}), - Bcc => _mkemail($args->{bcc}), - Subject => $args->{subject}, - %{$args->{headers} || {}}, - ], - body => $args->{textBody} || $args->{htmlBody}, + my $header = [ + From => _mkone($args->{from}), + To => _mkemail($args->{to}), + Cc => _mkemail($args->{cc}), + Bcc => _mkemail($args->{bcc}), + Subject => $args->{subject}, + %{$args->{headers} || {}}, + ]; + + # massive switch + my $MIME; + my $htmlpart; + my $text = $args->{textBody} ? $args->{textBody} : JMAP::DB::htmltotext($args->{htmlBody}); + my $textpart = Email::MIME->create( + attributes => { + content_type => 'text/plain', + charset => 'UTF-8', + }, + body => $text, ); - # XXX - attachments + if ($args->{htmlBody}) { + $htmlpart = Email::MIME->create( + attributes => { + content_type => 'text/html', + charset => 'UTF-8', + }, + body => $args->{htmlBody}, + ); + } + + my @attachments = $args->{attachments} ? @{$args->{attachments}} : (); + + if (@attachments and not $isDraft) { + my $encoded = encode_base64(@attachments, ''); + push @$header, "X-JMAP-Draft-Attachments" => $encoded; + @attachments = (); + } + + if (@attachments) { + # most complex case + if ($htmlpart) { + my $msgparts = Email::MIME->create( + attributes => { + content_type => 'multipart/alternative' + }, + parts => [$textpart, $htmlpart], + ); + # XXX - attachments + $MIME = Email::MIME->create( + header_str => [@$header, 'Content-Type' => 'multipart/mixed'], + parts => [$msgparts], + ); + } + else { + # XXX - attachments + $MIME = Email::MIME->create( + header_str => [@$header, 'Content-Type' => 'multipart/mixed'], + parts => [$textpart], + ); + } + } + else { + if ($htmlpart) { + $MIME = Email::MIME->create( + attributes => { + content_type => 'multipart/alternative', + }, + header_str => $header, + parts => [$textpart, $htmlpart], + ); + } + else { + $MIME = Email::MIME->create( + attributes => { + content_type => 'text/plain', + charset => 'UTF-8', + }, + header_str => $header, + body => $args->{textBody}, + ); + } + } + + my $res = $MIME->as_string(); + $res =~ s/\r?\n/\r\n/gs; - return $MIME->as_string(); + return $res; } # NOTE: this can ONLY be used to create draft messages diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2f02af9..50c23f9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -801,6 +801,7 @@ sub update_messages { my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } @$folderdata; + my %labelmap = map { $_->[3] => $_ } @$folderdata; my @changed; foreach my $ifolderid (keys %updatemap) { @@ -835,7 +836,10 @@ sub update_messages { } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one - my $newfolder = $foldermap{$id}[1]; + if ($id eq 'outbox') { + $id = $labelmap{'sent'}[3]; + } + my $newfolder = $jmailmap{$id}[1]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } # XXX - handle errors from backend commands diff --git a/bin/server.pl b/bin/server.pl index 9e8c86d..03d644b 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -596,6 +596,8 @@ sub prod_backfill { send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { $timer = undef; prod_backfill($accountid, @_); + # keep the idler running while we're backfilling + $idler{$accountid}{lastused} = time(); }); }); } From f09a813ac9fca95bd2f3acb61a317358e346a665 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 15:29:12 -0400 Subject: [PATCH 131/331] label sent area --- JMAP/ImapDB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 50c23f9..07f1627 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -838,6 +838,7 @@ sub update_messages { my $id = $action->{mailboxIds}->[0]; # there can be only one if ($id eq 'outbox') { $id = $labelmap{'sent'}[3]; + } my $newfolder = $jmailmap{$id}[1]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); From 47641451d5f2f53ba328f39b2df1e02fdedfddeb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 06:46:32 +1000 Subject: [PATCH 132/331] insert a magic 'Outbox' folder in the list Since it never gets updated, it's all good :) --- JMAP/API.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 2abe7e8..a9fa9ec 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -154,6 +154,9 @@ sub getMailboxes { my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnlyMailbox, mayReadItems, mayAddItems, mayRemoveItems, mayCreateChild, mayRename, mayDelete FROM jmailboxes WHERE active = 1"); + # outbox - magic + push @$data, [ 'outbox', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; + my %ids; if ($args->{ids}) { %ids = map { $_ => 1 } @{$args->{ids}}; From 7465898d1cc96707597bd78b187166f37c019a30 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 07:29:00 +1000 Subject: [PATCH 133/331] send email and setMailboxes --- JMAP/API.pm | 37 ++++++++++++++++- JMAP/ImapDB.pm | 106 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index a9fa9ec..4dcb7a2 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -329,7 +329,42 @@ sub getMailboxUpdates { sub setMailboxes { my $Self = shift; - return ['error', {type => 'notImplemented'}]; + my $args = shift; + + $Self->begin(); + + my $user = $Self->{db}->get_user(); + my $accountid = $Self->{db}->accountid(); + return $Self->_transError(['error', {type => 'accountNotFound'}]) + if ($args->{accountId} and $args->{accountId} ne $accountid); + + $Self->commit(); + + my $create = $args->{create} || {}; + my $update = $args->{update} || {}; + my $destroy = $args->{destroy} || []; + + # XXX - idmap support + my ($created, $notCreated) = $Self->{db}->create_mailboxes($create); + my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update); + my ($destroyed, $notDestroyed) = $Self->{db}->destroy_mailboxes($destroy); + + $Self->{db}->sync(); + + my @res; + push @res, ['mailboxesSet', { + accountId => $accountid, + oldState => undef, # proxy can't guarantee the old state + newState => undef, # or give a new state + created => $created, + notCreated => $notCreated, + updated => $updated, + notUpdated => $notUpdated, + destroyed => $destroyed, + notDestroyed => $notDestroyed, + }]; + + return @res; } sub _build_sort { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 07f1627..7c0efc8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -837,11 +837,19 @@ sub update_messages { if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one if ($id eq 'outbox') { - $id = $labelmap{'sent'}[3]; - + my $sentid = $labelmap{'sent'}[3]; + my $newfolder = $jmailmap{$id}[1]; + my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); + my $msg = $res->{data}{$uid}; + $Self->backend_cmd('send_email', $msg); + # strip the \Draft flag + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\draft"]); + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); + } + else { + my $newfolder = $jmailmap{$id}[1]; + $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } - my $newfolder = $jmailmap{$id}[1]; - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } # XXX - handle errors from backend commands push @changed, $msgid; @@ -1034,6 +1042,96 @@ sub fill_messages { return \%result; } +sub create_mailboxes { + my $Self = shift; + my $new = shift; + + my $dbh = $Self->{dbh}; + + my %idmap; + my %notcreated; + foreach my $cid (keys %$new) { + my $mailbox = $new->{$cid}; + + my $imapname = $mailbox->{name}; + if ($mailbox->{parentId}) { + my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $mailbox->{parentId}); + # XXX - errors + $imapname = "$parentName$sep$imapname"; + } + else { + my ($prefix) = $dbh->selectrow_array("SELECT prefix FROM iserver"); + $imapname = "$prefix$imapname"; + } + + my $res = $Self->backend_cmd('create_mailbox', $imapname); + # XXX - handle errors... + $idmap{$imapname} = $cid; # need to resolve this after the sync + } + + # (in theory we could save this until the end and resolve the names in after the renames and deletes... but it does mean + # we can't use ids as referenes...) + $Self->sync_folders(); + + my %createdmap; + foreach my $imapname (keys %idmap) { + my $cid = $idmap{$imapname}; + my ($jid) = $dbh->selectrow_array("SELECT jmailboxid FROM ifolders WHERE imapname = ?", {}, $imapname); + $createmap{$cid} = $jid; + } + + return (\%createmap, \%notcreated); +} + +sub update_mailboxes { + my $Self = shift; + my $update = shift; + + my $dbh = $Self->{dbh}; + + my @updated; + my %notupdated; + # XXX - reorder the crap out of this if renaming multiple mailboxes due to deep rename + foreach my $id (keys %$update) { + my $mailbox = $update->{$id}; + my $imapname = $mailbox->{name}; + if ($mailbox->{parentId}) { + my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $mailbox->{parentId}); + # XXX - errors + $imapname = "$parentName$sep$imapname"; + } + else { + my ($prefix) = $dbh->selectrow_array("SELECT prefix FROM iserver"); + $imapname = "$prefix$imapname"; + } + + my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + + $Self->backend_cmd('rename_mailbox', $oldname, $imapname); + push @updated, $uid; + } + + return (\@updated, \%notupdated); +} + +sub destroy_mailboxes { + my $Self = shift; + my $destroy = shift; + + my $dbh = $Self->{dbh}; + + my @destroyed; + my %notdestroyed; + foreach my $id (@$destroy) { + my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + + $Self->backend_cmd('delete_mailbox', $oldname); + push @destroyed, $uid; + } + + return (\@destroyed, \%notdestroyed); +} + sub create_calendar_events { my $Self = shift; my $new = shift; From cca1eff44eacefd7f359e299e1a662e5d872e062 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 19:54:49 -0400 Subject: [PATCH 134/331] things that are things --- JMAP/API.pm | 7 ++++--- JMAP/ImapDB.pm | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 4dcb7a2..8e181bd 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -212,12 +212,13 @@ sub getMailboxes { if (_prop_wanted($args, 'unreadThreads')) { my $folderlimit = ''; if ($ONLY_MAILBOXES{$item->[0]}) { - $folderlimit = "AND jmessagemap.jmailboxid = $item->[0]"; + $folderlimit = "AND jmessagemap.jmailboxid = " . $dbh->quote($item->[0]); } else { my @ids = grep { $ONLY_MAILBOXES{$_} } sort keys %ONLY_MAILBOXES; - $folderlimit = "AND jmessagemap.jmailboxid NOT IN (" . join(',', @ids) . ")" if @ids; + $folderlimit = "AND jmessagemap.jmailboxid NOT IN (" . join(',', map { $dbh->quote($_) } @ids) . ")" if @ids; } - ($rec{unreadThreads}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT thrid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = $item->[0] AND jmessages.active = 1 AND jmessagemap.active = 1 AND thrid IN (SELECT thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE isUnread = 1 AND jmessages.active = 1 AND jmessagemap.active = 1 $folderlimit)"); + my $sql ="SELECT COUNT(DISTINCT thrid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1 AND thrid IN (SELECT thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE isUnread = 1 AND jmessages.active = 1 AND jmessagemap.active = 1 $folderlimit)"; + ($rec{unreadThreads}) = $dbh->selectrow_array($sql, {}, $item->[0]); $rec{unreadThreads} += 0; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7c0efc8..9a986cd 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1073,7 +1073,7 @@ sub create_mailboxes { # we can't use ids as referenes...) $Self->sync_folders(); - my %createdmap; + my %createmap; foreach my $imapname (keys %idmap) { my $cid = $idmap{$imapname}; my ($jid) = $dbh->selectrow_array("SELECT jmailboxid FROM ifolders WHERE imapname = ?", {}, $imapname); @@ -1108,7 +1108,7 @@ sub update_mailboxes { my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); $Self->backend_cmd('rename_mailbox', $oldname, $imapname); - push @updated, $uid; + push @updated, $id; } return (\@updated, \%notupdated); @@ -1126,7 +1126,7 @@ sub destroy_mailboxes { my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); $Self->backend_cmd('delete_mailbox', $oldname); - push @destroyed, $uid; + push @destroyed, $id; } return (\@destroyed, \%notdestroyed); From 1a0728b9917fce8a0184e42c6f1ea860816b7acb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 10:43:16 +1000 Subject: [PATCH 135/331] have calendars, html sig --- JMAP/API.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 8e181bd..fa5c11b 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -39,8 +39,8 @@ sub getAccounts { isPrimary => $JSON::true, isReadOnly => $JSON::false, hasMail => $JSON::true, - hasContacts => $JSON::false, - hasCalendars => $JSON::false, + hasContacts => $JSON::true, + hasCalendars => $JSON::true, }; return ['accounts', { @@ -96,7 +96,7 @@ sub getPersonalities { email => $user->{email}, name => $user->{displayname} || $user->{email}, textSignature => "-- \ntext sig", - textSignature => "-- \nhtml sig", + htmlSignature => "-- \nhtml sig", replyTo => $user->{email}, autoBcc => "", addBccOnSMTP => $JSON::false, From 1c34700157796b529181d3b384d5abae51d2bac2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 20:46:14 -0400 Subject: [PATCH 136/331] move sent email to sent folder --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9a986cd..dcda1e1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -838,7 +838,7 @@ sub update_messages { my $id = $action->{mailboxIds}->[0]; # there can be only one if ($id eq 'outbox') { my $sentid = $labelmap{'sent'}[3]; - my $newfolder = $jmailmap{$id}[1]; + my $newfolder = $jmailmap{$sentid}[1]; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); my $msg = $res->{data}{$uid}; $Self->backend_cmd('send_email', $msg); From 127bc86f7a4b31a2d2e253ef78a31c29f110b10c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 20:51:03 -0400 Subject: [PATCH 137/331] fix save sent, fiddle Outbox label --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index fa5c11b..a3f6bf7 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -155,7 +155,7 @@ sub getMailboxes { my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnlyMailbox, mayReadItems, mayAddItems, mayRemoveItems, mayCreateChild, mayRename, mayDelete FROM jmailboxes WHERE active = 1"); # outbox - magic - push @$data, [ 'outbox', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; + push @$data, [ '$', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; my %ids; if ($args->{ids}) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index dcda1e1..24893a2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -836,9 +836,9 @@ sub update_messages { } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one - if ($id eq 'outbox') { - my $sentid = $labelmap{'sent'}[3]; - my $newfolder = $jmailmap{$sentid}[1]; + my $label = $jmailmap{$id}[3]; + if ($label eq 'outbox') { + my $newfolder = $labelmap{'sent'}[1]; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); my $msg = $res->{data}{$uid}; $Self->backend_cmd('send_email', $msg); From cc171d9af2bb5b1675a357772a1b3304fb724098 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 21:09:12 -0400 Subject: [PATCH 138/331] tricky finding the right folder --- JMAP/ImapDB.pm | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 24893a2..0f88de4 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -801,7 +801,9 @@ sub update_messages { my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } @$folderdata; - my %labelmap = map { $_->[3] => $_ } @$folderdata; + my $jmapdata = $dbh->selectall_arrayref("SELECT jmailboxid, role FROM jmailboxes"); + my %jidmap = map { $_->[0] => $_->[1] } @$jmapdata; + my %jrolemap = map { $_->[1] => $_->[0] } @$jmapdata; my @changed; foreach my $ifolderid (keys %updatemap) { @@ -836,13 +838,14 @@ sub update_messages { } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one - my $label = $jmailmap{$id}[3]; + my $label = $jidmap{$id}; if ($label eq 'outbox') { - my $newfolder = $labelmap{'sent'}[1]; + my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); my $msg = $res->{data}{$uid}; $Self->backend_cmd('send_email', $msg); # strip the \Draft flag + warn "SENDING $imapname $uidvalidity and moving to $newfolder"; $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\draft"]); $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } From 375b12c5fbdc6ccc167b020ffb3b19362c95441e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 21:20:19 -0400 Subject: [PATCH 139/331] outbox duh --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index a3f6bf7..fa5c11b 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -155,7 +155,7 @@ sub getMailboxes { my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnlyMailbox, mayReadItems, mayAddItems, mayRemoveItems, mayCreateChild, mayRename, mayDelete FROM jmailboxes WHERE active = 1"); # outbox - magic - push @$data, [ '$', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; + push @$data, [ 'outbox', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; my %ids; if ($args->{ids}) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 0f88de4..4462b46 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -838,8 +838,7 @@ sub update_messages { } if (exists $action->{mailboxIds}) { my $id = $action->{mailboxIds}->[0]; # there can be only one - my $label = $jidmap{$id}; - if ($label eq 'outbox') { + if ($id eq 'outbox') { my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); my $msg = $res->{data}{$uid}; From ff4a7aa8242e837ee42770d08a3a9bbaf5ec2b93 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 18 Jul 2015 21:34:22 -0400 Subject: [PATCH 140/331] remove pointless error --- bin/apiendpoint.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 6c065ff..673ca29 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -5,6 +5,11 @@ package JMAP::Backend; use Mail::IMAPTalk qw(:trace); +# stuff complains otherwise - twice for luck +use IO::Socket::SSL; +$IO::Socket::SSL::DEBUG = 0; +$IO::Socket::SSL::DEBUG = 0; + use Carp qw(verbose); use strict; use warnings; From b81775b55d94a80e72e45f335f67549e1af0de31 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 13:50:27 -0400 Subject: [PATCH 141/331] imap search --- JMAP/Sync/Common.pm | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1e95b5d..d31bb9b 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -426,4 +426,33 @@ sub imap_append { return ['append', $imapname, @$uid]; } +sub imap_search { + my $Self = shift; + my $folders = shift; + my @expr = @_; + + my $imap = $Self->connect_imap(); + + my %data; + foreach my $imapname (@$folders) { + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $imapname" unless $r; + + # XXX - check uidvalidity + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; + my $uidnext = $imap->get_response_code('uidnext') + 0; + my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; + my $exists = $imap->get_response_code('exists') || 0; + + if ($imap->capability->{'search=fuzzy'}) { + @expr = ('fuzzy', [@expr]); + } + + my $uids = $imap->search('charset', 'utf-8', @expr); + $data{$imapname} = [$uidvalidity, $uids]; + } + + return \%data; +} + 1; From 133d1dfcdc51639b2ce60a182fb319fc40766e58 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 04:47:44 +1000 Subject: [PATCH 142/331] almost complete search semantics --- JMAP/API.pm | 174 +++++++++++++++++++++++++++++++++----------- JMAP/ImapDB.pm | 23 ++++++ JMAP/Sync/Common.pm | 32 ++++---- 3 files changed, 168 insertions(+), 61 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index fa5c11b..02abd73 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -408,17 +408,107 @@ sub _match { # XXX - condition handling code if ($condition->{inMailboxes}) { - my $match = 0; + my $inall = 1; foreach my $id (@{$condition->{inMailboxes}}) { $storage->{mailbox}{$id} ||= $Self->_load_mailbox($id); - next unless $storage->{mailbox}{$id}{$item->[0]}[2]; #active - $match = 1; + next if $storage->{mailbox}{$id}{$item->{msgid}}[2]; #active + $inall = 0; } - return 0 unless $match; + return 0 unless $inall; + } + + if ($condition->{notInMailboxes}) { + my $inany = 0; + foreach my $id (@{$condition->{notInMailboxes}}) { + $storage->{mailbox}{$id} ||= $Self->_load_mailbox($id); + next unless $storage->{mailbox}{$id}{$item->{msgid}}[2]; #active + $inany = 1; + } + return 0 if $inany; + } + + if ($condition->{before}) { + my $time = str2time($condition->{before})->epoch(); + return 0 unless $time < $item->{internaldate}; + } + + if ($condition->{after}) { + my $time = str2time($condition->{before})->epoch(); + return 0 unless $time >= $item->{internaldate}; + } + + if ($condition->{minSize}) { + return 0 unless $item->{msgsize} >= $condition->{minSize}; + } + + if ($condition->{maxSize}) { + return 0 unless $item->{msgsize} < $condition->{maxSize}; + } + + if ($condition->{isFlagged}) { + # XXX - threaded versions? + return 0 unless $item->{isFlagged}; } if ($condition->{isUnseen}) { # XXX - threaded versions? + return 0 unless $item->{isUnseen}; + } + + if ($condition->{isAnswered}) { + # XXX - threaded versions? + return 0 unless $item->{isAnswered}; + } + + if ($condition->{isDraft}) { + # XXX - threaded versions? + return 0 unless $item->{isDraft}; + } + + if ($condition->{hasAttachment}) { + # XXX - hasAttachment + } + + if ($condition->{text}) { + $storage->{textsearch}{$condition->{text}} ||= $Self->{db}->imap_search('text', $condition->{text}); + return 0 unless $storage->{textsearch}{$condition->{text}}{$item->{msgid}}; + } + + if ($condition->{from}) { + $storage->{fromsearch}{$condition->{from}} ||= $Self->{db}->imap_search('from', $condition->{from}); + return 0 unless $storage->{fromsearch}{$condition->{from}}{$item->{msgid}}; + } + + if ($condition->{to}) { + $storage->{tosearch}{$condition->{to}} ||= $Self->{db}->imap_search('to', $condition->{to}); + return 0 unless $storage->{tosearch}{$condition->{to}}{$item->{msgid}}; + } + + if ($condition->{cc}) { + $storage->{ccsearch}{$condition->{cc}} ||= $Self->{db}->imap_search('cc', $condition->{cc}); + return 0 unless $storage->{ccsearch}{$condition->{cc}}{$item->{msgid}}; + } + + if ($condition->{bcc}) { + $storage->{bccsearch}{$condition->{bcc}} ||= $Self->{db}->imap_search('bcc', $condition->{bcc}); + return 0 unless $storage->{bccsearch}{$condition->{bcc}}{$item->{msgid}}; + } + + if ($condition->{subject}) { + $storage->{subjectsearch}{$condition->{subject}} ||= $Self->{db}->imap_search('subject', $condition->{subject}); + return 0 unless $storage->{subjectsearch}{$condition->{subject}}{$item->{msgid}}; + } + + if ($condition->{body}) { + $storage->{bodysearch}{$condition->{body}} ||= $Self->{db}->imap_search('body', $condition->{body}); + return 0 unless $storage->{bodysearch}{$condition->{body}}{$item->{msgid}}; + } + + if ($condition->{header}) { + my $cond = $condition->{header}; + $cond->[1] = '' if @$cond == 1; + $storage->{headersearch}{"@$cond"} ||= $Self->{db}->imap_search('header', @$cond) + return 0 unless $storage->{headersearch}{"@$cond"}{$item->{msgid}}; } return 1; @@ -494,7 +584,10 @@ sub getMessageList { return $Self->_transError(['error', {type => 'invalidArguments'}]) if $start < 0; my $sort = $Self->_build_sort($args->{sort}); - my $data = $dbh->selectall_arrayref("SELECT DISTINCT msgid,thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmessages.active = 1 AND jmessagemap.active = 1 ORDER BY $sort"); + my $data = $dbh->selectall_arrayref("SELECT * FROM jmessages WHERE active = 1 ORDER BY $sort", {Slice => {}}); + + # commit before applying the filter, because it might call out for searches + $Self->commit(); $data = $Self->_filter($data, $args->{filter}, {}) if $args->{filter}; $data = $Self->_collapse($data) if $args->{collapseThreads}; @@ -512,8 +605,6 @@ sub getMessageList { gotit: - $Self->commit(); - my $end = $args->{limit} ? $start + $args->{limit} - 1 : $#$data; $end = $#$data if $end > $#$data; @@ -573,7 +664,9 @@ sub getMessageListUpdates { if $start < 0; my $sort = $Self->_build_sort($args->{sort}); - my $data = $dbh->selectall_arrayref("SELECT msgid,thrid,jmodseq,active FROM jmessages ORDER BY $sort"); + my $data = $dbh->selectall_arrayref("SELECT * FROM jmessages ORDER BY $sort", {Slice => {}}); + + $Self->commit(); # now we have the same sorted data set. What we DON'T have is knowing that a message used to be in the filter, # but no longer is (aka isUnseen). There's no good way to do this :( So we have to assume that every message @@ -596,18 +689,16 @@ sub getMessageListUpdates { # non-deleted, unchanged message) my %finished; foreach my $item (@$data) { - my ($msgid, $thrid, $jmodseq, $active) = @$item; - # we don't have to tell anything about finished threads, not even check them for membership in the search - next if $finished{$thrid}; + next if $finished{$item->{thrid}}; # deleted is the same as not in filter for our purposes my $isin = $active ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; # only exemplars count for the total - we need to know total even if not telling any more - if ($isin and not $exemplar{$thrid}) { + if ($isin and not $exemplar{$item->{thrid}}) { $total++; - $exemplar{$thrid} = $msgid; + $exemplar{$item->{thrid}} = $item->{msgid}; } next unless $tell; @@ -616,33 +707,33 @@ sub getMessageListUpdates { if ($changed) { # if it's in AND it's the exemplar, it's been added - if ($isin and $exemplar{$thrid} eq $msgid) { - push @added, {messageId => "$msgid", threadId => "$thrid", index => $total-1}; - push @removed, {messageId => "$msgid", threadId => "$thrid"}; + if ($isin and $exemplar{$item->{thrid}} eq $item->{msgid}) { + push @added, {messageId => "$item->{msgid}", threadId => "$item->{thrid}", index => $total-1}; + push @removed, {messageId => "$item->{msgid}", threadId => "$item->{thrid}"}; $changes++; } # otherwise it's removed else { - push @removed, {messageId => "$msgid", threadId => "$thrid"}; + push @removed, {messageId => "$item->{msgid}", threadId => "$item->{thrid}"}; $changes++; } } # unchanged and isin, final candidate for old exemplar! elsif ($isin) { # remove it unless it's also the current exemplar - if ($exemplar{$thrid} ne $msgid) { - push @removed, {messageId => "$msgid", threadId => "$thrid"}; + if ($exemplar{$item->{thrid}} ne $item->{msgid}) { + push @removed, {messageId => "$item->{msgid}", threadId => "$item->{thrid}"}; $changes++; } # and we're done - $finished{$thrid} = 1; + $finished{$item->{thrid}} = 1; } if ($args->{maxChanges} and $changes > $args->{maxChanges}) { return $Self->_transError(['error', {type => 'tooManyChanges'}]); } - if ($args->{upToMessageId} and $args->{upToMessageId} eq $msgid) { + if ($args->{upToMessageId} and $args->{upToMessageId} eq $item->{msgid}) { # stop mentioning changes $tell = 0; } @@ -652,7 +743,6 @@ sub getMessageListUpdates { # non-collapsed case else { foreach my $item (@$data) { - my ($msgid, $thrid, $jmodseq, $active) = @$item; # deleted is the same as not in filter for our purposes my $isin = $active ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; @@ -665,12 +755,12 @@ sub getMessageListUpdates { if ($changed) { if ($isin) { - push @added, {messageId => "$msgid", threadId => "$thrid", index => $total-1}; - push @removed, {messageId => "$msgid", threadId => "$thrid"}; + push @added, {messageId => "$item->{msgid}", threadId => "$item->{thrid}", index => $total-1}; + push @removed, {messageId => "$item->{msgid}", threadId => "$item->{thrid}"}; $changes++; } else { - push @removed, {messageId => "$msgid", threadId => "$thrid"}; + push @removed, {messageId => "$item->{msgid}", threadId => "$item->{thrid}"}; $changes++; } } @@ -679,15 +769,13 @@ sub getMessageListUpdates { return $Self->_transError(['error', {type => 'tooManyChanges'}]); } - if ($args->{upToMessageId} and $args->{upToMessageId} eq $msgid) { + if ($args->{upToMessageId} and $args->{upToMessageId} eq $item->{msgid}) { # stop mentioning changes $tell = 0; } } } - $Self->commit(); - my @res; push @res, ['messageListUpdates', { accountId => $accountid, @@ -1102,7 +1190,7 @@ sub getThreads { foreach my $thrid (@{$args->{ids}}) { next if $seenids{$thrid}; $seenids{$thrid} = 1; - my $data = $dbh->selectall_arrayref("SELECT msgid,isDraft,msgmessageid,msginreplyto FROM jmessages WHERE thrid = ? AND active = 1 ORDER BY internaldate", {}, $thrid); + my $data = $dbh->selectall_arrayref("SELECT * FROM jmessages WHERE thrid = ? AND active = 1 ORDER BY internaldate", {Slice => {}}, $thrid); unless (@$data) { $missingids{$thrid} = 1; next; @@ -1111,24 +1199,24 @@ sub getThreads { my @msgs; my %seenmsgs; foreach my $item (@$data) { - next unless $item->[1]; - next unless $item->[3]; # push the rest of the drafts to the end - push @{$drafts{$item->[3]}}, $item->[0]; + next unless $item->{isDraft}; + next unless $item->{msginreplyto}; # push the rest of the drafts to the end + push @{$drafts{$item->{msginreplyto}}}, $item->{msgid}; } foreach my $item (@$data) { - next if $item->[1]; - push @msgs, $item->[0]; - $seenmsgs{$item->[0]} = 1; - if (my $draftmsgs = $drafts{$item->[2]}) { + next if $item->{isDraft}; + push @msgs, $item->{msgid}; + $seenmsgs{$item->{msgid}} = 1; + if (my $draftmsgs = $drafts{$item->{msgmessageid}}) { push @msgs, @$draftmsgs; $seenmsgs{$_} = 1 for @$draftmsgs; } } # make sure unlinked drafts aren't forgotten! foreach my $item (@$data) { - next if $seenmsgs{$item->[0]}; - push @msgs, $item->[0]; - $seenmsgs{$item->[0]} = 1; + next if $seenmsgs{$item->{msgid}}; + push @msgs, $item->{msgid}; + $seenmsgs{$item->{msgid}} = 1; } push @list, { id => "$thrid", @@ -1176,13 +1264,13 @@ sub getThreadUpdates { return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); - my $sql = "SELECT thrid,active FROM jmessages WHERE jmodseq > ?"; + my $sql = "SELECT * FROM jmessages WHERE jmodseq > ?"; if ($args->{maxChanges}) { $sql .= " LIMIT " . (int($args->{maxChanges}) + 1); } - my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); + my $data = $dbh->selectall_arrayref($sql, {Slice => {}}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { return $Self->_transError(['error', {type => 'tooManyChanges'}]); @@ -1191,8 +1279,8 @@ sub getThreadUpdates { my %threads; my %delcheck; foreach my $row (@$data) { - $threads{$row->[0]} = 1; - $delcheck{$row->[0]} = 1 unless $row->[1]; + $threads{$row->{msgid}} = 1; + $delcheck{$row->{msgid}} = 1 unless $row->{active}; } my @removed; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 4462b46..210911c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -730,6 +730,29 @@ sub do_folder { return $didold; } +sub imap_search { + my $Self = shift; + my @search = @_; + + my $dbh = $Self->dbh(); + my $data = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + + my %matches; + foreach my $item (@$data) { + my $from = $item->{uidfirst}; + my $to = $item->{uidnext}-1; + my $res = $Self->backend_cmd('imap_search', $item->{imapname}, 'uid', "$from:$to", @search); + # XXX - uidvaldity changed + next unless $res->[2] == $item->{uidvalidity}; + foreach my $uid (@{$res->[3]}) { + my ($msgid) = $dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? and uid = ?", {}, $item->{ifolderid}, $uid); + $matches{$msgid} = 1; + } + } + + return \%matches; +} + sub changed_record { my $Self = shift; my ($folder, $uid, $flaglist, $labellist) = @_; diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index d31bb9b..04707ff 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -418,7 +418,7 @@ sub imap_append { my $r = $imap->append($imapname, $flags, $internaldate, {'Literal' => $rfc822}); die "APPEND FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'appenduid'); # what's with that?? - my $uid = $imap->get_response_code('appenduid'); + my $uid = $imap->get_response_code('appenduid'); # returns [uidvalidity uid] # XXX - fetch the x-gm-msgid or envelope from the server so we know the # the ID that the server gave this message @@ -428,31 +428,27 @@ sub imap_append { sub imap_search { my $Self = shift; - my $folders = shift; + my $imapname = shift; my @expr = @_; my $imap = $Self->connect_imap(); - my %data; - foreach my $imapname (@$folders) { - my $r = $imap->examine($imapname); - die "EXAMINE FAILED $imapname" unless $r; - - # XXX - check uidvalidity - my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; - my $uidnext = $imap->get_response_code('uidnext') + 0; - my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; - my $exists = $imap->get_response_code('exists') || 0; + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $imapname" unless $r; - if ($imap->capability->{'search=fuzzy'}) { - @expr = ('fuzzy', [@expr]); - } + # XXX - check uidvalidity + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; + my $uidnext = $imap->get_response_code('uidnext') + 0; + my $highestmodseq = $imap->get_response_code('highestmodseq') || 0; + my $exists = $imap->get_response_code('exists') || 0; - my $uids = $imap->search('charset', 'utf-8', @expr); - $data{$imapname} = [$uidvalidity, $uids]; + if ($imap->capability->{'search=fuzzy'}) { + @expr = ('fuzzy', [@expr]); } - return \%data; + my $uids = $imap->search('charset', 'utf-8', @expr); + + return ['search', $imapname, $uidvalidity, $uids]; } 1; From 4c04f66e805a4f297ac75ad327423f0950ef2c62 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 14:50:21 -0400 Subject: [PATCH 143/331] API fixes --- JMAP/API.pm | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 02abd73..b84a89f 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -507,7 +507,7 @@ sub _match { if ($condition->{header}) { my $cond = $condition->{header}; $cond->[1] = '' if @$cond == 1; - $storage->{headersearch}{"@$cond"} ||= $Self->{db}->imap_search('header', @$cond) + $storage->{headersearch}{"@$cond"} ||= $Self->{db}->imap_search('header', @$cond); return 0 unless $storage->{headersearch}{"@$cond"}{$item->{msgid}}; } @@ -693,7 +693,7 @@ sub getMessageListUpdates { next if $finished{$item->{thrid}}; # deleted is the same as not in filter for our purposes - my $isin = $active ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; + my $isin = $item->{active} ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; # only exemplars count for the total - we need to know total even if not telling any more if ($isin and not $exemplar{$item->{thrid}}) { @@ -703,7 +703,7 @@ sub getMessageListUpdates { next unless $tell; # jmodseq greater than sinceState is a change - my $changed = ($jmodseq > $args->{sinceState}); + my $changed = ($item->{jmodseq} > $args->{sinceState}); if ($changed) { # if it's in AND it's the exemplar, it's been added @@ -744,14 +744,14 @@ sub getMessageListUpdates { else { foreach my $item (@$data) { # deleted is the same as not in filter for our purposes - my $isin = $active ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; + my $isin = $item->{active} ? ($args->{filter} ? $Self->_match($item, $args->{filter}, $storage) : 1) : 0; # all active messages count for the total $total++ if $isin; next unless $tell; # jmodseq greater than sinceState is a change - my $changed = ($jmodseq > $args->{sinceState}); + my $changed = ($item->{jmodseq} > $args->{sinceState}); if ($changed) { if ($isin) { From f6d936b77f9f7d7ce2ecb088944a019ef1a68abf Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 15:15:51 -0400 Subject: [PATCH 144/331] add refreshSyncedCalendars from FM API for demo --- JMAP/API.pm | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index b84a89f..728cb3b 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -49,6 +49,15 @@ sub getAccounts { }]; } +sub refreshSyncedCalendars { + my $Self = shift; + + $Self->{db}->sync_calendars(); + + # no response + return (); +} + sub getPreferences { my $Self = shift; my $args = shift; @@ -552,9 +561,9 @@ sub _collapse { my @res; my %seen; foreach my $item (@$data) { - next if $seen{$item->[1]}; + next if $seen{$item->{thrid}}; push @res, $item; - $seen{$item->[1]} = 1; + $seen{$item->{thrid}} = 1; } return \@res; } @@ -608,8 +617,8 @@ gotit: my $end = $args->{limit} ? $start + $args->{limit} - 1 : $#$data; $end = $#$data if $end > $#$data; - my @result = map { $data->[$_][0] } $start..$end; - my @thrid = map { $data->[$_][1] } $start..$end; + my @result = map { $data->[$_]{msgid} } $start..$end; + my @thrid = map { $data->[$_]{thrid} } $start..$end; my @res; push @res, ['messageList', { From f6379afccadce8ed219c2dbb8dd9805dc11e7a51 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 15:23:12 -0400 Subject: [PATCH 145/331] noisy changes --- JMAP/DB.pm | 11 ++++++++--- bin/apiendpoint.pl | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index cb4e381..24891cc 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -87,10 +87,15 @@ sub commit { # push an update if anything to tell.. if ($t->{modseq} and $Self->{change_cb}) { + my $state = $t->{modseq}; $Self->{change_cb}->($Self, { - Mailbox => "$t->{modseq}", - Thread => "$t->{modseq}", - Message => "$t->{modseq}", + Mailbox => "$state", + Thread => "$state", + Message => "$state", + Contact => "$state", + ContactGroup => "$state", + Calendar => "$state", + CalendarEvent => "$state", }); } } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 673ca29..8b06196 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -148,6 +148,10 @@ sub handle_getstate { Mailbox => "$state", Thread => "$state", Message => "$state", + Contact => "$state", + ContactGroup => "$state", + Calendar => "$state", + CalendarEvent => "$state", }, }, }; From dccef6ead0477860ee5ccdb24f5ae1c9e0dc86bc Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 07:48:44 +1000 Subject: [PATCH 146/331] capture IMAP prefix --- JMAP/DB.pm | 13 +++++++------ JMAP/ImapDB.pm | 10 +++++++--- JMAP/Sync/Common.pm | 2 +- JMAP/Sync/Standard.pm | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 24891cc..aa6435e 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -884,7 +884,7 @@ sub dupdate { $values->{mtime} = time(); my @keys = sort keys %$values; - my @lkeys = sort keys %$limit; + my @lkeys = $limit ? sort keys %$limit : (); my $sql = "UPDATE $table SET " . join (', ', map { "$_ = ?" } @keys); $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; @@ -898,13 +898,14 @@ sub filter_values { my $Self = shift; my ($table, $values, $limit) = @_; - # copy so we don't edit the original - my %values = %$values; + # copy so we don't edit the originals + my %values = $values ? %$values : (); - my @keys = sort keys %$values; - my @lkeys = sort keys %$limit; + my @keys = sort keys %values; + my @lkeys = $limit ? sort keys %$limit : (); - my $sql = "SELECT " . join(', ', @keys) . " FROM $table WHERE " . join(' AND ', map { "$_ = ?" } @lkeys); + my $sql = "SELECT " . join(', ', @keys) . " FROM $table"; + $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; my $data = $Self->dbh->selectrow_hashref($sql, {}, map { $limit->{$_} } @lkeys); foreach my $key (@keys) { delete $values{$key} if $limit->{$key}; # in the limit, no point setting again diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 210911c..f709c23 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -64,7 +64,7 @@ sub setuser { $Self->begin(); my $data = $Self->dbh->selectrow_arrayref("SELECT hostname, username, password FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}, {hostname => $data->[0]}); + $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}); } else { $Self->dinsert('iserver', { @@ -75,7 +75,7 @@ sub setuser { } my $user = $Self->dbh->selectrow_arrayref("SELECT email FROM account"); if ($user and $user->[0]) { - $Self->dmaybeupdate('account', {email => $username}, {email => $username}); + $Self->dmaybeupdate('account', {email => $username}); } else { $Self->dinsert('account', { @@ -131,7 +131,8 @@ sub backend_cmd { sub sync_folders { my $Self = shift; - my $folders = $Self->backend_cmd('folders', []); + my $data = $Self->backend_cmd('folders', []); + my ($prefix, $folders) = @$data; $Self->begin(); my $dbh = $Self->dbh(); @@ -165,6 +166,8 @@ sub sync_folders { $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); } + $Self->dmaybeupdate('iserver', {prefix => $prefix, lastfolderupdate => time()}); + $Self->commit(); if (keys %getstatus) { @@ -1417,6 +1420,7 @@ CREATE TABLE IF NOT EXISTS iserver ( username TEXT PRIMARY KEY, password TEXT, hostname TEXT, + prefix TEXT, lastfoldersync DATE, mtime DATE NOT NULL ); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 04707ff..46ee4c6 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -161,7 +161,7 @@ sub delete_card { sub folders { my $Self = shift; $Self->connect_imap(); - return $Self->{folders}; + return [$Self->{prefix}, $Self->{folders}]; } sub capability { diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 4d2b2f5..75e8941 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -79,7 +79,7 @@ sub connect_imap { next unless $Self->{imap}; $Self->{lastused} = time(); my $namespace = $Self->{imap}->namespace(); - my $prefix = $namespace->[0][0][0]; + my $prefix = $Self->{prefix} = $namespace->[0][0][0]; my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; my @folders = $Self->{imap}->$list('', '*'); From 2fbf724bfda2e5f785aac467b1119be7eefe330c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 18:48:56 -0400 Subject: [PATCH 147/331] lots of changes - hopefully folder mgmt --- JMAP/API.pm | 77 ++++++++++++++++++++++------------------------ JMAP/ImapDB.pm | 11 +++---- bin/apiendpoint.pl | 9 ++++-- 3 files changed, 48 insertions(+), 49 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 728cb3b..aa40875 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -161,41 +161,48 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, parentId, name, role, sortOrder, mustBeOnlyMailbox, mayReadItems, mayAddItems, mayRemoveItems, mayCreateChild, mayRename, mayDelete FROM jmailboxes WHERE active = 1"); + my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE active = 1", {Slice => {}}); # outbox - magic - push @$data, [ 'outbox', 0, "Outbox", 'outbox', 1, 1, 1, 1, 1, 0, 0, 0 ]; + push @$data, { + jmailboxid => 'outbox', + parentId => 0, + name => 'Outbox', + role => 'outbox', + sortOrder => 1, + mustBeOnlyMailbox => 1, + mayReadItems => 1, + mayAddItems => 1, + mayRemoveItems => 1, + mayCreateChild => 0, + mayRename => 0, + mayDelete => 0, + }; my %ids; if ($args->{ids}) { %ids = map { $_ => 1 } @{$args->{ids}}; } else { - %ids = map { $_->[0] => 1 } @$data; + %ids = map { $_->{jmailboxid} => 1 } @$data; } - my %byrole = map { $_->[3] => $_->[0] } grep { $_->[3] } @$data; + my %byrole = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$data; my @list; my %ONLY_MAILBOXES; foreach my $item (@$data) { - next unless delete $ids{$item->[0]}; - $ONLY_MAILBOXES{$item->[0]} = $item->[5]; + next unless delete $ids{$item->{jmailboxid}}; + $ONLY_MAILBOXES{$item->{jmailboxid}} = $item->{mustBeOnlyMailbox}; my %rec = ( - id => "$item->[0]", - parentId => ($item->[1] ? "$item->[1]" : undef), - name => $item->[2], - role => $item->[3], - sortOrder => $item->[4], - mustBeOnlyMailbox => $item->[5] ? $JSON::true : $JSON::false, - mayReadItems => $item->[6] ? $JSON::true : $JSON::false, - mayAddItems => $item->[7] ? $JSON::true : $JSON::false, - mayRemoveItems => $item->[8] ? $JSON::true : $JSON::false, - mayCreateChild => $item->[9] ? $JSON::true : $JSON::false, - mayRename => $item->[10] ? $JSON::true : $JSON::false, - mayDelete => $item->[11] ? $JSON::true : $JSON::false, + id => "$item->{jmailboxid}", + parentId => ($item->{parentId} ? "$item->{parentId}" : undef), + name => $item->{name}, + role => $item->{role}, + sortOrder => $item->{sortOrder}, + (map { $_ => ($item->{$_} ? $JSON::true : $JSON::false) } qw(mustBeOnlyMailbox mayReadItems mayAddItems mayRemoveItems mayCreateChild mayRename mayDelete)), ); foreach my $key (keys %rec) { @@ -203,16 +210,16 @@ sub getMailboxes { } if (_prop_wanted($args, 'totalMessages')) { - ($rec{totalMessages}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT msgid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->[0]); + ($rec{totalMessages}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT msgid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->{jmailboxid}); $rec{totalMessages} += 0; } if (_prop_wanted($args, 'unreadMessages')) { - ($rec{unreadMessages}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT msgid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.isUnread = 1 AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->[0]); + ($rec{unreadMessages}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT msgid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.isUnread = 1 AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->{jmailboxid}); $rec{unreadMessages} += 0; } if (_prop_wanted($args, 'totalThreads')) { - ($rec{totalThreads}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT thrid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->[0]); + ($rec{totalThreads}) = $dbh->selectrow_array("SELECT COUNT(DISTINCT thrid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1", {}, $item->{jmailboxid}); $rec{totalThreads} += 0; } @@ -220,14 +227,14 @@ sub getMailboxes { # so long as they aren't in an ONLY_MAILBOXES folder if (_prop_wanted($args, 'unreadThreads')) { my $folderlimit = ''; - if ($ONLY_MAILBOXES{$item->[0]}) { - $folderlimit = "AND jmessagemap.jmailboxid = " . $dbh->quote($item->[0]); + if ($ONLY_MAILBOXES{$item->{jmailboxid}}) { + $folderlimit = "AND jmessagemap.jmailboxid = " . $dbh->quote($item->{jmailboxid}); } else { my @ids = grep { $ONLY_MAILBOXES{$_} } sort keys %ONLY_MAILBOXES; $folderlimit = "AND jmessagemap.jmailboxid NOT IN (" . join(',', map { $dbh->quote($_) } @ids) . ")" if @ids; } my $sql ="SELECT COUNT(DISTINCT thrid) FROM jmessages JOIN jmessagemap USING (msgid) WHERE jmailboxid = ? AND jmessages.active = 1 AND jmessagemap.active = 1 AND thrid IN (SELECT thrid FROM jmessages JOIN jmessagemap USING (msgid) WHERE isUnread = 1 AND jmessages.active = 1 AND jmessagemap.active = 1 $folderlimit)"; - ($rec{unreadThreads}) = $dbh->selectrow_array($sql, {}, $item->[0]); + ($rec{unreadThreads}) = $dbh->selectrow_array($sql, {}, $item->{jmailboxid}); $rec{unreadThreads} += 0; } @@ -284,28 +291,18 @@ sub getMailboxUpdates { return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); - my $data = $dbh->selectall_arrayref("SELECT jmailboxid, jmodseq, jcountsmodseq, active FROM jmailboxes ORDER BY jmailboxid"); + my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE jmodseq > ?1 OR jcountsmodseq > ?1", {Slice => {}}, $sinceState); my @changed; my @removed; my $onlyCounts = 1; foreach my $item (@$data) { - if ($item->[1] > $sinceState) { - if ($item->[3]) { - push @changed, $item->[0]; - $onlyCounts = 0; - } - else { - push @removed, $item->[0]; - } + if ($item->{active}) { + push @changed, $item->{jmailboxid}; + $onlyCounts = 0 if $item->{jmodseq} > $sinceState; } - elsif (($item->[2] || 0) > $sinceState) { - if ($item->[3]) { - push @changed, $item->[0]; - } - else { - push @removed, $item->[0]; - } + else { + push @removed, $item->{jmailboxid}; } } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index f709c23..a501078 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -166,7 +166,7 @@ sub sync_folders { $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); } - $Self->dmaybeupdate('iserver', {prefix => $prefix, lastfolderupdate => time()}); + $Self->dmaybeupdate('iserver', {prefix => $prefix, lastfoldersync => time()}); $Self->commit(); @@ -239,13 +239,12 @@ sub sync_jmailboxes { parentId => $parentId, sortOrder => $sortOrder, mustBeOnlyMailbox => 1, - mayDelete => 0, mayReadItems => 1, mayAddItems => 1, mayRemoveItems => 1, - mayCreateChild => 0, - mayRename => 0, - mayDelete => 0, + mayCreateChild => 1, + mayRename => $role ? 1 : 0, + mayDelete => $role ? 1 : 0, ); if ($id) { if ($role and $roletoid{$role} and $roletoid{$role} != $id) { @@ -1135,7 +1134,7 @@ sub update_mailboxes { my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); - $Self->backend_cmd('rename_mailbox', $oldname, $imapname); + $Self->backend_cmd('rename_mailbox', $oldname, $imapname) if $oldname ne $imapname; push @updated, $id; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 8b06196..da0e186 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -106,7 +106,7 @@ sub process_request { my $handle = shift; set_accountid(shift); warn "Connected $accountid\n"; - $handle->push_read(json => mk_handler($accountid, 1)); + $handle->push_read(json => mk_handler(1)); }); EV::run; @@ -160,7 +160,7 @@ sub handle_getstate { } sub mk_handler { - my ($db, $n) = @_; + my ($n) = @_; $hdl->{killer} = AnyEvent->timer(after => 600, cb => sub { warn "SHUTTING DOWN $accountid ON TIMEOUT\n"; @@ -172,7 +172,7 @@ sub mk_handler { return sub { my ($hdl, $json) = @_; - $hdl->push_read(json => mk_handler($db, $n+1)); + $hdl->push_read(json => mk_handler($n+1)); # make sure we have a connection @@ -225,6 +225,9 @@ sub mk_handler { unless ($res) { $res = ['error', "$@"] } + if ($db and $db->in_transaction()) { + $res = ['error', "STILL IN TRANSACTION " . Dumper($res, $args, $tag)] + } $res->[2] = $tag; $hdl->push_write(json => $res) if $res->[0]; warn "HANDLED $cmd ($tag) => $res->[0] ($accountid)\n" ; From 099d1dba28bae656c0b51e59a590b9f3865f38b6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 10:01:28 +1000 Subject: [PATCH 148/331] shorter IDs with m and t --- JMAP/ImapDB.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index a501078..9963436 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -618,12 +618,13 @@ sub calcmsgid { my $envelope = $data->{envelope}; my $json = JSON::XS->new->allow_nonref->canonical; my $coded = $json->encode([$envelope, $data->{'rfc822.size'}]); - my $msgid = 's' . substr(sha1_hex($coded), 0, 11); + my $base = substr(sha1_hex($coded), 0, 9); + my $msgid = "m$base"; my $replyto = lc($envelope->{'In-Reply-To'} || ''); my $messageid = lc($envelope->{'Message-ID'} || ''); my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?)", {}, $replyto, $messageid); - $thrid ||= $msgid; + $thrid ||= "t$base"; foreach my $id ($replyto, $messageid) { next if $id eq ''; $Self->dbh->do("INSERT OR IGNORE INTO ithread (messageid, thrid) VALUES (?, ?)", {}, $id, $thrid); From 74b458321b7d97522e476ad85c94e3b065ec1521 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 20:08:06 -0400 Subject: [PATCH 149/331] don't send NULL date, it makes the client sad --- JMAP/DB.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index aa6435e..9dfd4fa 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -273,8 +273,7 @@ sub parse_date { sub isodate { my $Selft = shift; - my $epoch = shift; - return unless $epoch; # no 1970, punk + my $epoch = shift || time(); my $date = DateTime->from_epoch( epoch => $epoch ); return $date->iso8601(); From 79940673ecdee5f9211858b876fffd783587f8db Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 20:12:53 -0400 Subject: [PATCH 150/331] no warning for non-role folders --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9963436..eb1c133 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -829,7 +829,7 @@ sub update_messages { my %jmailmap = map { $_->[4] => $_ } @$folderdata; my $jmapdata = $dbh->selectall_arrayref("SELECT jmailboxid, role FROM jmailboxes"); my %jidmap = map { $_->[0] => $_->[1] } @$jmapdata; - my %jrolemap = map { $_->[1] => $_->[0] } @$jmapdata; + my %jrolemap = map { $_->[1] => $_->[0] } grep { $_-> [1] } @$jmapdata; my @changed; foreach my $ifolderid (keys %updatemap) { From 304ef3d63414a64a0e093a9e179c25266376f68d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 12:46:13 +1000 Subject: [PATCH 151/331] idmap support for all the things --- JMAP/API.pm | 50 +++++++++++++++++++++++++++----------------------- JMAP/ImapDB.pm | 21 +++++++++++++++++---- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index aa40875..420386c 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -21,6 +21,7 @@ sub idmap { if (@_) { $Self->{idmap}{$key} = shift; } + return $Self->{idmap} unless $key; return exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; } @@ -181,7 +182,7 @@ sub getMailboxes { my %ids; if ($args->{ids}) { - %ids = map { $_ => 1 } @{$args->{ids}}; + %ids = map { $Self->idmap($_) => 1 } @{$args->{ids}}; } else { %ids = map { $_->{jmailboxid} => 1 } @$data; @@ -351,9 +352,9 @@ sub setMailboxes { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_mailboxes($create); - my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_mailboxes($destroy); $Self->{db}->sync(); @@ -415,7 +416,7 @@ sub _match { # XXX - condition handling code if ($condition->{inMailboxes}) { my $inall = 1; - foreach my $id (@{$condition->{inMailboxes}}) { + foreach my $id (map { $Self->idmap($_) } @{$condition->{inMailboxes}}) { $storage->{mailbox}{$id} ||= $Self->_load_mailbox($id); next if $storage->{mailbox}{$id}{$item->{msgid}}[2]; #active $inall = 0; @@ -425,7 +426,7 @@ sub _match { if ($condition->{notInMailboxes}) { my $inany = 0; - foreach my $id (@{$condition->{notInMailboxes}}) { + foreach my $id (map { $Self->idmap($_) } @{$condition->{notInMailboxes}}) { $storage->{mailbox}{$id} ||= $Self->_load_mailbox($id); next unless $storage->{mailbox}{$id}{$item->{msgid}}[2]; #active $inany = 1; @@ -824,7 +825,7 @@ sub getMessages { } $need_content = 1 if ($args->{properties} and grep { m/^headers\./ } @{$args->{properties}}); my %msgidmap; - foreach my $msgid (@{$args->{ids}}) { + foreach my $msgid (map { $Self->idmap($_) } @{$args->{ids}}) { next if $seenids{$msgid}; $seenids{$msgid} = 1; my $data = $dbh->selectrow_hashref("SELECT * FROM jmessages WHERE msgid = ?", {}, $msgid); @@ -1070,9 +1071,9 @@ sub setMessages { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_messages($create); - my ($updated, $notUpdated) = $Self->{db}->update_messages($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_messages($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_messages($destroy); $Self->{db}->sync_imap(); @@ -1122,7 +1123,8 @@ sub importMessage { $Self->commit(); # import to a normal mailbox (or boxes) - my ($msgid, $thrid) = $Self->import_message($message, $args->{mailboxIds}, + my @ids = map { $Self->idmap($_) } @{$args->{mailboxIds}}; + my ($msgid, $thrid) = $Self->import_message($message, \@ids, isUnread => $args->{isUnread}, isFlagged => $args->{isFlagged}, isAnswered => $args->{isAnswered}, @@ -1162,7 +1164,8 @@ sub reportMessages { $Self->commit(); - my ($reported, $notfound) = $Self->report_messages($args->{messageIds}, $args->{asSpam}); + my @ids = map { $Self->idmap($_) } @{$args->{messageIds}}; + my ($reported, $notfound) = $Self->report_messages(\@ids, $args->{asSpam}); my @res; push @res, ['messagesReported', { @@ -1193,7 +1196,7 @@ sub getThreads { my %seenids; my %missingids; my @allmsgs; - foreach my $thrid (@{$args->{ids}}) { + foreach my $thrid (map { $Self->idmap($_) } @{$args->{ids}}) { next if $seenids{$thrid}; $seenids{$thrid} = 1; my $data = $dbh->selectall_arrayref("SELECT * FROM jmessages WHERE thrid = ? AND active = 1 ORDER BY internaldate", {Slice => {}}, $thrid); @@ -1369,7 +1372,7 @@ sub getCalendars { my %ids; if ($args->{ids}) { - %ids = map { $_ => 1 } @{$args->{ids}}; + %ids = map { $Self->idmap($_) => 1 } @{$args->{ids}}; } else { %ids = map { $_->[0] => 1 } @$data; @@ -1571,7 +1574,7 @@ sub getCalendarEvents { my %seenids; my %missingids; my @list; - foreach my $eventid (@{$args->{ids}}) { + foreach my $eventid (map { $Self->idmap($_) } @{$args->{ids}}) { next if $seenids{$eventid}; $seenids{$eventid} = 1; my $data = $dbh->selectrow_hashref("SELECT * FROM jevents WHERE eventuid = ?", {}, $eventid); @@ -1678,7 +1681,7 @@ sub getAddressbooks { my %ids; if ($args->{ids}) { - %ids = map { $_ => 1 } @{$args->{ids}}; + %ids = map { $Self->($_) => 1 } @{$args->{ids}}; } else { %ids = map { $_->[0] => 1 } @$data; @@ -1876,7 +1879,7 @@ sub getContacts { my %want; if ($args->{ids}) { - %want = map { $_ => 1 } @{$args->{ids}}; + %want = map { $Self->idmap($_) => 1 } @{$args->{ids}}; } else { %want = %$data; @@ -1983,7 +1986,7 @@ sub getContactGroups { my %want; if ($args->{ids}) { - %want = map { $_ => 1 } @{$args->{ids}}; + %want = map { $Self->idmap($_) => 1 } @{$args->{ids}}; } else { %want = %$data; @@ -2093,9 +2096,9 @@ sub setContactGroups { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_contact_groups($create); - my ($updated, $notUpdated) = $Self->{db}->update_contact_groups($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_contact_groups($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contact_groups($destroy); $Self->{db}->sync_addressbooks(); @@ -2133,9 +2136,9 @@ sub setContacts { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_contacts($create); - my ($updated, $notUpdated) = $Self->{db}->update_contacts($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_contacts($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contacts($destroy); $Self->{db}->sync_addressbooks(); @@ -2173,9 +2176,9 @@ sub setCalendarEvents { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_calendar_events($create); - my ($updated, $notUpdated) = $Self->{db}->update_calendar_events($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_calendar_events($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendar_events($destroy); $Self->{db}->sync_calendars(); @@ -2215,7 +2218,8 @@ sub setCalendars { # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_calendars($create); - my ($updated, $notUpdated) = $Self->{db}->update_calendars($update); + $Self->idmap($_, $created->{$_}{id}) for keys %$created; + my ($updated, $notUpdated) = $Self->{db}->update_calendars($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendars($destroy); $Self->{db}->sync_calendars(); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index eb1c133..3d40ea5 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -809,6 +809,7 @@ sub import_message { sub update_messages { my $Self = shift; my $changes = shift; + my $idmap = shift; my $dbh = $Self->{dbh}; @@ -863,7 +864,8 @@ sub update_messages { $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { - my $id = $action->{mailboxIds}->[0]; # there can be only one + my @mboxes = map { $idmap->($_) } @{$action->{mailboxIds}}; + my $id = $mboxes[0]; # there can be only one if ($id eq 'outbox') { my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); @@ -1114,6 +1116,7 @@ sub create_mailboxes { sub update_mailboxes { my $Self = shift; my $update = shift; + my $idmap = shift; my $dbh = $Self->{dbh}; @@ -1124,7 +1127,8 @@ sub update_mailboxes { my $mailbox = $update->{$id}; my $imapname = $mailbox->{name}; if ($mailbox->{parentId}) { - my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $mailbox->{parentId}); + my $parentId = $idmap->($mailbox->{parentId}); + my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $parentId); # XXX - errors $imapname = "$parentName$sep$imapname"; } @@ -1187,6 +1191,7 @@ sub create_calendar_events { sub update_calendar_events { my $Self = shift; my $update = shift; + my $idmap = shift; my $dbh = $Self->{dbh}; @@ -1250,7 +1255,10 @@ sub create_contact_groups { $card->uid($uid); $card->VKind('group'); $card->VName($contact->{name}) if exists $contact->{name}; - $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; + if (exists $contact->{memberIds}) { + my @ids = @{$contact->{memberIds}}; + $card->VGroupContactUIDs(\@ids); + } $Self->backend_cmd('new_card', $href, $card); $createmap{$cid} = { id => $uid }; @@ -1262,6 +1270,7 @@ sub create_contact_groups { sub update_contact_groups { my $Self = shift; my $changes = shift; + my $idmap = shift; my $dbh = $Self->{dbh}; @@ -1277,7 +1286,10 @@ sub update_contact_groups { my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); $card->VKind('group'); $card->VName($contact->{name}) if exists $contact->{name}; - $card->VGroupContactUIDs($contact->{memberIds}) if exists $contact->{memberIds}; + if (exists $contact->{memberIds}) { + my @ids = map { $idmap->($_) } @{$contact->{memberIds}}; + $card->VGroupContactUIDs(\@ids); + } $Self->backend_cmd('update_card', $resource, $card); push @updated, $carduid; @@ -1351,6 +1363,7 @@ sub create_contacts { sub update_contacts { my $Self = shift; my $changes = shift; + my $idmap = shift; my $dbh = $Self->{dbh}; From c9c6bbd0669ccf1e7f6e70d01e61c0a403a625be Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 22:56:54 -0400 Subject: [PATCH 152/331] suppress some undef warnings --- JMAP/API.pm | 1 + JMAP/ImapDB.pm | 1 + 2 files changed, 2 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 420386c..a075345 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1216,6 +1216,7 @@ sub getThreads { next if $item->{isDraft}; push @msgs, $item->{msgid}; $seenmsgs{$item->{msgid}} = 1; + next unless $item->{msgmessageid}; if (my $draftmsgs = $drafts{$item->{msgmessageid}}) { push @msgs, @$draftmsgs; $seenmsgs{$_} = 1 for @$draftmsgs; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 3d40ea5..4fc907c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1134,6 +1134,7 @@ sub update_mailboxes { } else { my ($prefix) = $dbh->selectrow_array("SELECT prefix FROM iserver"); + $prefix = '' unless $prefix; $imapname = "$prefix$imapname"; } From 4a13b422ee9d91ea4fe11563cf0e207d98c5669a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 19 Jul 2015 23:24:51 -0400 Subject: [PATCH 153/331] it's called contactIds --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index a075345..5a33192 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1876,7 +1876,7 @@ sub getContacts { #properties: String[] A list of properties to fetch for each message. - my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts", 'contactuid', {Slice => {}}); + my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts WHERE active = 1", 'contactuid', {Slice => {}}); my %want; if ($args->{ids}) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 4fc907c..58d75bd 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1256,8 +1256,8 @@ sub create_contact_groups { $card->uid($uid); $card->VKind('group'); $card->VName($contact->{name}) if exists $contact->{name}; - if (exists $contact->{memberIds}) { - my @ids = @{$contact->{memberIds}}; + if (exists $contact->{contactIds}) { + my @ids = @{$contact->{contactIds}}; $card->VGroupContactUIDs(\@ids); } @@ -1286,9 +1286,9 @@ sub update_contact_groups { } my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); $card->VKind('group'); - $card->VName($contact->{name}) if exists $contact->{name}; - if (exists $contact->{memberIds}) { - my @ids = map { $idmap->($_) } @{$contact->{memberIds}}; + $card->VFN($contact->{name}) if exists $contact->{name}; + if (exists $contact->{contactIds}) { + my @ids = map { $idmap->($_) } @{$contact->{contactIds}}; $card->VGroupContactUIDs(\@ids); } From 10d17c9bd9ed2c2c735eb11c24e6e162cac48ad6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 00:10:50 -0400 Subject: [PATCH 154/331] various contact fixes --- JMAP/API.pm | 9 ++++++--- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 2 +- bin/apiendpoint.pl | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 5a33192..b976f9f 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -19,10 +19,13 @@ sub idmap { my $Self = shift; my $key = shift; if (@_) { - $Self->{idmap}{$key} = shift; + my $val = shift; + warn "SET $key = $val"; + $Self->{idmap}{$key} = $val; } - return $Self->{idmap} unless $key; - return exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; + my $val = exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; + warn "GET $key = $val"; + return $val; } sub getAccounts { diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 9dfd4fa..00afefd 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -810,7 +810,7 @@ sub delete_card { $Self->dupdate('jcontacts', {active => 0}, {contactuid => $carduid, jaddressbookid => $jaddressbookid}); } else { - $Self->dupdate('jcontactgroupmap', {active => 0}, {groupuid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dupdate('jcontactgroups', {active => 0}, {groupuid => $carduid, jaddressbookid => $jaddressbookid}); } } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 58d75bd..14287e8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1255,7 +1255,7 @@ sub create_contact_groups { my $uid = new_uuid_string(); $card->uid($uid); $card->VKind('group'); - $card->VName($contact->{name}) if exists $contact->{name}; + $card->VFN($contact->{name}) if exists $contact->{name}; if (exists $contact->{contactIds}) { my @ids = @{$contact->{contactIds}}; $card->VGroupContactUIDs(\@ids); diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index da0e186..a1ad118 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -392,6 +392,7 @@ sub handle_jmap { my ($command, $args, $tag) = @$item; my @items; my $FuncRef = $api->can($command); + warn "JMAP CMD $command"; if ($FuncRef) { @items = eval { $api->$command($args, $tag) }; if ($@) { From ed92f0315ee1b3f871247e4c42641987ed4cd7ef Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 00:25:12 -0400 Subject: [PATCH 155/331] fix idmap and getContactGroups active --- JMAP/API.pm | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index b976f9f..141c2f4 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -20,11 +20,9 @@ sub idmap { my $key = shift; if (@_) { my $val = shift; - warn "SET $key = $val"; - $Self->{idmap}{$key} = $val; + $Self->{idmap}{"#$key"} = $val; } my $val = exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; - warn "GET $key = $val"; return $val; } @@ -1986,7 +1984,7 @@ sub getContactGroups { #properties: String[] A list of properties to fetch for each message. - my $data = $dbh->selectall_hashref("SELECT * FROM jcontactgroups", 'groupuid', {Slice => {}}); + my $data = $dbh->selectall_hashref("SELECT * FROM jcontactgroups WHERE active = 1", 'groupuid', {Slice => {}}); my %want; if ($args->{ids}) { From 54a2038a5d6b17a6adb431d516bb558f5905c2a3 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 00:27:35 -0400 Subject: [PATCH 156/331] separate setid API --- JMAP/API.pm | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 141c2f4..6af6f98 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -15,13 +15,16 @@ sub new { return bless {db => $db}, ref($class) || $class; } +sub setid { + my $Self = shift; + my $key = shift; + my $val = shift; + $Self->{idmap}{"#$key"} = $val; +} + sub idmap { my $Self = shift; my $key = shift; - if (@_) { - my $val = shift; - $Self->{idmap}{"#$key"} = $val; - } my $val = exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; return $val; } @@ -354,7 +357,7 @@ sub setMailboxes { my $destroy = $args->{destroy} || []; my ($created, $notCreated) = $Self->{db}->create_mailboxes($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_mailboxes($destroy); @@ -1073,7 +1076,7 @@ sub setMessages { my $destroy = $args->{destroy} || []; my ($created, $notCreated) = $Self->{db}->create_messages($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_messages($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_messages($destroy); @@ -2099,7 +2102,7 @@ sub setContactGroups { my $destroy = $args->{destroy} || []; my ($created, $notCreated) = $Self->{db}->create_contact_groups($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_contact_groups($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contact_groups($destroy); @@ -2139,7 +2142,7 @@ sub setContacts { my $destroy = $args->{destroy} || []; my ($created, $notCreated) = $Self->{db}->create_contacts($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_contacts($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_contacts($destroy); @@ -2179,7 +2182,7 @@ sub setCalendarEvents { my $destroy = $args->{destroy} || []; my ($created, $notCreated) = $Self->{db}->create_calendar_events($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_calendar_events($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendar_events($destroy); @@ -2218,9 +2221,8 @@ sub setCalendars { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; - # XXX - idmap support my ($created, $notCreated) = $Self->{db}->create_calendars($create); - $Self->idmap($_, $created->{$_}{id}) for keys %$created; + $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_calendars($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_calendars($destroy); From f00fb4f79e8a62561d46b2f41621f4f0975ef843 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 00:36:01 -0400 Subject: [PATCH 157/331] add a messageid to drafts --- JMAP/DB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 00afefd..8deff3a 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -567,6 +567,7 @@ sub _makemsg { Cc => _mkemail($args->{cc}), Bcc => _mkemail($args->{bcc}), Subject => $args->{subject}, + 'Message-Id' => $args->{msgmessageid}, %{$args->{headers} || {}}, ]; @@ -664,6 +665,7 @@ sub create_messages { foreach my $cid (keys %$args) { my $item = $args->{$cid}; + $item->{msgmessageid} = new_uuid_string() . '@proxy.jmap.io'; my $message = $Self->_makemsg($item); # XXX - let's just assume goodness for now - lots of error handling to add my ($msgid, $thrid) = $Self->import_message($message, [$draftid], From e092d8df487e061bb78054c5c9943c3ba61af215 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 00:39:57 -0400 Subject: [PATCH 158/331] add the Date too --- JMAP/DB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 8deff3a..b5e8567 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -567,6 +567,7 @@ sub _makemsg { Cc => _mkemail($args->{cc}), Bcc => _mkemail($args->{bcc}), Subject => $args->{subject}, + Date => Date::Format::time2str("%a, %d %b %Y %H:%M:%S %z", $args->{msgdate}), 'Message-Id' => $args->{msgmessageid}, %{$args->{headers} || {}}, ]; @@ -665,6 +666,7 @@ sub create_messages { foreach my $cid (keys %$args) { my $item = $args->{$cid}; + $item->{msgdate} = time(); $item->{msgmessageid} = new_uuid_string() . '@proxy.jmap.io'; my $message = $Self->_makemsg($item); # XXX - let's just assume goodness for now - lots of error handling to add From fffa600982b2cf6de71f63db207dd7b3bf955936 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:20:51 -0400 Subject: [PATCH 159/331] role folders are the non-deletable --- JMAP/ImapDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 14287e8..15e027b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -243,8 +243,8 @@ sub sync_jmailboxes { mayAddItems => 1, mayRemoveItems => 1, mayCreateChild => 1, - mayRename => $role ? 1 : 0, - mayDelete => $role ? 1 : 0, + mayRename => $role ? 0 : 1, + mayDelete => $role ? 0 : 1, ); if ($id) { if ($role and $roletoid{$role} and $roletoid{$role} != $id) { From 6bf2e57005f2e92b40ee1987376e4c28e66f868b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:36:10 -0400 Subject: [PATCH 160/331] actually implement create/rename/delete mailboxes --- JMAP/ImapDB.pm | 1 + JMAP/Sync/Common.pm | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 15e027b..7e101a9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1126,6 +1126,7 @@ sub update_mailboxes { foreach my $id (keys %$update) { my $mailbox = $update->{$id}; my $imapname = $mailbox->{name}; + next unless (defined $imapname and $imapname ne ''); if ($mailbox->{parentId}) { my $parentId = $idmap->($mailbox->{parentId}); my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $parentId); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 46ee4c6..07980f2 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -451,4 +451,38 @@ sub imap_search { return ['search', $imapname, $uidvalidity, $uids]; } +sub create_mailbox { + my $Self = shift; + my $imapname = shift; + + my $imap = $Self->connect_imap(); + + $imap->create($imapname); + + return []; +} + +sub rename_mailbox { + my $Self = shift; + my $oldname = shift; + my $imapname = shift; + + my $imap = $Self->connect_imap(); + + $imap->rename($oldname, $imapname); + + return []; +} + +sub delete_mailbox { + my $Self = shift; + my $imapname = shift; + + my $imap = $Self->connect_imap(); + + $imap->delete($imapname); + + return []; +} + 1; From 732476d49603dabc592bfea9101f5ab17a7151f9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:46:47 -0400 Subject: [PATCH 161/331] update on folder change --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 5 +++-- JMAP/Sync/Common.pm | 2 +- JMAP/Sync/Standard.pm | 6 +++++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 6af6f98..711f11e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -361,7 +361,7 @@ sub setMailboxes { my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update, sub { $Self->idmap(shift) }); my ($destroyed, $notDestroyed) = $Self->{db}->destroy_mailboxes($destroy); - $Self->{db}->sync(); + $Self->{db}->sync_imap(); my @res; push @res, ['mailboxesSet', { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7e101a9..222c4e8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -130,8 +130,9 @@ sub backend_cmd { # call in transaction sub sync_folders { my $Self = shift; + my $force = shift; - my $data = $Self->backend_cmd('folders', []); + my $data = $Self->backend_cmd('folders', $force); my ($prefix, $folders) = @$data; $Self->begin(); @@ -1101,7 +1102,7 @@ sub create_mailboxes { # (in theory we could save this until the end and resolve the names in after the renames and deletes... but it does mean # we can't use ids as referenes...) - $Self->sync_folders(); + $Self->sync_folders(1); my %createmap; foreach my $imapname (keys %idmap) { diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 07980f2..c4bd812 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -160,7 +160,7 @@ sub delete_card { # read folder list from the server sub folders { my $Self = shift; - $Self->connect_imap(); + $Self->connect_imap(1); return [$Self->{prefix}, $Self->{folders}]; } diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 75e8941..8988bd1 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -58,12 +58,16 @@ sub connect_contacts { sub connect_imap { my $Self = shift; + my $force = shift; - if ($Self->{imap}) { + if ($Self->{imap} and not $force) { $Self->{lastused} = time(); return $Self->{imap}; } + $Self->{imap}->disconnect() if $Self->{imap}; + delete $Self->{imap}; + for (1..3) { my $port = 993; my $usessl = $port != 143; # we use SSL for anything except default From 77346aa5eea5870c92d2f3b2d3df3bf270813f4d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:53:50 -0400 Subject: [PATCH 162/331] get folders every time --- JMAP/Sync/Common.pm | 31 +++++++++++++++++++++++-------- JMAP/Sync/Standard.pm | 19 ------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index c4bd812..1807b77 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -160,8 +160,29 @@ sub delete_card { # read folder list from the server sub folders { my $Self = shift; - $Self->connect_imap(1); - return [$Self->{prefix}, $Self->{folders}]; + my $force = shift; + + my $imap = $Self->connect_imap(); + + my $namespace = $imap->namespace(); + my $prefix = $namespace->[0][0][0]; + my $listcmd = $imap->capability()->{xlist} ? 'xlist' : 'list'; + my @folders = $imap->$listcmd('', '*'); + + my %folders; + foreach my $folder (@folders) { + my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; + my $name = $folder->[2]; + my $label = $role; + unless ($label) { + $label = $folder->[2]; + $label =~ s{^$prefix}{}; + $label =~ s{^[$folder->[1]]}{}; # just in case prefix was missing sep + } + $folders{$name} = [$folder->[1], $label]; + } + + return [$prefix, \%folders]; } sub capability { @@ -170,12 +191,6 @@ sub capability { return $imap->capability(); } -sub labels { - my $Self = shift; - $Self->connect_imap(); - return $Self->{labels}; -} - sub imap_noop { my $Self = shift; my $folders = shift; diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 8988bd1..7a6fc0d 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -82,25 +82,6 @@ sub connect_imap { ); next unless $Self->{imap}; $Self->{lastused} = time(); - my $namespace = $Self->{imap}->namespace(); - my $prefix = $Self->{prefix} = $namespace->[0][0][0]; - my $list = $Self->{imap}->capability()->{xlist} ? 'xlist' : 'list'; - my @folders = $Self->{imap}->$list('', '*'); - - delete $Self->{folders}; - delete $Self->{labels}; - foreach my $folder (@folders) { - my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; - my $name = $folder->[2]; - my $label = $role; - unless ($label) { - $label = $folder->[2]; - $label =~ s{^$prefix}{}; - $label =~ s{^[$folder->[1]]}{}; # just in case prefix was missing sep - } - $Self->{folders}{$name} = [$folder->[1], $label]; - $Self->{labels}{$label} = $name; - } return $Self->{imap}; } From 46727ca7b630af0d6a6565dfe946902e2144cb5e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:54:37 -0400 Subject: [PATCH 163/331] more sync folders cleanup --- JMAP/ImapDB.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 222c4e8..96647f8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -130,9 +130,8 @@ sub backend_cmd { # call in transaction sub sync_folders { my $Self = shift; - my $force = shift; - my $data = $Self->backend_cmd('folders', $force); + my $data = $Self->backend_cmd('folders'); my ($prefix, $folders) = @$data; $Self->begin(); @@ -1102,7 +1101,7 @@ sub create_mailboxes { # (in theory we could save this until the end and resolve the names in after the renames and deletes... but it does mean # we can't use ids as referenes...) - $Self->sync_folders(1); + $Self->sync_folders(); my %createmap; foreach my $imapname (keys %idmap) { From bf30ab2a7da6adc687f3daeb4e86da921c260999 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 01:56:47 -0400 Subject: [PATCH 164/331] sync when needed --- JMAP/ImapDB.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 96647f8..f2d10fa 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1101,7 +1101,7 @@ sub create_mailboxes { # (in theory we could save this until the end and resolve the names in after the renames and deletes... but it does mean # we can't use ids as referenes...) - $Self->sync_folders(); + $Self->sync_folders() if keys %idmap; my %createmap; foreach my $imapname (keys %idmap) { @@ -1145,6 +1145,8 @@ sub update_mailboxes { push @updated, $id; } + $Self->sync_folders() if @updated; + return (\@updated, \%notupdated); } From 96b9e60e0679f2de02a09b433f408e58e0054abf Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:06:03 -0400 Subject: [PATCH 165/331] handle where parentId is not set in the update of a subfolder --- JMAP/ImapDB.pm | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index f2d10fa..058815e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1127,8 +1127,11 @@ sub update_mailboxes { my $mailbox = $update->{$id}; my $imapname = $mailbox->{name}; next unless (defined $imapname and $imapname ne ''); - if ($mailbox->{parentId}) { - my $parentId = $idmap->($mailbox->{parentId}); + my $parentId = $mailbox->{parentId}; + ($parentId) = $dbh->selectrow_array("SELECT parentId FROM jmailboxes WHERE jmailboxid = ?", {}, $id) + unless exists $mailbox->{parentId}; + if ($parentId) { + $parentId = $idmap->($parentId); my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $parentId); # XXX - errors $imapname = "$parentName$sep$imapname"; @@ -1165,6 +1168,8 @@ sub destroy_mailboxes { push @destroyed, $id; } + $Self->sync_folders() if @destroyed; + return (\@destroyed, \%notdestroyed); } From d9f05545ecda1ed0bb3fbdbcc1d373bf4dcfe38b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:07:20 -0400 Subject: [PATCH 166/331] getMailboxes: just get them always --- JMAP/API.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 711f11e..6416913 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -157,6 +157,8 @@ sub getMailboxes { my $Self = shift; my $args = shift; + $Self->sync_folders(); # why not, on startup get data + # XXX - ideally this is transacted inside the DB $Self->begin(); my $dbh = $Self->{db}->dbh(); From 1061e6b4d4ef1aa24804b875256d80919b68a4e1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:08:20 -0400 Subject: [PATCH 167/331] thing back --- JMAP/API.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 6416913..711f11e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -157,8 +157,6 @@ sub getMailboxes { my $Self = shift; my $args = shift; - $Self->sync_folders(); # why not, on startup get data - # XXX - ideally this is transacted inside the DB $Self->begin(); my $dbh = $Self->{db}->dbh(); From 4d82a4854c95cff9894cca73c233690db807ea54 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:19:29 -0400 Subject: [PATCH 168/331] make client page do a background syncall --- bin/apiendpoint.pl | 12 ++++++++++++ bin/server.pl | 1 + 2 files changed, 13 insertions(+) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index a1ad118..551f509 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -211,6 +211,9 @@ sub mk_handler { if ($cmd eq 'sync') { return handle_sync(getdb(), $args, $tag); } + if ($cmd eq 'syncall') { + return handle_syncall(getdb(), $args, $tag); + } if ($cmd eq 'davsync') { return handle_davsync(getdb(), $args, $tag); } @@ -246,6 +249,15 @@ sub handle_sync { return ['sync', $JSON::true]; } +sub handle_syncall { + my $db = shift; + $db->sync_folders(); + $db->sync_imap(); + $db->sync_addressbooks(); + $db->sync_calendars(); + return ['syncall', $JSON::true]; +} + sub handle_backfill { my $db = shift; my $res = $db->backfill(); diff --git a/bin/server.pl b/bin/server.pl index 03d644b..142a20e 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -438,6 +438,7 @@ sub client_page { my $data = shift; prod_idler($accountid); + send_backend_request($accountid, 'syncall'); open(FH, "/home/jmap/jmap-perl/htdocs/landing.html"); local $/ = undef; From 542f085dc1e387a90dfa38ec512ffc0a5452778f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:28:26 -0400 Subject: [PATCH 169/331] don't pointlessly update old entries --- JMAP/ImapDB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 058815e..fc557f7 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -275,6 +275,7 @@ sub sync_jmailboxes { foreach my $mailbox (@$jmailboxes) { my $id = $mailbox->[0]; + next unless $mailbox->[4]; next if $seen{$id}; $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); } From 2b9c0b27377c823a52ba58812017ea69cb29560d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 02:55:20 -0400 Subject: [PATCH 170/331] DB: use UUID lib --- JMAP/DB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index b5e8567..401faab 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -9,6 +9,7 @@ use Data::Dumper; use DBI; use Carp qw(confess); +use Data::UUID::LibUUID; use IO::LockedFile; use JSON::XS qw(encode_json decode_json); use Email::MIME; @@ -24,7 +25,6 @@ use Date::Parse; use Net::CalDAVTalk; use Net::CardDAVTalk::VCard; use MIME::Base64 qw(encode_base64); - sub new { my $class = shift; my $accountid = shift || die; From a10ca6d617333cc613d21598404e1076e5cb27c5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 17:30:23 +1000 Subject: [PATCH 171/331] getSearchSnippets --- JMAP/API.pm | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 711f11e..76e4cbf 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -803,6 +803,48 @@ sub getMessageListUpdates { return @res; } +sub _extract_terms { + my $filter = shift; + return () unless $filter; + my @list; + push @list, _extract_terms($filter->{conditions}); + push @list, $filter->{body} if $filter->{body}; + push @list, $filter->{text} if $filter->{text}; + push @list, $filter->{subject} if $filter->{subject}; + return @list; +} + +sub getSearchSnippets { + my $Self = shift; + my $args = shift; + + my $messages = $Self->getMessages({ + accountId => $args->{accountId}, + ids => $args->{messageIds}, + properties => ['subject', 'textBody', 'preview']; + } + + return $messages unless $messages->[0] eq 'messages'; + $messages->[0] = 'searchSnippets'; + delete $messages->[1]{state}; + $messages->[1]{filter} = $args->{filter}; + + my @terms = _extract_terms($args->{filter}); + my $str = '\\b(\\Q' . join('\\E|\\Q', @terms) . '\\E)\\b'; + my $tag = 'b'; # XXX - wrap + foreach my $item (@{$messages->[1]{list}}) { + $item->{messageId} = delete $item->{id}; + $item->{subject} =~ s{$str}{<$tag>$1}gs; + my $text = delete $item->{textBody}; + if ($text =~ m{(.{,20}$str.*)}) { + $item->{preview} = $1; + $item->{preview} =~ s{$str}{<$tag>$1}gs; + } + } + + return $messages; +} + sub getMessages { my $Self = shift; my $args = shift; From 4471629322fa1cf4d87f55b89e55da4861301164 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 18:38:02 +1000 Subject: [PATCH 172/331] DB: do reply references properly --- JMAP/DB.pm | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 401faab..c1aed2e 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -556,11 +556,6 @@ sub _makemsg { my $args = shift; my $isDraft = shift; - my %replyHeaders; - if ($args->{inReplyToMessageId}) { - # XXX - get replyheaders - } - my $header = [ From => _mkone($args->{from}), To => _mkemail($args->{to}), @@ -666,6 +661,16 @@ sub create_messages { foreach my $cid (keys %$args) { my $item = $args->{$cid}; + if ($args->{inReplyToMessageId}) { + my ($replymessageid) = $dbh->selectrow_array("SELECT msgmessageid FROM jmessages WHERE msgid = ?", {}, $args->{inReplyToMessageId}); + unless ($replymessageid) { + $notCreated{$cid} = 'inReplyToNotFound'; + next; + } + $args->{headers}{'In-Reply-To'} = $replymessageid; + $args->{headers}{'References'} = $replymessageid; + # XXX - references + } $item->{msgdate} = time(); $item->{msgmessageid} = new_uuid_string() . '@proxy.jmap.io'; my $message = $Self->_makemsg($item); From e1e9391103bfe278625a5da20ab239c2c621118c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 19:17:25 +1000 Subject: [PATCH 173/331] API: sort by subject and from --- JMAP/API.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index 76e4cbf..0bc0da9 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -389,6 +389,9 @@ sub _build_sort { size => 'msgsize', isflagged => 'isFlagged', isunread => 'isUnread', + subject => 'msgsubject', + from => 'msgfrom', + to => 'msgto', ); my @items; $sortargs = [$sortargs] unless ref $sortargs; From efd2dcb20d69af60133c05b9cb702257069a8065 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 19:18:40 +1000 Subject: [PATCH 174/331] API: magic --- JMAP/API.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 0bc0da9..bd63c61 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -402,8 +402,7 @@ sub _build_sort { die unless $fieldmap{$field}; push @items, "$fieldmap{$field} $dir"; } - # XXX - sort by the from/subject fields - # XXX - threads? + push @items, "msgid desc"; # guarantee stable return join(', ', @items); } From d22e331ceb7d552a578ec18c7456c7c61f780b8c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 03:31:41 -0400 Subject: [PATCH 175/331] fix syntax --- JMAP/API.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index bd63c61..8d74df6 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -823,8 +823,8 @@ sub getSearchSnippets { my $messages = $Self->getMessages({ accountId => $args->{accountId}, ids => $args->{messageIds}, - properties => ['subject', 'textBody', 'preview']; - } + properties => ['subject', 'textBody', 'preview'], + }); return $messages unless $messages->[0] eq 'messages'; $messages->[0] = 'searchSnippets'; From f79293defc3088da86d357d99a8081b16f8de3fb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 04:02:59 -0400 Subject: [PATCH 176/331] make previews really work, HTML sig update --- JMAP/API.pm | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 8d74df6..3073b24 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -7,6 +7,7 @@ use JSON; use strict; use warnings; use Encode; +use HTML::GenerateUtil qw(escape_html); sub new { my $class = shift; @@ -110,7 +111,7 @@ sub getPersonalities { email => $user->{email}, name => $user->{displayname} || $user->{email}, textSignature => "-- \ntext sig", - htmlSignature => "-- \nhtml sig", + htmlSignature => "--
html sig", replyTo => $user->{email}, autoBcc => "", addBccOnSMTP => $JSON::false, @@ -832,15 +833,23 @@ sub getSearchSnippets { $messages->[1]{filter} = $args->{filter}; my @terms = _extract_terms($args->{filter}); - my $str = '\\b(\\Q' . join('\\E|\\Q', @terms) . '\\E)\\b'; + my $str = join("|", @terms); my $tag = 'b'; # XXX - wrap foreach my $item (@{$messages->[1]{list}}) { $item->{messageId} = delete $item->{id}; - $item->{subject} =~ s{$str}{<$tag>$1}gs; my $text = delete $item->{textBody}; - if ($text =~ m{(.{,20}$str.*)}) { - $item->{preview} = $1; - $item->{preview} =~ s{$str}{<$tag>$1}gs; + $item->{subject} = escape_html($item->{subject}); + $item->{preview} = escape_html($item->{preview}); + next unless @terms; + $item->{subject} =~ s{\b($str)\b}{<$tag>$1}gsi; + if ($text =~ m{(.{0,20}\b(?:$str)\b.*)}gsi) { + $item->{preview} = substr($1, 0, 200); + $item->{preview} =~ s{^\s+}{}gs; + $item->{preview} =~ s{\s+$}{}gs; + $item->{preview} =~ s{[\r\n]+}{ -- }gs; + $item->{preview} =~ s{\s+}{ }gs; + $item->{preview} = escape_html($item->{preview}); + $item->{preview} =~ s{\b($str)\b}{<$tag>$1}gsi; } } From 43dc9d05f2ad55c850ec09442a70c2259cf27ef5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 04:09:38 -0400 Subject: [PATCH 177/331] more debug --- bin/server.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/server.pl b/bin/server.pl index 142a20e..d791285 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -629,7 +629,9 @@ sub PushToHandle { sub PushEvent { my $Channel = shift; my %vals = @_; + print "PUSH EVENT $Channel " . encode_json(\%vals) . "\n"; foreach my $Fd (keys %{$PushMap{$Channel}{handles}}) { + warn " - to $Fd\n"; my $ToHandle = $PushMap{$Channel}{handles}{$Fd}; PushToHandle($ToHandle, %vals); } From 5989c744fbd548fd7fdcb4e2a4a6e2889e513274 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 04:49:37 -0400 Subject: [PATCH 178/331] it's on item --- JMAP/DB.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index c1aed2e..271a02b 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -661,14 +661,14 @@ sub create_messages { foreach my $cid (keys %$args) { my $item = $args->{$cid}; - if ($args->{inReplyToMessageId}) { - my ($replymessageid) = $dbh->selectrow_array("SELECT msgmessageid FROM jmessages WHERE msgid = ?", {}, $args->{inReplyToMessageId}); + if ($item->{inReplyToMessageId}) { + my ($replymessageid) = $dbh->selectrow_array("SELECT msgmessageid FROM jmessages WHERE msgid = ?", {}, $item->{inReplyToMessageId}); unless ($replymessageid) { $notCreated{$cid} = 'inReplyToNotFound'; next; } - $args->{headers}{'In-Reply-To'} = $replymessageid; - $args->{headers}{'References'} = $replymessageid; + $item->{headers}{'In-Reply-To'} = $replymessageid; + $item->{headers}{'References'} = $replymessageid; # XXX - references } $item->{msgdate} = time(); From 8ccddcbee494c6adf0d2e0a9b426c9f19d5d54fd Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 05:09:55 -0400 Subject: [PATCH 179/331] fix selection and update Also - don't re-download uploaded messages --- JMAP/ImapDB.pm | 19 ++++++++++++++----- JMAP/Sync/Common.pm | 23 ++++++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index fc557f7..01eaf23 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -773,7 +773,7 @@ sub changed_record { sub import_message { my $Self = shift; - my $message = shift; + my $rfc822 = shift; my $mailboxIds = shift; my %flags = @_; @@ -794,7 +794,7 @@ sub import_message { my $internaldate = time(); # XXX - allow setting? my $date = Date::Format::time2str('%e-%b-%Y %T %z', $internaldate); - my $data = $Self->backend_cmd('imap_append', $imapname, "(@flags)", $date, $message); + my $data = $Self->backend_cmd('imap_append', $imapname, "(@flags)", $date, $rfc822); warn Dumper($data); # XXX - compare $data->[2] with uidvalidity my $uid = $data->[3]; @@ -804,6 +804,9 @@ sub import_message { my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$mailboxIds->[0]}[0], $uid); + # save us having to download it again + $Self->add_raw_message($msgid, $rfc822); + return ($msgid, $thrid); } @@ -834,6 +837,7 @@ sub update_messages { my %jrolemap = map { $_->[1] => $_->[0] } grep { $_-> [1] } @$jmapdata; my @changed; + my %dirty; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; @@ -846,6 +850,7 @@ sub update_messages { $notchanged{$msgid} = "No folder found"; next; } + $dirty{$imapname} = 1; if (exists $action->{isUnread}) { my $bool = !$action->{isUnread}; my @flags = ("\\Seen"); @@ -869,17 +874,19 @@ sub update_messages { my $id = $mboxes[0]; # there can be only one if ($id eq 'outbox') { my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; - my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uid); - my $msg = $res->{data}{$uid}; - $Self->backend_cmd('send_email', $msg); + $Self->fill_messages($msgid); + my ($rfc822) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); + $Self->backend_cmd('send_email', $rfc822); # strip the \Draft flag warn "SENDING $imapname $uidvalidity and moving to $newfolder"; $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\draft"]); $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); + $dirty{$newfolder} = 1; } else { my $newfolder = $jmailmap{$id}[1]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); + $dirty{$newfolder} = 1; } } # XXX - handle errors from backend commands @@ -887,6 +894,8 @@ sub update_messages { } } + $Self->do_folder($_) for keys %dirty; + return (\@changed, \%notchanged); } diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 1807b77..28b857f 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -29,6 +29,16 @@ sub DESTROY { } } +sub _unselect { + my $imap = shift; + if ($imap->capability->{unselect}) { + $imap->unselect(); + } + else { + $imap->close(); + } +} + sub disconnect { my $Self = shift; if ($Self->{imap}) { @@ -206,13 +216,6 @@ sub imap_status { my $imap = $Self->connect_imap(); - if ($imap->capability->{unselect}) { - $imap->unselect(); - } - else { - $imap->close(); - } - my @fields = qw(uidvalidity uidnext messages); push @fields, "highestmodseq" if ($imap->capability->{condstore} or $imap->capability->{xymhighestmodseq}); my $data = $imap->multistatus("(@fields)", @$folders); @@ -247,6 +250,7 @@ sub imap_update { } $imap->store($uids, $isAdd ? "+flags" : "-flags", "(@$flags)"); + _unselect($imap); $res{updated} = $uids; @@ -277,6 +281,7 @@ sub imap_fill { } my $data = $imap->fetch($uids, "rfc822"); + _unselect($imap); my %ids; foreach my $uid (keys %$data) { @@ -310,6 +315,7 @@ sub imap_count { } my $data = $imap->fetch($uids, "UID"); + _unselect($imap); $res{data} = [sort { $a <=> $b } keys %$data]; return \%res; @@ -364,6 +370,7 @@ sub imap_move { $imap->store($uids, "+flags", "(\\seen \\deleted)"); $imap->uidexpunge($uids); } + _unselect($imap); $res{moved} = $uids; @@ -417,6 +424,7 @@ sub imap_fetch { my $data = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; $res{$key} = [$item, $data]; } + _unselect($imap); return \%res; } @@ -462,6 +470,7 @@ sub imap_search { } my $uids = $imap->search('charset', 'utf-8', @expr); + _unselect($imap); return ['search', $imapname, $uidvalidity, $uids]; } From 08ef0e125d1a22ddba228413d11f83f0bc070d5f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 05:19:12 -0400 Subject: [PATCH 180/331] index messageid --- JMAP/DB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 271a02b..fda2361 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -996,6 +996,7 @@ CREATE TABLE IF NOT EXISTS jmessages ( EOF $dbh->do("CREATE INDEX IF NOT EXISTS jthrid ON jmessages (thrid)"); + $dbh->do("CREATE INDEX IF NOT EXISTS jmsgmessageid ON jmessages (msgmessageid)"); $dbh->do(< Date: Mon, 20 Jul 2015 05:28:35 -0400 Subject: [PATCH 181/331] answered flag --- JMAP/ImapDB.pm | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 01eaf23..2e9fa73 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -879,9 +879,22 @@ sub update_messages { $Self->backend_cmd('send_email', $rfc822); # strip the \Draft flag warn "SENDING $imapname $uidvalidity and moving to $newfolder"; - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\draft"]); + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\Draft"]); $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); $dirty{$newfolder} = 1; + + # add the \Answered flag + my ($updateid) = $dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); + next unless $updateid; + my ($updatemsgid) = $dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgmessageid = ?", {}, $updateid); + next unless $updatemsgid; + my ($ifolderid, $updateuid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); + next unless $ifolderid; + my $updatename = $foldermap{$ifolderid}[1]; + my $updatevalidity = $foldermap{$ifolderid}[2]; + next unless $updatename; + $Self->backend_cmd('imap_update', $updatename, $updatevalidity, $updateuid, 1, ["\\Answered"]); + $dirty{$updatename} = 1; } else { my $newfolder = $jmailmap{$id}[1]; From 9542498dcb7402ee8ef51662a5d9e97fcadaeb76 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 05:34:57 -0400 Subject: [PATCH 182/331] transact the raw --- JMAP/ImapDB.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2e9fa73..ccf65a7 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -907,8 +907,6 @@ sub update_messages { } } - $Self->do_folder($_) for keys %dirty; - return (\@changed, \%notchanged); } From b23f89a839dcd5c44e70523e702ec7b861b8f4e7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 00:17:07 +1000 Subject: [PATCH 183/331] implement hasAttachment --- JMAP/API.pm | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 3073b24..7e5b8c5 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -415,12 +415,17 @@ sub _load_mailbox { return { map { $_->[0] => $_ } @$data }; } +sub _load_hasatt { + my $Self = shift; + my $data = $Self->{db}->dbh->selectcol_arrayref("SELECT msgid FROM jrawmessage WHERE hasAttachment = 1"); + return { map { $_ => 1 } @$data }; +} + sub _match { my $Self = shift; my ($item, $condition, $storage) = @_; return $Self->_match_operator($item, $condition, $storage) if $condition->{operator}; - # XXX - condition handling code if ($condition->{inMailboxes}) { my $inall = 1; foreach my $id (map { $Self->idmap($_) } @{$condition->{inMailboxes}}) { @@ -480,6 +485,8 @@ sub _match { } if ($condition->{hasAttachment}) { + $storage->{hasatt} ||= $Self->_load_hasatt(); + return 0 unless $storage->{hasatt}{$item->{msgid}}; # XXX - hasAttachment } From 2b66981205e2983e0ea2edd2d94558ba32d94b02 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 10:20:28 -0400 Subject: [PATCH 184/331] transaction for parse message --- JMAP/ImapDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ccf65a7..f5708ae 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -802,10 +802,12 @@ sub import_message { # make sure we're up to date: XXX - imap only $Self->do_folder($jmailmap{$mailboxIds->[0]}[0], $jmailmap{$mailboxIds->[0]}[2]); + $Self->begin(); my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$mailboxIds->[0]}[0], $uid); # save us having to download it again $Self->add_raw_message($msgid, $rfc822); + $Self->commit(); return ($msgid, $thrid); } From 6cb808c8478bfaa02fd084d877478bf29321983c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 10:22:46 -0400 Subject: [PATCH 185/331] hasAttachment is stored in parsed! --- JMAP/API.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 7e5b8c5..e0dc76b 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -417,8 +417,9 @@ sub _load_mailbox { sub _load_hasatt { my $Self = shift; - my $data = $Self->{db}->dbh->selectcol_arrayref("SELECT msgid FROM jrawmessage WHERE hasAttachment = 1"); - return { map { $_ => 1 } @$data }; + my $data = $Self->{db}->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage"); + my %parsed = map { $_->[0] => decode_json($_->[1]) } @$data; + return { map { $_ => 1 } grep { $parsed{$_}{hasAttachment} } keys %parsed }; } sub _match { From 267e68f065f3944bd6b3b2d1d5a1da99756f55ef Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 01:45:16 +1000 Subject: [PATCH 186/331] it's called isUnread --- JMAP/API.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index e0dc76b..c480a71 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -470,9 +470,9 @@ sub _match { return 0 unless $item->{isFlagged}; } - if ($condition->{isUnseen}) { + if ($condition->{isUnread}) { # XXX - threaded versions? - return 0 unless $item->{isUnseen}; + return 0 unless $item->{isUnread}; } if ($condition->{isAnswered}) { @@ -691,7 +691,7 @@ sub getMessageListUpdates { $Self->commit(); # now we have the same sorted data set. What we DON'T have is knowing that a message used to be in the filter, - # but no longer is (aka isUnseen). There's no good way to do this :( So we have to assume that every message + # but no longer is (aka isUnread). There's no good way to do this :( So we have to assume that every message # which is changed and NOT in the dataset used to be... # we also have to assume that it MIGHT have been the exemplar... From 73c1391950af833e8f359fd39cfb45af140ccef9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 12:04:58 -0400 Subject: [PATCH 187/331] return collapseThreads from getSearchSnippers for now --- JMAP/API.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/API.pm b/JMAP/API.pm index c480a71..aa6ff8f 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -839,6 +839,7 @@ sub getSearchSnippets { $messages->[0] = 'searchSnippets'; delete $messages->[1]{state}; $messages->[1]{filter} = $args->{filter}; + $messages->[1]{collapseThreads} = $args->{collapseThreads}, # work around client bug my @terms = _extract_terms($args->{filter}); my $str = join("|", @terms); From 47ebae41dbe9cea8b12875d30b6a852d04d0fe2a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 13:28:22 -0400 Subject: [PATCH 188/331] trim headers1 --- JMAP/API.pm | 1 + JMAP/ImapDB.pm | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index aa6ff8f..eb979f1 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -860,6 +860,7 @@ sub getSearchSnippets { $item->{preview} = escape_html($item->{preview}); $item->{preview} =~ s{\b($str)\b}{<$tag>$1}gsi; } + $item->{body} = $item->{preview}; # work around client bug } return $messages; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index f5708ae..321e91c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -611,6 +611,14 @@ sub firstsync { $Self->fill_messages(@$msgids); } +sub _trimh { + my $val = shift; + return '' unless defined $val; + $val =~ s{\s+$}{}; + $val =~ s{^\s+}{}; + return $val; +} + sub calcmsgid { my $Self = shift; my $imapname = shift; @@ -622,8 +630,8 @@ sub calcmsgid { my $base = substr(sha1_hex($coded), 0, 9); my $msgid = "m$base"; - my $replyto = lc($envelope->{'In-Reply-To'} || ''); - my $messageid = lc($envelope->{'Message-ID'} || ''); + my $replyto = _trimh($envelope->{'In-Reply-To'}); + my $messageid = _trimh($envelope->{'Message-ID'}); my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?)", {}, $replyto, $messageid); $thrid ||= "t$base"; foreach my $id ($replyto, $messageid) { @@ -878,23 +886,28 @@ sub update_messages { my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; $Self->fill_messages($msgid); my ($rfc822) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); + warn "SENDING $imapname $uidvalidity and moving to $newfolder"; $Self->backend_cmd('send_email', $rfc822); # strip the \Draft flag - warn "SENDING $imapname $uidvalidity and moving to $newfolder"; $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\Draft"]); $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); $dirty{$newfolder} = 1; + warn "LOOKING FOR message to update"; # add the \Answered flag my ($updateid) = $dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); - next unless $updateid; + goto done unless $updateid; + warn "FOUND ($updateid)"; my ($updatemsgid) = $dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgmessageid = ?", {}, $updateid); - next unless $updatemsgid; + warn "MAPPED TO $updatemsgid"; + goto done unless $updatemsgid; my ($ifolderid, $updateuid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); - next unless $ifolderid; + warn "MAPPED TO $ifolderid ($updateuid)"; + goto done unless $ifolderid; my $updatename = $foldermap{$ifolderid}[1]; my $updatevalidity = $foldermap{$ifolderid}[2]; - next unless $updatename; + goto done unless $updatename; + warn "MARKING $updatename $updatevalidity $updateuid as Answered"; $Self->backend_cmd('imap_update', $updatename, $updatevalidity, $updateuid, 1, ["\\Answered"]); $dirty{$updatename} = 1; } @@ -903,6 +916,7 @@ sub update_messages { $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); $dirty{$newfolder} = 1; } + done: } # XXX - handle errors from backend commands push @changed, $msgid; @@ -1043,8 +1057,8 @@ sub _envelopedata { msgcc => $envelope->{Cc}, msgbcc => $envelope->{Bcc}, msgdate => str2time($envelope->{Date}), - msginreplyto => $envelope->{'In-Reply-To'}, - msgmessageid => $envelope->{'Message-ID'}, + msginreplyto => _trimh($envelope->{'In-Reply-To'}), + msgmessageid => _trimh($envelope->{'Message-ID'}), ); } From 506d6ce77682ba6c5366431e03f19b6feffbbd21 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 13:51:25 -0400 Subject: [PATCH 189/331] only push changed things --- JMAP/DB.pm | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index fda2361..47bb1f3 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -25,6 +25,21 @@ use Date::Parse; use Net::CalDAVTalk; use Net::CardDAVTalk::VCard; use MIME::Base64 qw(encode_base64); + +my %TABLE2GROUPS = ( + jmessages => ['Messages', 'Threads'], + jmailboxes => ['Mailboxes'], + jmessagemap => ['Mailboxes'], + jrawmessage => [], + jfiles => [], # for now + jcalendars => ['Calendars'], + jevents => ['CalendarEvents'], + jaddressbooks => [], # not directly + jcontactgroups => ['ContactGroups'], + jcontactgroupmap => ['ContactGroups'], + jcontacts => ['Contacts'], +) + sub new { my $class = shift; my $accountid = shift || die; @@ -87,16 +102,14 @@ sub commit { # push an update if anything to tell.. if ($t->{modseq} and $Self->{change_cb}) { + my %map; my $state = $t->{modseq}; - $Self->{change_cb}->($Self, { - Mailbox => "$state", - Thread => "$state", - Message => "$state", - Contact => "$state", - ContactGroup => "$state", - Calendar => "$state", - CalendarEvent => "$state", - }); + foreach my $table (keys %{$t->{tables}}) { + foreach my $group (@{$TABLE2GROUPS{$table}}) { + $map{$group} = state; + } + } + $Self->{change_cb}->($Self, \%map); } } @@ -117,6 +130,7 @@ sub reset { sub dirty { my $Self = shift; + my $table = shift || die 'need to have a table to dirty'; confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{modseq}) { my $user = $Self->get_user(); @@ -125,6 +139,7 @@ sub dirty { $Self->{t}{modseq} = $user->{jhighestmodseq}; $Self->log('debug', "dirty at $user->{jhighestmodseq}"); } + $Self->{t}{tables}{$table} = $Self->{t}{modseq}; return $Self->{t}{modseq}; } @@ -878,7 +893,7 @@ sub dinsert { sub dmake { my $Self = shift; my ($table, $values, $backfilling) = @_; - $values->{jmodseq} = $backfilling ? 1 : $Self->dirty(); + $values->{jmodseq} = $backfilling ? 1 : $Self->dirty($table); $values->{active} = 1; return $Self->dinsert($table, $values); } @@ -937,7 +952,7 @@ sub dmaybeupdate { sub ddirty { my $Self = shift; my ($table, $values, $limit) = @_; - $values->{jmodseq} = $Self->dirty(); + $values->{jmodseq} = $Self->dirty($table); return $Self->dupdate($table, $values, $limit); } @@ -948,7 +963,7 @@ sub dmaybedirty { my $filtered = $Self->filter_values($table, $values, $limit); return unless %$filtered; - $filtered->{jmodseq} = $values->{jmodseq} = $Self->dirty(); + $filtered->{jmodseq} = $values->{jmodseq} = $Self->dirty($table); return $Self->dupdate($table, $filtered, $limit); } From 19d0d9cf9974ba7c5c4ef0a02f3d3b88c8de47d9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 13:57:14 -0400 Subject: [PATCH 190/331] Jstate magic --- JMAP/DB.pm | 13 ++++++++++--- bin/server.pl | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 47bb1f3..86040ee 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -38,7 +38,7 @@ my %TABLE2GROUPS = ( jcontactgroups => ['ContactGroups'], jcontactgroupmap => ['ContactGroups'], jcontacts => ['Contacts'], -) +); sub new { my $class = shift; @@ -103,10 +103,10 @@ sub commit { # push an update if anything to tell.. if ($t->{modseq} and $Self->{change_cb}) { my %map; - my $state = $t->{modseq}; + my $state = "$t->{modseq}"; foreach my $table (keys %{$t->{tables}}) { foreach my $group (@{$TABLE2GROUPS{$table}}) { - $map{$group} = state; + $map{$group} = $state; } } $Self->{change_cb}->($Self, \%map); @@ -1054,6 +1054,13 @@ CREATE TABLE IF NOT EXISTS account ( picture TEXT, jdeletedmodseq INTEGER, jhighestmodseq INTEGER, + jstateMailboxes TEXT, + jstateThreads TEXT, + jstateMessages TEXT, + jstateContacts TEXT, + jstateContactGroups TEXT, + jstateCalendars TEXT, + jstateCalendarEvents TEXT, mtime DATE ); EOF diff --git a/bin/server.pl b/bin/server.pl index d791285..049f484 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -628,6 +628,7 @@ sub PushToHandle { sub PushEvent { my $Channel = shift; + $Channel =~ s/:.*//; my %vals = @_; print "PUSH EVENT $Channel " . encode_json(\%vals) . "\n"; foreach my $Fd (keys %{$PushMap{$Channel}{handles}}) { From d20250f936f5e904b26913b33e2dc6a163b623f3 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 14:03:40 -0400 Subject: [PATCH 191/331] full state management --- JMAP/DB.pm | 7 +++++-- bin/apiendpoint.pl | 16 +++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 86040ee..3ca9d57 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -103,12 +103,16 @@ sub commit { # push an update if anything to tell.. if ($t->{modseq} and $Self->{change_cb}) { my %map; + my %dbdata = (jhighestmodseq => $t->{modseq}); my $state = "$t->{modseq}"; foreach my $table (keys %{$t->{tables}}) { foreach my $group (@{$TABLE2GROUPS{$table}}) { $map{$group} = $state; + $dbdata{"jstate$group"} = $state; } } + + $Self->dupdate('account', \%dbdata); $Self->{change_cb}->($Self, \%map); } } @@ -135,7 +139,6 @@ sub dirty { unless ($Self->{t}{modseq}) { my $user = $Self->get_user(); $user->{jhighestmodseq}++; - $Self->dbh->do("UPDATE account SET jhighestmodseq = ?", {}, $user->{jhighestmodseq}); $Self->{t}{modseq} = $user->{jhighestmodseq}; $Self->log('debug', "dirty at $user->{jhighestmodseq}"); } @@ -147,7 +150,7 @@ sub get_user { my $Self = shift; confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{user}) { - $Self->{t}{user} = $Self->dbh->selectrow_hashref("SELECT email,displayname,picture,jhighestmodseq,jdeletedmodseq FROM account"); + $Self->{t}{user} = $Self->dbh->selectrow_hashref("SELECT * FROM account"); } # bootstrap unless ($Self->{t}{user}) { diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 551f509..2a3d7e6 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -142,17 +142,15 @@ sub handle_getstate { my $state = "$user->{jhighestmodseq}"; $db->commit(); + my %map; + foreach my $key (keys %$user) { + next unless $key =~ m/^jstate(.*)/; + $map{$1} = $user->{$key} || "1"; + } + my $data = { changed => { - $db->accountid() => { - Mailbox => "$state", - Thread => "$state", - Message => "$state", - Contact => "$state", - ContactGroup => "$state", - Calendar => "$state", - CalendarEvent => "$state", - }, + $db->accountid() => \%map, }, }; From f13da7d408ef601d165daec5dd31beceef5439b6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 14:07:26 -0400 Subject: [PATCH 192/331] more push event detail --- bin/server.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/server.pl b/bin/server.pl index 049f484..4586efa 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -622,6 +622,7 @@ sub prod_idler { sub PushToHandle { my $Handle = shift; my %vals = @_; + print "PUSH EVENT $Handle " . encode_json(\%vals) . "\n"; my @Lines = map { "$_: " . (ref($vals{$_}) ? encode_json($vals{$_}) : $vals{$_}) } keys %vals; $Handle->push_write(join("\r\n", @Lines) . "\r\n\r\n"); } @@ -630,9 +631,7 @@ sub PushEvent { my $Channel = shift; $Channel =~ s/:.*//; my %vals = @_; - print "PUSH EVENT $Channel " . encode_json(\%vals) . "\n"; foreach my $Fd (keys %{$PushMap{$Channel}{handles}}) { - warn " - to $Fd\n"; my $ToHandle = $PushMap{$Channel}{handles}{$Fd}; PushToHandle($ToHandle, %vals); } From 31e4d5e37cda62ce0c096f22fa02a5d65233ebc0 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 14:09:35 -0400 Subject: [PATCH 193/331] gotta remember we're in the transaction still --- JMAP/DB.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 3ca9d57..0bf9b57 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -97,10 +97,9 @@ sub begin { sub commit { my $Self = shift; confess("NOT IN TRANSACTION") unless $Self->{t}; - $Self->dbh->commit(); - my $t = delete $Self->{t}; # push an update if anything to tell.. + my $t = $Self->{t}; if ($t->{modseq} and $Self->{change_cb}) { my %map; my %dbdata = (jhighestmodseq => $t->{modseq}); @@ -115,6 +114,9 @@ sub commit { $Self->dupdate('account', \%dbdata); $Self->{change_cb}->($Self, \%map); } + + $Self->dbh->commit(); + delete $Self->{t}; } sub rollback { From 74280a158db78a11a5e857f98988e001316d914d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 14:24:38 -0400 Subject: [PATCH 194/331] deplural --- JMAP/DB.pm | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 0bf9b57..9aaba03 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -27,17 +27,17 @@ use Net::CardDAVTalk::VCard; use MIME::Base64 qw(encode_base64); my %TABLE2GROUPS = ( - jmessages => ['Messages', 'Threads'], - jmailboxes => ['Mailboxes'], - jmessagemap => ['Mailboxes'], + jmessages => ['Message', 'Thread'], + jmailboxes => ['Mailbox'], + jmessagemap => ['Mailbox'], jrawmessage => [], jfiles => [], # for now - jcalendars => ['Calendars'], - jevents => ['CalendarEvents'], + jcalendars => ['Calendar'], + jevents => ['CalendarEvent'], jaddressbooks => [], # not directly - jcontactgroups => ['ContactGroups'], - jcontactgroupmap => ['ContactGroups'], - jcontacts => ['Contacts'], + jcontactgroups => ['ContactGroup'], + jcontactgroupmap => ['ContactGroup'], + jcontacts => ['Contact'], ); sub new { @@ -1059,13 +1059,13 @@ CREATE TABLE IF NOT EXISTS account ( picture TEXT, jdeletedmodseq INTEGER, jhighestmodseq INTEGER, - jstateMailboxes TEXT, - jstateThreads TEXT, - jstateMessages TEXT, - jstateContacts TEXT, - jstateContactGroups TEXT, - jstateCalendars TEXT, - jstateCalendarEvents TEXT, + jstateMailbox TEXT, + jstateThread TEXT, + jstateMessage TEXT, + jstateContact TEXT, + jstateContactGroup TEXT, + jstateCalendar TEXT, + jstateCalendarEvent TEXT, mtime DATE ); EOF From 564d70c29575bca0316066c3acd298e3df2bcbb8 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 17:14:00 -0400 Subject: [PATCH 195/331] wrapper is 'mark' --- JMAP/API.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index eb979f1..5ff750e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -843,7 +843,7 @@ sub getSearchSnippets { my @terms = _extract_terms($args->{filter}); my $str = join("|", @terms); - my $tag = 'b'; # XXX - wrap + my $tag = 'mark'; foreach my $item (@{$messages->[1]{list}}) { $item->{messageId} = delete $item->{id}; my $text = delete $item->{textBody}; From 3624f1431a994a5abc2ddd9d29079db7e8bcdf96 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 12:14:39 +1000 Subject: [PATCH 196/331] update main.css, add Config.pm --- JMAP/Config.pm | 27 +++++++++++++++++++++++++++ htdocs/main.css | 31 ++++++++++++++++--------------- 2 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 JMAP/Config.pm diff --git a/JMAP/Config.pm b/JMAP/Config.pm new file mode 100644 index 0000000..e85fa81 --- /dev/null +++ b/JMAP/Config.pm @@ -0,0 +1,27 @@ +#!/usr/bin/perl -c + +use strict; +use warnings; + +package JMAP::Config; + +use base 'Exporter'; + +our @EXPORT = qw(config); + +use JSON::XS qw(encode_json decode_json); +use IO::All; + +our $CONFIG; + +sub config { + unless ($CONFIG) { + my $file = "/etc/jmap_proxy.conf"; + $CONFIG = eval { decode_json(io->file($ENV{JMAPCONF} || $file)->slurp) }; + die "NEED config $file or ENV JMAPCONF" unless $CONFIG; + } + my $name = shift; + return $name ? $CONFIG->{$name} : $CONFIG; +} + +1; diff --git a/htdocs/main.css b/htdocs/main.css index 6c01802..90125df 100644 --- a/htdocs/main.css +++ b/htdocs/main.css @@ -13,7 +13,7 @@ blockquote { html { background: #fff; color: #1f1f1f; - font: 15px/1.55 "Open Sans", "Helvetica Neue", Arial, sans-serif; + font: 16px/1.55 "Source Sans Pro", "Helvetica Neue", Arial, sans-serif; } table { margin: 10px 0 15px 0; @@ -45,9 +45,9 @@ h3, h4, h5, h6 { - color: #404040; - line-height: 1.2; margin: 15px 0; + line-height: 1.2; + color: #404040; } h1 { border-bottom: 1px solid #eee; @@ -76,24 +76,24 @@ li { margin: 5px 0; } blockquote { - padding: 13px 13px 21px 15px; margin-bottom: 18px; + padding: 13px 13px 21px 15px; font-family:georgia,serif; font-style: italic; } blockquote:before { content:"\201C"; - font-size:40px; margin-left:-10px; - font-family:georgia,serif; color:#eee; + font-family:georgia,serif; + font-size:40px; } blockquote p { - font-size: 14px; - font-weight: 300; - line-height: 18px; margin-bottom: 0; + line-height: 18px; + font-size: 15px; font-style: italic; + font-weight: 300; } code, pre { font-family: Menlo, Monaco, Andale Mono, Courier New, monospace; @@ -150,8 +150,8 @@ sup { top: 0; left: 0; right: 0; - height: 44px; - line-height: 44px; + height: 49px; + line-height: 49px; border-bottom: 1px solid #1f1f1f; background: #366683; color: #fff; @@ -159,7 +159,7 @@ sup { #logo { padding: 0 30px; font-size: 32px; - font-weight: bold; + font-weight: 600; } #nav { position: absolute; @@ -182,6 +182,7 @@ sup { margin: 0 3px; padding: 0 10px; color: #fff; + font-size: 17px; text-decoration: none; } .nav-link:hover { @@ -198,14 +199,14 @@ sup { } #last-update { margin: -7px 0 15px; - font-size: 13px; + font-size: 14px; text-align: center; } /* Sidebar (Table of Contents) */ aside { position: fixed; overflow: auto; - top: 45px; + top: 50px; bottom: 0; width: 229px; border-right: 1px solid #eee; @@ -216,7 +217,7 @@ sup { margin: 0; padding: 15px 0; list-style: none; - font-size: 13px; + font-size: 14px; } .toc-selected { position: absolute; From 7278b4a0189880ead60af6fd3d97f78c5d980a42 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 12:16:16 +1000 Subject: [PATCH 197/331] update fonts --- htdocs/error.html | 2 +- htdocs/index.html | 2 +- htdocs/landing.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/htdocs/error.html b/htdocs/error.html index fc6b7f4..7f23194 100644 --- a/htdocs/error.html +++ b/htdocs/error.html @@ -3,7 +3,7 @@ - + JMAP Proxy Error diff --git a/htdocs/index.html b/htdocs/index.html index 3a39f0a..d99a108 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -3,7 +3,7 @@ - + JMAP Proxy diff --git a/htdocs/landing.html b/htdocs/landing.html index 2838445..c970c96 100644 --- a/htdocs/landing.html +++ b/htdocs/landing.html @@ -3,7 +3,7 @@ - + JMAP Proxy From c3324279946b60bb7b4ca28514b638f0f339c049 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 12:19:05 +1000 Subject: [PATCH 198/331] headings --- htdocs/error.html | 14 +++++++++++++- htdocs/index.html | 14 +++++++++++++- htdocs/landing.html | 14 +++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/htdocs/error.html b/htdocs/error.html index 7f23194..bff94dd 100644 --- a/htdocs/error.html +++ b/htdocs/error.html @@ -7,7 +7,19 @@ JMAP Proxy Error -
+ +
+ + +Fork me on GitHub +
+

ERROR

diff --git a/htdocs/index.html b/htdocs/index.html index d99a108..ec5e7b7 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -7,7 +7,19 @@ JMAP Proxy -
+ +
+ + +Fork me on GitHub +
+

JMAP Proxy

diff --git a/htdocs/landing.html b/htdocs/landing.html index c970c96..b35946f 100644 --- a/htdocs/landing.html +++ b/htdocs/landing.html @@ -7,7 +7,19 @@ JMAP Proxy -
+ +
+ + +Fork me on GitHub +
+

Good work, your JMAP proxy account is now set up

From 497941e5ccc022c32cdb8ee0d5e5004df00a4cfe Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 18:23:44 -0400 Subject: [PATCH 199/331] U upload --- bin/server.pl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/bin/server.pl b/bin/server.pl index 4586efa..855a156 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -148,6 +148,7 @@ sub read_idle_line { '/A' => \&do_A, '/J' => \&do_J, '/U' => \&do_U, + '/upload' => \&do_upload, '/raw' => \&do_raw, '/register' => \&do_register, '/delete' => \&do_delete, @@ -398,6 +399,35 @@ sub do_U { }, mkerr($req)); } +sub do_upload { + my ($httpd, $req) = @_; + + my $uri = $req->url(); + my $path = $uri->path(); + + return not_found($req) unless $path =~ m{^/upload/([^/]+)}; + + my $accountid = $1; + + return client_page($req, $accountid) unless lc $req->method eq 'post'; + + prod_idler($accountid); + + my $content = $req->content(); + return invalid_request($req) unless $content; + + my $type = $req->headers->{"content-type"}; + + $httpd->stop_request(); + + send_backend_request($accountid, 'upload', [$type, $content], sub { + my $res = shift; + my $html = encode_utf8($json->encode($res)); + $req->respond ({ content => ['application/json', $html] }); + return 1; + }, mkerr($req)); +} + sub do_jmap { my ($httpd, $req) = @_; From 0b659d8c57668411740c902c1742c7c818af5763 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 22:20:12 -0400 Subject: [PATCH 200/331] file upload --- JMAP/API.pm | 12 +++++++++++- bin/server.pl | 7 ++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 5ff750e..9d46d61 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1044,12 +1044,22 @@ sub getRawMessage { return ($type, $data, $filename); } +sub get_file { + my $Self = shift; + my $jfileid = shift; + + my $dbh = $Self->{db}->dbh(); + my ($type, $content) = $dbh->selectrow_array("SELECT type, content FROM jfiles WHERE jfileid = ?", {}, $jfileid); + return unless $content; + return ($type, $content); +} + # or this sub uploadFile { my $Self = shift; my ($type, $content) = @_; # XXX filehandle? - return $Self->upload_file($type, $content); + return $Self->put_file($type, $content); } sub downloadFile { diff --git a/bin/server.pl b/bin/server.pl index 855a156..c6bc38f 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -422,8 +422,8 @@ sub do_upload { send_backend_request($accountid, 'upload', [$type, $content], sub { my $res = shift; - my $html = encode_utf8($json->encode($res)); - $req->respond ({ content => ['application/json', $html] }); + my $response = encode_utf8($json->encode($res)); + $req->respond ({ content => ['application/json', $response] }); return 1; }, mkerr($req)); } @@ -652,7 +652,8 @@ sub prod_idler { sub PushToHandle { my $Handle = shift; my %vals = @_; - print "PUSH EVENT $Handle " . encode_json(\%vals) . "\n"; + my $Fd = fileno($Handle->fd); + print "PUSH EVENT $Fd " . encode_json(\%vals) . "\n"; my @Lines = map { "$_: " . (ref($vals{$_}) ? encode_json($vals{$_}) : $vals{$_}) } keys %vals; $Handle->push_write(join("\r\n", @Lines) . "\r\n\r\n"); } From 7242458ea390699622588514a797b78a6922e7d3 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 22:30:47 -0400 Subject: [PATCH 201/331] update HTML pages --- htdocs/error.html | 2 +- htdocs/index.html | 65 ++++++++------------------------------------- htdocs/landing.html | 11 ++++---- htdocs/signup.html | 57 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 61 deletions(-) create mode 100644 htdocs/signup.html diff --git a/htdocs/error.html b/htdocs/error.html index bff94dd..68463a7 100644 --- a/htdocs/error.html +++ b/htdocs/error.html @@ -10,7 +10,7 @@
-
+

The web client hooked up for this demo is a simple example (no compose, contacts, calendars). It is also MIT licenced and available on GitHub. The code is unminified, so takes much longer to initially load than it would in a production setting, but can be viewed with standard web-dev tools. These can also show the data exchanges being made to load the data and get delta updates.

+ +

The service is not running with all of the security measures employed on a production site such as FastMail, so DO NOT USE THIS PROXY FOR ACCOUNTS WITH SENSITIVE DATA.

+ +

When you create your account, the most recent 50 emails will be downloaded in their entirety, so the first page should be snappy immediately. After that, you are redirected to the landing page. A background task will continue to pull in batches of messages and add them to your account, so you will see older messages appear while you are using the interface.

+
+ diff --git a/htdocs/landing.html b/htdocs/landing.html index b35946f..19f16b9 100644 --- a/htdocs/landing.html +++ b/htdocs/landing.html @@ -10,7 +10,7 @@
-
diff --git a/htdocs/signup.html b/htdocs/signup.html new file mode 100644 index 0000000..2e449a6 --- /dev/null +++ b/htdocs/signup.html @@ -0,0 +1,57 @@ + + + + + + + +JMAP Proxy + + +
+ + +Fork me on GitHub +
+
+ +

Set hosts

+ +Please confirm hosts below. IMAP must be on port 993, SMTP on port 587, CalDAV and CardDAV on port 443. + +
+ + + + + + + + + + + + + + + + + + + + + +
IMAP
SMTP
CalDAV
CardDAV
Submit
+ + +
+ +
+ + From b75ae8151cb12a0d23576285e0ca7b1a23b791f1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 20 Jul 2015 22:37:41 -0400 Subject: [PATCH 202/331] server fix FD --- bin/server.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/server.pl b/bin/server.pl index c6bc38f..a50822d 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -652,8 +652,7 @@ sub prod_idler { sub PushToHandle { my $Handle = shift; my %vals = @_; - my $Fd = fileno($Handle->fd); - print "PUSH EVENT $Fd " . encode_json(\%vals) . "\n"; + print "PUSH EVENT " . encode_json(\%vals) . "\n"; my @Lines = map { "$_: " . (ref($vals{$_}) ? encode_json($vals{$_}) : $vals{$_}) } keys %vals; $Handle->push_write(join("\r\n", @Lines) . "\r\n\r\n"); } From f92372351c49484168d63857642772a3b77f188b Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 17:37:10 +1000 Subject: [PATCH 203/331] The giant GMail rewrite --- JMAP/GmailDB.pm | 1057 +------------------------------------------- JMAP/ImapDB.pm | 165 ++++--- JMAP/Sync/Gmail.pm | 49 +- 3 files changed, 143 insertions(+), 1128 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 7cabaab..bdf7849 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -4,7 +4,8 @@ use strict; use warnings; package JMAP::GmailDB; -use base qw(JMAP::DB); + +use base qw(JMAP::ImapDB); use DBI; use Date::Parse; @@ -19,1055 +20,11 @@ use AnyEvent::Socket; use Data::Dumper; use IO::All; -our $TAG = 1; - -# special use or name magic -my %ROLE_MAP = ( - 'inbox' => 'inbox', - 'drafts' => 'drafts', - 'junk' => 'spam', - 'deleted messages' => 'trash', - 'archive' => 'archive', - 'sent messages' => 'sent', - 'sent items' => 'sent', - 'trash' => 'trash', - '\\inbox' => 'inbox', - '\\trash' => 'trash', - '\\sent' => 'sent', - '\\junk' => 'junk', - '\\archive' => 'archive', - '\\drafts' => 'drafts', - '\\allmail' => 'allmail', -); - -my $O; -sub O { - unless ($O) { - my $data = io->file("/home/jmap/jmap-perl/config.json")->slurp; - my $config = decode_json($data); - $O = OAuth2::Tiny->new(%$config); - } - return $O; -} - -sub access_token { - my $Self = shift; - my $username = shift; - my $refresh_token = shift; - - unless ($refresh_token) { - ($username, $refresh_token) = $Self->dbh->selectrow_array("SELECT username, refresh_token FROM iserver"); - } - - my $O = $Self->O(); - my $data = $O->refresh($refresh_token); - - return ['gmail', $username, $data->{access_token}]; -} - - -sub setuser { - my $Self = shift; - my ($username, $refresh_token, $displayname, $picture) = @_; - my $data = $Self->dbh->selectrow_arrayref("SELECT username, refresh_token FROM iserver"); - if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {username => $username, refresh_token => $refresh_token}, {username => $data->[0]}); - } - else { - $Self->dinsert('iserver', { - username => $username, - refresh_token => $refresh_token, - }); - } - my $user = $Self->dbh->selectrow_arrayref("SELECT email, displayname FROM account"); - if ($user and $user->[0]) { - $Self->dmaybeupdate('account', {email => $username, displayname => $displayname, picture => $picture}); - } - else { - $Self->dinsert('account', { - email => $username, - displayname => $displayname, - picture => $picture, - jdeletedmodseq => 0, - jhighestmodseq => 1, - }); - } -} - -# synchronous backend for now -sub backend_cmd { - my $Self = shift; - my $cmd = shift; - my $cb; - if (ref($cmd)) { - $cb = $cmd; - $cmd = shift; - } - my @args = @_; - - my $w = AnyEvent->condvar; - - my $action = sub { - my $handle = shift; - my $tag = "T" . $TAG++; - $handle->push_write(json => [$cmd, \@args, $tag]); # whatever - $handle->push_write("\012"); - $handle->push_read(json => sub { - my $hdl = shift; - my $json = shift; - die "INVALID RESPONSE" unless $json->[2] eq $tag; - if ($cb) { - $cb->($json->[1]); - } - else { - $w->send($json->[1]); - } - }); - }; - if ($Self->{backend}) { - $action->($Self->{backend}); - } - else { - my $h = AnyEvent->condvar; - my $auth = $Self->access_token(); - tcp_connect('localhost', 5005, sub { - my $fh = shift; - my $handle; - $handle = AnyEvent::Handle->new(fh => $fh, - on_disconnect => sub { - delete $Self->{backend}; - undef $h; - }, - on_disconnect => sub { - delete $Self->{backend}; - undef $h; - }, - ); - $handle->push_write(json => {hostname => $auth->[0], username => $auth->[1], password => $auth->[2]}); - $handle->push_write("\012"); - $handle->push_read(json => sub { - my $hdl = shift; - my $json = shift; - die "Failed to setup " . Dumper($json) unless $json->[0] eq 'setup'; - $action->($handle); - $h->send($handle); - }); - # XXX - handle destroy correctly - # handle backend going away, etc - }); - - # synchronous startup to avoid race condition on setting up channel - $Self->{backend} = $h->recv; - } - - return if $cb; # async usage - - my $res = $w->recv; - #warn Dumper ($cmd, \@args, $res); - return $res; -} - -# synchronise list from IMAP server to local folder cache -# call in transaction -sub sync_folders { - my $Self = shift; - - my $dbh = $Self->dbh(); - - my $folders = $Self->backend_cmd('folders', []); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); - my %ibylabel = map { $_->[4] => $_ } @$ifolders; - my %seen; - - my %getstatus; - foreach my $name (sort keys %$folders) { - my $sep = $folders->{$name}[0]; - my $role = $ROLE_MAP{lc $folders->{$name}[1]}; - my $label = $role || $folders->{$name}[1]; - my $id = $ibylabel{$label}[0]; - if ($id) { - $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); - } - else { - $id = $Self->dinsert('ifolders', {sep => $sep, imapname => $name, label => $label}); - } - $seen{$id} = 1; - unless ($ibylabel{$label}[2]) { - # no uidvalidity, we need to get status for this one - next unless ($label eq 'allmail' or $label eq 'trash'); - $getstatus{$name} = $id; - } - } - - if (keys %getstatus) { - my $data = $Self->backend_cmd('imap_status', [keys %getstatus]); - foreach my $name (keys %$data) { - my $status = $data->{$name}; - $Self->dmaybeupdate('ifolders', { - uidvalidity => $status->{uidvalidity}, - uidnext => $status->{uidnext}, - uidfirst => $status->{uidnext}, - highestmodseq => $status->{highestmodseq}, - }, {ifolderid => $getstatus{$name}}); - } - } - - foreach my $folder (@$ifolders) { - my $id = $folder->[0]; - next if $seen{$id}; - $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); - } - - $Self->sync_jmailboxes(); -} - -our %PROTECTED_MAILBOXES = map { $_ => 1 } qw(inbox trash archive junk); -our %ONLY_MAILBOXES = map { $_ => 1 } qw(trash); -our %NO_RENAME = map { $_ => 1 } qw(inbox); - -# synchronise from the imap folder cache to the jmap mailbox listing -# call in transaction -sub sync_jmailboxes { - my $Self = shift; - my $dbh = $Self->dbh(); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); - my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); - - my %jbyid; - my %roletoid; - my %byname; - foreach my $mailbox (@$jmailboxes) { - $jbyid{$mailbox->[0]} = $mailbox; - $roletoid{$mailbox->[3]} = $mailbox->[0] if $mailbox->[3]; - $byname{$mailbox->[2]}{$mailbox->[1]} = $mailbox->[0]; - } - - my %seen; - foreach my $folder (@$ifolders) { - my $fname = $folder->[2]; - warn " MAPPING $fname ($folder->[1])"; - $fname =~ s/^INBOX\.//; - # check for roles first - my @bits = split "[$folder->[1]]", $fname; - my $role = $ROLE_MAP{lc $folder->[3]} || $ROLE_MAP{lc $fname}; - my $id = 0; - my $parentId = 0; - my $name; - my $sortOrder = 3; - $sortOrder = 2 if $role; - $sortOrder = 1 if ($role||'') eq 'inbox'; - while (my $item = shift @bits) { - $seen{$id} = 1 if $id; - $name = $item; - $parentId = $id; - $id = $byname{$parentId}{$name}; - unless ($id) { - if (@bits) { - # need to create intermediate folder ... - # XXX - label noselect? - $id = $Self->dmake('jmailboxes', {name => $name, sortOrder => 4, parentId => $parentId}); - $byname{$parentId}{$name} = $id; - } - } - } - next unless $name; - my %details = ( - name => $name, - parentId => $parentId, - sortOrder => $sortOrder, - mustBeOnly => $ONLY_MAILBOXES{$role||''}, - mayDelete => (not $PROTECTED_MAILBOXES{$role||''}), - mayRename => (not $NO_RENAME{$role||''}), - mayAdd => 1, - mayRemove => 1, - mayChild => 0, - mayRead => 1, - ); - if ($id) { - if ($role and $roletoid{$role} and $roletoid{$role} != $id) { - # still gotta move it - $id = $roletoid{$role}; - $Self->ddirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); - } - elsif (not $folder->[4]) { - # reactivate! - $Self->ddirty('jmailboxes', {active => 1}, {jmailboxid => $id}); - } - } - else { - # case: role - we need to see if there's a case for moving this thing - if ($role and $roletoid{$role}) { - $id = $roletoid{$role}; - $Self->ddirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); - } - else { - $id = $Self->dmake('jmailboxes', {role => $role, %details}); - $byname{$parentId}{$name} = $id; - $roletoid{$role} = $id if $role; - } - } - $seen{$id} = 1; - $Self->dmaybeupdate('ifolders', {jmailboxid => $id}, {ifolderid => $folder->[0]}); - } - - foreach my $mailbox (@$jmailboxes) { - my $id = $mailbox->[0]; - next if $seen{$id}; - $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); - } -} - -# synchronise list from CalDAV server to local folder cache -# call in transaction -sub sync_calendars { - my $Self = shift; - - my $dbh = $Self->dbh(); - - my $calendars = $Self->backend_cmd('calendars', []); - return unless $calendars; - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); - my %byhref = map { $_->[1] => $_ } @$icalendars; - - my %seen; - my @todo; - foreach my $calendar (@$calendars) { - my $id = $byhref{$calendar->{href}}[0]; - my $data = { - isReadOnly => $calendar->{isReadOnly}, - href => $calendar->{href}, - color => $calendar->{color}, - name => $calendar->{name}, - syncToken => $calendar->{syncToken}, - }; - if ($id) { - $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); - my $token = $byhref{$calendar->{href}}[5]; - if ($token ne $calendar->{syncToken}) { - push @todo, $id; - $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); - } - } - else { - $id = $Self->dinsert('icalendars', $data); - } - $seen{$id} = 1; - } - - foreach my $calendar (@$icalendars) { - my $id = $calendar->[0]; - next if $seen{$id}; - $dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); - } - - $Self->sync_jcalendars(); - - foreach my $id (@todo) { - $Self->do_calendar($id); - } -} - -# synchronise from the imap folder cache to the jmap mailbox listing -# call in transaction -sub sync_jcalendars { - my $Self = shift; - my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); - my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); - - my %jbyid; - foreach my $calendar (@$jcalendars) { - $jbyid{$calendar->[0]} = $calendar; - } - - my %seen; - foreach my $calendar (@$icalendars) { - my $data = { - name => $calendar->[1], - color => $calendar->[2], - isVisible => 1, - mayReadFreeBusy => 1, - mayReadItems => 1, - mayAddItems => 0, - mayModifyItems => 0, - mayRemoveItems => 0, - mayDelete => 0, - mayRename => 0, - }; - if ($jbyid{$calendar->[3]}) { - $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); - $seen{$calendar->[3]} = 1; - } - else { - my $id = $Self->dmake('jcalendars', $data); - $Self->dupdate('icalendars', {jcalendarid => $id}, {icalendarid => $calendar->[0]}); - $seen{$id} = 1; - } - } - - foreach my $calendar (@$jcalendars) { - my $id = $calendar->[0]; - next if $seen{$id}; - $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); - } -} - -sub do_calendar { - my $Self = shift; - my $calendarid = shift; - - my $dbh = $Self->dbh(); - - my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); - my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); - my %res = map { $_->[1] => $_ } @$exists; - - my $events = $Self->backend_cmd('events', {href => $href}); - - foreach my $resource (keys %$events) { - my $data = delete $res{$resource}; - my $raw = $events->{$resource}; - if ($data) { - my $id = $data->[0]; - next if $raw eq $data->[2]; - $Self->dmaybeupdate('ievents', {content => $raw, resource => $resource}, {ieventid => $id}); - } - else { - $Self->dinsert('ievents', {content => $raw, resource => $resource}); - } - my $event = $Self->parse_event($raw); - $Self->set_event($jcalendarid, $event); - } - - foreach my $resource (keys %res) { - my $data = delete $res{$resource}; - my $id = $data->[0]; - $Self->ddelete('ievents', {ieventid => $id}); - my $event = $Self->parse_event($data->[2]); - $Self->delete_event($jcalendarid, $event->{uid}); - } -} - -# synchronise list from CardDAV server to local folder cache -# call in transaction -sub sync_addressbooks { - my $Self = shift; - - my $dbh = $Self->dbh(); - - my $addressbooks = $Self->backend_cmd('addressbooks', []); - return unless $addressbooks; - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); - my %byhref = map { $_->[1] => $_ } @$iaddressbooks; - - my %seen; - my @todo; - foreach my $addressbook (@$addressbooks) { - my $id = $byhref{$addressbook->{href}}[0]; - my $data = { - isReadOnly => $addressbook->{isReadOnly}, - href => $addressbook->{href}, - name => $addressbook->{name}, - syncToken => $addressbook->{syncToken}, - }; - if ($id) { - my $token = $byhref{$addressbook->{href}}[4]; - if ($token ne $addressbook->{syncToken}) { - push @todo, $id; - $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); - } - } - else { - $id = $Self->dinsert('iaddressbooks', $data); - } - $seen{$id} = 1; - } - - foreach my $addressbook (@$iaddressbooks) { - my $id = $addressbook->[0]; - next if $seen{$id}; - $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); - } - - $Self->sync_jaddressbooks(); - - foreach my $id (@todo) { - $Self->do_addressbook($id); - } -} - -# synchronise from the imap folder cache to the jmap mailbox listing -# call in transaction -sub sync_jaddressbooks { - my $Self = shift; - my $dbh = $Self->dbh(); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks"); - my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks"); - - my %jbyid; - foreach my $addressbook (@$jaddressbooks) { - $jbyid{$addressbook->[0]} = $addressbook; - } - - my %seen; - foreach my $addressbook (@$iaddressbooks) { - my $data = { - name => $addressbook->[1], - isVisible => 1, - mayReadItems => 1, - mayAddItems => 0, - mayModifyItems => 0, - mayRemoveItems => 0, - mayDelete => 0, - mayRename => 0, - }; - if ($jbyid{$addressbook->[2]}) { - $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $addressbook->[2]}); - $seen{$addressbook->[2]} = 1; - } - else { - my $id = $Self->dmake('jaddressbooks', $data); - $Self->dupdate('iaddressbooks', {jaddressbookid => $id}, {iaddressbookid => $addressbook->[0]}); - $seen{$id} = 1; - } - } - - foreach my $addressbook (@$jaddressbooks) { - my $id = $addressbook->[0]; - next if $seen{$id}; - $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $id}); - } -} - -sub do_addressbook { - my $Self = shift; - my $addressbookid = shift; - - my $dbh = $Self->dbh(); - - my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); - my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); - my %res = map { $_->[1] => $_ } @$exists; - - my $cards = $Self->backend_cmd('cards', {href => $href}); - - foreach my $resource (keys %$cards) { - my $data = delete $res{$resource}; - my $raw = $cards->{$resource}; - if ($data) { - my $id = $data->[0]; - next if $raw eq $data->[2]; - $Self->dmaybeupdate('icards', {content => $raw, resource => $resource}, {icardid => $id}); - } - else { - $Self->dinsert('icards', {content => $raw, resource => $resource}); - } - my $card = $Self->parse_card($raw); - $Self->set_card($jaddressbookid, $card); - } - - foreach my $resource (keys %res) { - my $data = delete $res{$resource}; - my $id = $data->[0]; - $Self->ddelete('icards', {icardid => $id}); - my $card = $Self->parse_card($data->[2]); - $Self->delete_card($jaddressbookid, $card->{uid}, $card->{kind}); - } -} - -sub labels { - my $Self = shift; - unless ($Self->{t}{labels}) { - my $data = $Self->dbh->selectall_arrayref("SELECT label, ifolderid, jmailboxid, imapname FROM ifolders"); - $Self->{t}{labels} = { map { lc $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; - } - return $Self->{t}{labels}; -} - -sub sync_imap { - my $Self = shift; - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); - my @imapnames = map { $_->[1] } grep { $_->[2] } @$data; - my $status = $Self->backend_cmd('imap_status', \@imapnames); - - foreach my $row (@$data) { - # XXX - better handling of UIDvalidity change? - next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); - $Self->do_folder($row->[0], $row->[4]); - } -} - -sub backfill { - my $Self = shift; - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime"); - my $rest = 500; - foreach my $row (@$data) { - $rest -= $Self->do_folder(@$row, $rest); - last if $rest < 10; - } -} - -sub firstsync { - my $Self = shift; - - $Self->sync_folders(); - $Self->sync_calendars(); - $Self->sync_addressbooks(); - - my $labels = $Self->labels(); - - my $ifolderid = $labels->{"allmail"}[0]; - $Self->do_folder($ifolderid, undef, 50); - - my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); - - # pre-load the INBOX! - $Self->fill_messages(@$msgids); -} - -sub calcmsgid { - my $Self = shift; - my $envelope = shift; - my $json = JSON::XS->new->allow_nonref->canonical; - my $coded = $json->encode($envelope); - my $msgid = sha1_hex($coded); - - my $replyto = lc($envelope->{'In-Reply-To'} || ''); - my $messageid = lc($envelope->{'Message-ID'} || ''); - my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?)", {}, $replyto, $messageid); - $thrid ||= $msgid; - foreach my $id ($replyto, $messageid) { - next if $id eq ''; - $Self->dbh->do("INSERT OR IGNORE INTO ithread (messageid, thrid) VALUES (?, ?)", {}, $id, $thrid); - } - - return ($msgid, $thrid); -} - -sub do_folder { - my $Self = shift; - my $ifolderid = shift; - my $forcelabel = shift; - my $batchsize = shift; - - Carp::confess("NO FOLDERID") unless $ifolderid; - my $imap = $Self->{imap}; - my $dbh = $Self->dbh(); - - my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = - $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); - die "NO SUCH FOLDER $ifolderid" unless $imapname; - - my %fetches; - $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size x-gm-msgid x-gm-thrid)]]; - $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; - - if ($uidfirst > 1 and $batchsize) { - my $end = $uidfirst - 1; - $uidfirst -= $batchsize; - $uidfirst = 1 if $uidfirst < 1; - $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size x-gm-msgid x-gm-thrid)]]; - } - - my $res = $Self->backend_cmd('imap_fetch', $imapname, { - uidvalidity => $uidvalidity, - highestmodseq => $highestmodseq, - uidnext => $uidnext, - },\%fetches); - - if ($res->{newstate}{uidvalidity} != $uidvalidity) { - # going to want to nuke everything for the existing folder and create this - but for now, just die - die "UIDVALIDITY CHANGED $imapname: $uidvalidity => $res->{newstate}{uidvalidity}"; - } - - my $didold = 0; - if ($res->{backfill}) { - my $new = $res->{backfill}[1]; - $Self->{backfilling} = 1; - foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = ($new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}); - $didold++; - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); - } - delete $Self->{backfilling}; - } - - if ($res->{update}) { - my $changed = $res->{update}[1]; - foreach my $uid (sort { $a <=> $b } keys %$changed) { - $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, [$forcelabel]); - } - } - - if ($res->{new}) { - my $new = $res->{new}[1]; - foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = ($new->{$uid}{'x-gm-msgid'}, $new->{$uid}{'x-gm-thrid'}); - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); - } - } - - # need to make changes before counting - if ($uidfirst == 1) { - my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - if ($count != $res->{newstate}{exists}) { - my $to = $uidnext - 1; - $Self->log('debug', "COUNTING $imapname: $uidfirst:$to (something deleted)"); - my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); - my $uids = $res->{data}; - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ?", {}, $ifolderid); - my %exists = map { $_ => 1 } @$uids; - foreach my $uid (@$data) { - next if $exists{$uid}; - $Self->deleted_record($ifolderid, $uid); - } - } - } - - $Self->dupdate('ifolders', {highestmodseq => $res->{newstate}{highestmodseq}, uidfirst => $uidfirst, uidnext => $res->{newstate}{uidnext}}, {ifolderid => $ifolderid}); - - return $didold; -} - -sub changed_record { - my $Self = shift; - my ($folder, $uid, $flaglist, $labellist) = @_; - - my $flags = encode_json([sort @$flaglist]); - my $labels = encode_json([sort @$labellist]); - - my ($msgid) = $Self->{dbh}->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); - - $Self->dmaybeupdate('imessages', {flags => $flags, labels => $labels}, {ifolderid => $folder, uid => $uid}); - - $Self->apply_data($msgid, $flaglist, $labellist); -} - -sub update_messages { - my $Self = shift; - my $changes = shift; - - my $dbh = $Self->{dbh}; - - my %updatemap; - foreach my $msgid (keys %$changes) { - my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $updatemap{$ifolderid}{$uid} = $msgid; - } - - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); - my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[4] => $_ } @$folderdata; - - my %notchanged; - my @changed; - foreach my $ifolderid (keys %updatemap) { - # XXX - merge similar actions? - my $imapname = $foldermap{$ifolderid}[1]; - my $uidvalidity = $foldermap{$ifolderid}[2]; - - foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { - my $msgid = $updatemap{$ifolderid}{$uid}; - my $action = $changes->{$msgid}; - unless ($imapname and $uidvalidity) { - $notchanged{$msgid} = "No folder found"; - next; - } - if (exists $action->{isUnread}) { - my $bool = !$action->{isUnread}; - my @flags = ("\\Seen"); - $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); - } - if (exists $action->{isFlagged}) { - my $bool = $action->{isFlagged}; - my @flags = ("\\Flagged"); - $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); - } - if (exists $action->{isAnswered}) { - my $bool = $action->{isAnswered}; - my @flags = ("\\Answered"); - $Self->log('debug', "STORING $bool @flags for $uid"); - $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); - } - if (exists $action->{mailboxIds}) { - my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @{$action->{mailboxIds}}; - $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); - } - # XXX - handle errors from backend commands - push @changed, $msgid; - } - } - - return (\@changed, \%notchanged); -} - -sub delete_messages { - my $Self = shift; - my $ids = shift; - - my $dbh = $Self->{dbh}; - - my %deletemap; - foreach my $msgid (@$ids) { - my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); - $deletemap{$ifolderid}{$uid} = $msgid; - } - - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); - my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; - - my (@deleted, %notdeleted); - foreach my $ifolderid (keys %deletemap) { - # XXX - merge similar actions? - my $imapname = $foldermap{$ifolderid}[1]; - my $uidvalidity = $foldermap{$ifolderid}[2]; - unless ($imapname) { - $notdeleted{$_} = "No folder" for values %{$deletemap{$ifolderid}}; - } - my $uids = [sort keys %{$deletemap{$ifolderid}}]; - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder - push @deleted, values %{$deletemap{$ifolderid}}; - } - return (\@deleted, \%notdeleted); -} - -sub deleted_record { - my $Self = shift; - my ($folder, $uid) = @_; - - my ($msgid) = $Self->{dbh}->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); - - $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - - $Self->apply_data($msgid, [], []); -} - -sub new_record { - my $Self = shift; - my ($ifolderid, $uid, $flaglist, $labellist, $envelope, $internaldate, $msgid, $thrid, $size) = @_; - - my $flags = encode_json([sort @$flaglist]); - my $labels = encode_json([sort @$labellist]); - - my $data = { - ifolderid => $ifolderid, - uid => $uid, - flags => $flags, - labels => $labels, - internaldate => $internaldate, - msgid => $msgid, - thrid => $thrid, - envelope => encode_json($envelope), - size => $size, - }; - - # XXX - what about dupes? - $Self->dinsert('imessages', $data); - - $Self->apply_data($msgid, $flaglist, $labellist); -} - -sub apply_data { - my $Self = shift; - my ($msgid, $flaglist, $labellist) = @_; - - my %flagdata = ( - isUnread => 1, - isFlagged => 0, - isAnswered => 0, - isDraft => 0, - ); - foreach my $flag (@$flaglist) { - $flagdata{isUnread} = 0 if lc $flag eq '\\seen'; - $flagdata{isFlagged} = 1 if lc $flag eq '\\flagged'; - $flagdata{isAnswered} = 1 if lc $flag eq '\\answered'; - $flagdata{isDraft} = 1 if lc $flag eq '\\draft'; - } - - my $labels = $Self->labels(); - my @jmailboxids = grep { $_ } map { $labels->{lc $_}[1] } @$labellist; - - my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); - - $Self->log('debug', "DATA (@jmailboxids) for $msgid"); - - if ($old) { - $Self->log('debug', "changing $msgid"); - return $Self->change_message($msgid, \%flagdata, \@jmailboxids); - } - else { - $Self->log('debug', "adding $msgid"); - my $data = $Self->dbh->selectrow_hashref("SELECT thrid,internaldate,size,envelope FROM imessages WHERE msgid = ?", {}, $msgid); - return $Self->add_message({ - msgid => $msgid, - internaldate => $data->{internaldate}, - thrid => $data->{thrid}, - msgsize => $data->{size}, - _envelopedata($data->{envelope}), - %flagdata, - }, \@jmailboxids); - } -} - -sub _envelopedata { - my $data = shift; - my $envelope = decode_json($data); - my $encsub = decode('MIME-Header', $envelope->{Subject}); - return ( - msgsubject => $encsub, - msgfrom => $envelope->{From}, - msgto => $envelope->{To}, - msgcc => $envelope->{Cc}, - msgbcc => $envelope->{Bcc}, - msgdate => str2time($envelope->{Date}), - msginreplyto => $envelope->{'In-Reply-To'}, - msgmessageid => $envelope->{'Message-ID'}, - ); -} - -sub fill_messages { - my $Self = shift; - my @ids = @_; - - my $data = $Self->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage WHERE msgid IN (" . join(', ', map { "?" } @ids) . ")", {}, @ids); - my %result; - foreach my $line (@$data) { - $result{$line->[0]} = decode_json($line->[1]); - } - my @need = grep { not $result{$_} } @ids; - - return \%result unless @need; - - my $uids = $Self->dbh->selectall_arrayref("SELECT ifolderid, uid, msgid FROM imessages WHERE msgid IN (" . join(', ', map { "?" } @need) . ")", {}, @need); - my %udata; - foreach my $row (@$uids) { - $udata{$row->[0]}{$row->[1]} = $row->[2]; - } - - foreach my $ifolderid (sort keys %udata) { - my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); - my $uhash = $udata{$ifolderid}; - - my $uids = join(',', sort { $a <=> $b } keys %$uhash); - my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); - - foreach my $uid (keys %{$res->{data}}) { - warn "FETCHED BODY FOR $uid\n"; - my $rfc822 = $res->{data}{$uid}; - my $msgid = $uhash->{$uid}; - $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); - } - } - - my @stillneed = grep { not $result{$_} } @ids; - - return \%result; -} - -sub _initdb { - my $Self = shift; - my $dbh = shift; - - $Self->SUPER::_initdb($dbh); - - # XXX - password encryption? - $dbh->do(<do(<do(<do(<do(<do(<do(<do(<SUPER::new(@_); + $Self->{is_gmail} = 1; + return $Self; } 1; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 321e91c..e825bf2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -60,39 +60,39 @@ my %ROLE_MAP = ( sub setuser { my $Self = shift; - my ($hostname, $username, $password) = @_; + my %args = @_; + $Self->begin(); - my $data = $Self->dbh->selectrow_arrayref("SELECT hostname, username, password FROM iserver"); + + my $data = $Self->dbh->selectrow_arrayref("SELECT username FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', {hostname => $hostname, username => $username, password => $password}); + $Self->dmaybeupdate('iserver', \%args); } else { - $Self->dinsert('iserver', { - hostname => $hostname, - username => $username, - password => $password, - }); + $Self->dinsert('iserver', \%args); } + my $user = $Self->dbh->selectrow_arrayref("SELECT email FROM account"); if ($user and $user->[0]) { - $Self->dmaybeupdate('account', {email => $username}); + $Self->dmaybeupdate('account', {email => $args{username}}); } else { $Self->dinsert('account', { - email => $username, + email => $args{username}, jdeletedmodseq => 0, jhighestmodseq => 1, }); } + $Self->commit(); } -sub access_token { +sub access_data { my $Self = shift; - my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT hostname, username, password FROM iserver"); + my $config = $Self->dbh->selectrow_array("SELECT * FROM iserver", {Slice => {}}); - return [$hostname, $username, $password]; + return $config } # synchronous backend for now @@ -105,8 +105,7 @@ sub backend_cmd { Carp::confess("in transaction") if $Self->in_transaction(); unless ($Self->{backend}) { - my $auth = $Self->access_token(); - my $config = { hostname => $auth->[0], username => $auth->[1], password => $auth->[2] }; + my $config = $Self->access_token(); my $backend; if ($config->{hostname} eq 'gmail') { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $auth->[1]"; @@ -573,25 +572,40 @@ sub labels { sub sync_imap { my $Self = shift; my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); - my @imapnames = map { $_->[1] } @$data; + if ($Self->{is_gmail}) { + $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; + } + + my @imapnames = map { $_->{imapname} } @$data; my $status = $Self->backend_cmd('imap_status', \@imapnames); foreach my $row (@$data) { # XXX - better handling of UIDvalidity change? - next if ($status->{$row->[1]}{uidvalidity} == $row->[2] and $status->{$row->[1]}{highestmodseq} and $status->{$row->[1]}{highestmodseq} == $row->[3]); - $Self->do_folder($row->[0], $row->[4]); + next if ($status->{$row->{imapname}}{uidvalidity} == $row->{uidvalidity} and $status->{$row->{imapname}}{highestmodseq} and $status->{$row->{imapname}}{highestmodseq} == $row->{highestmodseq}); + my $label = $row->{label}; + $label = undef if lc $label eq '\\allmail'; + $Self->do_folder($row->{ifolderid}, $label); } } sub backfill { my $Self = shift; - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid,label FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime"); + my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime", {Slice => {}}); + if ($Self->{is_gmail}) { + $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; + } + return unless @$data; - my $rest = 50; + + my $rest = 500; foreach my $row (@$data) { - $rest -= $Self->do_folder(@$row, $rest); + my $id = $row->{ifolderid}; + my $label = $row->{label}; + $label = undef if lc $label eq '\\allmail'; + $rest -= $Self->do_folder($id, $label, $rest); last if $rest < 10; } + return 1; } @@ -657,17 +671,21 @@ sub do_folder { die "NO SUCH FOLDER $ifolderid" unless $imapname; my %fetches; + my @immutable = qw(internaldate envelope rfc822.size); + if ($Self->{is_gmail}) { + push @immutable, qw(x-gm-msgid x-gm-thrid x-gm-labels); + } if ($batchsize) { if ($uidfirst > 1) { my $end = $uidfirst - 1; $uidfirst -= $batchsize; $uidfirst = 1 if $uidfirst < 1; - $fetches{backfill} = [$uidfirst, $end, [qw(internaldate envelope rfc822.size)]]; + $fetches{backfill} = [$uidfirst, $end, \@immutable]; } } else { - $fetches{new} = [$uidnext, '*', [qw(internaldate envelope rfc822.size)]]; + $fetches{new} = [$uidnext, '*', \@immutable]; $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; } @@ -693,9 +711,17 @@ sub do_folder { my $new = $res->{backfill}[1]; $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); + my ($msgid, $thrid, @labels); + if ($Self->{is_gmail}) { + ($msgid, $thrid) = ($new->{"x-gm-msgid"}, $new->{"x-gm-thrid"}); + @labels = $forcelabel ? ($forcelabel) : @{$new->{"x-gm-labels"}}; + } + else { + ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); + @labels = ($forcelabel); + } $didold++; - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); + $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, \@labels, $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } delete $Self->{backfilling}; } @@ -710,7 +736,14 @@ sub do_folder { if ($res->{new}) { my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { - my ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); + if ($Self->{is_gmail}) { + ($msgid, $thrid) = ($new->{"x-gm-msgid"}, $new->{"x-gm-thrid"}); + @labels = $forcelabel ? ($forcelabel) : @{$new->{"x-gm-labels"}}; + } + else { + ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); + @labels = ($forcelabel); + } $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } } @@ -749,6 +782,10 @@ sub imap_search { my $dbh = $Self->dbh(); my $data = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + if ($Self->{is_gmail}) { + $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; + } + my %matches; foreach my $item (@$data) { my $from = $item->{uidfirst}; @@ -786,12 +823,13 @@ sub import_message { my %flags = @_; my $dbh = $Self->dbh(); - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, label, jmailboxid FROM ifolders"); - my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[3] => $_ } grep { $_->[3] } @$folderdata; + my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; + my %jmailmap = map { $_->{jmailboxid} => $_ } grep { $_->{jmailboxid} } @$folderdata; # store to the first named folder - we can use labels on gmail to add to other folders later. - my $imapname = $jmailmap{$mailboxIds->[0]}[1]; + my ($id, @others) = @$mailboxIds; + my $imapname = $jmailmap{$id}{imapname}; my @flags; push @flags, "\\Seen" unless $flags{isUnread}; @@ -803,15 +841,21 @@ sub import_message { my $date = Date::Format::time2str('%e-%b-%Y %T %z', $internaldate); my $data = $Self->backend_cmd('imap_append', $imapname, "(@flags)", $date, $rfc822); - warn Dumper($data); # XXX - compare $data->[2] with uidvalidity my $uid = $data->[3]; # make sure we're up to date: XXX - imap only - $Self->do_folder($jmailmap{$mailboxIds->[0]}[0], $jmailmap{$mailboxIds->[0]}[2]); + if ($Self->{is_gmail}) { + my ($am) = grep { lc $_->{label} eq '\\allmail' } @$folderdata; + $Self->do_folder($am->{ifolderid}, undef); + } + else { + my $fdata = $jmailmap{$mailboxIds->[0]}; + $Self->do_folder($fdata->{ifolderid}, $fdata->{label}); + } $Self->begin(); - my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$mailboxIds->[0]}[0], $uid); + my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$id}[0], $uid); # save us having to download it again $Self->add_raw_message($msgid, $rfc822); @@ -847,7 +891,6 @@ sub update_messages { my %jrolemap = map { $_->[1] => $_->[0] } grep { $_-> [1] } @$jmapdata; my @changed; - my %dirty; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? my $imapname = $foldermap{$ifolderid}[1]; @@ -860,7 +903,6 @@ sub update_messages { $notchanged{$msgid} = "No folder found"; next; } - $dirty{$imapname} = 1; if (exists $action->{isUnread}) { my $bool = !$action->{isUnread}; my @flags = ("\\Seen"); @@ -881,42 +923,40 @@ sub update_messages { } if (exists $action->{mailboxIds}) { my @mboxes = map { $idmap->($_) } @{$action->{mailboxIds}}; - my $id = $mboxes[0]; # there can be only one - if ($id eq 'outbox') { - my $newfolder = $jmailmap{$jrolemap{'sent'}}[1]; + my ($has_outbox) = grep { $_ eq 'outbox' } @mboxes; + my (@others) = grep { $_ ne 'outbox' } @mboxes; + if ($has_outbox) { + # move to sent when we're done + push @others, $jmailmap{$jrolemap{'sent'}}[0]; $Self->fill_messages($msgid); my ($rfc822) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); - warn "SENDING $imapname $uidvalidity and moving to $newfolder"; $Self->backend_cmd('send_email', $rfc822); + # strip the \Draft flag $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\Draft"]); - $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); - $dirty{$newfolder} = 1; - warn "LOOKING FOR message to update"; - # add the \Answered flag + # add the \Answered flag to our in-reply-to my ($updateid) = $dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); goto done unless $updateid; - warn "FOUND ($updateid)"; my ($updatemsgid) = $dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgmessageid = ?", {}, $updateid); - warn "MAPPED TO $updatemsgid"; goto done unless $updatemsgid; my ($ifolderid, $updateuid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); - warn "MAPPED TO $ifolderid ($updateuid)"; - goto done unless $ifolderid; + goto done unless $ifolderid; my $updatename = $foldermap{$ifolderid}[1]; my $updatevalidity = $foldermap{$ifolderid}[2]; goto done unless $updatename; - warn "MARKING $updatename $updatevalidity $updateuid as Answered"; $Self->backend_cmd('imap_update', $updatename, $updatevalidity, $updateuid, 1, ["\\Answered"]); - $dirty{$updatename} = 1; + } + done: + if ($Self->{is_gmail}) { + my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @others; + $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); } else { + my $id = $others[0]; my $newfolder = $jmailmap{$id}[1]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); - $dirty{$newfolder} = 1; } - done: } # XXX - handle errors from backend commands push @changed, $msgid; @@ -972,7 +1012,7 @@ sub deleted_record { $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - $Self->ddirty('jmessages', {}, {msgid => $msgid}); # dump modeseq + $Self->ddirty('jmessages', {}, {msgid => $msgid}); # bump modeseq $Self->delete_message_from_mailbox($msgid, $jmailboxid); } @@ -1127,7 +1167,7 @@ sub create_mailboxes { $imapname = "$parentName$sep$imapname"; } else { - my ($prefix) = $dbh->selectrow_array("SELECT prefix FROM iserver"); + my ($prefix) = $dbh->selectrow_array("SELECT imapPrefix FROM iserver"); $imapname = "$prefix$imapname"; } @@ -1174,7 +1214,7 @@ sub update_mailboxes { $imapname = "$parentName$sep$imapname"; } else { - my ($prefix) = $dbh->selectrow_array("SELECT prefix FROM iserver"); + my ($prefix) = $dbh->selectrow_array("SELECT imapPrefix FROM iserver"); $prefix = '' unless $prefix; $imapname = "$prefix$imapname"; } @@ -1478,8 +1518,15 @@ sub _initdb { CREATE TABLE IF NOT EXISTS iserver ( username TEXT PRIMARY KEY, password TEXT, - hostname TEXT, - prefix TEXT, + imapHost TEXT, + imapPort INTEGER, + imapSSL INTEGER, + imapPrefix TEXT, + smtpHost TEXT, + smtpPort INTEGER, + smtpSSL INTEGER, + caldavURL TEXT, + carddavURL TEXT, lastfoldersync DATE, mtime DATE NOT NULL ); @@ -1517,6 +1564,8 @@ CREATE TABLE IF NOT EXISTS imessages ( mtime DATE NOT NULL ); EOF + +# not used for Gmail, but it doesn't hurt to have it $dbh->do(<do(< 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); +my $O; +sub O { + unless ($O) { + my $data = io->file("/home/jmap/jmap-perl/config.json")->slurp; + my $config = decode_json($data); + $O = OAuth2::Tiny->new(%$config); + } + return $O; +} + +sub access_token { + my $Self = shift; + unless ($Self->{access_token}) { + my $refresh_token = $Self->{auth}{password}; + my $O = $Self->O(); + my $data = $O->refresh($refresh_token); + $Self->{access_token} = $data->{access_token}; + } + return $Self->{access_token}; +} + sub connect_calendars { my $Self = shift; @@ -26,7 +49,7 @@ sub connect_calendars { $Self->{calendars} = Net::GmailCalendars->new( user => $Self->{auth}{username}, - access_token => $Self->{auth}{access_token}, + access_token => $Self->access_token(), url => "https://apidata.googleusercontent.com/caldav/v2", is_google => 1, expandurl => 1, @@ -45,7 +68,7 @@ sub connect_contacts { $Self->{contacts} = Net::GmailContacts->new( user => $Self->{auth}{username}, - access_token => $Self->{auth}{access_token}, + access_token => $Self->access_token(), url => "https://www.googleapis.com/.well-known/carddav", expandurl => 1, ); @@ -70,7 +93,7 @@ sub connect_imap { Server => 'imap.gmail.com', Port => $port, Username => $Self->{auth}{username}, - Password => $Self->{auth}{access_token}, + Password => $Self->access_token(), # not configurable right now... UseSSL => $usessl, UseBlocking => $usessl, @@ -78,17 +101,6 @@ sub connect_imap { next unless $Self->{imap}; $Self->log('debug', "Connected as $Self->{auth}{username}"); $Self->{lastused} = time(); - my @folders = $Self->{imap}->xlist('', '*'); - - delete $Self->{folders}; - delete $Self->{labels}; - foreach my $folder (@folders) { - my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; - my $name = $folder->[2]; - my $label = $role || $folder->[2]; - $Self->{folders}{$name} = $label; - $Self->{labels}{$label} = $name; - } return $Self->{imap}; } @@ -108,15 +120,8 @@ sub send_email { port => 465, ssl => 1, sasl_username => $Self->{auth}{username}, - access_token => $Self->{auth}{access_token}, + access_token => $Self->access_token(), }) }); } -sub update_folder { - my $Self = shift; - my $imapname = shift; - my $state = shift; - - return $Self->SUPER::update_folder($imapname, $state, ['x-gm-labels'], ['x-gm-msgid', 'x-gm-thrid']); -} From 0638d4639091923c9e154e9e50a990f45ae2d87c Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 03:53:39 -0400 Subject: [PATCH 204/331] fix up lots of create --- JMAP/ImapDB.pm | 19 ++++++++++--------- JMAP/Sync/Gmail.pm | 1 + bin/apiendpoint.pl | 25 +++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index e825bf2..85de7f8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -107,16 +107,16 @@ sub backend_cmd { unless ($Self->{backend}) { my $config = $Self->access_token(); my $backend; - if ($config->{hostname} eq 'gmail') { - $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $auth->[1]"; - } elsif ($config->{hostname} eq 'imap.mail.me.com') { - $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $auth->[1]"; - } elsif ($config->{hostname} eq 'mail.messagingengine.com') { - $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $auth->[1]"; - } elsif ($config->{hostname} eq 'imap.mail.yahoo.com') { - $backend = JMAP::Sync::Yahoo->new($config) || die "failed to setup $auth->[1]"; + if ($config->{imapHost} eq 'imap.gmail.com') { + $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $config->{username}"; + } elsif ($config->{imapHost} eq 'imap.mail.me.com') { + $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $config->{username}"; + } elsif ($config->{imapHost} eq 'mail.messagingengine.com') { + $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $config->{username}"; + } elsif ($config->{imapHost} eq 'imap.mail.yahoo.com') { + $backend = JMAP::Sync::Yahoo->new($config) || die "failed to setup $config->{username}"; } else { - $backend = JMAP::Sync::Standard->new($config) || die "failed to setup $auth->[1]"; + $backend = JMAP::Sync::Standard->new($config) || die "failed to setup $config->{username}"; } $Self->{backend} = $backend; } @@ -736,6 +736,7 @@ sub do_folder { if ($res->{new}) { my $new = $res->{new}[1]; foreach my $uid (sort { $a <=> $b } keys %$new) { + my ($msgid, $thrid, @labels); if ($Self->{is_gmail}) { ($msgid, $thrid) = ($new->{"x-gm-msgid"}, $new->{"x-gm-thrid"}); @labels = $forcelabel ? ($forcelabel) : @{$new->{"x-gm-labels"}}; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index aeef243..4c2472a 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -125,3 +125,4 @@ sub send_email { }); } +1; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 2a3d7e6..ae0f4af 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -312,7 +312,19 @@ sub handle_cb_google { } getdb(); - $db->setuser($email, $gmaildata->{refresh_token}, $data->{name}, $data->{picture}); + #$db->setuser(username => $email, password => $gmaildata->{refresh_token}, email => $data->{name}, picture => $data->{picture}); + $db->setuser( + username => $email, + password => $gmaildata->{refresh_token}, + imapHost => 'imap.gmail.com', + imapPort => '993', + imapSSL => 1, + smtpHost => 'smtp.gmail.com', + smtpPort => 465, + smtpSSL => 1, + caldavURL => "https://apidata.googleusercontent.com/caldav/v2", + carddavURL => "https://www.googleapis.com/.well-known/carddav", + ); $db->firstsync(); return ['registered', [$accountid, $email]]; @@ -343,7 +355,16 @@ sub handle_signup { $dbh->do("INSERT INTO accounts (email, accountid, type) VALUES (?, ?, ?)", {}, $detail->[1], $accountid, 'imap'); } getdb(); - $db->setuser(@$detail); + $db->setuser( + username => $detail->[1], + password => $detail->[2], + imapHost => $detail->[0], + imapPort => 993, + imapSSL => 1, + smtpHost => $detail->[0], + smtpPort => 465, + smtpSSL => 1, + ); $db->firstsync(); return ['signedup', [$accountid, $detail->[1]]]; From 994ee873ff6e8b61dfac7695fd2c4235aeadc61a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 03:54:27 -0400 Subject: [PATCH 205/331] add back host for now --- htdocs/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/htdocs/index.html b/htdocs/index.html index 8266ca2..4d491c0 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -29,6 +29,10 @@

JMAP Proxy

+ + + + From 11e19404b05fe745b42235a86def927edeadfa4d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 04:13:45 -0400 Subject: [PATCH 206/331] some fixes --- JMAP/ImapDB.pm | 12 ++++++------ JMAP/Sync/Gmail.pm | 3 --- bin/apiendpoint.pl | 3 ++- bin/server.pl | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 85de7f8..c226a85 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -90,9 +90,9 @@ sub setuser { sub access_data { my $Self = shift; - my $config = $Self->dbh->selectrow_array("SELECT * FROM iserver", {Slice => {}}); + my $config = $Self->dbh->selectall_arrayref("SELECT * FROM iserver", {Slice => {}}); - return $config + return $config->[0]; } # synchronous backend for now @@ -105,7 +105,7 @@ sub backend_cmd { Carp::confess("in transaction") if $Self->in_transaction(); unless ($Self->{backend}) { - my $config = $Self->access_token(); + my $config = $Self->access_data(); my $backend; if ($config->{imapHost} eq 'imap.gmail.com') { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $config->{username}"; @@ -165,7 +165,7 @@ sub sync_folders { $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); } - $Self->dmaybeupdate('iserver', {prefix => $prefix, lastfoldersync => time()}); + $Self->dmaybeupdate('iserver', {imapPrefix => $prefix, lastfoldersync => time()}); $Self->commit(); @@ -571,7 +571,7 @@ sub labels { sub sync_imap { my $Self = shift; - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, highestmodseq, label FROM ifolders"); + my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } @@ -1089,7 +1089,7 @@ sub apply_data { sub _envelopedata { my $data = shift; - my $envelope = decode_json($data); + my $envelope = decode_json($data || "{}"); my $encsub = Encode::decode('MIME-Header', $envelope->{Subject}); return ( msgsubject => $encsub, diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 4c2472a..27f78a7 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -85,10 +85,8 @@ sub connect_imap { } for (1..3) { - $Self->log('debug', "Looking for server for $Self->{auth}{username}"); my $port = 993; my $usessl = $port != 143; # we use SSL for anything except default - $Self->log('debug', "getting imaptalk"); $Self->{imap} = Mail::GmailTalk->new( Server => 'imap.gmail.com', Port => $port, @@ -99,7 +97,6 @@ sub connect_imap { UseBlocking => $usessl, ); next unless $Self->{imap}; - $Self->log('debug', "Connected as $Self->{auth}{username}"); $Self->{lastused} = time(); return $Self->{imap}; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index ae0f4af..72b007c 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -18,6 +18,7 @@ package JMAP::Backend; use Mail::IMAPTalk; use Data::Dumper; use AnyEvent::HTTPD; +use JMAP::Sync::Gmail; use JMAP::GmailDB; use JMAP::ImapDB; use JMAP::DB; @@ -284,7 +285,7 @@ sub accountsdb { sub handle_cb_google { my $code = shift; - my $O = JMAP::GmailDB::O(); + my $O = JMAP::Sync::Gmail::O(); die "NO ACCESS CODE PROVIDED (did you hit cancel?)\n" unless $code; my $gmaildata = $O->finish($code); diff --git a/bin/server.pl b/bin/server.pl index a50822d..94992f4 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -18,7 +18,7 @@ use AnyEvent::Socket; use AnyEvent::Util; use AnyEvent::HTTP; -use JMAP::GmailDB; +use JMAP::Sync::Gmail; use JSON::XS qw(encode_json decode_json); use Encode qw(encode_utf8); @@ -714,7 +714,7 @@ sub HandleKeepAlive { sub do_register { my ($httpd, $req) = @_; - my $O = JMAP::GmailDB::O(); + my $O = JMAP::Sync::Gmail::O(); $req->respond({redirect => $O->start(new_uuid_string())}); }; From e2c47da70ebd80e75c2b6828f53674463be18044 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 10:12:25 -0400 Subject: [PATCH 207/331] Google fix --- JMAP/GmailDB.pm | 11 +++++++++++ JMAP/ImapDB.pm | 10 +++++----- bin/apiendpoint.pl | 3 ++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index bdf7849..b920192 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -19,6 +19,7 @@ use AnyEvent; use AnyEvent::Socket; use Data::Dumper; use IO::All; +use JMAP::Sync::Gmail; sub new { my $class = shift; @@ -27,4 +28,14 @@ sub new { return $Self; } +sub access_token { + my $Self = shift; + my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT imapHost, username, password FROM iserver"); + + my $O = JMAP::Sync::Gmail::O(); + my $data = $O->refresh($password); + + return [$hostname, $username, $data->{access_token}]; +} + 1; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c226a85..c2770b0 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -713,8 +713,8 @@ sub do_folder { foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid, @labels); if ($Self->{is_gmail}) { - ($msgid, $thrid) = ($new->{"x-gm-msgid"}, $new->{"x-gm-thrid"}); - @labels = $forcelabel ? ($forcelabel) : @{$new->{"x-gm-labels"}}; + ($msgid, $thrid) = ($new->{$uid}{"x-gm-msgid"}, $new->{$uid}{"x-gm-thrid"}); + @labels = $forcelabel ? ($forcelabel) : @{$new->{$uid}{"x-gm-labels"}}; } else { ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); @@ -738,14 +738,14 @@ sub do_folder { foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid, @labels); if ($Self->{is_gmail}) { - ($msgid, $thrid) = ($new->{"x-gm-msgid"}, $new->{"x-gm-thrid"}); - @labels = $forcelabel ? ($forcelabel) : @{$new->{"x-gm-labels"}}; + ($msgid, $thrid) = ($new->{$uid}{"x-gm-msgid"}, $new->{$uid}{"x-gm-thrid"}); + @labels = $forcelabel ? ($forcelabel) : @{$new->{$uid}{"x-gm-labels"}}; } else { ($msgid, $thrid) = $Self->calcmsgid($imapname, $uid, $new->{$uid}); @labels = ($forcelabel); } - $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, [$forcelabel], $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); + $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, \@labels, $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 72b007c..0df7163 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -324,7 +324,8 @@ sub handle_cb_google { smtpPort => 465, smtpSSL => 1, caldavURL => "https://apidata.googleusercontent.com/caldav/v2", - carddavURL => "https://www.googleapis.com/.well-known/carddav", + # still broken + #carddavURL => "https://www.googleapis.com/.well-known/carddav", ); $db->firstsync(); From 8f14bcd262efffc8d7c279afe107d88013c09a25 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 10:21:32 -0400 Subject: [PATCH 208/331] Gmail: sync allmail, never inbox --- JMAP/ImapDB.pm | 15 ++++++++------- JMAP/Sync/Common.pm | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c2770b0..c450c52 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -616,13 +616,14 @@ sub firstsync { my $labels = $Self->labels(); - my $ifolderid = $labels->{"inbox"}[0]; - $Self->do_folder($ifolderid, "inbox", 50); - - my $msgids = $Self->dbh->selectcol_arrayref("SELECT msgid FROM imessages WHERE ifolderid = ? ORDER BY uid DESC LIMIT 50", {}, $ifolderid); - - # pre-load the INBOX! - $Self->fill_messages(@$msgids); + if ($Self->{is_gmail}) { + my $ifolderid = $labels->{"\\allmail"}[0]; + $Self->do_folder($ifolderid, undef, 50); + } + else { + my $ifolderid = $labels->{"inbox"}[0]; + $Self->do_folder($ifolderid, "inbox", 50); + } } sub _trimh { diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 28b857f..e4631c0 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -189,7 +189,7 @@ sub folders { $label =~ s{^$prefix}{}; $label =~ s{^[$folder->[1]]}{}; # just in case prefix was missing sep } - $folders{$name} = [$folder->[1], $label]; + $folders{$name} = [$folder->[1], lc $label]; } return [$prefix, \%folders]; From bc0e5377ae26ce8b6ecadf23c4d4509d3d4aa8f4 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 00:24:57 +1000 Subject: [PATCH 209/331] fix labels --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c450c52..22b740e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -143,7 +143,7 @@ sub sync_folders { my %getstatus; foreach my $name (sort keys %$folders) { my $sep = $folders->{$name}[0]; - my $role = $ROLE_MAP{lc $folders->{$name}[1]}; + my $role = lc $ROLE_MAP{lc $folders->{$name}[4]}; my $label = $role || $folders->{$name}[1]; my $id = $ibylabel{$label}[0]; if ($id) { From ff376b695abdd868cf9a9ea9b0d104a22543cf33 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 10:37:50 -0400 Subject: [PATCH 210/331] fix role map for gmail --- JMAP/ImapDB.pm | 4 ++-- JMAP/Sync/Common.pm | 9 ++++++--- JMAP/Sync/Gmail.pm | 3 +++ bin/server.pl | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 22b740e..694894d 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -143,7 +143,7 @@ sub sync_folders { my %getstatus; foreach my $name (sort keys %$folders) { my $sep = $folders->{$name}[0]; - my $role = lc $ROLE_MAP{lc $folders->{$name}[4]}; + my $role = $ROLE_MAP{$folders->{$name}[1]}; my $label = $role || $folders->{$name}[1]; my $id = $ibylabel{$label}[0]; if ($id) { @@ -211,7 +211,7 @@ sub sync_jmailboxes { $fname =~ s/^INBOX\.//; # check for roles first my @bits = split "[$folder->[1]]", $fname; - my $role = $ROLE_MAP{lc $fname}; + my $role = $ROLE_MAP{lc $folder->[3]}; my $id = 0; my $parentId = 0; my $name; diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index e4631c0..9c264af 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -183,13 +183,16 @@ sub folders { foreach my $folder (@folders) { my ($role) = grep { not $KNOWN_SPECIALS{lc $_} } @{$folder->[0]}; my $name = $folder->[2]; - my $label = $role; - unless ($label) { + my $label; + if ($role) { + $label = lc $role; + } + else { $label = $folder->[2]; $label =~ s{^$prefix}{}; $label =~ s{^[$folder->[1]]}{}; # just in case prefix was missing sep } - $folders{$name} = [$folder->[1], lc $label]; + $folders{$name} = [$folder->[1], $label]; } return [$prefix, \%folders]; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 27f78a7..6734128 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -61,6 +61,9 @@ sub connect_calendars { sub connect_contacts { my $Self = shift; + # XXX - until we fix it + return; + if ($Self->{contacts}) { $Self->{lastused} = time(); return $Self->{contacts}; diff --git a/bin/server.pl b/bin/server.pl index 94992f4..3be6eee 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -619,6 +619,7 @@ sub HandleEventSource { sub prod_backfill { my $accountid = shift; my $force = shift; + return; return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; From 5b81aa2bb7bbf09a99ab48cb33f7a684f7c197bb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 10:39:48 -0400 Subject: [PATCH 211/331] re-enable backfill --- bin/server.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/server.pl b/bin/server.pl index 3be6eee..94992f4 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -619,7 +619,6 @@ sub HandleEventSource { sub prod_backfill { my $accountid = shift; my $force = shift; - return; return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; From 20afb10af8eef6eb86f663967a32127ea67becfb Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 10:53:08 -0400 Subject: [PATCH 212/331] don't lose track on label change --- JMAP/ImapDB.pm | 10 ++++++---- bin/apiendpoint.pl | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 694894d..f09abda 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -143,8 +143,7 @@ sub sync_folders { my %getstatus; foreach my $name (sort keys %$folders) { my $sep = $folders->{$name}[0]; - my $role = $ROLE_MAP{$folders->{$name}[1]}; - my $label = $role || $folders->{$name}[1]; + my $label = $folders->{$name}[1]; my $id = $ibylabel{$label}[0]; if ($id) { $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); @@ -673,8 +672,10 @@ sub do_folder { my %fetches; my @immutable = qw(internaldate envelope rfc822.size); + my @mutable; if ($Self->{is_gmail}) { push @immutable, qw(x-gm-msgid x-gm-thrid x-gm-labels); + push @mutable, qw(x-gm-labels); } if ($batchsize) { @@ -687,7 +688,7 @@ sub do_folder { } else { $fetches{new} = [$uidnext, '*', \@immutable]; - $fetches{update} = [$uidfirst, $uidnext - 1, [], $highestmodseq]; + $fetches{update} = [$uidfirst, $uidnext - 1, \@mutable, $highestmodseq]; } $Self->commit(); @@ -730,7 +731,8 @@ sub do_folder { if ($res->{update}) { my $changed = $res->{update}[1]; foreach my $uid (sort { $a <=> $b } keys %$changed) { - $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, [$forcelabel]); + @labels = $forcelabel ? ($forcelabel) : @{$new->{$uid}{"x-gm-labels"}}; + $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, \@labels); } } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 0df7163..e567219 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -250,6 +250,7 @@ sub handle_sync { sub handle_syncall { my $db = shift; + return ['syncall', $JSON::true]; $db->sync_folders(); $db->sync_imap(); $db->sync_addressbooks(); @@ -259,7 +260,8 @@ sub handle_syncall { sub handle_backfill { my $db = shift; - my $res = $db->backfill(); + #my $res = $db->backfill(); + my $res = 0; return ['sync', $res ? $JSON::true : $JSON::false]; } From 4337a81324318fd26e05459028394a72fb247f08 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 11:13:33 -0400 Subject: [PATCH 213/331] fix labels --- JMAP/ImapDB.pm | 5 ++++- JMAP/Sync/Gmail.pm | 33 +++++++++++++++++++++++++++++++++ bin/apiendpoint.pl | 4 +--- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index f09abda..bee61e1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -731,7 +731,10 @@ sub do_folder { if ($res->{update}) { my $changed = $res->{update}[1]; foreach my $uid (sort { $a <=> $b } keys %$changed) { - @labels = $forcelabel ? ($forcelabel) : @{$new->{$uid}{"x-gm-labels"}}; + my @labels = ($forcelabel); + if ($Self->{is_gmail}) { + @labels = $forcelabel ? ($forcelabel) : @{$changed->{$uid}{"x-gm-labels"}}; + } $Self->changed_record($ifolderid, $uid, $changed->{$uid}{'flags'}, \@labels); } } diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 6734128..86e9c50 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -125,4 +125,37 @@ sub send_email { }); } +sub imap_labels { + my $Self = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uids = shift; + my $labels = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->select($imapname); + die "SELECT FAILED $imapname" unless $r; + + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; + + my %res = ( + imapname => $imapname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + ); + + if ($olduidvalidity != $uidvalidity) { + return \%res; + } + + $imap->store($uids, "x-gm-labels" "(@$labels)"); + _unselect($imap); + + $res{updated} = $uids; + + return \%res; +} + + 1; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index e567219..0df7163 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -250,7 +250,6 @@ sub handle_sync { sub handle_syncall { my $db = shift; - return ['syncall', $JSON::true]; $db->sync_folders(); $db->sync_imap(); $db->sync_addressbooks(); @@ -260,8 +259,7 @@ sub handle_syncall { sub handle_backfill { my $db = shift; - #my $res = $db->backfill(); - my $res = 0; + my $res = $db->backfill(); return ['sync', $res ? $JSON::true : $JSON::false]; } From d5a92714bb7bfe20b1af9d4f75532c9765ce1994 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 21 Jul 2015 11:28:24 -0400 Subject: [PATCH 214/331] more wip --- JMAP/ImapDB.pm | 5 +++-- JMAP/Sync/Gmail.pm | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index bee61e1..fb1b24c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -620,8 +620,9 @@ sub firstsync { $Self->do_folder($ifolderid, undef, 50); } else { - my $ifolderid = $labels->{"inbox"}[0]; - $Self->do_folder($ifolderid, "inbox", 50); + my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname FROM ifolders"); + my ($folder) = grep { lc $_->[1] eq 'inbox' } @$data; + $Self->do_folder($folder->[0], "inbox", 50); } } diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 86e9c50..95d0ffd 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -149,7 +149,7 @@ sub imap_labels { return \%res; } - $imap->store($uids, "x-gm-labels" "(@$labels)"); + $imap->store($uids, "x-gm-labels", "(@$labels)"); _unselect($imap); $res{updated} = $uids; From 612bc2880209328f9f7b4c2e7180855f154a81a9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 04:37:45 +1000 Subject: [PATCH 215/331] use ENV for config directly --- JMAP/API.pm | 3 ++- JMAP/Config.pm | 27 --------------------------- JMAP/DB.pm | 5 +++-- JMAP/Sync/Gmail.pm | 3 ++- JMAP/Sync/Standard.pm | 3 ++- bin/server.pl | 11 ++++++----- 6 files changed, 15 insertions(+), 37 deletions(-) delete mode 100644 JMAP/Config.pm diff --git a/JMAP/API.pm b/JMAP/API.pm index 9d46d61..0784ae9 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -3,6 +3,7 @@ package JMAP::API; use JMAP::DB; +use JMAP::Config; use JSON; use strict; use warnings; @@ -954,7 +955,7 @@ sub getMessages { } if (_prop_wanted($args, 'rawUrl')) { - $item->{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; + $item->{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/$msgid"; } if (_prop_wanted($args, 'blobId')) { diff --git a/JMAP/Config.pm b/JMAP/Config.pm deleted file mode 100644 index e85fa81..0000000 --- a/JMAP/Config.pm +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/perl -c - -use strict; -use warnings; - -package JMAP::Config; - -use base 'Exporter'; - -our @EXPORT = qw(config); - -use JSON::XS qw(encode_json decode_json); -use IO::All; - -our $CONFIG; - -sub config { - unless ($CONFIG) { - my $file = "/etc/jmap_proxy.conf"; - $CONFIG = eval { decode_json(io->file($ENV{JMAPCONF} || $file)->slurp) }; - die "NEED config $file or ENV JMAPCONF" unless $CONFIG; - } - my $name = shift; - return $name ? $CONFIG->{$name} : $CONFIG; -} - -1; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 9aaba03..81a463c 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -9,6 +9,7 @@ use Data::Dumper; use DBI; use Carp qw(confess); +use JSON::Config; use Data::UUID::LibUUID; use IO::LockedFile; use JSON::XS qw(encode_json decode_json); @@ -425,7 +426,7 @@ sub attachments { push @res, { id => $id, type => $type, - url => "https://proxy.jmap.io/raw/$accountid/$messageid/$id/$filename", # XXX dep + url => "https://$ENV{jmaphost}/raw/$accountid/$messageid/$id/$filename", # XXX dep blobId => "$messageid/$id", name => $filename, size => length($body), @@ -692,7 +693,7 @@ sub create_messages { # XXX - references } $item->{msgdate} = time(); - $item->{msgmessageid} = new_uuid_string() . '@proxy.jmap.io'; + $item->{msgmessageid} = new_uuid_string() . "\@$ENV{jmaphost}"; my $message = $Self->_makemsg($item); # XXX - let's just assume goodness for now - lots of error handling to add my ($msgid, $thrid) = $Self->import_message($message, [$draftid], diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 95d0ffd..a9aede1 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -6,6 +6,7 @@ use warnings; package JMAP::Sync::Gmail; use base qw(JMAP::Sync::Common); +use JMAP::Config; use Mail::GmailTalk; use JSON::XS qw(encode_json decode_json); use Email::Simple; @@ -115,7 +116,7 @@ sub send_email { sendmail($email, { from => $Self->{auth}{username}, transport => Email::Sender::Transport::GmailSMTP->new({ - helo => 'proxy.jmap.io', + helo => $ENV{jmaphost}, host => 'smtp.gmail.com', port => 465, ssl => 1, diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 7a6fc0d..3885571 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -6,6 +6,7 @@ use warnings; package JMAP::Sync::Standard; use base qw(JMAP::Sync::Common); +use JSON::Config; use Mail::IMAPTalk; use JSON::XS qw(encode_json decode_json); use Email::Simple; @@ -96,7 +97,7 @@ sub send_email { sendmail($email, { from => $Self->{auth}{username}, transport => Email::Sender::Transport::SMTPS->new({ - helo => 'proxy.jmap.io', + helo => $ENV{jmaphost}, host => $Self->{auth}{smtpserver}, port => 587, ssl => 'starttls', diff --git a/bin/server.pl b/bin/server.pl index 94992f4..0fe79b2 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -3,6 +3,7 @@ use lib '/home/jmap/jmap-perl'; #use Mail::IMAPTalk qw(:trace); +use JMAP::Config; use HTML::GenerateUtil qw(escape_html escape_uri); use strict; use warnings; @@ -480,7 +481,7 @@ sub client_page { $req->respond ({ content => ['text/html', $html] }); }, sub { my $cookie = bake_cookie("jmap_$accountid", {value => '', path => '/'}); - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://proxy.jmap.io/" }, "Redirected"]); + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/" }, "Redirected"]); }); } @@ -509,7 +510,7 @@ sub home_page { EOF foreach my $key (sort keys %ids) { - $sessiontext .= qq{\n \n\n}; + $sessiontext .= qq{\n \n\n}; } $sessiontext .= "
Host
User
$ids{$key}\n
$ids{$key}\n
"; } @@ -730,7 +731,7 @@ sub do_delete { if ($accountid) { send_backend_request($accountid, 'delete', $accountid, sub { my $cookie = bake_cookie("jmap_$accountid", {value => '', path => '/'}); - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://proxy.jmap.io/" }, "Redirected"]); + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/" }, "Redirected"]); }, mkerr($req)); } }; @@ -751,7 +752,7 @@ sub do_signup { path => '/', expires => '+3M', }); - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://proxy.jmap.io/jmap/$data->[0]" }, + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[0]" }, "Redirected"]); delete $backend{$accountid} unless $data->[0] eq $accountid; send_backend_request($data->[0], 'sync', $data->[0]); @@ -777,7 +778,7 @@ sub do_cb_google { path => '/', expires => '+3M', }); - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://proxy.jmap.io/jmap/$data->[0]" }, + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[0]" }, "Redirected"]); delete $backend{$accountid} unless $data->[0] eq $accountid; send_backend_request($data->[0], 'sync', $data->[0]); From ae23de318f0eb8fcb516f2aae0a9984166e9b9d7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 04:46:14 +1000 Subject: [PATCH 216/331] remove JMAP::Config --- JMAP/API.pm | 1 - JMAP/Sync/Gmail.pm | 1 - bin/server.pl | 1 - 3 files changed, 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 0784ae9..49e2763 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -3,7 +3,6 @@ package JMAP::API; use JMAP::DB; -use JMAP::Config; use JSON; use strict; use warnings; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index a9aede1..a1d1c48 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -6,7 +6,6 @@ use warnings; package JMAP::Sync::Gmail; use base qw(JMAP::Sync::Common); -use JMAP::Config; use Mail::GmailTalk; use JSON::XS qw(encode_json decode_json); use Email::Simple; diff --git a/bin/server.pl b/bin/server.pl index 0fe79b2..dafc8c2 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -3,7 +3,6 @@ use lib '/home/jmap/jmap-perl'; #use Mail::IMAPTalk qw(:trace); -use JMAP::Config; use HTML::GenerateUtil qw(escape_html escape_uri); use strict; use warnings; From 322f1e1a4f5317229ec002fb75fd63530982dcf3 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 21 Jul 2015 15:13:25 -0400 Subject: [PATCH 217/331] remove more JMAP::Config --- JMAP/DB.pm | 1 - JMAP/Sync/Standard.pm | 1 - 2 files changed, 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 81a463c..97e9c91 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -9,7 +9,6 @@ use Data::Dumper; use DBI; use Carp qw(confess); -use JSON::Config; use Data::UUID::LibUUID; use IO::LockedFile; use JSON::XS qw(encode_json decode_json); diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 3885571..6efe2ad 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -6,7 +6,6 @@ use warnings; package JMAP::Sync::Standard; use base qw(JMAP::Sync::Common); -use JSON::Config; use Mail::IMAPTalk; use JSON::XS qw(encode_json decode_json); use Email::Simple; From a0d08dd50bc6a23b07aca95f9683e7d48ccc27e0 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 21 Jul 2015 17:54:44 -0400 Subject: [PATCH 218/331] fix access token, proxy name, all the things --- JMAP/API.pm | 2 +- JMAP/ImapDB.pm | 9 +++++++++ JMAP/Sync/Gmail.pm | 21 +++++++++------------ bin/server.pl | 1 + htdocs/landing.html | 6 +++--- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 49e2763..064bf62 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1157,7 +1157,7 @@ sub setMessages { foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; - $created->{$cid}{rawUrl} = "https://proxy.jmap.io/raw/$accountid/$msgid"; + $created->{$cid}{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/$msgid"; $created->{$cid}{blobId} = "$msgid"; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index fb1b24c..98b69e8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -87,6 +87,15 @@ sub setuser { $Self->commit(); } +sub access_token { + my $Self = shift; + my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT imapHost, username, password FROM iserver"); + + return [$hostname, $username, $password]; +} + + + sub access_data { my $Self = shift; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index a1d1c48..2f7ed4c 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -50,7 +50,7 @@ sub connect_calendars { $Self->{calendars} = Net::GmailCalendars->new( user => $Self->{auth}{username}, access_token => $Self->access_token(), - url => "https://apidata.googleusercontent.com/caldav/v2", + url => $Self->{auth}{caldavURL}, is_google => 1, expandurl => 1, ); @@ -61,9 +61,6 @@ sub connect_calendars { sub connect_contacts { my $Self = shift; - # XXX - until we fix it - return; - if ($Self->{contacts}) { $Self->{lastused} = time(); return $Self->{contacts}; @@ -72,7 +69,7 @@ sub connect_contacts { $Self->{contacts} = Net::GmailContacts->new( user => $Self->{auth}{username}, access_token => $Self->access_token(), - url => "https://www.googleapis.com/.well-known/carddav", + url => $Self->{auth}{carddavURL}, expandurl => 1, ); @@ -88,11 +85,11 @@ sub connect_imap { } for (1..3) { - my $port = 993; - my $usessl = $port != 143; # we use SSL for anything except default + my $port = $Self->{auth}{imapPort}; + my $usessl = $Self->{auth}{imapSSL}; $Self->{imap} = Mail::GmailTalk->new( - Server => 'imap.gmail.com', - Port => $port, + Server => $Self->{auth}{imapHost}, + Port => $port; Username => $Self->{auth}{username}, Password => $Self->access_token(), # not configurable right now... @@ -116,9 +113,9 @@ sub send_email { from => $Self->{auth}{username}, transport => Email::Sender::Transport::GmailSMTP->new({ helo => $ENV{jmaphost}, - host => 'smtp.gmail.com', - port => 465, - ssl => 1, + host => $Self->{auth}{smtpHost}, + port => $Self->{auth}{smtpPort}, + ssl => $Self->{auth}{smtpSSL}, sasl_username => $Self->{auth}{username}, access_token => $Self->access_token(), }) diff --git a/bin/server.pl b/bin/server.pl index dafc8c2..997322a 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -477,6 +477,7 @@ sub client_page { $html =~ s{\$INFO}{Account: $data->[0] ($data->[1])}gs; $html =~ s/\$UUID/$accountid/gs; + $html =~ s/\$JMAPHOST/$ENV{jmaphost}/gs; $req->respond ({ content => ['text/html', $html] }); }, sub { my $cookie = bake_cookie("jmap_$accountid", {value => '', path => '/'}); diff --git a/htdocs/landing.html b/htdocs/landing.html index 19f16b9..5d43c63 100644 --- a/htdocs/landing.html +++ b/htdocs/landing.html @@ -25,13 +25,13 @@

Good work, your JMAP proxy account is now set up

$INFO - (back to proxy login page)

-

Access your account with the open source client.

+

Access your account with the open source client.

-

You can also POST with the JMAP protocol to https://proxy.jmap.io/jmap/$UUID/ – there is no need to use any other form of authentication, the UUID is used by itself for privacy. Yes, we know this isn't enough security for a public service (and we need to implement the JMAP auth spec!), but it will do for testing.

+

You can also POST with the JMAP protocol to https://$JMAPHOST/jmap/$UUID/ – there is no need to use any other form of authentication, the UUID is used by itself for privacy. Yes, we know this isn't enough security for a public service (and we need to implement the JMAP auth spec!), but it will do for testing.

Please note: it may take a few minutes before the API responds to requests, because it needs to do an initial sync of mailbox envelope data.

-

If you need to reset your account or want to delete all your data, please go to https://proxy.jmap.io/delete/$UUID. It will redirect you to the homepage when the account is wiped.

+

If you need to reset your account or want to delete all your data, please go to https://$JMAPHOST/delete/$UUID. It will redirect you to the homepage when the account is wiped.

From c493cc4c52143a4d42f73dc664d2285a72b92231 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 21 Jul 2015 18:09:24 -0400 Subject: [PATCH 219/331] gmail: enable carddav again --- JMAP/Sync/Gmail.pm | 2 +- bin/apiendpoint.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 2f7ed4c..6d01155 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -89,7 +89,7 @@ sub connect_imap { my $usessl = $Self->{auth}{imapSSL}; $Self->{imap} = Mail::GmailTalk->new( Server => $Self->{auth}{imapHost}, - Port => $port; + Port => $port, Username => $Self->{auth}{username}, Password => $Self->access_token(), # not configurable right now... diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 0df7163..c21f7d0 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -325,7 +325,7 @@ sub handle_cb_google { smtpSSL => 1, caldavURL => "https://apidata.googleusercontent.com/caldav/v2", # still broken - #carddavURL => "https://www.googleapis.com/.well-known/carddav", + carddavURL => "https://www.googleapis.com/.well-known/carddav", ); $db->firstsync(); From 2e536f21ee1ea7ee0d9b383519ce77ddc0faab53 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 08:11:45 +1000 Subject: [PATCH 220/331] no google hacks --- JMAP/Sync/Gmail.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 6d01155..10d858f 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -51,7 +51,6 @@ sub connect_calendars { user => $Self->{auth}{username}, access_token => $Self->access_token(), url => $Self->{auth}{caldavURL}, - is_google => 1, expandurl => 1, ); From e805bc2bde6c162e4e0b132ff1cd6048d9734d85 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 08:32:58 +1000 Subject: [PATCH 221/331] ImapDB: don't do UID counts when backfilling --- JMAP/ImapDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 98b69e8..7c08d5a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -769,6 +769,8 @@ sub do_folder { $Self->commit(); + return $didold if $batchsize; + # need to make changes before counting my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); # if we don't know everything, we have to ALWAYS check or moves break @@ -788,8 +790,6 @@ sub do_folder { } $Self->commit(); } - - return $didold; } sub imap_search { From b0409d62f5ae0f12885aafd57c19c8768f0f8aae Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 23 Jul 2015 09:26:55 +1000 Subject: [PATCH 222/331] ImapDB: magic --- JMAP/ImapDB.pm | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7c08d5a..2bb4f6e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -900,18 +900,18 @@ sub update_messages { } } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); - my %foldermap = map { $_->[0] => $_ } @$folderdata; - my %jmailmap = map { $_->[4] => $_ } @$folderdata; - my $jmapdata = $dbh->selectall_arrayref("SELECT jmailboxid, role FROM jmailboxes"); - my %jidmap = map { $_->[0] => $_->[1] } @$jmapdata; - my %jrolemap = map { $_->[1] => $_->[0] } grep { $_-> [1] } @$jmapdata; + my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my %foldermap = map { $_->{folderid} => $_ } @$folderdata; + my %jmailmap = map { $_->{jmailboxid} => $_ } @$folderdata; + my $jmapdata = $dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); + my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; + my %jrolemap = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$jmapdata; my @changed; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? - my $imapname = $foldermap{$ifolderid}[1]; - my $uidvalidity = $foldermap{$ifolderid}[2]; + my $imapname = $foldermap{$ifolderid}{imapname}; + my $uidvalidity = $foldermap{$ifolderid}{uidvalidity}; foreach my $uid (sort keys %{$updatemap{$ifolderid}}) { my $msgid = $updatemap{$ifolderid}{$uid}; @@ -939,12 +939,13 @@ sub update_messages { $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, $bool, \@flags); } if (exists $action->{mailboxIds}) { + # jmailboxid my @mboxes = map { $idmap->($_) } @{$action->{mailboxIds}}; my ($has_outbox) = grep { $_ eq 'outbox' } @mboxes; my (@others) = grep { $_ ne 'outbox' } @mboxes; if ($has_outbox) { # move to sent when we're done - push @others, $jmailmap{$jrolemap{'sent'}}[0]; + push @others, $jmailmap{$jrolemap{'sent'}}{jmailboxid}; $Self->fill_messages($msgid); my ($rfc822) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); $Self->backend_cmd('send_email', $rfc822); @@ -959,19 +960,19 @@ sub update_messages { goto done unless $updatemsgid; my ($ifolderid, $updateuid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); goto done unless $ifolderid; - my $updatename = $foldermap{$ifolderid}[1]; - my $updatevalidity = $foldermap{$ifolderid}[2]; + my $updatename = $foldermap{$ifolderid}{imapname}; + my $updatevalidity = $foldermap{$ifolderid}{uidvalidity}; goto done unless $updatename; $Self->backend_cmd('imap_update', $updatename, $updatevalidity, $updateuid, 1, ["\\Answered"]); } done: if ($Self->{is_gmail}) { - my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}[2] || $jmailmap{$_}[1] } @others; + my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}{label} } @others; $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); } else { my $id = $others[0]; - my $newfolder = $jmailmap{$id}[1]; + my $newfolder = $jmailmap{$id}{imapname}; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uid, $newfolder); } } From c5c0de8776f08fb668af7ab7db1018dfd929d7b5 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 23 Jul 2015 09:47:38 +1000 Subject: [PATCH 223/331] unselect magic --- JMAP/Sync/Common.pm | 13 +++++++------ JMAP/Sync/Gmail.pm | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 9c264af..9b2e22c 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -30,6 +30,7 @@ sub DESTROY { } sub _unselect { + my $Self = shift; my $imap = shift; if ($imap->capability->{unselect}) { $imap->unselect(); @@ -253,7 +254,7 @@ sub imap_update { } $imap->store($uids, $isAdd ? "+flags" : "-flags", "(@$flags)"); - _unselect($imap); + $Self->_unselect($imap); $res{updated} = $uids; @@ -284,7 +285,7 @@ sub imap_fill { } my $data = $imap->fetch($uids, "rfc822"); - _unselect($imap); + $Self->_unselect($imap); my %ids; foreach my $uid (keys %$data) { @@ -318,7 +319,7 @@ sub imap_count { } my $data = $imap->fetch($uids, "UID"); - _unselect($imap); + $Self->_unselect($imap); $res{data} = [sort { $a <=> $b } keys %$data]; return \%res; @@ -373,7 +374,7 @@ sub imap_move { $imap->store($uids, "+flags", "(\\seen \\deleted)"); $imap->uidexpunge($uids); } - _unselect($imap); + $Self->_unselect($imap); $res{moved} = $uids; @@ -427,7 +428,7 @@ sub imap_fetch { my $data = $imap->fetch("$from:$to", "(@flags)", @extra) || {}; $res{$key} = [$item, $data]; } - _unselect($imap); + $Self->_unselect($imap); return \%res; } @@ -473,7 +474,7 @@ sub imap_search { } my $uids = $imap->search('charset', 'utf-8', @expr); - _unselect($imap); + $Self->_unselect($imap); return ['search', $imapname, $uidvalidity, $uids]; } diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 10d858f..30f2814 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -146,7 +146,7 @@ sub imap_labels { } $imap->store($uids, "x-gm-labels", "(@$labels)"); - _unselect($imap); + $Self->_unselect($imap); $res{updated} = $uids; From 50ef4aa39f2e2c4ed726de7fe60b43db2e19b903 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 22 Jul 2015 19:48:20 -0400 Subject: [PATCH 224/331] fix gmail detection --- JMAP/ImapDB.pm | 2 +- bin/server.pl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2bb4f6e..2690adf 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -901,7 +901,7 @@ sub update_messages { } my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); - my %foldermap = map { $_->{folderid} => $_ } @$folderdata; + my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; my %jmailmap = map { $_->{jmailboxid} => $_ } @$folderdata; my $jmapdata = $dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; diff --git a/bin/server.pl b/bin/server.pl index 997322a..6f9f4aa 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -45,7 +45,7 @@ sub idler { send_backend_request("$accountid:sync", 'gettoken', $accountid, sub { my ($data) = @_; if ($data) { - my $imap = $data->[0] eq 'gmail' ? AnyEvent::Gmail->new( + my $imap = $data->[0] eq 'imap.gmail.com' ? AnyEvent::Gmail->new( host => 'imap.gmail.com', user => $data->[1], token => $data->[2], @@ -63,6 +63,7 @@ sub idler { connect => sub { $imap->login()->cb(sub { my ($ok, $line) = shift->recv; + warn "LOGIN $data->[1]: $ok @$line\n"; if ($ok) { setup_examine($edgecb, $imap); } @@ -130,6 +131,7 @@ sub read_idle_line { my $handle = shift; my $line = shift; if ($line =~ m/^\*/) { + warn "GOT IDLE LINE $line\n"; $timer = make_timer($imap); $edgecb->($line); } From 2f190fcd2aee323c1a64efb5b7ed1b28273a5b4a Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 22 Jul 2015 19:55:13 -0400 Subject: [PATCH 225/331] remove noise --- bin/server.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/server.pl b/bin/server.pl index 6f9f4aa..1c62cdc 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -131,7 +131,6 @@ sub read_idle_line { my $handle = shift; my $line = shift; if ($line =~ m/^\*/) { - warn "GOT IDLE LINE $line\n"; $timer = make_timer($imap); $edgecb->($line); } From b74f9655179bd08b71cf2e160c2d54c9830fbcab Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 23 Jul 2015 13:39:30 +1000 Subject: [PATCH 226/331] it's called put_file now --- JMAP/DB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 97e9c91..ee03ec0 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -843,7 +843,7 @@ sub delete_card { } } -sub create_file { +sub put_file { my $Self = shift; my $type = shift; my $content = shift; From bb3f4bba52ecc243795412a3d1e5b71169d696ad Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 23 Jul 2015 15:07:35 +1000 Subject: [PATCH 227/331] use Template::Toolkit - signup frontend part --- bin/server.pl | 70 ++++++++++++++++++++++++--------------------- htdocs/error.html | 2 +- htdocs/index.html | 14 ++++----- htdocs/landing.html | 8 +++--- htdocs/signup.html | 35 +++++++++++++++++------ 5 files changed, 74 insertions(+), 55 deletions(-) diff --git a/bin/server.pl b/bin/server.pl index 1c62cdc..202a68d 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -21,17 +21,16 @@ use JMAP::Sync::Gmail; use JSON::XS qw(encode_json decode_json); use Encode qw(encode_utf8); +use Template::Toolkit; +my $TT = Template->new(); sub mkerr { my $req = shift; return sub { my $error = shift; - open(FH, "/home/jmap/jmap-perl/htdocs/error.html"); - local $/ = undef; - my $html = ; - close(FH); + my $html = ''; $error =~ s{at (/|bin/).*}{}s; - $html =~ s/\$ERROR/$error/gs; + $TT->process("/home/jmap/jmap-perl/htdocs/error.html", { error => $error }, \$html); $req->respond({content => ['text/html', $html]}); }; } @@ -471,15 +470,13 @@ sub client_page { prod_idler($accountid); send_backend_request($accountid, 'syncall'); - open(FH, "/home/jmap/jmap-perl/htdocs/landing.html"); - local $/ = undef; - my $html = ; - close(FH); - - $html =~ s{\$INFO}{Account: $data->[0] ($data->[1])}gs; - $html =~ s/\$UUID/$accountid/gs; - $html =~ s/\$JMAPHOST/$ENV{jmaphost}/gs; - $req->respond ({ content => ['text/html', $html] }); + my $html = ''; + $TT->process("/home/jmap/jmap-perl/htdocs/landing.html", { + info => "Account: $data->[0] ($data->[1])", + uuid => $accountid, + jmaphost => $ENV{jmaphost}, + }, \$html); + $req->respond({content => ['text/html', $html]}); }, sub { my $cookie = bake_cookie("jmap_$accountid", {value => '', path => '/'}); $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/" }, "Redirected"]); @@ -489,11 +486,6 @@ sub client_page { sub home_page { my ($httpd, $req) = @_; - open(FH, "/home/jmap/jmap-perl/htdocs/index.html"); - local $/ = undef; - my $html = ; - close(FH); - my $sessiontext = ''; my @text; my $cookies = crush_cookie($req->headers->{cookie}); @@ -515,9 +507,12 @@ sub home_page { } $sessiontext .= ""; } - - $html =~ s/\$SESSIONS/$sessiontext/gs; - $req->respond ({ content => ['text/html', $html] }); + my $html = ''; + $TT->process("/home/jmap/jmap-perl/htdocs/index.html", { + sessions => $sessiontext, + jmaphost => $ENV{jmaphost}, + }, \$html); + $req->respond({content => ['text/html', $html]}); } sub need_auth { @@ -740,23 +735,32 @@ sub do_delete { sub do_signup { my ($httpd, $req) = @_; - my $host = $req->parm('host'); - my $user = $req->parm('user'); - my $pass = $req->parm('pass'); + my $uri = $req->url(); + my $path = $uri->path(); + + my %opts; + foreach my $key (qw(username password imapHost imapPort imapSSL smtpHost smtpPort smtpSSL caldavURL carddavURL)) { + $opts{$key} = $req->parm($key); + } my $accountid = new_uuid_string(); - send_backend_request($accountid, 'signup', [$host, $user, $pass], sub { + send_backend_request($accountid, 'signup', \%opts, sub { my ($data) = @_; - if ($data) { - my $cookie = bake_cookie("jmap_$data->[0]", { - value => $data->[1], + if ($data && $data->[0] eq 'done') { + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[1]" }, + "Redirected"]); + delete $backend{$accountid} unless $data->[1] eq $accountid; + send_backend_request($data->[0], 'sync', $data->[1]); + my $cookie = bake_cookie("jmap_$data->[1]", { + value => $data->[2], path => '/', expires => '+3M', }); - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[0]" }, - "Redirected"]); - delete $backend{$accountid} unless $data->[0] eq $accountid; - send_backend_request($data->[0], 'sync', $data->[0]); + } + else { + my $html = ''; + $TT->process("/home/jmap/jmap-perl/htdocs/signup.html", $data->[1], \$html); + $req->respond({content => ['text/html', $html]}); } else { not_found($req); diff --git a/htdocs/error.html b/htdocs/error.html index 68463a7..afd3321 100644 --- a/htdocs/error.html +++ b/htdocs/error.html @@ -23,7 +23,7 @@

ERROR

-

$ERROR

+

[% error %]

diff --git a/htdocs/index.html b/htdocs/index.html index 4d491c0..5d613d5 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -23,23 +23,19 @@

JMAP Proxy

-$SESSIONS +[% sessions %]

OAuth to a Gmail account or log in to an IMAP server below:

- - + + - - - - - - + + diff --git a/htdocs/landing.html b/htdocs/landing.html index 5d43c63..a8d4a73 100644 --- a/htdocs/landing.html +++ b/htdocs/landing.html @@ -23,15 +23,15 @@

Good work, your JMAP proxy account is now set up

-

$INFO - (back to proxy login page)

+

[% info %] - (back to proxy login page)

-

Access your account with the open source client.

+

Access your account with the open source client.

-

You can also POST with the JMAP protocol to https://$JMAPHOST/jmap/$UUID/ – there is no need to use any other form of authentication, the UUID is used by itself for privacy. Yes, we know this isn't enough security for a public service (and we need to implement the JMAP auth spec!), but it will do for testing.

+

You can also POST with the JMAP protocol to https://[% jmaphost %]/jmap/[% uuid %]/ – there is no need to use any other form of authentication, the UUID is used by itself for privacy. Yes, we know this isn't enough security for a public service (and we need to implement the JMAP auth spec!), but it will do for testing.

Please note: it may take a few minutes before the API responds to requests, because it needs to do an initial sync of mailbox envelope data.

-

If you need to reset your account or want to delete all your data, please go to https://$JMAPHOST/delete/$UUID. It will redirect you to the homepage when the account is wiped.

+

If you need to reset your account or want to delete all your data, please go to https://[% jmaphost %]/delete/[% uuid %]. It will redirect you to the homepage when the account is wiped.

diff --git a/htdocs/signup.html b/htdocs/signup.html index 2e449a6..fe0ea3d 100644 --- a/htdocs/signup.html +++ b/htdocs/signup.html @@ -23,33 +23,52 @@

Set hosts

-Please confirm hosts below. IMAP must be on port 993, SMTP on port 587, CalDAV and CardDAV on port 443. +Please confirm host details below.
HostEmail
User
PassPassword
Submit
- + - + - + - + - +
IMAP + + + +
SMTP + + + +
CalDAV
CardDAV
Submit + + + + +
- -
From b372e5f3c4c0a32640b2c3eac8a446b8a60f7962 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Thu, 23 Jul 2015 15:47:05 +1000 Subject: [PATCH 228/331] massive authen rewrite - user DNS lookups --- JMAP/ImapDB.pm | 9 --- JMAP/Sync/Fastmail.pm | 22 ------- JMAP/Sync/ICloud.pm | 22 ------- JMAP/Sync/Standard.pm | 25 +++---- JMAP/Sync/Yahoo.pm | 44 ------------- bin/apiendpoint.pl | 147 ++++++++++++++++++++++++++++++++++++------ htdocs/signup.html | 12 ++-- 7 files changed, 148 insertions(+), 133 deletions(-) delete mode 100644 JMAP/Sync/Fastmail.pm delete mode 100644 JMAP/Sync/ICloud.pm delete mode 100644 JMAP/Sync/Yahoo.pm diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2690adf..530bbe7 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -19,9 +19,6 @@ use AnyEvent::Socket; use Date::Format; use Data::Dumper; use JMAP::Sync::Gmail; -use JMAP::Sync::ICloud; -use JMAP::Sync::Fastmail; -use JMAP::Sync::Yahoo; use JMAP::Sync::Standard; our $TAG = 1; @@ -118,12 +115,6 @@ sub backend_cmd { my $backend; if ($config->{imapHost} eq 'imap.gmail.com') { $backend = JMAP::Sync::Gmail->new($config) || die "failed to setup $config->{username}"; - } elsif ($config->{imapHost} eq 'imap.mail.me.com') { - $backend = JMAP::Sync::ICloud->new($config) || die "failed to setup $config->{username}"; - } elsif ($config->{imapHost} eq 'mail.messagingengine.com') { - $backend = JMAP::Sync::Fastmail->new($config) || die "failed to setup $config->{username}"; - } elsif ($config->{imapHost} eq 'imap.mail.yahoo.com') { - $backend = JMAP::Sync::Yahoo->new($config) || die "failed to setup $config->{username}"; } else { $backend = JMAP::Sync::Standard->new($config) || die "failed to setup $config->{username}"; } diff --git a/JMAP/Sync/Fastmail.pm b/JMAP/Sync/Fastmail.pm deleted file mode 100644 index 9d39079..0000000 --- a/JMAP/Sync/Fastmail.pm +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/perl -c - -use strict; -use warnings; - -package JMAP::Sync::Fastmail; -use base qw(JMAP::Sync::Standard); - -sub new { - my $class = shift; - my $auth = shift; - my %a = ( - imapserver => 'mail.messagingengine.com', - smtpserver => 'mail.messagingengine.com', - calurl => 'https://caldav.messagingengine.com/', - addressbookurl => 'https://carddav.messagingengine.com/', - %$auth, - ); - return JMAP::Sync::Standard->new(\%a); -} - -1; diff --git a/JMAP/Sync/ICloud.pm b/JMAP/Sync/ICloud.pm deleted file mode 100644 index 218ce44..0000000 --- a/JMAP/Sync/ICloud.pm +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/perl -c - -use strict; -use warnings; - -package JMAP::Sync::ICloud; -use base qw(JMAP::Sync::Standard); - -sub new { - my $class = shift; - my $auth = shift; - my %a = ( - imapserver => 'imap.mail.me.com', - smtpserver => 'smtp.mail.me.com', - calurl => 'https://caldav.icloud.com/', - addressbookurl => 'https://contacts.icloud.com/', - %$auth, - ); - return JMAP::Sync::Standard->new(\%a); -} - -1; diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 6efe2ad..890abf3 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -19,7 +19,7 @@ my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSel sub connect_calendars { my $Self = shift; - return unless $Self->{auth}{calurl}; + return unless $Self->{auth}{caldavURL}; if ($Self->{calendars}) { $Self->{lastused} = time(); @@ -29,7 +29,7 @@ sub connect_calendars { $Self->{calendars} = Net::CalDAVTalk->new( user => $Self->{auth}{username}, password => $Self->{auth}{password}, - url => $Self->{auth}{calurl}, + url => $Self->{auth}{caldavURL}, expandurl => 1, ); @@ -39,7 +39,7 @@ sub connect_calendars { sub connect_contacts { my $Self = shift; - return unless $Self->{auth}{addressbookurl}; + return unless $Self->{auth}{carddavURL}; if ($Self->{contacts}) { $Self->{lastused} = time(); @@ -49,7 +49,7 @@ sub connect_contacts { $Self->{contacts} = Net::CardDAVTalk->new( user => $Self->{auth}{username}, password => $Self->{auth}{password}, - url => $Self->{auth}{addressbookurl}, + url => $Self->{auth}{carddavURL}, expandurl => 1, ); @@ -69,11 +69,10 @@ sub connect_imap { delete $Self->{imap}; for (1..3) { - my $port = 993; - my $usessl = $port != 143; # we use SSL for anything except default + my $usessl = $Self->{auth}{imapSSL} - 1; $Self->{imap} = Mail::IMAPTalk->new( - Server => $Self->{auth}{imapserver}, - Port => $port, + Server => $Self->{auth}{imapHost}, + Port => $Self->{auth}{imapPort}, Username => $Self->{auth}{username}, Password => $Self->{auth}{password}, # not configurable right now... @@ -88,18 +87,22 @@ sub connect_imap { die "Could not connect to IMAP server: $@"; } + sub send_email { my $Self = shift; my $rfc822 = shift; + my $ssl; + $ssl = 'ssl' if $Self->{auth}{smtpSSL} == 2; + $ssl = 'startls' if $Self->{auth}{smtpSSL} == 3; my $email = Email::Simple->new($rfc822); sendmail($email, { from => $Self->{auth}{username}, transport => Email::Sender::Transport::SMTPS->new({ helo => $ENV{jmaphost}, - host => $Self->{auth}{smtpserver}, - port => 587, - ssl => 'starttls', + host => $Self->{auth}{smtpHost}, + port => $Self->{auth}{smtpPort}, + ssl => $ssl, sasl_username => $Self->{auth}{username}, sasl_password => $Self->{auth}{password}, }), diff --git a/JMAP/Sync/Yahoo.pm b/JMAP/Sync/Yahoo.pm deleted file mode 100644 index fd00fce..0000000 --- a/JMAP/Sync/Yahoo.pm +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/perl -c - -use strict; -use warnings; - -package JMAP::Sync::Yahoo; -use base qw(JMAP::Sync::Standard); - -sub connect_contacts { - my $Self = shift; - - return unless $Self->{auth}{addressbookurl}; - - if ($Self->{contacts}) { - $Self->{lastused} = time(); - return $Self->{contacts}; - } - - $Self->{contacts} = Net::CardDAVTalk->new( - user => $Self->{auth}{username}, - password => $Self->{auth}{password}, - url => $Self->{auth}{addressbookurl}, - expandurl => 1, - is_google => 1, - ); - - return $Self->{contacts}; -} - - -sub new { - my $class = shift; - my $auth = shift; - my %a = ( - imapserver => 'imap.mail.yahoo.com', - smtpserver => 'smtp.mail.yahoo.com', - calurl => 'https://caldav.calendar.yahoo.com', - addressbookurl => 'https://carddav.address.yahoo.com', - %$auth, - ); - return $class->SUPER::new(\%a); -} - -1; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index c21f7d0..089a2e6 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -29,6 +29,8 @@ package JMAP::Backend; use AnyEvent::HTTP; use EV; use JSON::XS qw(encode_json decode_json); +use Net::DNS; +use Net::DNS::Resolver; use Net::Server::Fork; @@ -314,19 +316,18 @@ sub handle_cb_google { getdb(); #$db->setuser(username => $email, password => $gmaildata->{refresh_token}, email => $data->{name}, picture => $data->{picture}); - $db->setuser( + $db->setuser({ username => $email, password => $gmaildata->{refresh_token}, imapHost => 'imap.gmail.com', imapPort => '993', - imapSSL => 1, + imapSSL => 2, smtpHost => 'smtp.gmail.com', smtpPort => 465, - smtpSSL => 1, + smtpSSL => 2, caldavURL => "https://apidata.googleusercontent.com/caldav/v2", - # still broken carddavURL => "https://www.googleapis.com/.well-known/carddav", - ); + }, { picture => $data->{picture}); $db->firstsync(); return ['registered', [$accountid, $email]]; @@ -335,8 +336,125 @@ sub handle_cb_google { sub handle_signup { my $detail = shift; + $detail->{imapPort} ||= 993; + $detail->{imapSSL} ||= 2; + $detail->{smtpPort} ||= 587; + $detail->{smtpSSL} ||= 3; + + if ($detail->{username} =~ m/\@icloud\.com/) { + $detail->{imapHost} = 'imap.mail.me.com'; + $detail->{smtpHost} = 'smtp.mail.me.com'; + $detail->{caldavURL} = 'https://caldav.icloud.com/'; + $detail->{carddavURL} = 'https://contacts.icloud.com/'; + $detail->{force} = 1; + } + + elsif ($detail->{username} =~ m/\@yahoo\.com/) { + $detail->{imapHost} = 'imap.mail.yahoo.com', + $detail->{smtpHost} = 'smtp.mail.yahoo.com'; + $detail->{caldavURL} = 'https://caldav.calendar.yahoo.com'; + $detail->{carddavURL} = 'https://carddav.address.yahoo.com'; + $detail->{force} = 1; + } + + else { + my $Resolver = Net::DNS::Resolver->new; + my $domain = $detail->{username}; + $domain =~ s/\@.*//; + my $reply; + ($reply) = $Resolver->search("_imaps._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + $detail->{imapHost} = $reply->target(); + $detail->{imapPort} = $reply->port(); + } + } + else { + my ($reply) = $Resolver->search("_imap._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + $detail->{imapHost} = $reply->target(); + $detail->{imapPort} = $reply->port(); + $detail->{imapSSL} = 3; + } + } + } + ($reply) = $Resolver->search("_smtps._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + $detail->{smtpHost} = $reply->target(); + $detail->{smtpPort} = $reply->port(); + $detail->{smtpSSL} = 2; + } + } + else { + my ($reply) = $Resolver->search("_submission._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + $detail->{smtpHost} = $reply->target(); + $detail->{smtpPort} = $reply->port(); + $detail->{smtpSSL} = 3; + } + } + } + + ($reply) = $Resolver->search("_caldavs._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + my $host = $reply->target(); + my $port = $reply->port(); + $detail->{caldavURL} = "https://$host"; + $detail->{caldavURL} .= ":$port" unless $port eq 443; + } + } + else { + my ($reply) = $Resolver->search("_caldav._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + my $host = $reply->target(); + my $port = $reply->port(); + $detail->{caldavURL} = "http://$host"; + $detail->{caldavURL} .= ":$port" unless $port eq 80; + } + } + } + + ($reply) = $Resolver->search("_carddavs._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + my $host = $reply->target(); + my $port = $reply->port(); + $detail->{carddavURL} = "https://$host"; + $detail->{carddavURL} .= ":$port" unless $port eq 443; + } + } + else { + my ($reply) = $Resolver->search("_carddav._tcp.$domain", "srv"); + if ($reply) { + my @d = $reply->answer; + if (@d) { + my $host = $reply->target(); + my $port = $reply->port(); + $detail->{carddavURL} = "http://$host"; + $detail->{carddavURL} .= ":$port" unless $port eq 80; + } + } + } + } + + unless ($detail->{force}) { + return ['continue', $detail]; + } + my $imap = Mail::IMAPTalk->new( - Server => $detail->[0], + Server => $detail->, Port => 993, UseSSL => 1, UseBlocking => 1, @@ -349,27 +467,18 @@ sub handle_signup { $imap->logout(); my $dbh = accountsdb(); - my ($existing, $type) = $dbh->selectrow_array("SELECT accountid, type FROM accounts WHERE email = ?", {}, $detail->[1]); + my ($existing, $type) = $dbh->selectrow_array("SELECT accountid, type FROM accounts WHERE email = ?", {}, $detail->{username}); if ($existing) { set_accountid($existing); } else { - $dbh->do("INSERT INTO accounts (email, accountid, type) VALUES (?, ?, ?)", {}, $detail->[1], $accountid, 'imap'); + $dbh->do("INSERT INTO accounts (email, accountid, type) VALUES (?, ?, ?)", {}, $detail->{username}, $accountid, 'imap'); } getdb(); - $db->setuser( - username => $detail->[1], - password => $detail->[2], - imapHost => $detail->[0], - imapPort => 993, - imapSSL => 1, - smtpHost => $detail->[0], - smtpPort => 465, - smtpSSL => 1, - ); + $db->setuser(%$detail); $db->firstsync(); - return ['signedup', [$accountid, $detail->[1]]]; + return ['done', [$accountid, $detail->[1]]]; } sub handle_delete { diff --git a/htdocs/signup.html b/htdocs/signup.html index fe0ea3d..78e574a 100644 --- a/htdocs/signup.html +++ b/htdocs/signup.html @@ -33,9 +33,9 @@

Set hosts

@@ -45,9 +45,9 @@

Set hosts

From 220841993d9398087a92ce32a345d7f775e769c7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 23:41:49 -0400 Subject: [PATCH 229/331] put_file is a DB method --- JMAP/API.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 064bf62..3589bef 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1059,7 +1059,7 @@ sub uploadFile { my $Self = shift; my ($type, $content) = @_; # XXX filehandle? - return $Self->put_file($type, $content); + return $Self->{db}->put_file($type, $content); } sub downloadFile { From 041b6f57ddc04b3910ac45740d42592ff0fb4d84 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 22 Jul 2015 23:43:03 -0400 Subject: [PATCH 230/331] transaction file upload --- JMAP/DB.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index ee03ec0..8104c25 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -851,9 +851,13 @@ sub put_file { my $size = length($content); + $Self->begin(); + # XXX - no dedup on sha1 here yet my $id = $Self->dinsert('jfiles', { type => $type, size => $size, content => $content, expires => $expires }); + $Self->commit(); + return { id => $id, type => $type, From 684b11bb80db4dfc8fe5b02b9d6ac47407e5bd01 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 02:15:31 -0400 Subject: [PATCH 231/331] work around upgrdae --- JMAP/ImapDB.pm | 11 ++++++----- JMAP/Sync/Gmail.pm | 4 ++-- bin/apiendpoint.pl | 30 +++++++++++++++--------------- bin/server.pl | 28 +++++++++++++--------------- htdocs/signup.html | 12 ++++++------ 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 530bbe7..60ad68b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -57,25 +57,26 @@ my %ROLE_MAP = ( sub setuser { my $Self = shift; - my %args = @_; + my $args = shift; + # XXX - picture, etc $Self->begin(); my $data = $Self->dbh->selectrow_arrayref("SELECT username FROM iserver"); if ($data and $data->[0]) { - $Self->dmaybeupdate('iserver', \%args); + $Self->dmaybeupdate('iserver', $args); } else { - $Self->dinsert('iserver', \%args); + $Self->dinsert('iserver', $args); } my $user = $Self->dbh->selectrow_arrayref("SELECT email FROM account"); if ($user and $user->[0]) { - $Self->dmaybeupdate('account', {email => $args{username}}); + $Self->dmaybeupdate('account', {email => $args->{username}}); } else { $Self->dinsert('account', { - email => $args{username}, + email => $args->{username}, jdeletedmodseq => 0, jhighestmodseq => 1, }); diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 30f2814..990395a 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -85,7 +85,7 @@ sub connect_imap { for (1..3) { my $port = $Self->{auth}{imapPort}; - my $usessl = $Self->{auth}{imapSSL}; + my $usessl = 1; $Self->{imap} = Mail::GmailTalk->new( Server => $Self->{auth}{imapHost}, Port => $port, @@ -114,7 +114,7 @@ sub send_email { helo => $ENV{jmaphost}, host => $Self->{auth}{smtpHost}, port => $Self->{auth}{smtpPort}, - ssl => $Self->{auth}{smtpSSL}, + ssl => 1, sasl_username => $Self->{auth}{username}, access_token => $Self->access_token(), }) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 089a2e6..d3e5bb1 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -315,7 +315,6 @@ sub handle_cb_google { } getdb(); - #$db->setuser(username => $email, password => $gmaildata->{refresh_token}, email => $data->{name}, picture => $data->{picture}); $db->setuser({ username => $email, password => $gmaildata->{refresh_token}, @@ -327,7 +326,7 @@ sub handle_cb_google { smtpSSL => 2, caldavURL => "https://apidata.googleusercontent.com/caldav/v2", carddavURL => "https://www.googleapis.com/.well-known/carddav", - }, { picture => $data->{picture}); + }, { picture => $data->{picture} }); $db->firstsync(); return ['registered', [$accountid, $email]]; @@ -336,6 +335,7 @@ sub handle_cb_google { sub handle_signup { my $detail = shift; + my $force = delete $detail->{force}; $detail->{imapPort} ||= 993; $detail->{imapSSL} ||= 2; $detail->{smtpPort} ||= 587; @@ -346,7 +346,7 @@ sub handle_signup { $detail->{smtpHost} = 'smtp.mail.me.com'; $detail->{caldavURL} = 'https://caldav.icloud.com/'; $detail->{carddavURL} = 'https://contacts.icloud.com/'; - $detail->{force} = 1; + $force = 1; } elsif ($detail->{username} =~ m/\@yahoo\.com/) { @@ -354,7 +354,7 @@ sub handle_signup { $detail->{smtpHost} = 'smtp.mail.yahoo.com'; $detail->{caldavURL} = 'https://caldav.calendar.yahoo.com'; $detail->{carddavURL} = 'https://carddav.address.yahoo.com'; - $detail->{force} = 1; + $force = 1; } else { @@ -449,20 +449,20 @@ sub handle_signup { } } - unless ($detail->{force}) { - return ['continue', $detail]; + unless ($force) { + return ['signup', ['continue', $detail]]; } my $imap = Mail::IMAPTalk->new( - Server => $detail->, - Port => 993, - UseSSL => 1, - UseBlocking => 1, + Server => $detail->{imapHost}, + Port => $detail->{imapPort}, + UseSSL => ($detail->{imapSSL} > 1), + UseBlocking => ($detail->{imapSSL} > 1), ); - die "UNABLE TO CONNECT to $detail->[0]\n" unless $imap; + die "UNABLE TO CONNECT for $detail->{username}\n" unless $imap; - my $ok = $imap->login($detail->[1], $detail->[2]); - die "LOGIN FAILED FOR $detail->[1] on $detail->[0]" unless $ok; + my $ok = $imap->login($detail->{username}, $detail->{password}); + die "LOGIN FAILED FOR $detail->{username}" unless $ok; my $capa = $imap->capability(); $imap->logout(); @@ -475,10 +475,10 @@ sub handle_signup { $dbh->do("INSERT INTO accounts (email, accountid, type) VALUES (?, ?, ?)", {}, $detail->{username}, $accountid, 'imap'); } getdb(); - $db->setuser(%$detail); + $db->setuser($detail); $db->firstsync(); - return ['done', [$accountid, $detail->[1]]]; + return ['signup', ['done', $accountid, $detail->{username}]]; } sub handle_delete { diff --git a/bin/server.pl b/bin/server.pl index 202a68d..45e3abf 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -21,8 +21,8 @@ use JMAP::Sync::Gmail; use JSON::XS qw(encode_json decode_json); use Encode qw(encode_utf8); -use Template::Toolkit; -my $TT = Template->new(); +use Template; +my $TT = Template->new(INCLUDE_PATH => '/home/jmap/jmap-perl/htdocs'); sub mkerr { my $req = shift; @@ -30,7 +30,7 @@ sub mkerr { my $error = shift; my $html = ''; $error =~ s{at (/|bin/).*}{}s; - $TT->process("/home/jmap/jmap-perl/htdocs/error.html", { error => $error }, \$html); + $TT->process("error.html", { error => $error }, \$html) || die $Template::ERROR; $req->respond({content => ['text/html', $html]}); }; } @@ -471,11 +471,11 @@ sub client_page { send_backend_request($accountid, 'syncall'); my $html = ''; - $TT->process("/home/jmap/jmap-perl/htdocs/landing.html", { + $TT->process("landing.html", { info => "Account: $data->[0] ($data->[1])", uuid => $accountid, jmaphost => $ENV{jmaphost}, - }, \$html); + }, \$html) || die $Template::ERROR; $req->respond({content => ['text/html', $html]}); }, sub { my $cookie = bake_cookie("jmap_$accountid", {value => '', path => '/'}); @@ -508,10 +508,10 @@ sub home_page { $sessiontext .= ""; } my $html = ''; - $TT->process("/home/jmap/jmap-perl/htdocs/index.html", { + $TT->process("index.html", { sessions => $sessiontext, jmaphost => $ENV{jmaphost}, - }, \$html); + }, \$html) || die $Template::ERROR; $req->respond({content => ['text/html', $html]}); } @@ -739,32 +739,30 @@ sub do_signup { my $path = $uri->path(); my %opts; - foreach my $key (qw(username password imapHost imapPort imapSSL smtpHost smtpPort smtpSSL caldavURL carddavURL)) { + foreach my $key (qw(username password imapHost imapPort imapSSL smtpHost smtpPort smtpSSL caldavURL carddavURL force)) { $opts{$key} = $req->parm($key); } my $accountid = new_uuid_string(); send_backend_request($accountid, 'signup', \%opts, sub { my ($data) = @_; + warn Dumper($data); if ($data && $data->[0] eq 'done') { - $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[1]" }, - "Redirected"]); - delete $backend{$accountid} unless $data->[1] eq $accountid; send_backend_request($data->[0], 'sync', $data->[1]); my $cookie = bake_cookie("jmap_$data->[1]", { value => $data->[2], path => '/', expires => '+3M', }); + $req->respond([301, 'redirected', { 'Set-Cookie' => $cookie, Location => "https://$ENV{jmaphost}/jmap/$data->[1]" }, + "Redirected"]); + delete $backend{$accountid} unless $data->[1] eq $accountid; } else { my $html = ''; - $TT->process("/home/jmap/jmap-perl/htdocs/signup.html", $data->[1], \$html); + $TT->process("signup.html", $data->[1], \$html) || die $Template::ERROR; $req->respond({content => ['text/html', $html]}); } - else { - not_found($req); - } return 1; }, mkerr($req)); } diff --git a/htdocs/signup.html b/htdocs/signup.html index 78e574a..469ea52 100644 --- a/htdocs/signup.html +++ b/htdocs/signup.html @@ -33,9 +33,9 @@

Set hosts

@@ -45,9 +45,9 @@

Set hosts

From 5050871c6d0f7e6cda73fd07d5607e381894c8f5 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 11:38:14 -0400 Subject: [PATCH 232/331] fix detection, uploads --- JMAP/DB.pm | 6 +++--- bin/apiendpoint.pl | 37 ++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 8104c25..8ed5073 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -292,7 +292,7 @@ sub parse_date { } sub isodate { - my $Selft = shift; + my $Self = shift; my $epoch = shift || time(); my $date = DateTime->from_epoch( epoch => $epoch ); @@ -859,9 +859,9 @@ sub put_file { $Self->commit(); return { - id => $id, + id => "$id", type => $type, - expires => $expires, + expires => scalar($Self->isodate($expires)), size => $size, }; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index d3e5bb1..785bd29 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -360,14 +360,17 @@ sub handle_signup { else { my $Resolver = Net::DNS::Resolver->new; my $domain = $detail->{username}; - $domain =~ s/\@.*//; + $domain =~ s/.*\@//; my $reply; + warn "RESOLVING: $domain\n"; ($reply) = $Resolver->search("_imaps._tcp.$domain", "srv"); + use Data::Dumper; + warn Dumper($reply); if ($reply) { my @d = $reply->answer; if (@d) { - $detail->{imapHost} = $reply->target(); - $detail->{imapPort} = $reply->port(); + $detail->{imapHost} = $d[0]->target(); + $detail->{imapPort} = $d[0]->port(); } } else { @@ -375,8 +378,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - $detail->{imapHost} = $reply->target(); - $detail->{imapPort} = $reply->port(); + $detail->{imapHost} = $d[0]->target(); + $detail->{imapPort} = $d[0]->port(); $detail->{imapSSL} = 3; } } @@ -385,8 +388,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - $detail->{smtpHost} = $reply->target(); - $detail->{smtpPort} = $reply->port(); + $detail->{smtpHost} = $d[0]->target(); + $detail->{smtpPort} = $d[0]->port(); $detail->{smtpSSL} = 2; } } @@ -395,8 +398,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - $detail->{smtpHost} = $reply->target(); - $detail->{smtpPort} = $reply->port(); + $detail->{smtpHost} = $d[0]->target(); + $detail->{smtpPort} = $d[0]->port(); $detail->{smtpSSL} = 3; } } @@ -406,8 +409,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - my $host = $reply->target(); - my $port = $reply->port(); + my $host = $d[0]->target(); + my $port = $d[0]->port(); $detail->{caldavURL} = "https://$host"; $detail->{caldavURL} .= ":$port" unless $port eq 443; } @@ -417,8 +420,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - my $host = $reply->target(); - my $port = $reply->port(); + my $host = $d[0]->target(); + my $port = $d[0]->port(); $detail->{caldavURL} = "http://$host"; $detail->{caldavURL} .= ":$port" unless $port eq 80; } @@ -429,8 +432,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - my $host = $reply->target(); - my $port = $reply->port(); + my $host = $d[0]->target(); + my $port = $d[0]->port(); $detail->{carddavURL} = "https://$host"; $detail->{carddavURL} .= ":$port" unless $port eq 443; } @@ -440,8 +443,8 @@ sub handle_signup { if ($reply) { my @d = $reply->answer; if (@d) { - my $host = $reply->target(); - my $port = $reply->port(); + my $host = $d[0]->target(); + my $port = $d[0]->port(); $detail->{carddavURL} = "http://$host"; $detail->{carddavURL} .= ":$port" unless $port eq 80; } From 53e58c7569171cc5302b5b8a9d597132c8c0c2e6 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 12:08:56 -0400 Subject: [PATCH 233/331] api fixer --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 60ad68b..a01d865 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -864,7 +864,7 @@ sub import_message { } $Self->begin(); - my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$id}[0], $uid); + my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$id}{ifolderid}, $uid); # save us having to download it again $Self->add_raw_message($msgid, $rfc822); From 19235f3690e89a2ec95b2c4342092a581ec5ec5d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 02:11:00 +1000 Subject: [PATCH 234/331] whitespace --- JMAP/DB.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 8ed5073..418cfab 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -628,14 +628,14 @@ sub _makemsg { # XXX - attachments $MIME = Email::MIME->create( header_str => [@$header, 'Content-Type' => 'multipart/mixed'], - parts => [$msgparts], + parts => [$msgparts], ); } else { # XXX - attachments $MIME = Email::MIME->create( header_str => [@$header, 'Content-Type' => 'multipart/mixed'], - parts => [$textpart], + parts => [$textpart], ); } } @@ -646,7 +646,7 @@ sub _makemsg { content_type => 'multipart/alternative', }, header_str => $header, - parts => [$textpart, $htmlpart], + parts => [$textpart, $htmlpart], ); } else { @@ -656,7 +656,7 @@ sub _makemsg { charset => 'UTF-8', }, header_str => $header, - body => $args->{textBody}, + body => $args->{textBody}, ); } } From bae40d43fa1ef0f4209711ad5ad5da3007f1062d Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 02:16:04 +1000 Subject: [PATCH 235/331] add comment --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index a01d865..e443514 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -208,7 +208,7 @@ sub sync_jmailboxes { my %seen; foreach my $folder (@$ifolders) { my $fname = $folder->[2]; - $fname =~ s/^INBOX\.//; + $fname =~ s/^INBOX\.//; # XXX - this should be removing $prefix # check for roles first my @bits = split "[$folder->[1]]", $fname; my $role = $ROLE_MAP{lc $folder->[3]}; From 271c78da123e9e5f7f83975d8aad6eb426217177 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 02:22:53 +1000 Subject: [PATCH 236/331] read back draft attachments --- JMAP/DB.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 418cfab..5f3ada4 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -380,6 +380,13 @@ sub attachments { my $messages = shift; my $num = 0; my @res; + + my $draftatt = $eml->header('X-JMAP-Draft-Attachments'); + if ($draftatt) { + my $attach = decode_json($draftatt); + push @res, @$attach; + } + foreach my $sub ($eml->subparts()) { $num++; my $type = $sub->content_type(); @@ -433,6 +440,7 @@ sub attachments { %extra, }; } + return @res; } From cacf200b3e8babbb52b7883385231449850232e0 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 12:37:18 -0400 Subject: [PATCH 237/331] encode attachments - correctly --- JMAP/DB.pm | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 5f3ada4..bf9a45a 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -24,7 +24,7 @@ use DateTime; use Date::Parse; use Net::CalDAVTalk; use Net::CardDAVTalk::VCard; -use MIME::Base64 qw(encode_base64); +use MIME::Base64 qw(encode_base64 decode_base64); my %TABLE2GROUPS = ( jmessages => ['Message', 'Thread'], @@ -383,8 +383,14 @@ sub attachments { my $draftatt = $eml->header('X-JMAP-Draft-Attachments'); if ($draftatt) { - my $attach = decode_json($draftatt); - push @res, @$attach; + eval { + my $json = decode_base64($draftatt); + my $attach = decode_json($json); + push @res, @$attach; + }; + if ($@) { + warn "FAILED TO PARSE $draftatt => $@"; + } } foreach my $sub ($eml->subparts()) { @@ -619,7 +625,7 @@ sub _makemsg { my @attachments = $args->{attachments} ? @{$args->{attachments}} : (); if (@attachments and not $isDraft) { - my $encoded = encode_base64(@attachments, ''); + my $encoded = encode_base64(encode_json(\@attachments), ''); push @$header, "X-JMAP-Draft-Attachments" => $encoded; @attachments = (); } From f181bb944fbb7591442fbf0cfe8ba80232a00f18 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 13:09:53 -0400 Subject: [PATCH 238/331] X-JMAP: means it's got a header --- JMAP/DB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index bf9a45a..2145689 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -527,6 +527,7 @@ sub hasatt { my $eml = shift; my $type = $eml->content_type() || 'text/plain'; return 1 if $type =~ m{(image|video|application)/}; + return 1 if $eml->header('X-JMAP-Draft-Attachments'); foreach my $sub ($eml->subparts()) { my $res = hasatt($sub); return $res if $res; From 6b4471ea11750d293fa9a4b8c260a2c695853528 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 03:06:08 +1000 Subject: [PATCH 239/331] no slash in parts --- JMAP/API.pm | 8 +++++--- JMAP/DB.pm | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 3589bef..af17803 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1030,9 +1030,11 @@ sub getRawMessage { my $msgid = $selector; my $part; my $filename; - if ($msgid =~ s{/([^/]+)/?(.*)}{}) { - $part = $1; - $filename = $2; + if ($msgid =~ s{/?(.*)}{}) { + $filename = $1; + } + if ($msgid =~ s{-(.*)}{}) { + $part = $1; } # skipping transactions here diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 2145689..2da38b7 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -438,8 +438,8 @@ sub attachments { push @res, { id => $id, type => $type, - url => "https://$ENV{jmaphost}/raw/$accountid/$messageid/$id/$filename", # XXX dep - blobId => "$messageid/$id", + url => "https://$ENV{jmaphost}/raw/$accountid/$messageid-$id/$filename", # XXX dep + blobId => "$messageid-$id", name => $filename, size => length($body), isInline => $isInline, From 5a7d494f497d8a3bc88021bcba246ac6450c5ce7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 04:33:10 +1000 Subject: [PATCH 240/331] simplify get_events API call --- JMAP/ImapDB.pm | 2 +- JMAP/Sync/Common.pm | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index e443514..6002ca9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -389,7 +389,7 @@ sub do_calendar { my $dbh = $Self->dbh(); my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); - my $events = $Self->backend_cmd('get_events', {href => $href}); + my $events = $Self->backend_cmd('get_events', $href); $Self->begin(); my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 9b2e22c..0c74e61 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -60,12 +60,12 @@ sub get_calendars { sub get_events { my $Self = shift; - my $Args = shift; + my $href = shift; my $talk = $Self->connect_calendars(); return unless $talk; - $Args->{href} =~ s{/$}{}; - my $data = $talk->GetEvents($Args->{href}, Full => 1); + $href =~ s{/$}{}; + my $data = $talk->GetEvents($href, Full => 1); my %res; foreach my $item (@$data) { From 253c6cce9771244757d7e5a3ef495d0dfc27d312 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 04:48:48 +1000 Subject: [PATCH 241/331] naming changes --- JMAP/ImapDB.pm | 2 +- JMAP/Sync/Common.pm | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 6002ca9..c6bc9c7 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -527,7 +527,7 @@ sub do_addressbook { my $dbh = $Self->dbh(); my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); - my $cards = $Self->backend_cmd('get_cards', {href => $href}); + my $cards = $Self->backend_cmd('get_cards', $href); $Self->begin(); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 0c74e61..45f0849 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -60,12 +60,12 @@ sub get_calendars { sub get_events { my $Self = shift; - my $href = shift; + my $collection = shift; my $talk = $Self->connect_calendars(); return unless $talk; - $href =~ s{/$}{}; - my $data = $talk->GetEvents($href, Full => 1); + $collection =~ s{/$}{}; + my $data = $talk->GetEvents($collection, Full => 1); my %res; foreach my $item (@$data) { @@ -77,14 +77,14 @@ sub get_events { sub new_event { my $Self = shift; - my $href = shift; + my $collection = shift; # is collection of the calendar my $event = shift; - $href =~ s{/$}{}; + $collection =~ s{/$}{}; my $talk = $Self->connect_calendars(); return unless $talk; - $talk->NewEvent($href, $event); + $talk->NewEvent($collection, $event); } sub update_event { @@ -105,7 +105,7 @@ sub delete_event { my $talk = $Self->connect_calendars(); return unless $talk; - $talk->DeleteEvent({href => $resource}); + $talk->DeleteEvent($resource); # XXX - we pass more properties for no good reason to this API } sub get_addressbooks { @@ -120,12 +120,12 @@ sub get_addressbooks { sub get_cards { my $Self = shift; - my $Args = shift; + my $collection = shift; my $talk = $Self->connect_contacts(); return unless $talk; - $Args->{href} =~ s{/$}{}; - my $data = $talk->GetContacts($Args->{href}); + $collection =~ s{/$}{}; + my $data = $talk->GetContacts($collection); my %res; foreach my $item (@$data) { @@ -137,14 +137,14 @@ sub get_cards { sub new_card { my $Self = shift; - my $href = shift; + my $collection = shift; my $card = shift; - $href =~ s{/$}{}; + $collection =~ s{/$}{}; my $talk = $Self->connect_contacts(); return unless $talk; - $talk->NewContact($href, $card); + $talk->NewContact($collection, $card); } sub update_card { @@ -445,12 +445,13 @@ sub imap_append { my $r = $imap->append($imapname, $flags, $internaldate, {'Literal' => $rfc822}); die "APPEND FAILED $r" unless (lc($r) eq 'ok' or lc($r) eq 'appenduid'); # what's with that?? - my $uid = $imap->get_response_code('appenduid'); # returns [uidvalidity uid] + my $response = $imap->get_response_code('appenduid'); + my ($uidvalidity, $uid) = @$response; # XXX - fetch the x-gm-msgid or envelope from the server so we know the # the ID that the server gave this message - return ['append', $imapname, @$uid]; + return ['append', $imapname, $uidvalidity, $uid]; } sub imap_search { From 9d927c17c39b73993212e57f299b589a7f9f204f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 04:53:53 +1000 Subject: [PATCH 242/331] remove some noise --- JMAP/Sync/Gmail.pm | 2 -- JMAP/Sync/Standard.pm | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 990395a..2d1749c 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -16,8 +16,6 @@ use Net::GmailContacts; use OAuth2::Tiny; use IO::All; -my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect); - my $O; sub O { unless ($O) { diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 890abf3..eebf3b8 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -14,8 +14,6 @@ use Email::Sender::Transport::SMTPS; use Net::CalDAVTalk; use Net::CardDAVTalk; -my %KNOWN_SPECIALS = map { lc $_ => 1 } qw(\\HasChildren \\HasNoChildren \\NoSelect \\NoInferiors); - sub connect_calendars { my $Self = shift; @@ -69,13 +67,12 @@ sub connect_imap { delete $Self->{imap}; for (1..3) { - my $usessl = $Self->{auth}{imapSSL} - 1; + my $usessl = $Self->{auth}{imapSSL} - 1; # IDs for Mail::IMAPTalk are one lower than our internal format $Self->{imap} = Mail::IMAPTalk->new( Server => $Self->{auth}{imapHost}, Port => $Self->{auth}{imapPort}, Username => $Self->{auth}{username}, Password => $Self->{auth}{password}, - # not configurable right now... UseSSL => $usessl, UseBlocking => $usessl, ); From c7354bfc9751976c1e9303e45a7ceb104c03242e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 04:57:38 +1000 Subject: [PATCH 243/331] whitespace --- JMAP/GmailDB.pm | 2 +- JMAP/ImapDB.pm | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index b920192..2f7a64d 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -34,7 +34,7 @@ sub access_token { my $O = JMAP::Sync::Gmail::O(); my $data = $O->refresh($password); - + return [$hostname, $username, $data->{access_token}]; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c6bc9c7..7d6a470 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -92,8 +92,6 @@ sub access_token { return [$hostname, $username, $password]; } - - sub access_data { my $Self = shift; From 54290df6dabcb1098d0817a86f1f9bafb8b07c1f Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 16:06:37 -0400 Subject: [PATCH 244/331] strip [Gmail] from the jmap name --- JMAP/ImapDB.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7d6a470..02c7570 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -206,9 +206,10 @@ sub sync_jmailboxes { my %seen; foreach my $folder (@$ifolders) { my $fname = $folder->[2]; - $fname =~ s/^INBOX\.//; # XXX - this should be removing $prefix # check for roles first my @bits = split "[$folder->[1]]", $fname; + shift @bits if $bits[0] eq 'INBOX'; + shift @bits if $bits[0] eq '[Gmail]'; my $role = $ROLE_MAP{lc $folder->[3]}; my $id = 0; my $parentId = 0; From 6e826c0d895d65b33067868a82d7b235c70afa68 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 16:55:40 -0400 Subject: [PATCH 245/331] logic fixes --- JMAP/ImapDB.pm | 25 ++++++++++++------------- JMAP/Sync/Common.pm | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 02c7570..a7bec06 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -208,7 +208,7 @@ sub sync_jmailboxes { my $fname = $folder->[2]; # check for roles first my @bits = split "[$folder->[1]]", $fname; - shift @bits if $bits[0] eq 'INBOX'; + shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); # otehrwise we get none... shift @bits if $bits[0] eq '[Gmail]'; my $role = $ROLE_MAP{lc $folder->[3]}; my $id = 0; @@ -563,7 +563,7 @@ sub labels { my $Self = shift; unless ($Self->{labels}) { my $data = $Self->dbh->selectall_arrayref("SELECT label, ifolderid, jmailboxid, imapname FROM ifolders"); - $Self->{labels} = { map { lc $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; + $Self->{labels} = { map { $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; } return $Self->{labels}; } @@ -613,16 +613,15 @@ sub firstsync { $Self->sync_folders(); - my $labels = $Self->labels(); + my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); if ($Self->{is_gmail}) { - my $ifolderid = $labels->{"\\allmail"}[0]; - $Self->do_folder($ifolderid, undef, 50); + my ($folder) = grep { lc $_->{label} eq '\\allmail' } @$data; + $Self->do_folder($folder->{ifolderid}, undef, 50) if $folder; } else { - my $data = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname FROM ifolders"); - my ($folder) = grep { lc $_->[1] eq 'inbox' } @$data; - $Self->do_folder($folder->[0], "inbox", 50); + my ($folder) = grep { lc $_->{imapname} eq 'inbox' } @$data; + $Self->do_folder($folder->{ifolderid}, $folder->{label}, 50) if $folder; } } @@ -684,12 +683,12 @@ sub do_folder { my $end = $uidfirst - 1; $uidfirst -= $batchsize; $uidfirst = 1 if $uidfirst < 1; - $fetches{backfill} = [$uidfirst, $end, \@immutable]; + $fetches{backfill} = [$uidfirst, $end, [@immutable, @mutable]]; } } else { - $fetches{new} = [$uidnext, '*', \@immutable]; - $fetches{update} = [$uidfirst, $uidnext - 1, \@mutable, $highestmodseq]; + $fetches{new} = [$uidnext, '*', [@immutable, @mutable]]; + $fetches{update} = [$uidfirst, $uidnext - 1, [@mutable], $highestmodseq]; } $Self->commit(); @@ -893,7 +892,7 @@ sub update_messages { my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; - my %jmailmap = map { $_->{jmailboxid} => $_ } @$folderdata; + my %jmailmap = map { $_->{jmailboxid} => $_ } grep { $_->{jmailboxid} } @$folderdata; my $jmapdata = $dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; my %jrolemap = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$jmapdata; @@ -1071,7 +1070,7 @@ sub apply_data { } my $labels = $Self->labels(); - my @jmailboxids = grep { $_ } map { $labels->{lc $_}[1] } @$labellist; + my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @$labellist; my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 45f0849..ab01dde 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -186,7 +186,7 @@ sub folders { my $name = $folder->[2]; my $label; if ($role) { - $label = lc $role; + $label = $role; } else { $label = $folder->[2]; From 5f0195e362666d54d106c83187e15d844f436084 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 09:07:23 +1000 Subject: [PATCH 246/331] synthetic archive and outbox become slightly more real --- JMAP/API.pm | 16 --------------- JMAP/ImapDB.pm | 56 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index af17803..8419651 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -169,22 +169,6 @@ sub getMailboxes { my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE active = 1", {Slice => {}}); - # outbox - magic - push @$data, { - jmailboxid => 'outbox', - parentId => 0, - name => 'Outbox', - role => 'outbox', - sortOrder => 1, - mustBeOnlyMailbox => 1, - mayReadItems => 1, - mayAddItems => 1, - mayRemoveItems => 1, - mayCreateChild => 0, - mayRename => 0, - mayDelete => 0, - }; - my %ids; if ($args->{ids}) { %ids = map { $Self->idmap($_) => 1 } @{$args->{ids}}; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index a7bec06..3c9ebfa 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -271,12 +271,59 @@ sub sync_jmailboxes { $Self->dmaybeupdate('ifolders', {jmailboxid => $id}, {ifolderid => $folder->[0]}); } + if ($roletoid{'outbox'}) { + $seen{$roletoid{'outbox'}} = 1; + } + else { + # outbox - magic + my $outbox = { + parentId => 0, + name => 'Outbox', + role => 'outbox', + sortOrder => 2, + mustBeOnlyMailbox => 1, + mayReadItems => 1, + mayAddItems => 1, + mayRemoveItems => 1, + mayCreateChild => 0, + mayRename => 0, + mayDelete => 0, + }; + my $id = $Self->dmake('mailboxes', $outbox); + $seen{$id} = 1; + $roletoid{'outbox'} = $id; + } + + if ($roletoid{'archive'}) { + $seen{$roletoid{'archive'}} = 1; + } + else { + # archive - magic + my $archive = { + parentId => 0, + name => 'Archive', + role => 'archive', + sortOrder => 2, + mustBeOnlyMailbox => not $Self->{db}->{is_gmail}, + mayReadItems => 1, + mayAddItems => 1, + mayRemoveItems => 1, + mayCreateChild => 0, + mayRename => 0, + mayDelete => 0, + }; + my $id = $Self->dmake('mailboxes', $archive); + $seen{$id} = 1; + $roletoid{'archive'} = $id; + } + foreach my $mailbox (@$jmailboxes) { my $id = $mailbox->[0]; next unless $mailbox->[4]; next if $seen{$id}; $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); } + $Self->commit(); } @@ -957,7 +1004,9 @@ sub update_messages { } done: if ($Self->{is_gmail}) { - my @labels = grep { lc $_ ne '\\allmail' } map { $jmailmap{$_}{label} } @others; + # because 'archive' is synthetic on gmail it will appear as a non-present record here, + # and be stripped - so we store back labels == [] + my @labels = grep { $_ and lc $_ ne '\\allmail' } map { $jmailmap{$_}{label} } @others; $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); } else { @@ -1070,7 +1119,10 @@ sub apply_data { } my $labels = $Self->labels(); - my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @$labellist; + my @list = @$labellist; + # gmail empty list means archive at our end + @list = ('archive') if ($Self->{is_gmail} and not @list); + my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @list; my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); From 0bd4ae9cf5548c292c2a1f400370ce1c3dddb18e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 09:10:14 +1000 Subject: [PATCH 247/331] fix jidmap for outbox/archive --- JMAP/ImapDB.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 3c9ebfa..8e7eba9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -978,8 +978,8 @@ sub update_messages { if (exists $action->{mailboxIds}) { # jmailboxid my @mboxes = map { $idmap->($_) } @{$action->{mailboxIds}}; - my ($has_outbox) = grep { $_ eq 'outbox' } @mboxes; - my (@others) = grep { $_ ne 'outbox' } @mboxes; + my ($has_outbox) = grep { $jidmap{$_} eq 'outbox' } @mboxes; + my (@others) = grep { $jidmap{$_} ne 'outbox' } @mboxes; if ($has_outbox) { # move to sent when we're done push @others, $jmailmap{$jrolemap{'sent'}}{jmailboxid}; @@ -1004,8 +1004,8 @@ sub update_messages { } done: if ($Self->{is_gmail}) { - # because 'archive' is synthetic on gmail it will appear as a non-present record here, - # and be stripped - so we store back labels == [] + # because 'archive' is synthetic on gmail we strip it here + (@others) = grep { $jidmap{$_} ne 'archive' } @others; my @labels = grep { $_ and lc $_ ne '\\allmail' } map { $jmailmap{$_}{label} } @others; $Self->backend_cmd('imap_labels', $imapname, $uidvalidity, $uid, \@labels); } From c84ceb3e654cd9075f8cf7038c8fbff47fdf2563 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Fri, 24 Jul 2015 09:28:12 +1000 Subject: [PATCH 248/331] fix archive naming --- JMAP/ImapDB.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 8e7eba9..5077bdb 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1121,7 +1121,9 @@ sub apply_data { my $labels = $Self->labels(); my @list = @$labellist; # gmail empty list means archive at our end - @list = ('archive') if ($Self->{is_gmail} and not @list); + if ($Self->{is_gmail} and not @list) { + @list = $Self->{dbh}->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = 'archive'"); + } my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @list; my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); From edca329ec74b0088bc9669704de9dc2d8bc12a0f Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 19:11:17 -0400 Subject: [PATCH 249/331] suppress debug --- JMAP/DB.pm | 2 +- bin/apiendpoint.pl | 2 +- bin/server.pl | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 2da38b7..47f8fe2 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -68,7 +68,7 @@ sub log { } else { my ($level, @items) = @_; - #return if $level eq 'debug'; + return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 785bd29..d79b236 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -3,7 +3,7 @@ use lib '/home/jmap/jmap-perl'; package JMAP::Backend; -use Mail::IMAPTalk qw(:trace); +#use Mail::IMAPTalk qw(:trace); # stuff complains otherwise - twice for luck use IO::Socket::SSL; diff --git a/bin/server.pl b/bin/server.pl index 45e3abf..ebfd970 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -616,6 +616,7 @@ sub HandleEventSource { sub prod_backfill { my $accountid = shift; my $force = shift; + return; return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; From 24c5d72e92f483eec855df6db4a65d6a1e4b7dc0 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 19:29:06 -0400 Subject: [PATCH 250/331] debugging and stuff --- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 6 +++--- bin/apiendpoint.pl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 47f8fe2..2da38b7 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -68,7 +68,7 @@ sub log { } else { my ($level, @items) = @_; - return if $level eq 'debug'; + #return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 5077bdb..142d95b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -289,7 +289,7 @@ sub sync_jmailboxes { mayRename => 0, mayDelete => 0, }; - my $id = $Self->dmake('mailboxes', $outbox); + my $id = $Self->dmake('jmailboxes', $outbox); $seen{$id} = 1; $roletoid{'outbox'} = $id; } @@ -304,7 +304,7 @@ sub sync_jmailboxes { name => 'Archive', role => 'archive', sortOrder => 2, - mustBeOnlyMailbox => not $Self->{db}->{is_gmail}, + mustBeOnlyMailbox => 1, mayReadItems => 1, mayAddItems => 1, mayRemoveItems => 1, @@ -312,7 +312,7 @@ sub sync_jmailboxes { mayRename => 0, mayDelete => 0, }; - my $id = $Self->dmake('mailboxes', $archive); + my $id = $Self->dmake('jmailboxes', $archive); $seen{$id} = 1; $roletoid{'archive'} = $id; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index d79b236..785bd29 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -3,7 +3,7 @@ use lib '/home/jmap/jmap-perl'; package JMAP::Backend; -#use Mail::IMAPTalk qw(:trace); +use Mail::IMAPTalk qw(:trace); # stuff complains otherwise - twice for luck use IO::Socket::SSL; From 230614686f23909117f07571c3fd8992e10643f0 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 19:46:19 -0400 Subject: [PATCH 251/331] fix the ordering of archive folder adding --- JMAP/ImapDB.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 142d95b..e7cefe1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1121,10 +1121,12 @@ sub apply_data { my $labels = $Self->labels(); my @list = @$labellist; # gmail empty list means archive at our end + my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @list; + + # check for archive folder for gmail if ($Self->{is_gmail} and not @list) { - @list = $Self->{dbh}->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = 'archive'"); + @jmailboxids = $Self->{dbh}->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = 'archive'"); } - my @jmailboxids = grep { $_ } map { $labels->{$_}[1] } @list; my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); From ca2bc1f50c40716fb48536fabf5a56f588b16cb5 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 20:40:29 -0400 Subject: [PATCH 252/331] re-run firstsync on main page view --- bin/apiendpoint.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 785bd29..bdaeb7b 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -252,6 +252,7 @@ sub handle_sync { sub handle_syncall { my $db = shift; + $db->firstsync(); $db->sync_folders(); $db->sync_imap(); $db->sync_addressbooks(); From 221709f68130700c4e40a6853a12883d7ceae2bf Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 20:41:25 -0400 Subject: [PATCH 253/331] backfill again --- bin/server.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/server.pl b/bin/server.pl index ebfd970..45e3abf 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -616,7 +616,6 @@ sub HandleEventSource { sub prod_backfill { my $accountid = shift; my $force = shift; - return; return if (not $force and $idler{$accountid}{backfilling}); $idler{$accountid}{backfilling} = 1; From b22ac1b81817e5bb040f910051a5542cb754d4d4 Mon Sep 17 00:00:00 2001 From: jmap Date: Thu, 23 Jul 2015 20:42:48 -0400 Subject: [PATCH 254/331] cleanaccounts: simple tool so we don't lose auth --- bin/cleanaccounts.pl | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 bin/cleanaccounts.pl diff --git a/bin/cleanaccounts.pl b/bin/cleanaccounts.pl new file mode 100644 index 0000000..0754cb7 --- /dev/null +++ b/bin/cleanaccounts.pl @@ -0,0 +1,33 @@ +#!/usr/bin/perl -w + +use DBI; + +sub accountsdb { + my $dbh = DBI->connect("dbi:SQLite:dbname=/home/jmap/data/accounts.sqlite3"); + $dbh->do(<selectcol_arrayref("SELECT accountid FROM accounts"); + +foreach my $id (@$ids) { + my $file = "/home/jmap/data/$id.sqlite3"; + next unless -f $file; + my $adbh = DBI->connect("dbi:SQLite:dbname=$file"); + my $tables = $adbh->selectcol_arrayref("SELECT name FROM sqlite_master"); + foreach my $table (@$tables) { + next if $table eq 'account'; + next if $table eq 'iserver'; + next if $table =~ m/^sqlite_/; + $adbh->do("DROP TABLE $table"); + } +} + From 6e3595330001088d286adf6376d7223264e953da Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 02:38:26 +1000 Subject: [PATCH 255/331] parse calendars/contacts BEFORE locking DB --- JMAP/ImapDB.pm | 95 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index e7cefe1..2cd2784 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -436,33 +436,39 @@ sub do_calendar { my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); my $events = $Self->backend_cmd('get_events', $href); + # parse events before we lock + my %parsed = map { $Self->parse_event($events->{$_}) } keys %$events; $Self->begin(); - my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, content FROM ievents WHERE icalendarid = ?", {}, $calendarid); - my %res = map { $_->[1] => $_ } @$exists; + my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $calendarid); + my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %$events) { my $data = delete $res{$resource}; my $raw = $events->{$resource}; - my $event = $Self->parse_event($raw); + my $event = $parsed{$resource}; my $uid = $event->{uid}; + my $item = { + icalendarid => $calendarid, + uid => $uid, + resource => $resource, + content => $raw, + }; if ($data) { - my $id = $data->[0]; - next if $raw eq $data->[2]; - $Self->dmaybeupdate('ievents', {icalendarid => $calendarid, uid => $uid, content => $raw, resource => $resource}, {ieventid => $id}); + my $id = $data->{ieventid}; + $Self->dmaybeupdate('ievents', $item, {ieventid => $id}); } else { - $Self->dinsert('ievents', {icalendarid => $calendarid, uid => $uid, content => $raw, resource => $resource}); + $Self->dinsert('ievents', $item); } $Self->set_event($jcalendarid, $event); } foreach my $resource (keys %res) { my $data = delete $res{$resource}; - my $id = $data->[0]; + my $id = $data->{ieventid}; $Self->ddelete('ievents', {ieventid => $id}); - my $event = $Self->parse_event($data->[2]); - $Self->delete_event($jcalendarid, $event->{uid}); + $Self->delete_event($jcalendarid, $data->{uid}); } $Self->commit(); @@ -479,13 +485,13 @@ sub sync_addressbooks { $Self->begin(); my $dbh = $Self->dbh(); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, name, isReadOnly, syncToken FROM iaddressbooks"); - my %byhref = map { $_->[1] => $_ } @$iaddressbooks; + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, syncToken FROM iaddressbooks", {Slice => {}}); + my %byhref = map { $_->{href} => $_ } @$iaddressbooks; my %seen; my @todo; foreach my $addressbook (@$addressbooks) { - my $id = $byhref{$addressbook->{href}}[0]; + my $id = $byhref{$addressbook->{href}}{iaddressbookid}; my $data = { isReadOnly => $addressbook->{isReadOnly}, href => $addressbook->{href}, @@ -493,7 +499,7 @@ sub sync_addressbooks { syncToken => $addressbook->{syncToken}, }; if ($id) { - my $token = $byhref{$addressbook->{href}}[4]; + my $token = $byhref{$addressbook->{href}}{syncToken}; if ($token ne $addressbook->{syncToken}) { push @todo, $id; $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); @@ -507,7 +513,7 @@ sub sync_addressbooks { } foreach my $addressbook (@$iaddressbooks) { - my $id = $addressbook->[0]; + my $id = $addressbook->{iaddressbookid}; next if $seen{$id}; $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); } @@ -525,20 +531,23 @@ sub sync_addressbooks { # call in transaction sub sync_jaddressbooks { my $Self = shift; + $Self->begin(); my $dbh = $Self->dbh(); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks"); - my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks"); + + my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks", {Slice => {}}); + my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks", {Slice => {}}); my %jbyid; foreach my $addressbook (@$jaddressbooks) { - $jbyid{$addressbook->[0]} = $addressbook; + $jbyid{$addressbook->{jaddressbookid}} = $addressbook; } my %seen; foreach my $addressbook (@$iaddressbooks) { + my $aid = $addressbook->{iaddressbooks}; my $data = { - name => $addressbook->[1], + name => $addressbook->{name}, isVisible => 1, mayReadItems => 1, mayAddItems => 1, @@ -547,21 +556,22 @@ sub sync_jaddressbooks { mayDelete => 0, mayRename => 0, }; - if ($addressbook->[2] && $jbyid{$addressbook->[2]}) { - $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $addressbook->[2]}); - $seen{$addressbook->[2]} = 1; + my $jid = $addressbook->{jaddressbookid}; + if ($jid && $jbyid{$jid}) { + $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $jid}); + $seen{$jid} = 1; } else { - my $id = $Self->dmake('jaddressbooks', $data); - $Self->dupdate('iaddressbooks', {jaddressbookid => $id}, {iaddressbookid => $addressbook->[0]}); - $seen{$id} = 1; + $jid = $Self->dmake('jaddressbooks', $data); + $Self->dupdate('iaddressbooks', {jaddressbookid => $jid}, {iaddressbookid => $aid}); + $seen{$jid} = 1; } } foreach my $addressbook (@$jaddressbooks) { - my $id = $addressbook->[0]; - next if $seen{$id}; - $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $id}); + my $jid = $addressbook->{jaddressbookid}; + next if $seen{$jid}; + $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $jid}); } $Self->commit(); } @@ -574,35 +584,43 @@ sub do_addressbook { my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); my $cards = $Self->backend_cmd('get_cards', $href); + # parse before locking + my %parsed = map { $Self->parse_card($cards->{$_}) } keys %$cards; $Self->begin(); - my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, content FROM icards WHERE iaddressbookid = ?", {}, $addressbookid); - my %res = map { $_->[1] => $_ } @$exists; + my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $addressbookid); + my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; my $raw = $cards->{$resource}; - my $card = $Self->parse_card($raw); my $uid = $card->{uid}; + my $kind = $card->{kind}; + my $item = { + iaddressbookid => $addressbookid, + resource => $resource, + uid => $uid, + kind => $kind, + content => $raw, + }; if ($data) { - my $id = $data->[0]; - next if $raw eq $data->[2]; - $Self->dmaybeupdate('icards', {iaddressbookid => $addressbookid, uid => $uid, content => $raw, resource => $resource}, {icardid => $id}); + my $id = $data->{icardid}; + $Self->dmaybeupdate('icards', $item, {icardid => $id}); } else { - $Self->dinsert('icards', {iaddressbookid => $addressbookid, uid => $uid, content => $raw, resource => $resource}); + $Self->dinsert('icards', $item); } $Self->set_card($jaddressbookid, $card); } foreach my $resource (keys %res) { my $data = delete $res{$resource}; - my $id = $data->[0]; + my $id = $data->{icardid}; $Self->ddelete('icards', {icardid => $id}); - my $card = $Self->parse_card($data->[2]); - $Self->delete_card($jaddressbookid, $card->{uid}, $card->{kind}); + $Self->delete_card($jaddressbookid, $data->{uid}, $data->{kind}); } + $Self->commit(); } @@ -1685,6 +1703,7 @@ CREATE TABLE IF NOT EXISTS icards ( iaddressbookid INTEGER, resource TEXT, uid TEXT, + kind TEXT, content TEXT, mtime DATE NOT NULL ); From cbf5b1431ceba0dea504cbf473962dbc91ea970a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 03:25:54 +1000 Subject: [PATCH 256/331] don't parse email under lock --- JMAP/DB.pm | 21 --------------------- JMAP/ImapDB.pm | 33 ++++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 2da38b7..14af19b 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -264,27 +264,6 @@ sub get_raw_message { return find_part($eml, $part); } -sub add_raw_message { - my $Self = shift; - my $msgid = shift; - my $rfc822 = shift; - - my $eml = Email::MIME->new($rfc822); - my $message = $Self->parse_message($msgid, $eml); - my $parsed = encode_json($message); - - # fiddle the top-level fields - my $data = { - msgid => $msgid, - rfc822 => $rfc822, - parsed => $parsed, - }; - - my $jid = $Self->dinsert('jrawmessage', $data); - - return decode_json($parsed); -} - sub parse_date { my $Self = shift; my $date = shift; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2cd2784..49095aa 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -926,11 +926,18 @@ sub import_message { $Self->do_folder($fdata->{ifolderid}, $fdata->{label}); } - $Self->begin(); my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$id}{ifolderid}, $uid); # save us having to download it again - $Self->add_raw_message($msgid, $rfc822); + my $eml = Email::MIME->new($rfc822); + my $message = $Self->parse_message($msgid, $eml); + + $Self->begin(); + $Self->dinsert('jrawmessage', { + msgid => $msgid, + rfc822 => $rfc822, + parsed => encode_json($message), + }); $Self->commit(); return ($msgid, $thrid); @@ -1212,20 +1219,32 @@ sub fill_messages { next unless $imapname; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); + my %parsed; + foreach my $uid (keys %{$res->{data}}) { + my $rfc822 = $res->{data}{$uid}; + next unless $rfc822; + my $msgid = $uhash->{$uid}; + next if $result{$msgid}; + my $eml = Email::MIME->new($rfc822); + $parsed{$uid} = $Self->parse_message($msgid, $eml); + } $Self->begin(); - - foreach my $uid (sort { $a <=> $b } keys %{$res->{data}}) { + foreach my $uid (sort { $a <=> $b } keys %parsed) { my $msgid = $uhash->{$uid}; - next if $result{$msgid}; my $rfc822 = $res->{data}{$uid}; next unless $rfc822; - $result{$msgid} = $Self->add_raw_message($msgid, $rfc822); + $Self->dinsert('jrawmessage', { + msgid => $msgid, + rfc822 => $rfc822, + parsed => encode_json($parsed{$uid}), + }); + $result{$msgid} = $parsed{$uid} } - $Self->commit(); } + # XXX - handle not getting data that we need? my @stillneed = grep { not $result{$_} } @ids; return \%result; From 10ff9f911122f2d9b5d37f694be0a22ab8b6a460 Mon Sep 17 00:00:00 2001 From: jmap Date: Fri, 24 Jul 2015 13:43:02 -0400 Subject: [PATCH 257/331] fix parsing --- JMAP/ImapDB.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 49095aa..8c7a1e1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -437,7 +437,7 @@ sub do_calendar { my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); my $events = $Self->backend_cmd('get_events', $href); # parse events before we lock - my %parsed = map { $Self->parse_event($events->{$_}) } keys %$events; + my %parsed = map { $_ => $Self->parse_event($events->{$_}) } keys %$events; $Self->begin(); my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $calendarid); @@ -585,7 +585,7 @@ sub do_addressbook { my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); my $cards = $Self->backend_cmd('get_cards', $href); # parse before locking - my %parsed = map { $Self->parse_card($cards->{$_}) } keys %$cards; + my %parsed = map { $_ => $Self->parse_card($cards->{$_}) } keys %$cards; $Self->begin(); @@ -595,6 +595,7 @@ sub do_addressbook { foreach my $resource (keys %$cards) { my $data = delete $res{$resource}; my $raw = $cards->{$resource}; + my $card = $parsed{$resource}; my $uid = $card->{uid}; my $kind = $card->{kind}; my $item = { From 920a28ec1c135d82cd9c474bd777422c8088bfd7 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 03:51:18 +1000 Subject: [PATCH 258/331] remove backfilling logic and add a 'jcreated' to everything --- JMAP/DB.pm | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 14af19b..48664df 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -238,7 +238,7 @@ sub add_message { return unless @$mailboxes; # no mailboxes, no message - $Self->dmake('jmessages', $data, $Self->{backfilling}); + $Self->dmake('jmessages', $data); foreach my $mailbox (@$mailboxes) { $Self->add_message_to_mailbox($data->{msgid}, $mailbox); } @@ -895,8 +895,10 @@ sub dinsert { # dinsert with a modseq sub dmake { my $Self = shift; - my ($table, $values, $backfilling) = @_; - $values->{jmodseq} = $backfilling ? 1 : $Self->dirty($table); + my ($table, $values) = @_; + my $modseq = $Self->dirty($table); + $values->{jcreated} = $modseq; + $values->{jmodseq} = $modseq; $values->{active} = 1; return $Self->dinsert($table, $values); } @@ -1007,6 +1009,7 @@ CREATE TABLE IF NOT EXISTS jmessages ( msgmessageid TEXT, msgdate INTEGER, msgsize INTEGER, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN @@ -1030,6 +1033,7 @@ CREATE TABLE IF NOT EXISTS jmailboxes ( mayCreateChild BOOLEAN, mayRename BOOLEAN, mayDelete BOOLEAN, + jcreated INTEGER, jmodseq INTEGER, jcountsmodseq INTEGER, mtime DATE, @@ -1041,6 +1045,7 @@ EOF CREATE TABLE IF NOT EXISTS jmessagemap ( jmailboxid INTEGER, msgid TEXT, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN, @@ -1102,6 +1107,7 @@ CREATE TABLE IF NOT EXISTS jcalendars ( mayRemoveItems BOOLEAN, mayDelete BOOLEAN, mayRename BOOLEAN, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN @@ -1115,6 +1121,7 @@ CREATE TABLE IF NOT EXISTS jevents ( firststart DATE, lastend DATE, payload TEXT, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN @@ -1132,6 +1139,7 @@ CREATE TABLE IF NOT EXISTS jaddressbooks ( mayRemoveItems BOOLEAN, mayDelete BOOLEAN, mayRename BOOLEAN, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN @@ -1143,6 +1151,7 @@ CREATE TABLE IF NOT EXISTS jcontactgroups ( groupuid TEXT PRIMARY KEY, jaddressbookid INTEGER, name TEXT, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN @@ -1166,6 +1175,7 @@ CREATE TABLE IF NOT EXISTS jcontacts ( jaddressbookid INTEGER, isFlagged BOOLEAN, payload TEXT, + jcreated INTEGER, jmodseq INTEGER, mtime DATE, active BOOLEAN From 844c497f304aff914ea75db0c191e0e2ff663f89 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 06:41:13 +1000 Subject: [PATCH 259/331] wip on jcreated and remove dead code --- JMAP/API.pm | 2 ++ bin/server.pl | 82 --------------------------------------------------- 2 files changed, 2 insertions(+), 82 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 8419651..14a242a 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -710,6 +710,7 @@ sub getMessageListUpdates { # jmodseq greater than sinceState is a change my $changed = ($item->{jmodseq} > $args->{sinceState}); + my $isnew = ($item->{jcreated} > $args->{sinceState}); if ($changed) { # if it's in AND it's the exemplar, it's been added @@ -758,6 +759,7 @@ sub getMessageListUpdates { # jmodseq greater than sinceState is a change my $changed = ($item->{jmodseq} > $args->{sinceState}); + my $isnew = ($item->{jcreated} > $args->{sinceState}); if ($changed) { if ($isin) { diff --git a/bin/server.pl b/bin/server.pl index 45e3abf..bef8031 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -145,9 +145,6 @@ sub read_idle_line { $httpd->reg_cb ( '/jmap' => \&do_jmap, - '/A' => \&do_A, - '/J' => \&do_J, - '/U' => \&do_U, '/upload' => \&do_upload, '/raw' => \&do_raw, '/register' => \&do_register, @@ -320,85 +317,6 @@ sub do_raw { }, mkerr($req)); } -sub _getaccountid { - my $req = shift; - my $header = $req->headers->{"authorization"}; - $header =~ s/^Token //; - return $header; -} - -sub do_A { - my ($httpd, $req) = @_; - - return not_found($req) unless $req->method eq 'post'; - - my $content = $req->content(); - return invalid_request($req) unless $content; - my $request = eval { $json->decode($content) }; - return invalid_request($req) unless ($request and ref($request) eq 'HASH'); - - # more validation? - - $httpd->stop_request(); - - send_backend_request("A", 'authenticate', $request, sub { - my $res = shift; - my $html = encode_utf8($json->encode($res)); - $req->respond ({ content => ['application/json', $html] }); - return 1; - }, mkerr($req)); -} - -sub do_J { - my ($httpd, $req) = @_; - - return not_found($req) unless $req->method eq 'post'; - - my $accountid = _getaccountid($req); - return need_auth($req) unless $accountid; - - prod_idler($accountid); - - my $content = $req->content(); - return invalid_request($req) unless $content; - my $request = eval { $json->decode($content) }; - return invalid_request($req) unless ($request and ref($request) eq 'ARRAY'); - - $httpd->stop_request(); - - send_backend_request($accountid, 'jmap', $request, sub { - my $res = shift; - my $html = encode_utf8($json->encode($res)); - $req->respond ({ content => ['application/json', $html] }); - return 1; - }, mkerr($req)); -} - -sub do_U { - my ($httpd, $req) = @_; - - return not_found($req) unless $req->method eq 'post'; - - my $accountid = _getaccountid($req); - return need_auth($req) unless $accountid; - - prod_idler($accountid); - - my $content = $req->content(); - return invalid_request($req) unless $content; - - my $type = $req->headers->{"content-type"}; - - $httpd->stop_request(); - - send_backend_request($accountid, 'upload', [$type, $content], sub { - my $res = shift; - my $html = encode_utf8($json->encode($res)); - $req->respond ({ content => ['application/json', $html] }); - return 1; - }, mkerr($req)); -} - sub do_upload { my ($httpd, $req) = @_; From 39cda172ebba026fe9193b11d7d6dc37b11d4d04 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 06:47:13 +1000 Subject: [PATCH 260/331] apiendpoint: more robust against issues --- bin/apiendpoint.pl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index bdaeb7b..3e694ae 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -62,7 +62,7 @@ sub getdb { die "no accountid" unless $accountid; $dbh ||= accountsdb(); my ($email, $type) = $dbh->selectrow_array("SELECT email,type FROM accounts WHERE accountid = ?", {}, $accountid); - die "no type for $accountid" unless $type; + die "Account $accountid does not exist, please re-create\n" unless $type; warn "CONNECTING: $email $type\n"; if ($type eq 'gmail') { $db = JMAP::GmailDB->new($accountid); @@ -71,7 +71,7 @@ sub getdb { $db = JMAP::ImapDB->new($accountid); } else { - die "Weird type $type"; + die "Weird type $type for $accountid"; } $db->{change_cb} = \&change_cb; return $db; @@ -141,9 +141,10 @@ sub handle_getstate { $db->begin(); my $user = $db->get_user(); + $db->commit(); + die "Failed to get user" unless $user; my $state = "$user->{jhighestmodseq}"; - $db->commit(); my %map; foreach my $key (keys %$user) { @@ -230,7 +231,7 @@ sub mk_handler { $res = ['error', "$@"] } if ($db and $db->in_transaction()) { - $res = ['error', "STILL IN TRANSACTION " . Dumper($res, $args, $tag)] + $db->rollback(); } $res->[2] = $tag; $hdl->push_write(json => $res) if $res->[0]; @@ -263,7 +264,7 @@ sub handle_syncall { sub handle_backfill { my $db = shift; my $res = $db->backfill(); - return ['sync', $res ? $JSON::true : $JSON::false]; + return ['backfill', $res ? $JSON::true : $JSON::false]; } sub handle_davsync { @@ -365,8 +366,6 @@ sub handle_signup { my $reply; warn "RESOLVING: $domain\n"; ($reply) = $Resolver->search("_imaps._tcp.$domain", "srv"); - use Data::Dumper; - warn Dumper($reply); if ($reply) { my @d = $reply->answer; if (@d) { @@ -543,7 +542,7 @@ sub handle_jmap { @items = eval { $api->$command($args, $tag) }; if ($@) { @items = ['error', { type => "serverError", message => "$@" }]; - eval { $api->rollback() }; + eval { $api->rollback() }; } } else { From dd61b9c490237600647999ada12616f03dbb16a6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 06:49:07 +1000 Subject: [PATCH 261/331] don't raise-error --- JMAP/API.pm | 2 +- JMAP/DB.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 14a242a..89256c3 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -59,7 +59,7 @@ sub refreshSyncedCalendars { my $Self = shift; $Self->{db}->sync_calendars(); - + # no response return (); } diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 48664df..d68ba2e 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -43,7 +43,7 @@ my %TABLE2GROUPS = ( sub new { my $class = shift; my $accountid = shift || die; - my $dbh = DBI->connect("dbi:SQLite:dbname=/home/jmap/data/$accountid.sqlite3", undef, undef, { RaiseError => 1 }); + my $dbh = DBI->connect("dbi:SQLite:dbname=/home/jmap/data/$accountid.sqlite3"); my $Self = bless { accountid => $accountid, dbh => $dbh, start => time() }, ref($class) || $class; $Self->_initdb($dbh); return $Self; From e7124c466cddb67eabefa4ef14ea5ab0bc681d86 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 11:01:56 +1000 Subject: [PATCH 262/331] imap_getpart API for fetching parts from the backend --- JMAP/Sync/Common.pm | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index ab01dde..bbb1ad4 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -295,6 +295,40 @@ sub imap_fill { return \%res; } +sub imap_getpart { + my $Self = shift; + my $imapname = shift; + my $olduidvalidity = shift || 0; + my $uid = shift; + my $partnum = shift; + + my $imap = $Self->connect_imap(); + + my $r = $imap->examine($imapname); + die "EXAMINE FAILED $imapname" unless $r; + + my $uidvalidity = $imap->get_response_code('uidvalidity') + 0; + + my %res = ( + imapname => $imapname, + olduidvalidity => $olduidvalidity, + newuidvalidity => $uidvalidity, + ); + + if ($olduidvalidity != $uidvalidity) { + return \%res; + } + + my $key = $part ? "BINARY[$part]" : "RFC822"; + my $data = $imap->fetch($uid, $key); + $Self->_unselect($imap); + + ($res{data}) = values %{$data->{$uid}}; # ignore which key we got + return \%res; +} + +} + sub imap_count { my $Self = shift; my $imapname = shift; From 48135617147f7d9acaa1403fadcddeb87a0b6c78 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sat, 25 Jul 2015 11:13:18 +1000 Subject: [PATCH 263/331] change DB schema to not keep raw message --- JMAP/API.pm | 5 ++--- JMAP/DB.pm | 32 +------------------------------- JMAP/ImapDB.pm | 33 ++++++++++++++++++++++----------- 3 files changed, 25 insertions(+), 45 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 89256c3..6dfa98d 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -401,9 +401,8 @@ sub _load_mailbox { sub _load_hasatt { my $Self = shift; - my $data = $Self->{db}->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage"); - my %parsed = map { $_->[0] => decode_json($_->[1]) } @$data; - return { map { $_ => 1 } grep { $parsed{$_}{hasAttachment} } keys %parsed }; + my $data = $Self->{db}->dbh->selectcol_arrayref("SELECT msgid FROM jrawmessage WHERE hasAttachment = 1"); + return { map { $_ => 1 } @$data }; } sub _match { diff --git a/JMAP/DB.pm b/JMAP/DB.pm index d68ba2e..c8695c4 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -253,17 +253,6 @@ sub add_message_to_mailbox { $Self->dmaybeupdate('jmailboxes', {jcountsmodseq => $data->{jmodseq}}, {jmailboxid => $jmailboxid}); } -sub get_raw_message { - my $Self = shift; - my $rfc822 = shift; - my $part = shift; - - return ('message/rfc822', $rfc822) unless $part; - - my $eml = Email::MIME->new($rfc822); - return find_part($eml, $part); -} - sub parse_date { my $Self = shift; my $date = shift; @@ -332,25 +321,6 @@ sub headers { return \%data; } -sub find_part { - my $eml = shift; - my $target = shift; - my $part = shift; - my $num = 0; - foreach my $sub ($eml->subparts()) { - $num++; - my $id = $part ? "$part.$num" : $num; - my $type = $sub->content_type(); - $type =~ s/;.*//; - return ($type, $sub->body()) if ($id eq $target); - if ($type =~ m{^multipart/}) { - my @res = find_part($sub, $target, $id); - return @res if @res; - } - } - return (); -} - sub attachments { my $Self = shift; my $messageid = shift; @@ -1076,8 +1046,8 @@ EOF $dbh->do(<begin(); $Self->dinsert('jrawmessage', { msgid => $msgid, - rfc822 => $rfc822, parsed => encode_json($message), + hasAttachment => $message->{hasAttachment}, }); $Self->commit(); @@ -1009,8 +1009,8 @@ sub update_messages { if ($has_outbox) { # move to sent when we're done push @others, $jmailmap{$jrolemap{'sent'}}{jmailboxid}; - $Self->fill_messages($msgid); - my ($rfc822) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); + my ($rfc822) = $Self->get_raw_message($msgid); + # XXX - add attachments - we might actually want the parsed message and then realise the attachments... $Self->backend_cmd('send_email', $rfc822); # strip the \Draft flag @@ -1227,20 +1227,18 @@ sub fill_messages { my $msgid = $uhash->{$uid}; next if $result{$msgid}; my $eml = Email::MIME->new($rfc822); - $parsed{$uid} = $Self->parse_message($msgid, $eml); + $parsed{$msgid} = $Self->parse_message($msgid, $eml); } $Self->begin(); - foreach my $uid (sort { $a <=> $b } keys %parsed) { - my $msgid = $uhash->{$uid}; - my $rfc822 = $res->{data}{$uid}; - next unless $rfc822; + foreach my $msgid (sort keys %parsed) { + my $message = $parsed{$msgid}; $Self->dinsert('jrawmessage', { msgid => $msgid, - rfc822 => $rfc822, - parsed => encode_json($parsed{$uid}), + parsed => $message, + hasAttachment => $message->{hasAttachment}, }); - $result{$msgid} = $parsed{$uid} + $result{$msgid} = $parsed{$msgid}; } $Self->commit(); } @@ -1251,6 +1249,19 @@ sub fill_messages { return \%result; } +sub get_raw_message { + my $Self = shift; + my $msgid = shift; + my $part = shift; + + my ($imapname, $uidvalidity, $uid) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity, uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid = ?", {}, $msgid); + return unless $imapname; + + my $res = $Self->backend_cmd('imap_getpart', $imapname, $uidvalidity, $uid, $part); + + return $res->{data}; +} + sub create_mailboxes { my $Self = shift; my $new = shift; From fac84899ed7d86ec23270de9a3ab8bc9a462f060 Mon Sep 17 00:00:00 2001 From: jmap Date: Fri, 24 Jul 2015 21:18:23 -0400 Subject: [PATCH 264/331] fix broken Common module --- JMAP/Sync/Common.pm | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index bbb1ad4..7eb395d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -300,7 +300,7 @@ sub imap_getpart { my $imapname = shift; my $olduidvalidity = shift || 0; my $uid = shift; - my $partnum = shift; + my $part = shift; my $imap = $Self->connect_imap(); @@ -327,8 +327,6 @@ sub imap_getpart { return \%res; } -} - sub imap_count { my $Self = shift; my $imapname = shift; From 023e685d0a3dfc84a80ae14c0fcf8e5fd7c493e5 Mon Sep 17 00:00:00 2001 From: jmap Date: Fri, 24 Jul 2015 21:26:50 -0400 Subject: [PATCH 265/331] encode the raw parsed --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7673416..4f5d26d 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1235,7 +1235,7 @@ sub fill_messages { my $message = $parsed{$msgid}; $Self->dinsert('jrawmessage', { msgid => $msgid, - parsed => $message, + parsed => encode_json($message), hasAttachment => $message->{hasAttachment}, }); $result{$msgid} = $parsed{$msgid}; From b51accf4d3e437d0f8b851f2f49d20d42c54e948 Mon Sep 17 00:00:00 2001 From: jmap Date: Fri, 24 Jul 2015 21:40:11 -0400 Subject: [PATCH 266/331] types from the server too --- JMAP/API.pm | 6 +----- JMAP/ImapDB.pm | 4 ++-- JMAP/Sync/Common.pm | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 6dfa98d..55eda61 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1022,12 +1022,8 @@ sub getRawMessage { $part = $1; } - # skipping transactions here - my $dbh = $Self->{db}->dbh(); - my ($content) = $dbh->selectrow_array("SELECT rfc822 FROM jrawmessage WHERE msgid = ?", {}, $msgid); - return unless $content; + my ($type, $data) = $Self->{db}->get_raw_message($msgid, $part); - my ($type, $data) = $Self->{db}->get_raw_message($content, $part); return ($type, $data, $filename); } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 4f5d26d..d9778a2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1009,7 +1009,7 @@ sub update_messages { if ($has_outbox) { # move to sent when we're done push @others, $jmailmap{$jrolemap{'sent'}}{jmailboxid}; - my ($rfc822) = $Self->get_raw_message($msgid); + my ($type, $rfc822) = $Self->get_raw_message($msgid); # XXX - add attachments - we might actually want the parsed message and then realise the attachments... $Self->backend_cmd('send_email', $rfc822); @@ -1259,7 +1259,7 @@ sub get_raw_message { my $res = $Self->backend_cmd('imap_getpart', $imapname, $uidvalidity, $uid, $part); - return $res->{data}; + return ($res->{type}, $res->{data}); } sub create_mailboxes { diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 7eb395d..0c1a298 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -319,11 +319,22 @@ sub imap_getpart { return \%res; } - my $key = $part ? "BINARY[$part]" : "RFC822"; + my $key = $part ? "(BINARY[$part] BODY[$part.header.fields (content-type)])" : "RFC822"; my $data = $imap->fetch($uid, $key); $Self->_unselect($imap); - ($res{data}) = values %{$data->{$uid}}; # ignore which key we got + my @keys = keys %{$data->{$uid}}; + if ($part) { + my ($datakey) = grep { m/binary/i } @keys; + my ($typekey) = grep { m/header/i } @keys; + my $type = $data->{$uid}{$typekey}; + $type =~ s/;.*//; + $res{type} = $type; + $res{type} = $data->{$uid}{$datakey}; + } + else { + ($res{data}) = values %{$data->{$uid}}; # ignore which key we got + } return \%res; } From 364eca0fe153bf77f96d361381c12ff5629681b1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 26 Jul 2015 14:50:10 +1000 Subject: [PATCH 267/331] change msgid format and handle blobs for raw --- JMAP/API.pm | 20 ++++++++++++++------ JMAP/DB.pm | 4 ++-- JMAP/ImapDB.pm | 20 +++++++++++++++++++- JMAP/Sync/Common.pm | 16 +++------------- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 55eda61..f841261 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -939,11 +939,11 @@ sub getMessages { } if (_prop_wanted($args, 'rawUrl')) { - $item->{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/$msgid"; + $item->{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/m-$msgid"; } if (_prop_wanted($args, 'blobId')) { - $item->{blobId} = "$msgid"; + $item->{blobId} = "m-$msgid"; } push @list, $item; @@ -1013,16 +1013,24 @@ sub getRawMessage { my $selector = shift; my $msgid = $selector; + return () unless $msgid =~ s/^([mf])-//; + my $source = $1; my $part; my $filename; - if ($msgid =~ s{/?(.*)}{}) { + if ($msgid =~ s{/(.*)}{}) { $filename = $1; } if ($msgid =~ s{-(.*)}{}) { $part = $1; } - my ($type, $data) = $Self->{db}->get_raw_message($msgid, $part); + my ($type, $data); + if ($source) eq 'f') { + ($type, $data) = $Self->get_file($msgid); + } + else { + ($type, $data) = $Self->{db}->get_raw_message($msgid, $part); + } return ($type, $data, $filename); } @@ -1140,8 +1148,8 @@ sub setMessages { foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; - $created->{$cid}{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/$msgid"; - $created->{$cid}{blobId} = "$msgid"; + $created->{$cid}{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/m-$msgid"; + $created->{$cid}{blobId} = "m-$msgid"; } my @res; diff --git a/JMAP/DB.pm b/JMAP/DB.pm index c8695c4..7c0a3ab 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -387,8 +387,8 @@ sub attachments { push @res, { id => $id, type => $type, - url => "https://$ENV{jmaphost}/raw/$accountid/$messageid-$id/$filename", # XXX dep - blobId => "$messageid-$id", + url => "https://$ENV{jmaphost}/raw/$accountid/m-$messageid-$id/$filename", # XXX dep + blobId => "m-$messageid-$id", name => $filename, size => length($body), isInline => $isInline, diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d9778a2..c5a3838 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1249,6 +1249,18 @@ sub fill_messages { return \%result; } +sub find_type { + my $message = shift; + my $part = shift; + + return $message->{type} if ($message->{part} || '') eq $part; + + foreach my $sub (@{$part->{attachments}}) { + my $type = find_type($sub, $part); + return $type if $type; + } +} + sub get_raw_message { my $Self = shift; my $msgid = shift; @@ -1257,9 +1269,15 @@ sub get_raw_message { my ($imapname, $uidvalidity, $uid) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity, uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid = ?", {}, $msgid); return unless $imapname; + my $type = 'message/rfc822'; + if ($part) { + my $parsed = $Self->fill_messages($msgid); + $type = find_type($parsed, $part); + } + my $res = $Self->backend_cmd('imap_getpart', $imapname, $uidvalidity, $uid, $part); - return ($res->{type}, $res->{data}); + return ($type, $res->{data}); } sub create_mailboxes { diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 0c1a298..11dc1f9 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -319,22 +319,12 @@ sub imap_getpart { return \%res; } - my $key = $part ? "(BINARY[$part] BODY[$part.header.fields (content-type)])" : "RFC822"; + my $key = $part ? "BINARY[$part]" : "RFC822"; my $data = $imap->fetch($uid, $key); $Self->_unselect($imap); - my @keys = keys %{$data->{$uid}}; - if ($part) { - my ($datakey) = grep { m/binary/i } @keys; - my ($typekey) = grep { m/header/i } @keys; - my $type = $data->{$uid}{$typekey}; - $type =~ s/;.*//; - $res{type} = $type; - $res{type} = $data->{$uid}{$datakey}; - } - else { - ($res{data}) = values %{$data->{$uid}}; # ignore which key we got - } + ($res{data}) = values %{$data->{$uid}}; # ignore which key we got + return \%res; } From d9ec1eb2c9442fb4aed4ce7ed354b4e1b1bd9734 Mon Sep 17 00:00:00 2001 From: jmap Date: Sun, 26 Jul 2015 20:46:07 -0400 Subject: [PATCH 268/331] add back backfilling --- JMAP/API.pm | 2 +- JMAP/DB.pm | 2 +- JMAP/ImapDB.pm | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index f841261..9527559 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -1025,7 +1025,7 @@ sub getRawMessage { } my ($type, $data); - if ($source) eq 'f') { + if ($source eq 'f') { ($type, $data) = $Self->get_file($msgid); } else { diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 7c0a3ab..f21ddc0 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -112,7 +112,7 @@ sub commit { } $Self->dupdate('account', \%dbdata); - $Self->{change_cb}->($Self, \%map); + $Self->{change_cb}->($Self, \%map) unless $Self->{t}->{backfilling}; } $Self->dbh->commit(); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c5a3838..879aaea 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -773,6 +773,7 @@ sub do_folder { } $Self->begin(); + $Self->{t}{backfilling} = 1 if $batchsize; my $didold = 0; if ($res->{backfill}) { @@ -1253,9 +1254,9 @@ sub find_type { my $message = shift; my $part = shift; - return $message->{type} if ($message->{part} || '') eq $part; + return $message->{type} if ($message->{id} || '') eq $part; - foreach my $sub (@{$part->{attachments}}) { + foreach my $sub (@{$message->{attachments}}) { my $type = find_type($sub, $part); return $type if $type; } From 27e8937793803daf11b6cb77221e2db5d0416784 Mon Sep 17 00:00:00 2001 From: jmap Date: Sun, 26 Jul 2015 23:26:18 -0400 Subject: [PATCH 269/331] make all maxChanges errors be fatal for now - TODO, hasMoreChanges --- JMAP/API.pm | 66 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 9527559..a8d4671 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -275,10 +275,12 @@ sub getMailboxUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + my $sinceState = $args->{sinceState}; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE jmodseq > ?1 OR jcountsmodseq > ?1", {Slice => {}}, $sinceState); @@ -301,7 +303,7 @@ sub getMailboxUpdates { my @res = (['mailboxUpdates', { accountId => $accountid, oldState => "$sinceState", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], onlyCountsChanged => $onlyCounts ? JSON::true : JSON::false, @@ -659,9 +661,11 @@ sub getMessageListUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $start = $args->{position} || 0; @@ -736,7 +740,7 @@ sub getMessageListUpdates { } if ($args->{maxChanges} and $changes > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } if ($args->{upToMessageId} and $args->{upToMessageId} eq $item->{msgid}) { @@ -773,7 +777,7 @@ sub getMessageListUpdates { } if ($args->{maxChanges} and $changes > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } if ($args->{upToMessageId} and $args->{upToMessageId} eq $item->{msgid}) { @@ -1075,9 +1079,11 @@ sub getMessageUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT msgid,active FROM jmessages WHERE jmodseq > ?"; @@ -1085,7 +1091,7 @@ sub getMessageUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } my @changed; @@ -1106,7 +1112,7 @@ sub getMessageUpdates { push @res, ['messageUpdates', { accountId => $accountid, oldState => "$args->{sinceState}", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]; @@ -1337,9 +1343,11 @@ sub getThreadUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT * FROM jmessages WHERE jmodseq > ?"; @@ -1351,7 +1359,7 @@ sub getThreadUpdates { my $data = $dbh->selectall_arrayref($sql, {Slice => {}}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } my %threads; @@ -1378,7 +1386,7 @@ sub getThreadUpdates { push @res, ['threadUpdates', { accountId => $accountid, oldState => $args->{sinceState}, - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => \@changed, removed => \@removed, }]; @@ -1497,10 +1505,12 @@ sub getCalendarUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + my $sinceState = $args->{sinceState}; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT jcalendarid, jmodseq, active FROM jcalendars ORDER BY jcalendarid"); @@ -1533,7 +1543,7 @@ sub getCalendarUpdates { my @res = (['calendarUpdates', { accountId => $accountid, oldState => "$sinceState", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]); @@ -1687,9 +1697,11 @@ sub getCalendarEventUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT eventuid,active FROM jevents WHERE jmodseq > ?"; @@ -1697,7 +1709,7 @@ sub getCalendarEventUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } $Self->commit(); @@ -1718,7 +1730,7 @@ sub getCalendarEventUpdates { push @res, ['calendarEventUpdates', { accountId => $accountid, oldState => "$args->{sinceState}", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]; @@ -1803,10 +1815,12 @@ sub getAddressbookUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + my $sinceState = $args->{sinceState}; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $sinceState <= $user->{jdeletedmodseq}); my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, jmodseq, active FROM jaddressbooks ORDER BY jaddressbookid"); @@ -1839,7 +1853,7 @@ sub getAddressbookUpdates { my @res = (['addressbookUpdates', { accountId => $accountid, oldState => "$sinceState", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]); @@ -1991,9 +2005,11 @@ sub getContactUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT contactuid,active FROM jcontacts WHERE jmodseq > ?"; @@ -2001,7 +2017,7 @@ sub getContactUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } $Self->commit(); @@ -2021,7 +2037,7 @@ sub getContactUpdates { push @res, ['contactUpdates', { accountId => $accountid, oldState => "$args->{sinceState}", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]; @@ -2102,9 +2118,11 @@ sub getContactGroupUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; - return $Self->_transError(['error', {type => 'cannotCalculateChanges'}]) + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]) if ($user->{jdeletedmodseq} and $args->{sinceState} <= $user->{jdeletedmodseq}); my $sql = "SELECT groupuid,active FROM jcontactgroups WHERE jmodseq > ?"; @@ -2112,7 +2130,7 @@ sub getContactGroupUpdates { my $data = $dbh->selectall_arrayref($sql, {}, $args->{sinceState}); if ($args->{maxChanges} and @$data > $args->{maxChanges}) { - return $Self->_transError(['error', {type => 'tooManyChanges'}]); + return $Self->_transError(['error', {type => 'cannotCalculateChanges', newState => $newState}]); } my @changed; @@ -2132,7 +2150,7 @@ sub getContactGroupUpdates { push @res, ['contactGroupUpdates', { accountId => $accountid, oldState => "$args->{sinceState}", - newState => "$user->{jhighestmodseq}", + newState => $newState, changed => [map { "$_" } @changed], removed => [map { "$_" } @removed], }]; From d9f6835c94d1173b25ec72c002522517a94bb302 Mon Sep 17 00:00:00 2001 From: jmap Date: Sun, 26 Jul 2015 23:49:02 -0400 Subject: [PATCH 270/331] fix attachment download --- JMAP/ImapDB.pm | 2 +- JMAP/Sync/Common.pm | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 879aaea..35a5bf3 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1273,7 +1273,7 @@ sub get_raw_message { my $type = 'message/rfc822'; if ($part) { my $parsed = $Self->fill_messages($msgid); - $type = find_type($parsed, $part); + $type = find_type($parsed->{$msgid}, $part); } my $res = $Self->backend_cmd('imap_getpart', $imapname, $uidvalidity, $uid, $part); diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 11dc1f9..11da0ec 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -323,7 +323,8 @@ sub imap_getpart { my $data = $imap->fetch($uid, $key); $Self->_unselect($imap); - ($res{data}) = values %{$data->{$uid}}; # ignore which key we got + my $datakey = $part ? 'binary' : 'rfc822'; + $res{data} = $data->{$uid}{$datakey}; return \%res; } From e00be17d7e96807e0972b043ca3d870eadee3ba0 Mon Sep 17 00:00:00 2001 From: jmap Date: Mon, 27 Jul 2015 01:20:59 -0400 Subject: [PATCH 271/331] backfill heaps and less often --- JMAP/ImapDB.pm | 2 +- bin/server.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 35a5bf3..ff74f91 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -662,7 +662,7 @@ sub backfill { return unless @$data; - my $rest = 500; + my $rest = 5000; foreach my $row (@$data) { my $id = $row->{ifolderid}; my $label = $row->{label}; diff --git a/bin/server.pl b/bin/server.pl index bef8031..de5d7cf 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -538,7 +538,7 @@ sub prod_backfill { $idler{$accountid}{backfilling} = 1; my $timer; - $timer = AnyEvent->timer(after => 5, cb => sub { + $timer = AnyEvent->timer(after => 60, cb => sub { send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { $timer = undef; prod_backfill($accountid, @_); From 234bab767d3f9c10fb1cfbd1bc39214c17e4620e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 14:16:25 +1000 Subject: [PATCH 272/331] enforce more about transactionality of DB calls --- JMAP/API.pm | 13 +++++-------- JMAP/DB.pm | 7 ++++--- JMAP/ImapDB.pm | 36 ++++++++++++++++++------------------ 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index a8d4671..5bbfef7 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -40,7 +40,7 @@ sub getAccounts { my @list; push @list, { - id => $Self->{db}->{accountId}, + id => $Self->{db}->accountid(), name => $user->{displayname} || $user->{email}, isPrimary => $JSON::true, isReadOnly => $JSON::false, @@ -158,7 +158,6 @@ sub getMailboxes { my $Self = shift; my $args = shift; - # XXX - ideally this is transacted inside the DB $Self->begin(); my $dbh = $Self->{db}->dbh(); @@ -167,6 +166,8 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jhighestmodseq}"; + my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE active = 1", {Slice => {}}); my %ids; @@ -237,7 +238,7 @@ sub getMailboxes { return ['mailboxes', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [map { "$_" } keys %missingids] : undef), }]; } @@ -267,9 +268,9 @@ sub getIdentities { sub getMailboxUpdates { my $Self = shift; my $args = shift; - my $dbh = $Self->{db}->dbh(); $Self->begin(); + my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); my $accountid = $Self->{db}->accountid(); return $Self->_transError(['error', {type => 'accountNotFound'}]) @@ -1071,7 +1072,6 @@ sub getMessageUpdates { my $args = shift; $Self->begin(); - my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1335,7 +1335,6 @@ sub getThreadUpdates { my $args = shift; $Self->begin(); - my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1497,7 +1496,6 @@ sub getCalendarUpdates { my $args = shift; $Self->begin(); - my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); @@ -1689,7 +1687,6 @@ sub getCalendarEventUpdates { my $args = shift; $Self->begin(); - my $dbh = $Self->{db}->dbh(); my $user = $Self->{db}->get_user(); diff --git a/JMAP/DB.pm b/JMAP/DB.pm index f21ddc0..0aecfc3 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -76,7 +76,8 @@ sub log { sub dbh { my $Self = shift; - return $Self->{dbh}; + confess("NOT IN TRANSACTION") unless $Self->{t}; + return $Self->{t}{dbh}; } sub in_transaction { @@ -88,7 +89,7 @@ sub begin { my $Self = shift; confess("ALREADY IN TRANSACTION") if $Self->{t}; my $accountid = $Self->accountid(); - $Self->{t} = {}; + $Self->{t} = {dbh => $Self->{dbh}}; # we need this because sqlite locking isn't as robust as you might hope $Self->{t}{lock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); $Self->dbh->begin_work(); @@ -110,7 +111,7 @@ sub commit { $dbdata{"jstate$group"} = $state; } } - + $Self->dupdate('account', \%dbdata); $Self->{change_cb}->($Self, \%map) unless $Self->{t}->{backfilling}; } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ff74f91..975457c 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -883,7 +883,7 @@ sub changed_record { my $flags = encode_json([grep { lc $_ ne '\\recent' } sort @$flaglist]); my $labels = encode_json([sort @$labellist]); - my ($msgid) = $Self->{dbh}->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); + my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); $Self->dmaybeupdate('imessages', {flags => $flags, labels => $labels}, {ifolderid => $folder, uid => $uid}); @@ -950,7 +950,7 @@ sub update_messages { my $changes = shift; my $idmap = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %updatemap; my %notchanged; @@ -1054,7 +1054,7 @@ sub destroy_messages { my $Self = shift; my $ids = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %destroymap; my %notdestroyed; @@ -1091,7 +1091,7 @@ sub deleted_record { my $Self = shift; my ($folder, $uid) = @_; - my ($msgid, $jmailboxid) = $Self->{dbh}->selectrow_array("SELECT msgid, jmailboxid FROM imessages JOIN ifolders USING (ifolderid) WHERE imessages.ifolderid = ? AND uid = ?", {}, $folder, $uid); + my ($msgid, $jmailboxid) = $Self->dbh->selectrow_array("SELECT msgid, jmailboxid FROM imessages JOIN ifolders USING (ifolderid) WHERE imessages.ifolderid = ? AND uid = ?", {}, $folder, $uid); return unless $msgid; $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); @@ -1152,10 +1152,10 @@ sub apply_data { # check for archive folder for gmail if ($Self->{is_gmail} and not @list) { - @jmailboxids = $Self->{dbh}->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = 'archive'"); + @jmailboxids = $Self->dbh->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = 'archive'"); } - my ($old) = $Self->{dbh}->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); + my ($old) = $Self->dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgid = ? AND active = 1", {}, $msgid); $Self->log('debug', "DATA (@jmailboxids) for $msgid"); @@ -1285,7 +1285,7 @@ sub create_mailboxes { my $Self = shift; my $new = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %idmap; my %notcreated; @@ -1327,7 +1327,7 @@ sub update_mailboxes { my $update = shift; my $idmap = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @updated; my %notupdated; @@ -1366,7 +1366,7 @@ sub destroy_mailboxes { my $Self = shift; my $destroy = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @destroyed; my %notdestroyed; @@ -1386,7 +1386,7 @@ sub create_calendar_events { my $Self = shift; my $new = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %createmap; my %notcreated; @@ -1411,7 +1411,7 @@ sub update_calendar_events { my $update = shift; my $idmap = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @updated; my %notupdated; @@ -1434,7 +1434,7 @@ sub destroy_calendar_events { my $Self = shift; my $destroy = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @destroyed; my %notdestroyed; @@ -1456,7 +1456,7 @@ sub create_contact_groups { my $Self = shift; my $new = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %createmap; my %notcreated; @@ -1490,7 +1490,7 @@ sub update_contact_groups { my $changes = shift; my $idmap = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @updated; my %notchanged; @@ -1520,7 +1520,7 @@ sub destroy_contact_groups { my $Self = shift; my $destroy = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @destroyed; my %notdestroyed; @@ -1541,7 +1541,7 @@ sub create_contacts { my $Self = shift; my $new = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my %createmap; my %notcreated; @@ -1583,7 +1583,7 @@ sub update_contacts { my $changes = shift; my $idmap = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @updated; my %notchanged; @@ -1622,7 +1622,7 @@ sub destroy_contacts { my $Self = shift; my $destroy = shift; - my $dbh = $Self->{dbh}; + my $dbh = $Self->dbh; my @destroyed; my %notdestroyed; From 70775f9036957fbbbd974f14b50ea7328cd9a7b8 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 15:10:04 +1000 Subject: [PATCH 273/331] Rewrite all the locking around reading from the DB --- JMAP/DB.pm | 15 +-- JMAP/ImapDB.pm | 270 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 183 insertions(+), 102 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 0aecfc3..4d3d879 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -92,7 +92,7 @@ sub begin { $Self->{t} = {dbh => $Self->{dbh}}; # we need this because sqlite locking isn't as robust as you might hope $Self->{t}{lock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); - $Self->dbh->begin_work(); + $Self->{t}{dbh}->begin_work(); } sub commit { @@ -116,14 +116,14 @@ sub commit { $Self->{change_cb}->($Self, \%map) unless $Self->{t}->{backfilling}; } - $Self->dbh->commit(); + $Self->{t}{dbh}->commit(); delete $Self->{t}; } sub rollback { my $Self = shift; confess("NOT IN TRANSACTION") unless $Self->{t}; - $Self->dbh->rollback(); + $Self->{t}{dbh}->rollback(); delete $Self->{t}; } @@ -131,14 +131,13 @@ sub rollback { sub reset { my $Self = shift; return unless $Self->{t}; - $Self->dbh->rollback(); + $Self->{t}{dbh}->rollback(); delete $Self->{t}; } sub dirty { my $Self = shift; my $table = shift || die 'need to have a table to dirty'; - confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{modseq}) { my $user = $Self->get_user(); $user->{jhighestmodseq}++; @@ -151,7 +150,6 @@ sub dirty { sub get_user { my $Self = shift; - confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{user}) { $Self->{t}{user} = $Self->dbh->selectrow_hashref("SELECT * FROM account"); } @@ -168,7 +166,6 @@ sub get_user { sub get_mailboxes { my $Self = shift; - confess("NOT IN TRANSACTION") unless $Self->{t}; unless ($Self->{t}{mailboxes}) { $Self->{t}{mailboxes} = $Self->dbh->selectall_hashref("SELECT jmailboxid, jmodseq, label, name, parentId, nummessages, numumessages, numthreads, numuthreads, active FROM jmailboxes", 'jmailboxid', {Slice => {}}); } @@ -848,8 +845,6 @@ sub dinsert { my $Self = shift; my ($table, $values) = @_; - confess("NOT IN TRANSACTION") unless $Self->{t}; - $values->{mtime} = time(); my @keys = sort keys %$values; @@ -947,8 +942,6 @@ sub ddelete { my $Self = shift; my ($table, $limit) = @_; - confess("NOT IN TRANSACTION") unless $Self->{t}; - my @lkeys = sort keys %$limit; my $sql = "DELETE FROM $table WHERE " . join(' AND ', map { "$_ = ?" } @lkeys); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 975457c..d2b94ba 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -133,9 +133,8 @@ sub sync_folders { my ($prefix, $folders) = @$data; $Self->begin(); - my $dbh = $Self->dbh(); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); + my $ifolders = $Self->dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); my %ibylabel = map { $_->[4] => $_ } @$ifolders; my %seen; @@ -160,7 +159,7 @@ sub sync_folders { foreach my $folder (@$ifolders) { my $id = $folder->[0]; next if $seen{$id}; - $dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); + $Self->dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); } $Self->dmaybeupdate('iserver', {imapPrefix => $prefix, lastfoldersync => time()}); @@ -190,9 +189,8 @@ sub sync_folders { sub sync_jmailboxes { my $Self = shift; $Self->begin(); - my $dbh = $Self->dbh(); - my $ifolders = $dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); - my $jmailboxes = $dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); + my $ifolders = $Self->dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); + my $jmailboxes = $Self->dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); my %jbyid; my %roletoid; @@ -335,9 +333,8 @@ sub sync_calendars { return unless $calendars; $Self->begin(); - my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); + my $icalendars = $Self->dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); my %byhref = map { $_->[1] => $_ } @$icalendars; my %seen; @@ -369,7 +366,7 @@ sub sync_calendars { foreach my $calendar (@$icalendars) { my $id = $calendar->[0]; next if $seen{$id}; - $dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); + $Self->dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); } $Self->commit(); @@ -386,9 +383,8 @@ sub sync_calendars { sub sync_jcalendars { my $Self = shift; $Self->begin(); - my $dbh = $Self->dbh(); - my $icalendars = $dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); - my $jcalendars = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); + my $icalendars = $Self->dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); + my $jcalendars = $Self->dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); my %jbyid; foreach my $calendar (@$jcalendars) { @@ -432,15 +428,17 @@ sub do_calendar { my $Self = shift; my $calendarid = shift; - my $dbh = $Self->dbh(); + $Self->begin(); + + my ($href, $jcalendarid) = $Self->dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + $Self->commit(); - my ($href, $jcalendarid) = $dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); my $events = $Self->backend_cmd('get_events', $href); # parse events before we lock my %parsed = map { $_ => $Self->parse_event($events->{$_}) } keys %$events; $Self->begin(); - my $exists = $dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $calendarid); + my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $calendarid); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %$events) { @@ -483,9 +481,8 @@ sub sync_addressbooks { return unless $addressbooks; $Self->begin(); - my $dbh = $Self->dbh(); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, href, syncToken FROM iaddressbooks", {Slice => {}}); + my $iaddressbooks = $Self->dbh->selectall_arrayref("SELECT iaddressbookid, href, syncToken FROM iaddressbooks", {Slice => {}}); my %byhref = map { $_->{href} => $_ } @$iaddressbooks; my %seen; @@ -515,7 +512,7 @@ sub sync_addressbooks { foreach my $addressbook (@$iaddressbooks) { my $id = $addressbook->{iaddressbookid}; next if $seen{$id}; - $dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + $Self->dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); } $Self->commit(); @@ -533,10 +530,9 @@ sub sync_jaddressbooks { my $Self = shift; $Self->begin(); - my $dbh = $Self->dbh(); - my $iaddressbooks = $dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks", {Slice => {}}); - my $jaddressbooks = $dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks", {Slice => {}}); + my $iaddressbooks = $Self->dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks", {Slice => {}}); + my $jaddressbooks = $Self->dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks", {Slice => {}}); my %jbyid; foreach my $addressbook (@$jaddressbooks) { @@ -580,16 +576,16 @@ sub do_addressbook { my $Self = shift; my $addressbookid = shift; - my $dbh = $Self->dbh(); - - my ($href, $jaddressbookid) = $dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); + $Self->begin(); + my ($href, $jaddressbookid) = $Self->dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); + $Self->commit(); my $cards = $Self->backend_cmd('get_cards', $href); # parse before locking my %parsed = map { $_ => $Self->parse_card($cards->{$_}) } keys %$cards; $Self->begin(); - my $exists = $dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $addressbookid); + my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $addressbookid); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %$cards) { @@ -636,10 +632,13 @@ sub labels { sub sync_imap { my $Self = shift; + + $Self->begin(); my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } + $Self->commit(); my @imapnames = map { $_->{imapname} } @$data; my $status = $Self->backend_cmd('imap_status', \@imapnames); @@ -730,10 +729,9 @@ sub do_folder { Carp::confess("NO FOLDERID") unless $ifolderid; $Self->begin(); - my $dbh = $Self->dbh(); my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = - $dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); + $Self->dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); die "NO SUCH FOLDER $ifolderid" unless $imapname; my %fetches; @@ -829,7 +827,9 @@ sub do_folder { return $didold if $batchsize; # need to make changes before counting - my ($count) = $dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + $Self->begin(); + my ($count) = $Self->dbh->selectrow_array("SELECT COUNT(*) FROM imessages WHERE ifolderid = ?", {}, $ifolderid); + $Self->commit(); # if we don't know everything, we have to ALWAYS check or moves break if ($uidfirst != 1 or $count != $res->{newstate}{exists}) { # welcome to the future @@ -839,7 +839,7 @@ sub do_folder { my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); $Self->begin(); my $uids = $res->{data}; - my $data = $dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ? AND uid >= ?", {}, $ifolderid, $uidfirst); + my $data = $Self->dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ? AND uid >= ?", {}, $ifolderid, $uidfirst); my %exists = map { $_ => 1 } @$uids; foreach my $uid (@$data) { next if $exists{$uid}; @@ -853,12 +853,12 @@ sub imap_search { my $Self = shift; my @search = @_; - my $dbh = $Self->dbh(); - my $data = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); - + $Self->begin(); + my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } + $Self->commit(); my %matches; foreach my $item (@$data) { @@ -867,10 +867,12 @@ sub imap_search { my $res = $Self->backend_cmd('imap_search', $item->{imapname}, 'uid', "$from:$to", @search); # XXX - uidvaldity changed next unless $res->[2] == $item->{uidvalidity}; + $Self->begin(); foreach my $uid (@{$res->[3]}) { - my ($msgid) = $dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? and uid = ?", {}, $item->{ifolderid}, $uid); + my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? and uid = ?", {}, $item->{ifolderid}, $uid); $matches{$msgid} = 1; } + $Self->commit(); } return \%matches; @@ -896,8 +898,10 @@ sub import_message { my $mailboxIds = shift; my %flags = @_; - my $dbh = $Self->dbh(); - my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + $Self->begin(); + my $folderdata = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + $Self->commit(); + my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; my %jmailmap = map { $_->{jmailboxid} => $_ } grep { $_->{jmailboxid} } @$folderdata; @@ -928,9 +932,11 @@ sub import_message { $Self->do_folder($fdata->{ifolderid}, $fdata->{label}); } + $Self->begin(); my ($msgid, $thrid) = $Self->dbh->selectrow_array("SELECT msgid, thrid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $jmailmap{$id}{ifolderid}, $uid); + $Self->commit(); - # save us having to download it again + # save us having to download it again - drop out of transaction so we don't wait on the parse my $eml = Email::MIME->new($rfc822); my $message = $Self->parse_message($msgid, $eml); @@ -950,12 +956,10 @@ sub update_messages { my $changes = shift; my $idmap = shift; - my $dbh = $Self->dbh; - my %updatemap; my %notchanged; foreach my $msgid (keys %$changes) { - my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); + my ($ifolderid, $uid) = $Self->dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); if ($ifolderid and $uid) { $updatemap{$ifolderid}{$uid} = $msgid; } @@ -964,10 +968,10 @@ sub update_messages { } } - my $folderdata = $dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $folderdata = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; my %jmailmap = map { $_->{jmailboxid} => $_ } grep { $_->{jmailboxid} } @$folderdata; - my $jmapdata = $dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); + my $jmapdata = $Self->dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; my %jrolemap = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$jmapdata; @@ -1018,11 +1022,11 @@ sub update_messages { $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\Draft"]); # add the \Answered flag to our in-reply-to - my ($updateid) = $dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); + my ($updateid) = $Self->dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); goto done unless $updateid; - my ($updatemsgid) = $dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgmessageid = ?", {}, $updateid); + my ($updatemsgid) = $Self->dbh->selectrow_array("SELECT msgid FROM jmessages WHERE msgmessageid = ?", {}, $updateid); goto done unless $updatemsgid; - my ($ifolderid, $updateuid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); + my ($ifolderid, $updateuid) = $Self->dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $updatemsgid); goto done unless $ifolderid; my $updatename = $foldermap{$ifolderid}{imapname}; my $updatevalidity = $foldermap{$ifolderid}{uidvalidity}; @@ -1054,12 +1058,11 @@ sub destroy_messages { my $Self = shift; my $ids = shift; - my $dbh = $Self->dbh; - + $Self->begin(); my %destroymap; my %notdestroyed; foreach my $msgid (@$ids) { - my ($ifolderid, $uid) = $dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); + my ($ifolderid, $uid) = $Self->dbh->selectrow_array("SELECT ifolderid, uid FROM imessages WHERE msgid = ?", {}, $msgid); if ($ifolderid and $uid) { $destroymap{$ifolderid}{$uid} = $msgid; } @@ -1068,10 +1071,12 @@ sub destroy_messages { } } - my $folderdata = $dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); + my $folderdata = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; + $Self->commit(); + my @destroyed; foreach my $ifolderid (keys %destroymap) { # XXX - merge similar actions? @@ -1084,6 +1089,7 @@ sub destroy_messages { $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder push @destroyed, values %{$destroymap{$ifolderid}}; } + return (\@destroyed, \%notdestroyed); } @@ -1285,8 +1291,7 @@ sub create_mailboxes { my $Self = shift; my $new = shift; - my $dbh = $Self->dbh; - + $Self->begin(); my %idmap; my %notcreated; foreach my $cid (keys %$new) { @@ -1294,30 +1299,36 @@ sub create_mailboxes { my $imapname = $mailbox->{name}; if ($mailbox->{parentId}) { - my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $mailbox->{parentId}); + my ($parentName, $sep) = $Self->dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $mailbox->{parentId}); # XXX - errors $imapname = "$parentName$sep$imapname"; } else { - my ($prefix) = $dbh->selectrow_array("SELECT imapPrefix FROM iserver"); + my ($prefix) = $Self->dbh->selectrow_array("SELECT imapPrefix FROM iserver"); $imapname = "$prefix$imapname"; } + $idmap{$imapname} = $cid; # need to resolve this after the sync + } - my $res = $Self->backend_cmd('create_mailbox', $imapname); + $Self->commit(); + + foreach my $imapname (sort keys %idmap) { # XXX - handle errors... - $idmap{$imapname} = $cid; # need to resolve this after the sync + my $res = $Self->backend_cmd('create_mailbox', $imapname); } # (in theory we could save this until the end and resolve the names in after the renames and deletes... but it does mean # we can't use ids as referenes...) $Self->sync_folders() if keys %idmap; + $Self->begin(); my %createmap; foreach my $imapname (keys %idmap) { my $cid = $idmap{$imapname}; - my ($jid) = $dbh->selectrow_array("SELECT jmailboxid FROM ifolders WHERE imapname = ?", {}, $imapname); + my ($jid) = $Self->dbh->selectrow_array("SELECT jmailboxid FROM ifolders WHERE imapname = ?", {}, $imapname); $createmap{$cid} = $jid; } + $Self->commit(); return (\%createmap, \%notcreated); } @@ -1327,36 +1338,45 @@ sub update_mailboxes { my $update = shift; my $idmap = shift; - my $dbh = $Self->dbh; + $Self->begin(); my @updated; my %notupdated; + my %namemap; # XXX - reorder the crap out of this if renaming multiple mailboxes due to deep rename foreach my $id (keys %$update) { my $mailbox = $update->{$id}; my $imapname = $mailbox->{name}; next unless (defined $imapname and $imapname ne ''); my $parentId = $mailbox->{parentId}; - ($parentId) = $dbh->selectrow_array("SELECT parentId FROM jmailboxes WHERE jmailboxid = ?", {}, $id) + ($parentId) = $Self->dbh->selectrow_array("SELECT parentId FROM jmailboxes WHERE jmailboxid = ?", {}, $id) unless exists $mailbox->{parentId}; if ($parentId) { $parentId = $idmap->($parentId); - my ($parentName, $sep) = $dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $parentId); + my ($parentName, $sep) = $Self->dbh->selectrow_array("SELECT imapname, sep FROM ifolders WHERE jmailboxid = ?", {}, $parentId); # XXX - errors $imapname = "$parentName$sep$imapname"; } else { - my ($prefix) = $dbh->selectrow_array("SELECT imapPrefix FROM iserver"); + my ($prefix) = $Self->dbh->selectrow_array("SELECT imapPrefix FROM iserver"); $prefix = '' unless $prefix; $imapname = "$prefix$imapname"; } - my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + my ($oldname) = $Self->dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + + $namemap{$oldname} = $newname; - $Self->backend_cmd('rename_mailbox', $oldname, $imapname) if $oldname ne $imapname; push @updated, $id; } + $Self->commit(); + + foreach my $oldname (sort keys %namemap) { + my $newname = $namemap{$oldname}; + $Self->backend_cmd('rename_mailbox', $oldname, $imapname) if $oldname ne $imapname; + } + $Self->sync_folders() if @updated; return (\@updated, \%notupdated); @@ -1366,15 +1386,23 @@ sub destroy_mailboxes { my $Self = shift; my $destroy = shift; - my $dbh = $Self->dbh; + $Self->begin(); my @destroyed; my %notdestroyed; + my %namemap; foreach my $id (@$destroy) { - my ($oldname) = $dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + my ($oldname) = $Self->dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); + $namemap{$oldname} = 1; + push @destroyed, $id; + } + $Self->commit(); + + # we reverse so we delete children before parents + foreach my $oldname (reverse sort keys %namemap) { + # XXX - handle errors $Self->backend_cmd('delete_mailbox', $oldname); - push @destroyed, $id; } $Self->sync_folders() if @destroyed; @@ -1386,23 +1414,31 @@ sub create_calendar_events { my $Self = shift; my $new = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my %createmap; my %notcreated; foreach my $cid (keys %$new) { my $calendar = $new->{$cid}; - my ($href) = $dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendar->{calendarId}); + my ($href) = $Self->dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendar->{calendarId}); unless ($href) { $notcreated{$cid} = "No such calendar on server"; next; } my $uid = new_uuid_string(); - $Self->backend_cmd('new_event', $href, {%$calendar, uid => $uid}); + $todo{$href} = {%$calendar, uid => $uid}; + $createmap{$cid} = { id => $uid }; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('new_event', $href, $todo{$href}); + } + return (\%createmap, \%notcreated); } @@ -1411,22 +1447,30 @@ sub update_calendar_events { my $update = shift; my $idmap = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @updated; my %notupdated; foreach my $uid (keys %$update) { my $calendar = $update->{$uid}; - my ($resource) = $dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); + my ($resource) = $Self->dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); unless ($resource) { $notupdated{$uid} = "No such event on server"; next; } - $Self->backend_cmd('update_event', $resource, $calendar); + $todo{$resource} = $calendar; + push @updated, $uid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('update_event', $href, $todo{$href}); + } + return (\@updated, \%notupdated); } @@ -1434,21 +1478,29 @@ sub destroy_calendar_events { my $Self = shift; my $destroy = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @destroyed; my %notdestroyed; foreach my $uid (@$destroy) { - my ($resource) = $dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); + my ($resource) = $Self->dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); unless ($resource) { $notdestroyed{$uid} = "No such event on server"; next; } - $Self->backend_cmd('delete_event', $resource); + $todo{$resource} = 1; + push @destroyed, $uid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('delete_event', $href); + } + return (\@destroyed, \%notdestroyed); } @@ -1456,14 +1508,15 @@ sub create_contact_groups { my $Self = shift; my $new = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my %createmap; my %notcreated; foreach my $cid (keys %$new) { my $contact = $new->{$cid}; - #my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); - my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks"); + #my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); + my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { $notcreated{$cid} = "No such addressbook on server"; next; @@ -1478,10 +1531,17 @@ sub create_contact_groups { $card->VGroupContactUIDs(\@ids); } - $Self->backend_cmd('new_card', $href, $card); + $todo{$href} = $card; + $createmap{$cid} = { id => $uid }; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('new_card', $href, $todo{$href}); + } + return (\%createmap, \%notcreated); } @@ -1490,13 +1550,14 @@ sub update_contact_groups { my $changes = shift; my $idmap = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @updated; my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; - my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { $notchanged{$carduid} = "No such card on server"; next; @@ -1509,10 +1570,16 @@ sub update_contact_groups { $card->VGroupContactUIDs(\@ids); } - $Self->backend_cmd('update_card', $resource, $card); + $todo{$resource} = $card; push @updated, $carduid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('update_card', $href, $todo{$href}); + } + return (\@updated, \%notchanged); } @@ -1520,20 +1587,27 @@ sub destroy_contact_groups { my $Self = shift; my $destroy = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @destroyed; my %notdestroyed; foreach my $carduid (@$destroy) { - my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { $notdestroyed{$carduid} = "No such card on server"; next; } - $Self->backend_cmd('delete_card', $resource); + $todo{$resource} = 1; push @destroyed, $carduid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('delete_card', $href); + } + return (\@destroyed, \%notdestroyed); } @@ -1541,13 +1615,13 @@ sub create_contacts { my $Self = shift; my $new = shift; - my $dbh = $Self->dbh; + $Self->begin(); my %createmap; my %notcreated; foreach my $cid (keys %$new) { my $contact = $new->{$cid}; - my ($href) = $dbh->selectrow_array("SELECT href FROM iaddressbooks"); + my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { $notcreated{$cid} = "No such addressbook on server"; next; @@ -1583,13 +1657,14 @@ sub update_contacts { my $changes = shift; my $idmap = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @updated; my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; - my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { $notchanged{$carduid} = "No such card on server"; next; @@ -1611,10 +1686,16 @@ sub update_contacts { $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; $card->VNotes($contact->{notes}) if exists $contact->{notes}; - $Self->backend_cmd('update_card', $resource, $card); + $todo{$resource} = $card; push @updated, $carduid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('update_card', $href, $todo{$href}); + } + return (\@updated, \%notchanged); } @@ -1622,20 +1703,27 @@ sub destroy_contacts { my $Self = shift; my $destroy = shift; - my $dbh = $Self->dbh; + $Self->begin(); + my %todo; my @destroyed; my %notdestroyed; foreach my $carduid (@$destroy) { - my ($resource, $content) = $dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); + my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { $notdestroyed{$carduid} = "No such card on server"; next; } - $Self->backend_cmd('delete_card', $resource); + $todo{$resource} = 1; push @destroyed, $carduid; } + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('delete_card', $href); + } + return (\@destroyed, \%notdestroyed); } From 9fb832d363a0f0a7ad4a42383f513be10a68edc8 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 15:51:42 +1000 Subject: [PATCH 274/331] remove rawUrl --- JMAP/API.pm | 5 ----- 1 file changed, 5 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 5bbfef7..c82bb80 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -943,10 +943,6 @@ sub getMessages { $item->{size} = $data->{msgsize}; } - if (_prop_wanted($args, 'rawUrl')) { - $item->{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/m-$msgid"; - } - if (_prop_wanted($args, 'blobId')) { $item->{blobId} = "m-$msgid"; } @@ -1154,7 +1150,6 @@ sub setMessages { foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; - $created->{$cid}{rawUrl} = "https://$ENV{jmaphost}/raw/$accountid/m-$msgid"; $created->{$cid}{blobId} = "m-$msgid"; } From da6d592de36df8e27aaddee7230359a40a8be3e1 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 16:11:40 +1000 Subject: [PATCH 275/331] transaction around move --- JMAP/ImapDB.pm | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d2b94ba..2ed76e8 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -956,6 +956,8 @@ sub update_messages { my $changes = shift; my $idmap = shift; + $Self->begin(); + my %updatemap; my %notchanged; foreach my $msgid (keys %$changes) { @@ -975,6 +977,8 @@ sub update_messages { my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; my %jrolemap = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$jmapdata; + $Self->commit(); + my @changed; foreach my $ifolderid (keys %updatemap) { # XXX - merge similar actions? @@ -1014,13 +1018,16 @@ sub update_messages { if ($has_outbox) { # move to sent when we're done push @others, $jmailmap{$jrolemap{'sent'}}{jmailboxid}; + my ($type, $rfc822) = $Self->get_raw_message($msgid); # XXX - add attachments - we might actually want the parsed message and then realise the attachments... $Self->backend_cmd('send_email', $rfc822); # strip the \Draft flag + $Self->backend_cmd('imap_update', $imapname, $uidvalidity, $uid, 0, ["\\Draft"]); + $Self->begin(); # add the \Answered flag to our in-reply-to my ($updateid) = $Self->dbh->selectrow_array("SELECT msginreplyto FROM jmessages WHERE msgid = ?", {}, $msgid); goto done unless $updateid; @@ -1031,9 +1038,11 @@ sub update_messages { my $updatename = $foldermap{$ifolderid}{imapname}; my $updatevalidity = $foldermap{$ifolderid}{uidvalidity}; goto done unless $updatename; + $Self->commit(); $Self->backend_cmd('imap_update', $updatename, $updatevalidity, $updateuid, 1, ["\\Answered"]); } done: + $Self->reset(); # bogus, but otherwise we need to commit on all the done commands if ($Self->{is_gmail}) { # because 'archive' is synthetic on gmail we strip it here (@others) = grep { $jidmap{$_} ne 'archive' } @others; From 2e52b7bb8915b2a3808195fd58171ab30bb53759 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 20:57:36 +1000 Subject: [PATCH 276/331] fix iaddressbookid handling --- JMAP/ImapDB.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 2ed76e8..73da06e 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -498,8 +498,8 @@ sub sync_addressbooks { if ($id) { my $token = $byhref{$addressbook->{href}}{syncToken}; if ($token ne $addressbook->{syncToken}) { - push @todo, $id; $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + push @todo, $id; } } else { @@ -536,12 +536,13 @@ sub sync_jaddressbooks { my %jbyid; foreach my $addressbook (@$jaddressbooks) { + next unless $addressbook->{jaddressbookid}; $jbyid{$addressbook->{jaddressbookid}} = $addressbook; } my %seen; foreach my $addressbook (@$iaddressbooks) { - my $aid = $addressbook->{iaddressbooks}; + my $aid = $addressbook->{iaddressbookid}; my $data = { name => $addressbook->{name}, isVisible => 1, From cdcafbaa26e985bc97afa9c51fcc6111d28435a9 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 21:15:17 +1000 Subject: [PATCH 277/331] calendars and addressbooks - single transaction for all cards --- JMAP/ImapDB.pm | 188 ++++++++++++++++++++++++------------------------- 1 file changed, 93 insertions(+), 95 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 73da06e..d2bbde6 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -338,7 +338,7 @@ sub sync_calendars { my %byhref = map { $_->[1] => $_ } @$icalendars; my %seen; - my @todo; + my %todo; foreach my $calendar (@$calendars) { my $id = $calendar->{href} ? $byhref{$calendar->{href}}[0] : 0; my $data = { @@ -352,14 +352,13 @@ sub sync_calendars { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); my $token = $byhref{$calendar->{href}}[5]; if ($token ne $calendar->{syncToken}) { - push @todo, $id; $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); } } else { $id = $Self->dinsert('icalendars', $data); - push @todo, $id; } + $todo{$id} = $calendar->{href}; $seen{$id} = 1; } @@ -369,20 +368,18 @@ sub sync_calendars { $Self->dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); } - $Self->commit(); - $Self->sync_jcalendars(); - foreach my $id (@todo) { - $Self->do_calendar($id); - } + $Self->commit(); + + $Self->do_calendars(\%todo); } # synchronise from the imap folder cache to the jmap mailbox listing # call in transaction sub sync_jcalendars { my $Self = shift; - $Self->begin(); + my $icalendars = $Self->dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); my $jcalendars = $Self->dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); @@ -421,52 +418,54 @@ sub sync_jcalendars { next if $seen{$id}; $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); } - $Self->commit(); } -sub do_calendar { +sub do_calendars { my $Self = shift; - my $calendarid = shift; - - $Self->begin(); + my $cals = shift; - my ($href, $jcalendarid) = $Self->dbh->selectrow_array("SELECT href, jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); - $Self->commit(); - - my $events = $Self->backend_cmd('get_events', $href); - # parse events before we lock - my %parsed = map { $_ => $Self->parse_event($events->{$_}) } keys %$events; + my %allparsed; + foreach my $href (sort values %$cals) { + my $events = $Self->backend_cmd('get_events', $href); + # parse events before we lock + my %parsed = map { $_ => $Self->parse_event($events->{$_}) } keys %$events; + $allparsed{$href} = \%parsed; + } $Self->begin(); - my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $calendarid); - my %res = map { $_->{resource} => $_ } @$exists; - - foreach my $resource (keys %$events) { - my $data = delete $res{$resource}; - my $raw = $events->{$resource}; - my $event = $parsed{$resource}; - my $uid = $event->{uid}; - my $item = { - icalendarid => $calendarid, - uid => $uid, - resource => $resource, - content => $raw, - }; - if ($data) { - my $id = $data->{ieventid}; - $Self->dmaybeupdate('ievents', $item, {ieventid => $id}); - } - else { - $Self->dinsert('ievents', $item); + foreach my $id (keys %$cals) { + my $href = $cals->{$id}; + my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); + my %res = map { $_->{resource} => $_ } @$exists; + + foreach my $resource (keys %$events) { + my $data = delete $res{$resource}; + my $raw = $events->{$resource}; + my $event = $parsed{$resource}; + my $uid = $event->{uid}; + my $item = { + icalendarid => $id, + uid => $uid, + resource => $resource, + content => $raw, + }; + if ($data) { + my $eid = $data->{ieventid}; + $Self->dmaybeupdate('ievents', $item, {ieventid => $eid}); + } + else { + $Self->dinsert('ievents', $item); + } + $Self->set_event($jcalendarid, $event); } - $Self->set_event($jcalendarid, $event); - } - foreach my $resource (keys %res) { - my $data = delete $res{$resource}; - my $id = $data->{ieventid}; - $Self->ddelete('ievents', {ieventid => $id}); - $Self->delete_event($jcalendarid, $data->{uid}); + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $id = $data->{ieventid}; + $Self->ddelete('ievents', {ieventid => $id}); + $Self->delete_event($jcalendarid, $data->{uid}); + } } $Self->commit(); @@ -486,7 +485,7 @@ sub sync_addressbooks { my %byhref = map { $_->{href} => $_ } @$iaddressbooks; my %seen; - my @todo; + my %todo; foreach my $addressbook (@$addressbooks) { my $id = $byhref{$addressbook->{href}}{iaddressbookid}; my $data = { @@ -499,13 +498,12 @@ sub sync_addressbooks { my $token = $byhref{$addressbook->{href}}{syncToken}; if ($token ne $addressbook->{syncToken}) { $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); - push @todo, $id; } } else { $id = $Self->dinsert('iaddressbooks', $data); - push @todo, $id; } + $todo{$id} = $addressbook->{href}; $seen{$id} = 1; } @@ -515,13 +513,11 @@ sub sync_addressbooks { $Self->dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); } - $Self->commit(); - $Self->sync_jaddressbooks(); - foreach my $id (@todo) { - $Self->do_addressbook($id); - } + $Self->commit(); + + $Self->do_addressbooks(\%todo); } # synchronise from the imap folder cache to the jmap mailbox listing @@ -529,8 +525,6 @@ sub sync_addressbooks { sub sync_jaddressbooks { my $Self = shift; - $Self->begin(); - my $iaddressbooks = $Self->dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks", {Slice => {}}); my $jaddressbooks = $Self->dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks", {Slice => {}}); @@ -570,53 +564,57 @@ sub sync_jaddressbooks { next if $seen{$jid}; $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $jid}); } - $Self->commit(); } -sub do_addressbook { +sub do_addressbooks { my $Self = shift; - my $addressbookid = shift; + my $books = shift; - $Self->begin(); - my ($href, $jaddressbookid) = $Self->dbh->selectrow_array("SELECT href, jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $addressbookid); - $Self->commit(); - my $cards = $Self->backend_cmd('get_cards', $href); - # parse before locking - my %parsed = map { $_ => $Self->parse_card($cards->{$_}) } keys %$cards; + my %allparsed; + foreach my $href (sort values %$books) { + my $cards = $Self->backend_cmd('get_cards', $href); + # parse before locking + my %parsed = map { $_ => $Self->parse_card($cards->{$_}) } keys %$cards; + $allparsed{$href} = \%parsed; + } $Self->begin(); - my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $addressbookid); - my %res = map { $_->{resource} => $_ } @$exists; - - foreach my $resource (keys %$cards) { - my $data = delete $res{$resource}; - my $raw = $cards->{$resource}; - my $card = $parsed{$resource}; - my $uid = $card->{uid}; - my $kind = $card->{kind}; - my $item = { - iaddressbookid => $addressbookid, - resource => $resource, - uid => $uid, - kind => $kind, - content => $raw, - }; - if ($data) { - my $id = $data->{icardid}; - $Self->dmaybeupdate('icards', $item, {icardid => $id}); - } - else { - $Self->dinsert('icards', $item); + foreach my $id (keys %$books) { + my $href = $books->{$id}; + my ($jaddresbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); + my %res = map { $_->{resource} => $_ } @$exists; + + foreach my $resource (keys %$cards) { + my $data = delete $res{$resource}; + my $raw = $cards->{$resource}; + my $card = $allparsed{$href}{$resource}; + my $uid = $card->{uid}; + my $kind = $card->{kind}; + my $item = { + iaddressbookid => $id, + resource => $resource, + uid => $uid, + kind => $kind, + content => $raw, + }; + if ($data) { + my $id = $data->{icardid}; + $Self->dmaybeupdate('icards', $item, {icardid => $id}); + } + else { + $Self->dinsert('icards', $item); + } + $Self->set_card($jaddressbookid, $card); } - $Self->set_card($jaddressbookid, $card); - } - foreach my $resource (keys %res) { - my $data = delete $res{$resource}; - my $id = $data->{icardid}; - $Self->ddelete('icards', {icardid => $id}); - $Self->delete_card($jaddressbookid, $data->{uid}, $data->{kind}); + foreach my $resource (keys %res) { + my $data = delete $res{$resource}; + my $cid = $data->{icardid}; + $Self->ddelete('icards', {icardid => $cid}); + $Self->delete_card($jaddressbookid, $data->{uid}, $data->{kind}); + } } $Self->commit(); From 6e4a565ec78b9f8cca911fca203c20adddb1a5b6 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 21:28:42 +1000 Subject: [PATCH 278/331] Deletion needs to set modseqs on things --- JMAP/ImapDB.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d2bbde6..ab4de74 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -403,7 +403,7 @@ sub sync_jcalendars { mayRename => 1, }; if ($calendar->[3] && $jbyid{$calendar->[3]}) { - $Self->dmaybeupdate('jcalendars', $data, {jcalendarid => $calendar->[3]}); + $Self->dmaybedirty('jcalendars', $data, {jcalendarid => $calendar->[3]}); $seen{$calendar->[3]} = 1; } else { @@ -416,7 +416,8 @@ sub sync_jcalendars { foreach my $calendar (@$jcalendars) { my $id = $calendar->[0]; next if $seen{$id}; - $Self->dupdate('jcalendars', {active => 0}, {jcalendarid => $id}); + $Self->dmaybedirty('jcalendars', {active => 0}, {jcalendarid => $id}); + $Self->dmaybedirty('jevents', {active => 0}, {jcalendarid => $id}); } } @@ -549,7 +550,7 @@ sub sync_jaddressbooks { }; my $jid = $addressbook->{jaddressbookid}; if ($jid && $jbyid{$jid}) { - $Self->dmaybeupdate('jaddressbooks', $data, {jaddressbookid => $jid}); + $Self->dmaybedirty('jaddressbooks', $data, {jaddressbookid => $jid}); $seen{$jid} = 1; } else { @@ -562,7 +563,9 @@ sub sync_jaddressbooks { foreach my $addressbook (@$jaddressbooks) { my $jid = $addressbook->{jaddressbookid}; next if $seen{$jid}; - $Self->dupdate('jaddressbooks', {active => 0}, {jaddressbookid => $jid}); + $Self->dmaybedirty('jaddressbooks', {active => 0}, {jaddressbookid => $jid}); + $Self->dmaybedirty('jcontactgroups', {active => 0}, {jaddressbookid => $jid}); + $Self->dmaybedirty('jcontacts', {active => 0}, {jaddressbookid => $jid}); } } From eef91f9f80700aa811acb090c233342e67d2503d Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 02:50:40 -0400 Subject: [PATCH 279/331] fixes for DB and [Gmail] magic folders --- JMAP/API.pm | 6 ++++- JMAP/ImapDB.pm | 67 +++++++++++++++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index c82bb80..974d1b0 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -398,13 +398,17 @@ sub _load_mailbox { my $Self = shift; my $id = shift; + $Self->begin(); my $data = $Self->{db}->dbh->selectall_arrayref("SELECT msgid,jmodseq,active FROM jmessagemap WHERE jmailboxid = ?", {}, $id); + $Self->commit(); return { map { $_->[0] => $_ } @$data }; } sub _load_hasatt { my $Self = shift; + $Self->begin(); my $data = $Self->{db}->dbh->selectcol_arrayref("SELECT msgid FROM jrawmessage WHERE hasAttachment = 1"); + $Self->commit(); return { map { $_ => 1 } @$data }; } @@ -954,7 +958,7 @@ sub getMessages { # need to load messages from the server if ($need_content) { - my $content = $Self->{db}->fill_messages('interactive', map { $_->{id} } @list); + my $content = $Self->{db}->fill_messages(map { $_->{id} } @list); foreach my $item (@list) { my $data = $content->{$item->{id}}; foreach my $prop (qw(preview textBody htmlBody)) { diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ab4de74..828eb57 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -87,7 +87,10 @@ sub setuser { sub access_token { my $Self = shift; + + $Self->begin(); my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT imapHost, username, password FROM iserver"); + $Self->commit(); return [$hostname, $username, $password]; } @@ -95,7 +98,9 @@ sub access_token { sub access_data { my $Self = shift; + $Self->begin(); my $config = $Self->dbh->selectall_arrayref("SELECT * FROM iserver", {Slice => {}}); + $Self->commit(); return $config->[0]; } @@ -203,11 +208,13 @@ sub sync_jmailboxes { my %seen; foreach my $folder (@$ifolders) { + next if lc $folder->[3] eq "\\allmail"; # we don't show this folder my $fname = $folder->[2]; # check for roles first my @bits = split "[$folder->[1]]", $fname; - shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); # otehrwise we get none... - shift @bits if $bits[0] eq '[Gmail]'; + shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); really we should be stripping the actual prefix, if any + shift @bits if $bits[0] eq '[Gmail]'; # we special case this GMail magic + next unless @bits; # also skip the magic '[Gmail]' top-level my $role = $ROLE_MAP{lc $folder->[3]}; my $id = 0; my $parentId = 0; @@ -1214,6 +1221,8 @@ sub fill_messages { my $Self = shift; my @ids = @_; + $Self->begin(); + my $data = $Self->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage WHERE msgid IN (" . join(', ', map { "?" } @ids) . ")", {}, @ids); my %result; foreach my $line (@$data) { @@ -1221,24 +1230,38 @@ sub fill_messages { } my @need = grep { not $result{$_} } @ids; - return \%result unless @need; - - my $uids = $Self->dbh->selectall_arrayref("SELECT ifolderid, uid, msgid FROM imessages WHERE msgid IN (" . join(', ', map { "?" } @need) . ")", {}, @need); my %udata; - foreach my $row (@$uids) { - $udata{$row->[0]}{$row->[1]} = $row->[2]; + if (@need) { + my $uids = $Self->dbh->selectall_arrayref("SELECT ifolderid, uid, msgid FROM imessages WHERE msgid IN (" . join(', ', map { "?" } @need) . ")", {}, @need); + foreach my $row (@$uids) { + $udata{$row->[0]}{$row->[1]} = $row->[2]; + } } + my %foldermap; foreach my $ifolderid (sort keys %udata) { my $uhash = $udata{$ifolderid}; my $uids = join(',', sort { $a <=> $b } grep { not $result{$uhash->{$_}} } keys %$uhash); next unless $uids; - my ($imapname, $uidvalidity) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); next unless $imapname; + $foldermap{$ifolderid} = [$imapname, $uidvalidity]; + } + + # drop out of transaction to actually fetch the data + $Self->commit(); + my %parsed; + foreach my $ifolderid (sort keys %udata) { + my $uhash = $udata{$ifolderid}; + my $uids = join(',', sort { $a <=> $b } grep { not $result{$uhash->{$_}} } keys %$uhash); + next unless $uids; + + my $data = $foldermap{$ifolderid}; + next unless $data; + + my ($imapname, $uidvalidity) = @$data; my $res = $Self->backend_cmd('imap_fill', $imapname, $uidvalidity, $uids); - my %parsed; foreach my $uid (keys %{$res->{data}}) { my $rfc822 = $res->{data}{$uid}; next unless $rfc822; @@ -1247,19 +1270,19 @@ sub fill_messages { my $eml = Email::MIME->new($rfc822); $parsed{$msgid} = $Self->parse_message($msgid, $eml); } + } - $Self->begin(); - foreach my $msgid (sort keys %parsed) { - my $message = $parsed{$msgid}; - $Self->dinsert('jrawmessage', { - msgid => $msgid, - parsed => encode_json($message), - hasAttachment => $message->{hasAttachment}, - }); - $result{$msgid} = $parsed{$msgid}; - } - $Self->commit(); + $Self->begin(); + foreach my $msgid (sort keys %parsed) { + my $message = $parsed{$msgid}; + $Self->dinsert('jrawmessage', { + msgid => $msgid, + parsed => encode_json($message), + hasAttachment => $message->{hasAttachment}, + }); + $result{$msgid} = $parsed{$msgid}; } + $Self->commit(); # XXX - handle not getting data that we need? my @stillneed = grep { not $result{$_} } @ids; @@ -1376,7 +1399,7 @@ sub update_mailboxes { my ($oldname) = $Self->dbh->selectrow_array("SELECT imapname FROM ifolders WHERE jmailboxid = ?", {}, $id); - $namemap{$oldname} = $newname; + $namemap{$oldname} = $imapname; push @updated, $id; } @@ -1384,7 +1407,7 @@ sub update_mailboxes { $Self->commit(); foreach my $oldname (sort keys %namemap) { - my $newname = $namemap{$oldname}; + my $imapname = $namemap{$oldname}; $Self->backend_cmd('rename_mailbox', $oldname, $imapname) if $oldname ne $imapname; } From 2b1571da3c26c03eda6a54d8f60688260a271edb Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 06:35:24 -0400 Subject: [PATCH 280/331] fix comment --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 828eb57..6c7efd1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -212,7 +212,7 @@ sub sync_jmailboxes { my $fname = $folder->[2]; # check for roles first my @bits = split "[$folder->[1]]", $fname; - shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); really we should be stripping the actual prefix, if any + shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); # really we should be stripping the actual prefix, if any shift @bits if $bits[0] eq '[Gmail]'; # we special case this GMail magic next unless @bits; # also skip the magic '[Gmail]' top-level my $role = $ROLE_MAP{lc $folder->[3]}; From 901431ea3bd619f86c6103e951fdc31d88c779e7 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 07:47:25 -0400 Subject: [PATCH 281/331] fix all the bits --- JMAP/ImapDB.pm | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 6c7efd1..223a027 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -433,24 +433,26 @@ sub do_calendars { my $cals = shift; my %allparsed; + my %allevents; foreach my $href (sort values %$cals) { my $events = $Self->backend_cmd('get_events', $href); # parse events before we lock my %parsed = map { $_ => $Self->parse_event($events->{$_}) } keys %$events; $allparsed{$href} = \%parsed; + $allevents{$href} = $events; } $Self->begin(); foreach my $id (keys %$cals) { my $href = $cals->{$id}; - my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $calendarid); + my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $id); my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; - foreach my $resource (keys %$events) { + foreach my $resource (keys %{$allparsed{$href}}) { my $data = delete $res{$resource}; - my $raw = $events->{$resource}; - my $event = $parsed{$resource}; + my $raw = $allevents{$href}{$resource}; + my $event = $allparsed{$href}{$resource}; my $uid = $event->{uid}; my $item = { icalendarid => $id, @@ -580,25 +582,27 @@ sub do_addressbooks { my $Self = shift; my $books = shift; + my %allcards; my %allparsed; foreach my $href (sort values %$books) { my $cards = $Self->backend_cmd('get_cards', $href); # parse before locking my %parsed = map { $_ => $Self->parse_card($cards->{$_}) } keys %$cards; $allparsed{$href} = \%parsed; + $allcards{$href} = $cards; } $Self->begin(); foreach my $id (keys %$books) { my $href = $books->{$id}; - my ($jaddresbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + my ($jaddressbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; - foreach my $resource (keys %$cards) { + foreach my $resource (keys %{$allparsed{$href}}) { my $data = delete $res{$resource}; - my $raw = $cards->{$resource}; + my $raw = $allcards{$href}{$resource}; my $card = $allparsed{$href}{$resource}; my $uid = $card->{uid}; my $kind = $card->{kind}; @@ -610,8 +614,8 @@ sub do_addressbooks { content => $raw, }; if ($data) { - my $id = $data->{icardid}; - $Self->dmaybeupdate('icards', $item, {icardid => $id}); + my $cid = $data->{icardid}; + $Self->dmaybeupdate('icards', $item, {icardid => $cid}); } else { $Self->dinsert('icards', $item); From c0308cb7ce5a229068d4254d6caee1edd652b78a Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 07:53:24 -0400 Subject: [PATCH 282/331] needs a transaction for folder list too --- JMAP/ImapDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 223a027..d0d9940 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -691,7 +691,9 @@ sub firstsync { $Self->sync_folders(); + $Self->begin(); my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + $Self->commit(); if ($Self->{is_gmail}) { my ($folder) = grep { lc $_->{label} eq '\\allmail' } @$data; From b95309e8faed98aad670707ed64bfffd836054be Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 08:02:24 -0400 Subject: [PATCH 283/331] GmailDB - access_token transaction --- JMAP/GmailDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 2f7a64d..9945321 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -30,7 +30,9 @@ sub new { sub access_token { my $Self = shift; + $Self->begin(); my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT imapHost, username, password FROM iserver"); + $Self->commit(); my $O = JMAP::Sync::Gmail::O(); my $data = $O->refresh($password); From 510ac444e4e4924e74873226a350b484c1f20614 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 08:14:57 -0400 Subject: [PATCH 284/331] backfill needs a transaction --- JMAP/ImapDB.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d0d9940..9ac6785 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -667,10 +667,13 @@ sub sync_imap { sub backfill { my $Self = shift; + + $Self->begin(); my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders WHERE uidnext > 1 AND uidfirst > 1 ORDER BY mtime", {Slice => {}}); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } + $Self->commit(); return unless @$data; From 23c29cca5fd2f87a4f5f1aeb7d2645573a199f42 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 22:26:17 +1000 Subject: [PATCH 285/331] transactional fix for create too --- JMAP/DB.pm | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 4d3d879..ca68259 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -636,15 +636,16 @@ sub create_messages { my %created; my %notCreated; - my $dbh = $Self->dbh(); + $Self->begin(); # XXX - get draft mailbox ID - my ($draftid) = $dbh->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = ?", {}, "drafts"); + my ($draftid) = $Self->dbh->selectrow_array("SELECT jmailboxid FROM jmailboxes WHERE role = ?", {}, "drafts"); + my %todo; foreach my $cid (keys %$args) { my $item = $args->{$cid}; if ($item->{inReplyToMessageId}) { - my ($replymessageid) = $dbh->selectrow_array("SELECT msgmessageid FROM jmessages WHERE msgid = ?", {}, $item->{inReplyToMessageId}); + my ($replymessageid) = $Self->dbh->selectrow_array("SELECT msgmessageid FROM jmessages WHERE msgid = ?", {}, $item->{inReplyToMessageId}); unless ($replymessageid) { $notCreated{$cid} = 'inReplyToNotFound'; next; @@ -657,6 +658,13 @@ sub create_messages { $item->{msgmessageid} = new_uuid_string() . "\@$ENV{jmaphost}"; my $message = $Self->_makemsg($item); # XXX - let's just assume goodness for now - lots of error handling to add + $todo{$cid} = $message; + } + + $Self->commit(); + + foreach my $cid (keys %todo) { + my $message = $todo{$cid}; my ($msgid, $thrid) = $Self->import_message($message, [$draftid], isUnread => 0, isAnswered => 0, From adcb40319d9be324f760367f65a750c4d9d51a51 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 22:38:08 +1000 Subject: [PATCH 286/331] protect against transactions if nothing to do --- JMAP/DB.pm | 2 ++ JMAP/ImapDB.pm | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index ca68259..03c6e76 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -636,6 +636,8 @@ sub create_messages { my %created; my %notCreated; + return ({}, {}) unless %$args; + $Self->begin(); # XXX - get draft mailbox ID diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9ac6785..b935052 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -974,6 +974,8 @@ sub update_messages { my $changes = shift; my $idmap = shift; + return ([], {}) unless %$changes; + $Self->begin(); my %updatemap; @@ -1085,6 +1087,8 @@ sub destroy_messages { my $Self = shift; my $ids = shift; + return ([], {}) unless @$ids; + $Self->begin(); my %destroymap; my %notdestroyed; @@ -1230,6 +1234,8 @@ sub fill_messages { my $Self = shift; my @ids = @_; + return {} unless @ids; + $Self->begin(); my $data = $Self->dbh->selectall_arrayref("SELECT msgid, parsed FROM jrawmessage WHERE msgid IN (" . join(', ', map { "?" } @ids) . ")", {}, @ids); @@ -1260,6 +1266,8 @@ sub fill_messages { # drop out of transaction to actually fetch the data $Self->commit(); + return \%result unless keys %udata; + my %parsed; foreach my $ifolderid (sort keys %udata) { my $uhash = $udata{$ifolderid}; @@ -1334,6 +1342,8 @@ sub create_mailboxes { my $Self = shift; my $new = shift; + return ({}, {}) unless keys %$new; + $Self->begin(); my %idmap; my %notcreated; @@ -1381,6 +1391,8 @@ sub update_mailboxes { my $update = shift; my $idmap = shift; + return ([], {}) unless %$update; + $Self->begin(); my @updated; @@ -1429,6 +1441,8 @@ sub destroy_mailboxes { my $Self = shift; my $destroy = shift; + return ([], {}) unless @$destroy; + $Self->begin(); my @destroyed; @@ -1457,6 +1471,8 @@ sub create_calendar_events { my $Self = shift; my $new = shift; + return ({}, {}) unless keys %$new; + $Self->begin(); my %todo; @@ -1490,6 +1506,8 @@ sub update_calendar_events { my $update = shift; my $idmap = shift; + return ([], {}) unless %$update; + $Self->begin(); my %todo; @@ -1521,6 +1539,8 @@ sub destroy_calendar_events { my $Self = shift; my $destroy = shift; + return ([], {}) unless @$destroy; + $Self->begin(); my %todo; @@ -1551,6 +1571,8 @@ sub create_contact_groups { my $Self = shift; my $new = shift; + return ({}, {}) unless keys %$new; + $Self->begin(); my %todo; @@ -1593,6 +1615,8 @@ sub update_contact_groups { my $changes = shift; my $idmap = shift; + return ([], {}) unless %$changes; + $Self->begin(); my %todo; @@ -1630,6 +1654,8 @@ sub destroy_contact_groups { my $Self = shift; my $destroy = shift; + return ([], {}) unless @$destroy; + $Self->begin(); my %todo; @@ -1658,6 +1684,8 @@ sub create_contacts { my $Self = shift; my $new = shift; + return ({}, {}) unless keys %$new; + $Self->begin(); my %createmap; @@ -1700,6 +1728,8 @@ sub update_contacts { my $changes = shift; my $idmap = shift; + return ([], {}) unless %$changes; + $Self->begin(); my %todo; @@ -1746,6 +1776,8 @@ sub destroy_contacts { my $Self = shift; my $destroy = shift; + return ([], {}) unless @$destroy; + $Self->begin(); my %todo; From d2208c26d2fd5f7db9d86a38ad869929f5fe9c81 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 08:44:54 -0400 Subject: [PATCH 287/331] more maybe --- JMAP/DB.pm | 13 +++++-------- JMAP/ImapDB.pm | 8 ++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 03c6e76..d3a8924 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -487,7 +487,7 @@ sub delete_message_from_mailbox { my ($msgid, $jmailboxid) = @_; my $data = {active => 0}; - $Self->ddirty('jmessagemap', $data, {msgid => $msgid, jmailboxid => $jmailboxid}); + $Self->dmaybedirty('jmessagemap', $data, {msgid => $msgid, jmailboxid => $jmailboxid}); $Self->dmaybeupdate('jmailboxes', {jcountsmodseq => $data->{jmodseq}}, {jmailboxid => $jmailboxid}); } @@ -495,10 +495,7 @@ sub change_message { my $Self = shift; my ($msgid, $data, $newids) = @_; - # doesn't work if only IDs have changed :( - #return unless $Self->dmaybedirty('jmessages', $data, {msgid => $msgid}); - - $Self->ddirty('jmessages', $data, {msgid => $msgid}); + $Self->dmaybedirty('jmessages', $data, {msgid => $msgid}); my $oldids = $Self->dbh->selectcol_arrayref("SELECT jmailboxid FROM jmessagemap WHERE msgid = ? AND active = 1", {}, $msgid); my %old = map { $_ => 1 } @$oldids; @@ -735,7 +732,7 @@ sub delete_event { my $Self = shift; my $jcalendarid = shift; # doesn't matter my $eventuid = shift; - return $Self->dupdate('jevents', {active => 0}, {eventuid => $eventuid}); + return $Self->dmaybedirty('jevents', {active => 0}, {eventuid => $eventuid}); } sub parse_card { @@ -808,10 +805,10 @@ sub delete_card { my $carduid = shift; my $kind = shift; if ($kind eq 'contact') { - $Self->dupdate('jcontacts', {active => 0}, {contactuid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dmaybedirty('jcontacts', {active => 0}, {contactuid => $carduid, jaddressbookid => $jaddressbookid}); } else { - $Self->dupdate('jcontactgroups', {active => 0}, {groupuid => $carduid, jaddressbookid => $jaddressbookid}); + $Self->dmaybedirty('jcontactgroups', {active => 0}, {groupuid => $carduid, jaddressbookid => $jaddressbookid}); } } diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b935052..d4e05b9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -253,18 +253,18 @@ sub sync_jmailboxes { if ($role and $roletoid{$role} and $roletoid{$role} != $id) { # still gotta move it $id = $roletoid{$role}; - $Self->ddirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); + $Self->dmaybedirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); } elsif (not $folder->[4]) { # reactivate! - $Self->ddirty('jmailboxes', {active => 1}, {jmailboxid => $id}); + $Self->dmaybedirty('jmailboxes', {active => 1}, {jmailboxid => $id}); } } else { # case: role - we need to see if there's a case for moving this thing if ($role and $roletoid{$role}) { $id = $roletoid{$role}; - $Self->ddirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); + $Self->dmaybedirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); } else { $id = $Self->dmake('jmailboxes', {role => $role, %details}); @@ -1133,7 +1133,7 @@ sub deleted_record { $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - $Self->ddirty('jmessages', {}, {msgid => $msgid}); # bump modeseq + $Self->dmaybedirty('jmessages', {}, {msgid => $msgid}); # bump modeseq $Self->delete_message_from_mailbox($msgid, $jmailboxid); } From 1a1b3fe2062899db842f48d7841eaa63e8e18283 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 22:55:14 +1000 Subject: [PATCH 288/331] sort json encoders --- JMAP/DB.pm | 10 ++++++---- JMAP/GmailDB.pm | 1 - JMAP/ImapDB.pm | 18 ++++++++++-------- JMAP/Sync/Common.pm | 1 - JMAP/Sync/Gmail.pm | 2 +- JMAP/Sync/Standard.pm | 1 - bin/apiendpoint.pl | 2 +- bin/server.pl | 8 +++++--- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index d3a8924..3760bb3 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -11,7 +11,7 @@ use Carp qw(confess); use Data::UUID::LibUUID; use IO::LockedFile; -use JSON::XS qw(encode_json decode_json); +use JSON::XS qw(decode_json); use Email::MIME; # seriously, it's parsable, get over it $Email::MIME::ContentType::STRICT_PARAMS = 0; @@ -26,6 +26,8 @@ use Net::CalDAVTalk; use Net::CardDAVTalk::VCard; use MIME::Base64 qw(encode_base64 decode_base64); +my $json = JSON::XS->new->utf8->canonical(); + my %TABLE2GROUPS = ( jmessages => ['Message', 'Thread'], jmailboxes => ['Mailbox'], @@ -570,7 +572,7 @@ sub _makemsg { my @attachments = $args->{attachments} ? @{$args->{attachments}} : (); if (@attachments and not $isDraft) { - my $encoded = encode_base64(encode_json(\@attachments), ''); + my $encoded = encode_base64($json->encode(\@attachments), ''); push @$header, "X-JMAP-Draft-Attachments" => $encoded; @attachments = (); } @@ -724,7 +726,7 @@ sub set_event { $Self->dmake('jevents', { eventuid => $eventuid, jcalendarid => $jcalendarid, - payload => encode_json($event), + payload => $json->encode($event), }); } @@ -780,7 +782,7 @@ sub set_card { $Self->dmake('jcontacts', { contactuid => $carduid, jaddressbookid => $jaddressbookid, - payload => encode_json($card), + payload => $json->encode($card), }); } else { diff --git a/JMAP/GmailDB.pm b/JMAP/GmailDB.pm index 9945321..8088a80 100644 --- a/JMAP/GmailDB.pm +++ b/JMAP/GmailDB.pm @@ -9,7 +9,6 @@ use base qw(JMAP::ImapDB); use DBI; use Date::Parse; -use JSON::XS qw(encode_json decode_json); use Data::UUID::LibUUID; use OAuth2::Tiny; use Encode; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index d4e05b9..7da0102 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -8,7 +8,7 @@ use base qw(JMAP::DB); use DBI; use Date::Parse; -use JSON::XS qw(encode_json decode_json); +use JSON::XS qw(decode_json); use Data::UUID::LibUUID; use OAuth2::Tiny; use Digest::SHA qw(sha1_hex); @@ -21,6 +21,8 @@ use Data::Dumper; use JMAP::Sync::Gmail; use JMAP::Sync::Standard; +my $json = JSON::XS->new->utf8->canonical(); + our $TAG = 1; # special use or name magic @@ -900,8 +902,8 @@ sub changed_record { my $Self = shift; my ($folder, $uid, $flaglist, $labellist) = @_; - my $flags = encode_json([grep { lc $_ ne '\\recent' } sort @$flaglist]); - my $labels = encode_json([sort @$labellist]); + my $flags = $json->encode([grep { lc $_ ne '\\recent' } sort @$flaglist]); + my $labels = $json->encode([sort @$labellist]); my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); @@ -961,7 +963,7 @@ sub import_message { $Self->begin(); $Self->dinsert('jrawmessage', { msgid => $msgid, - parsed => encode_json($message), + parsed => $json->encode($message), hasAttachment => $message->{hasAttachment}, }); $Self->commit(); @@ -1141,8 +1143,8 @@ sub new_record { my $Self = shift; my ($ifolderid, $uid, $flaglist, $labellist, $envelope, $internaldate, $msgid, $thrid, $size) = @_; - my $flags = encode_json([grep { lc $_ ne '\\recent' } sort @$flaglist]); - my $labels = encode_json([sort @$labellist]); + my $flags = $json->encode([grep { lc $_ ne '\\recent' } sort @$flaglist]); + my $labels = $json->encode([sort @$labellist]); my $data = { ifolderid => $ifolderid, @@ -1152,7 +1154,7 @@ sub new_record { internaldate => $internaldate, msgid => $msgid, thrid => $thrid, - envelope => encode_json($envelope), + envelope => $json->encode($envelope), size => $size, }; @@ -1294,7 +1296,7 @@ sub fill_messages { my $message = $parsed{$msgid}; $Self->dinsert('jrawmessage', { msgid => $msgid, - parsed => encode_json($message), + parsed => $json->encode($message), hasAttachment => $message->{hasAttachment}, }); $result{$msgid} = $parsed{$msgid}; diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 11da0ec..6ccd19d 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -6,7 +6,6 @@ use warnings; package JMAP::Sync::Common; use Mail::IMAPTalk; -use JSON::XS qw(encode_json decode_json); use Email::Simple; use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::SMTPS; diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index 2d1749c..b9f349b 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -7,7 +7,7 @@ package JMAP::Sync::Gmail; use base qw(JMAP::Sync::Common); use Mail::GmailTalk; -use JSON::XS qw(encode_json decode_json); +use JSON::XS qw(decode_json); use Email::Simple; use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::GmailSMTP; diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index eebf3b8..8bf3793 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -7,7 +7,6 @@ package JMAP::Sync::Standard; use base qw(JMAP::Sync::Common); use Mail::IMAPTalk; -use JSON::XS qw(encode_json decode_json); use Email::Simple; use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::SMTPS; diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 3e694ae..8a61eb7 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -28,7 +28,7 @@ package JMAP::Backend; use AnyEvent::Util; use AnyEvent::HTTP; use EV; -use JSON::XS qw(encode_json decode_json); +use JSON::XS qw(decode_json); use Net::DNS; use Net::DNS::Resolver; diff --git a/bin/server.pl b/bin/server.pl index de5d7cf..cb4fe45 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -19,11 +19,13 @@ use AnyEvent::Util; use AnyEvent::HTTP; use JMAP::Sync::Gmail; -use JSON::XS qw(encode_json decode_json); +use JSON::XS qw(decode_json); use Encode qw(encode_utf8); use Template; my $TT = Template->new(INCLUDE_PATH => '/home/jmap/jmap-perl/htdocs'); +my $json = JSON::XS->new->utf8->canonical(); + sub mkerr { my $req = shift; return sub { @@ -567,8 +569,8 @@ sub prod_idler { sub PushToHandle { my $Handle = shift; my %vals = @_; - print "PUSH EVENT " . encode_json(\%vals) . "\n"; - my @Lines = map { "$_: " . (ref($vals{$_}) ? encode_json($vals{$_}) : $vals{$_}) } keys %vals; + print "PUSH EVENT " . $json->encode(\%vals) . "\n"; + my @Lines = map { "$_: " . (ref($vals{$_}) ? $json->encode($vals{$_}) : $vals{$_}) } keys %vals; $Handle->push_write(join("\r\n", @Lines) . "\r\n\r\n"); } From b730273ec097ee7be9125d78fe7de998e57cffa2 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 28 Jul 2015 23:16:14 +1000 Subject: [PATCH 289/331] compare cards and don't update if unchanged --- JMAP/ImapDB.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 7da0102..c30a3d9 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -448,7 +448,7 @@ sub do_calendars { foreach my $id (keys %$cals) { my $href = $cals->{$id}; my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); + my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, raw, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -464,6 +464,7 @@ sub do_calendars { }; if ($data) { my $eid = $data->{ieventid}; + next if $raw eq $data->{raw}; $Self->dmaybeupdate('ievents', $item, {ieventid => $eid}); } else { @@ -599,7 +600,7 @@ sub do_addressbooks { foreach my $id (keys %$books) { my $href = $books->{$id}; my ($jaddressbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); + my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, raw, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -617,6 +618,7 @@ sub do_addressbooks { }; if ($data) { my $cid = $data->{icardid}; + next if $raw eq $data->{raw}; $Self->dmaybeupdate('icards', $item, {icardid => $cid}); } else { From d06cc4e9cb69395ea2c5efc26f7c69788af1a385 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 08:43:45 +1000 Subject: [PATCH 290/331] DB: remove stale unused early API --- JMAP/DB.pm | 66 ------------------------------------------------------ 1 file changed, 66 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 3760bb3..cc12582 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -166,72 +166,6 @@ sub get_user { return $Self->{t}{user}; } -sub get_mailboxes { - my $Self = shift; - unless ($Self->{t}{mailboxes}) { - $Self->{t}{mailboxes} = $Self->dbh->selectall_hashref("SELECT jmailboxid, jmodseq, label, name, parentId, nummessages, numumessages, numthreads, numuthreads, active FROM jmailboxes", 'jmailboxid', {Slice => {}}); - } - return $Self->{t}{mailboxes}; -} - -sub get_mailbox { - my $Self = shift; - my $jmailboxid = shift; - return $Self->get_mailboxes->{$jmailboxid}; -} - -sub add_mailbox { - my $Self = shift; - my ($name, $label, $parentId) = @_; - - my $mailboxes = $Self->get_mailboxes(); - - confess("ALREADY EXISTS $name in $parentId") if grep { $_->{name} eq $name and $_->{parentId} == $parentId } values %$mailboxes; - - my $data = { - name => $name, - label => $label, - parentId => $parentId, - nummessages => 0, - numthreads => 0, - numumessages => 0, - numuthreads => 0, - active => 1, - }; - - my $id = $data->{jmailboxid} = $Self->dinsert('jmailboxes', $data); - $Self->{t}{mailboxes}{$id} = $data; - - return $id; -} - -sub update_mailbox { - my $Self = shift; - my $jmailboxid = shift; - my $fields = shift; - - my $mailbox = $Self->get_mailbox($jmailboxid); - confess("INVALID ID $jmailboxid") unless $mailbox; - confess("NOT ACTIVE $jmailboxid") unless $mailbox->{active}; - - foreach my $key (keys %$fields) { - $mailbox->{$key} = $fields->{$key}; - } - - unless ($mailbox->{active}) { - # XXX - sanity check all? - confess("Still messages") if $mailbox->{nummessages}; - } - - $Self->dmaybedirty('jmailboxes', $mailbox, {jmailboxid => $jmailboxid}); -} - -sub delete_mailbox { - my $Self = shift; - my $jmailboxid = shift; - return $Self->update_mailbox($jmailboxid, {active => 0}); -} - sub add_message { my $Self = shift; my ($data, $mailboxes) = @_; From 75eeff3e2f000d0654ad93d6938cf05cfe4cf88e Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 08:56:16 +1000 Subject: [PATCH 291/331] use the correct state variables everywhere - don't bump stuff that isn't changed --- JMAP/API.pm | 64 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 974d1b0..e1b608e 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -166,7 +166,7 @@ sub getMailboxes { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateMailbox}"; my $data = $dbh->selectall_arrayref("SELECT * FROM jmailboxes WHERE active = 1", {Slice => {}}); @@ -276,7 +276,7 @@ sub getMailboxUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateMailbox}"; my $sinceState = $args->{sinceState}; return $Self->_transError(['error', {type => 'invalidArguments'}]) @@ -583,6 +583,8 @@ sub getMessageList { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateMailbox}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) if (exists $args->{position} and exists $args->{anchor}); return $Self->_transError(['error', {type => 'invalidArguments'}]) @@ -629,7 +631,7 @@ gotit: filter => $args->{filter}, sort => $args->{sort}, collapseThreads => $args->{collapseThreads}, - state => "$user->{jhighestmodseq}", + state => $newState, canCalculateUpdates => $JSON::true, position => $start, total => scalar(@$data), @@ -666,7 +668,7 @@ sub getMessageListUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateMailbox}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; @@ -799,7 +801,7 @@ sub getMessageListUpdates { sort => $args->{sort}, collapseThreads => $args->{collapseThreads}, oldState => "$args->{sinceState}", - newState => "$user->{jhighestmodseq}", + newState => $newState, removed => \@removed, added => \@added, total => $total, @@ -872,6 +874,8 @@ sub getMessages { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateMessage}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) unless $args->{ids}; #properties: String[] A list of properties to fetch for each message. @@ -1007,7 +1011,7 @@ sub getMessages { return ['messages', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [keys %missingids] : undef), }]; } @@ -1079,7 +1083,7 @@ sub getMessageUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateMessage}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; @@ -1263,6 +1267,8 @@ sub getThreads { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateThread}"; + # XXX - error if no IDs my @list; @@ -1314,7 +1320,7 @@ sub getThreads { push @res, ['threads', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [keys %missingids] : undef), }]; @@ -1341,7 +1347,7 @@ sub getThreadUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateThread}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; @@ -1443,6 +1449,8 @@ sub getCalendars { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateCalendar}"; + my $data = $dbh->selectall_arrayref("SELECT jcalendarid, name, color, isVisible, mayReadFreeBusy, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jcalendars WHERE active = 1"); my %ids; @@ -1485,7 +1493,7 @@ sub getCalendars { return ['calendars', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [map { "$_" } keys %missingids] : undef), }]; } @@ -1502,7 +1510,7 @@ sub getCalendarUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateCalendar}"; my $sinceState = $args->{sinceState}; return $Self->_transError(['error', {type => 'invalidArguments'}]) @@ -1596,6 +1604,8 @@ sub getCalendarEventList { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateCalendarEvent}"; + my $start = $args->{position} || 0; return $Self->_transError(['error', {type => 'invalidArguments'}]) if $start < 0; @@ -1615,7 +1625,7 @@ sub getCalendarEventList { push @res, ['calendarEventList', { accountId => $accountid, filter => $args->{filter}, - state => "$user->{jhighestmodseq}", + state => $newState, position => $start, total => scalar(@$data), calendarEventIds => [map { "$_" } @result], @@ -1643,6 +1653,8 @@ sub getCalendarEvents { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateCalendarEvent}"; + return $Self->_transError(['error', {type => 'invalidArguments'}]) unless $args->{ids}; #properties: String[] A list of properties to fetch for each message. @@ -1676,7 +1688,7 @@ sub getCalendarEvents { return ['calendarEvents', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [keys %missingids] : undef), }]; } @@ -1693,7 +1705,7 @@ sub getCalendarEventUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateCalendarEvent}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; @@ -1754,6 +1766,9 @@ sub getAddressbooks { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + # we have no datatype for this yet + my $newState = "$user->{jhighestmodseq}"; + my $data = $dbh->selectall_arrayref("SELECT jaddressbookid, name, isVisible, mayReadItems, mayAddItems, mayModifyItems, mayRemoveItems, mayDelete, mayRename FROM jaddressbooks WHERE active = 1"); my %ids; @@ -1788,13 +1803,13 @@ sub getAddressbooks { push @list, \%rec; } my %missingids = %ids; - + $Self->commit(); return ['addressbooks', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%missingids ? [map { "$_" } keys %missingids] : undef), }]; } @@ -1811,6 +1826,7 @@ sub getAddressbookUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + # we have no datatype for you yet my $newState = "$user->{jhighestmodseq}"; my $sinceState = $args->{sinceState}; @@ -1905,6 +1921,8 @@ sub getContactList { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateContact}"; + my $start = $args->{position} || 0; return $Self->_transError(['error', {type => 'invalidArguments'}]) if $start < 0; @@ -1924,7 +1942,7 @@ sub getContactList { push @res, ['contactList', { accountId => $accountid, filter => $args->{filter}, - state => "$user->{jhighestmodseq}", + state => $newState, position => $start, total => scalar(@$data), contactIds => [map { "$_" } @result], @@ -1952,6 +1970,8 @@ sub getContacts { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateContact}"; + #properties: String[] A list of properties to fetch for each message. my $data = $dbh->selectall_hashref("SELECT * FROM jcontacts WHERE active = 1", 'contactuid', {Slice => {}}); @@ -1984,7 +2004,7 @@ sub getContacts { return ['contacts', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%want ? [keys %want] : undef), }]; } @@ -2001,7 +2021,7 @@ sub getContactUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateContact}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; @@ -2061,6 +2081,8 @@ sub getContactGroups { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); + my $newState = "$user->{jstateContactGroup}"; + #properties: String[] A list of properties to fetch for each message. my $data = $dbh->selectall_hashref("SELECT * FROM jcontactgroups WHERE active = 1", 'groupuid', {Slice => {}}); @@ -2097,7 +2119,7 @@ sub getContactGroups { return ['contactGroups', { list => \@list, accountId => $accountid, - state => "$user->{jhighestmodseq}", + state => $newState, notFound => (%want ? [keys %want] : undef), }]; } @@ -2114,7 +2136,7 @@ sub getContactGroupUpdates { return $Self->_transError(['error', {type => 'accountNotFound'}]) if ($args->{accountId} and $args->{accountId} ne $accountid); - my $newState = "$user->{jhighestmodseq}"; + my $newState = "$user->{jstateContactGroup}"; return $Self->_transError(['error', {type => 'invalidArguments'}]) if not $args->{sinceState}; From d2710124c9aa39f98317f55a7920a8e197091f74 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 19:13:37 -0400 Subject: [PATCH 292/331] correct name for raw field --- JMAP/ImapDB.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c30a3d9..73aecfb 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -448,7 +448,7 @@ sub do_calendars { foreach my $id (keys %$cals) { my $href = $cals->{$id}; my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, raw, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); + my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, content, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -464,7 +464,7 @@ sub do_calendars { }; if ($data) { my $eid = $data->{ieventid}; - next if $raw eq $data->{raw}; + next if $raw eq $data->{content}; $Self->dmaybeupdate('ievents', $item, {ieventid => $eid}); } else { @@ -600,7 +600,7 @@ sub do_addressbooks { foreach my $id (keys %$books) { my $href = $books->{$id}; my ($jaddressbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, raw, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); + my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, content, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -618,7 +618,7 @@ sub do_addressbooks { }; if ($data) { my $cid = $data->{icardid}; - next if $raw eq $data->{raw}; + next if $raw eq $data->{content}; $Self->dmaybeupdate('icards', $item, {icardid => $cid}); } else { From a29c3aec7684cb3713e6a7c38e0690249ce9a72b Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 19:30:40 -0400 Subject: [PATCH 293/331] log filter --- JMAP/DB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index cc12582..b3e8bae 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -843,6 +843,7 @@ sub filter_values { my $sql = "SELECT " . join(', ', @keys) . " FROM $table"; $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; + $Self->log('debug', $sql, _dbl(map { $limit->{$_} } @lkeys)); my $data = $Self->dbh->selectrow_hashref($sql, {}, map { $limit->{$_} } @lkeys); foreach my $key (@keys) { delete $values{$key} if $limit->{$key}; # in the limit, no point setting again From fe01da5c7e4faf504c564cc94bfef3dd697b9b60 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 19:30:59 -0400 Subject: [PATCH 294/331] use utf8 encoding on raw calendar/cards --- JMAP/ImapDB.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 73aecfb..542ed2f 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -460,11 +460,11 @@ sub do_calendars { icalendarid => $id, uid => $uid, resource => $resource, - content => $raw, + content => encode_utf8($raw), }; if ($data) { my $eid = $data->{ieventid}; - next if $raw eq $data->{content}; + next if $raw eq decode_utf8($data->{content}); $Self->dmaybeupdate('ievents', $item, {ieventid => $eid}); } else { @@ -614,11 +614,11 @@ sub do_addressbooks { resource => $resource, uid => $uid, kind => $kind, - content => $raw, + content => encode_utf8($raw), }; if ($data) { my $cid = $data->{icardid}; - next if $raw eq $data->{content}; + next if $raw eq decode_utf8($data->{content}); $Self->dmaybeupdate('icards', $item, {icardid => $cid}); } else { From 4e499bc4f26687d67c1c51d8ac00867a677abf07 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 19:31:10 -0400 Subject: [PATCH 295/331] no maybe about dirty deletes --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 542ed2f..69d7057 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1137,7 +1137,7 @@ sub deleted_record { $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - $Self->dmaybedirty('jmessages', {}, {msgid => $msgid}); # bump modeseq + $Self->ddirty('jmessages', {}, {msgid => $msgid}); # bump modeseq $Self->delete_message_from_mailbox($msgid, $jmailboxid); } From 141f16173764ed54f5161ca26136dd6f54bcbd62 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:18:42 -0400 Subject: [PATCH 296/331] sort by msgdate, not internaldate --- JMAP/API.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index e1b608e..d266f4d 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -373,7 +373,7 @@ sub _build_sort { return 'internaldate desc' unless $sortargs; my %fieldmap = ( id => 'msgid', - date => 'internaldate', + date => 'msgdate', size => 'msgsize', isflagged => 'isFlagged', isunread => 'isUnread', From 73e8d6d8da51f78009c250c472bd2a0578379c86 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:19:56 -0400 Subject: [PATCH 297/331] sort: just msgid if nothing else passed --- JMAP/API.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index d266f4d..72f4609 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -369,8 +369,7 @@ sub setMailboxes { sub _build_sort { my $Self = shift; - my $sortargs = shift; - return 'internaldate desc' unless $sortargs; + my $sortargs = shift || []; my %fieldmap = ( id => 'msgid', date => 'msgdate', From a8dbe6591858a5fc73c7c6c2c1a20f1a15517ec9 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:29:59 -0400 Subject: [PATCH 298/331] dnuke API --- JMAP/DB.pm | 14 ++++++++++++++ JMAP/ImapDB.pm | 23 +++++++++++++---------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index b3e8bae..7b8eddf 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -882,6 +882,20 @@ sub dmaybedirty { return $Self->dupdate($table, $filtered, $limit); } +sub dnuke { + my $Self = shift; + my ($table, $limit) = @_; + + my $modseq = $Self->dirty($table); + + my @lkeys = sort keys %$limit; + my $sql = "UPDATE $table SET active = 0, jmodseq = ? WHERE active = 1 AND " . join(' AND ', map { "$_ = ?" } @lkeys); + + $Self->log('debug', $sql, _dbl($modseq), _dbl(map { $limit->{$_} } @lkeys)); + + $Self->dbh->do($sql, {}, $modseq, map { $limit->{$_} } @lkeys); +} + sub ddelete { my $Self = shift; my ($table, $limit) = @_; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 69d7057..12de7d1 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -360,8 +360,9 @@ sub sync_calendars { if ($id) { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); my $token = $byhref{$calendar->{href}}[5]; - if ($token ne $calendar->{syncToken}) { - $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); + if ($token eq $calendar->{syncToken}) { + $seen{$id} = 1; + next; } } else { @@ -374,7 +375,7 @@ sub sync_calendars { foreach my $calendar (@$icalendars) { my $id = $calendar->[0]; next if $seen{$id}; - $Self->dbh->do("DELETE FROM icalendars WHERE icalendarid = ?", {}, $id); + $Self->ddelete('icalendars', {icalendarid => $id}); } $Self->sync_jcalendars(); @@ -425,8 +426,8 @@ sub sync_jcalendars { foreach my $calendar (@$jcalendars) { my $id = $calendar->[0]; next if $seen{$id}; - $Self->dmaybedirty('jcalendars', {active => 0}, {jcalendarid => $id}); - $Self->dmaybedirty('jevents', {active => 0}, {jcalendarid => $id}); + $Self->dnuke('jcalendars', {jcalendarid => $id}); + $Self->dnuke('jevents', {jcalendarid => $id}); } } @@ -508,9 +509,11 @@ sub sync_addressbooks { syncToken => $addressbook->{syncToken}, }; if ($id) { + $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); my $token = $byhref{$addressbook->{href}}{syncToken}; - if ($token ne $addressbook->{syncToken}) { - $Self->dmaybeupdate('iaddressbooks', $data, {iaddressbookid => $id}); + if ($token eq $addressbook->{syncToken}) { + $seen{$id} = 1; + next; } } else { @@ -575,9 +578,9 @@ sub sync_jaddressbooks { foreach my $addressbook (@$jaddressbooks) { my $jid = $addressbook->{jaddressbookid}; next if $seen{$jid}; - $Self->dmaybedirty('jaddressbooks', {active => 0}, {jaddressbookid => $jid}); - $Self->dmaybedirty('jcontactgroups', {active => 0}, {jaddressbookid => $jid}); - $Self->dmaybedirty('jcontacts', {active => 0}, {jaddressbookid => $jid}); + $Self->dnuke('jaddressbooks', {jaddressbookid => $jid}); + $Self->dnuke('jcontactgroups', {jaddressbookid => $jid}); + $Self->dnuke('jcontacts', {jaddressbookid => $jid}); } } From bf5a259c5c00f1512548262c5eb2474908a12074 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:37:18 -0400 Subject: [PATCH 299/331] changed_record - don't update the jrecord if the irecord is unchanged --- JMAP/ImapDB.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 12de7d1..ef01230 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -910,9 +910,9 @@ sub changed_record { my $flags = $json->encode([grep { lc $_ ne '\\recent' } sort @$flaglist]); my $labels = $json->encode([sort @$labellist]); - my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); + return unless $Self->dmaybeupdate('imessages', {flags => $flags, labels => $labels}, {ifolderid => $folder, uid => $uid}); - $Self->dmaybeupdate('imessages', {flags => $flags, labels => $labels}, {ifolderid => $folder, uid => $uid}); + my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); $Self->apply_data($msgid, $flaglist, $labellist); } From e2ad6675714f34749d35ea2f81452972e93016e8 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:55:48 -0400 Subject: [PATCH 300/331] superlock: don't allow ANY changes to be made or pushed while we're doing an update --- JMAP/API.pm | 24 ++++++++++++++++++++++++ JMAP/DB.pm | 13 ++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 72f4609..8415e45 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -344,6 +344,8 @@ sub setMailboxes { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_mailboxes($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_mailboxes($update, sub { $Self->idmap(shift) }); @@ -351,6 +353,8 @@ sub setMailboxes { $Self->{db}->sync_imap(); + $Self->{db}->end_superlock(); + my @res; push @res, ['mailboxesSet', { accountId => $accountid, @@ -1148,6 +1152,8 @@ sub setMessages { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_messages($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_messages($update, sub { $Self->idmap(shift) }); @@ -1155,6 +1161,8 @@ sub setMessages { $Self->{db}->sync_imap(); + $Self->{db}->end_superlock(); + foreach my $cid (sort keys %$created) { my $msgid = $created->{$cid}{id}; $created->{$cid}{blobId} = "m-$msgid"; @@ -2200,6 +2208,8 @@ sub setContactGroups { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_contact_groups($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_contact_groups($update, sub { $Self->idmap(shift) }); @@ -2207,6 +2217,8 @@ sub setContactGroups { $Self->{db}->sync_addressbooks(); + $Self->{db}->end_superlock(); + my @res; push @res, ['contactGroupsSet', { accountId => $accountid, @@ -2240,6 +2252,8 @@ sub setContacts { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_contacts($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_contacts($update, sub { $Self->idmap(shift) }); @@ -2247,6 +2261,8 @@ sub setContacts { $Self->{db}->sync_addressbooks(); + $Self->{db}->end_superlock(); + my @res; push @res, ['contactsSet', { accountId => $accountid, @@ -2280,6 +2296,8 @@ sub setCalendarEvents { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_calendar_events($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_calendar_events($update, sub { $Self->idmap(shift) }); @@ -2287,6 +2305,8 @@ sub setCalendarEvents { $Self->{db}->sync_calendars(); + $Self->{db}->end_superlock(); + my @res; push @res, ['calendarEventsSet', { accountId => $accountid, @@ -2320,6 +2340,8 @@ sub setCalendars { my $update = $args->{update} || {}; my $destroy = $args->{destroy} || []; + $Self->{db}->begin_superlock(); + my ($created, $notCreated) = $Self->{db}->create_calendars($create); $Self->setid($_, $created->{$_}{id}) for keys %$created; my ($updated, $notUpdated) = $Self->{db}->update_calendars($update, sub { $Self->idmap(shift) }); @@ -2327,6 +2349,8 @@ sub setCalendars { $Self->{db}->sync_calendars(); + $Self->{db}->end_superlock(); + my @res; push @res, ['calendarsSet', { accountId => $accountid, diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 7b8eddf..ced47fc 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -87,13 +87,24 @@ sub in_transaction { return $Self->{t} ? 1 : 0; } +sub begin_superlock { + my $Self = shift; + my $accountid = $Self->accountid(); + $Self->{superlock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); +} + +sub end_superlock { + my $Self = shift; + delete $Self->{superlock}; +} + sub begin { my $Self = shift; confess("ALREADY IN TRANSACTION") if $Self->{t}; my $accountid = $Self->accountid(); $Self->{t} = {dbh => $Self->{dbh}}; # we need this because sqlite locking isn't as robust as you might hope - $Self->{t}{lock} = IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); + $Self->{t}{lock} = $Self->{superlock} || IO::LockedFile->new(">/home/jmap/data/$accountid.lock"); $Self->{t}{dbh}->begin_work(); } From 56046169cdf4ba359f3e8189b53478e5e791b3c9 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:57:09 -0400 Subject: [PATCH 301/331] fix role for \\junk folder --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ef01230..deac9ba 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -52,7 +52,7 @@ my %ROLE_MAP = ( '\\inbox' => 'inbox', '\\trash' => 'trash', '\\sent' => 'sent', - '\\junk' => 'junk', + '\\junk' => 'spam', '\\archive' => 'archive', '\\drafts' => 'drafts', ); From 52fb43af6b87212d81b78c0c08352d0ddb4837fb Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 20:58:22 -0400 Subject: [PATCH 302/331] handle no position on message list --- JMAP/API.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 8415e45..6f4fe19 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -590,8 +590,6 @@ sub getMessageList { return $Self->_transError(['error', {type => 'invalidArguments'}]) if (exists $args->{position} and exists $args->{anchor}); - return $Self->_transError(['error', {type => 'invalidArguments'}]) - if (not exists $args->{position} and not exists $args->{anchor}); return $Self->_transError(['error', {type => 'invalidArguments'}]) if (exists $args->{anchor} and not exists $args->{anchorOffset}); return $Self->_transError(['error', {type => 'invalidArguments'}]) From 091e04596c3f21bbe4d0d97fcaa0cd688b5a20b9 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 21:17:59 -0400 Subject: [PATCH 303/331] transaction on raw --- JMAP/ImapDB.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index deac9ba..6584120 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1331,7 +1331,9 @@ sub get_raw_message { my $msgid = shift; my $part = shift; + $Self->begin(); my ($imapname, $uidvalidity, $uid) = $Self->dbh->selectrow_array("SELECT imapname, uidvalidity, uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid = ?", {}, $msgid); + $Self->commit(); return unless $imapname; my $type = 'message/rfc822'; From 3cd41a9031626b246e1ba993e861df786f383b68 Mon Sep 17 00:00:00 2001 From: jmap Date: Tue, 28 Jul 2015 21:26:14 -0400 Subject: [PATCH 304/331] Yahoo creates bogus sizes --- JMAP/ImapDB.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 6584120..9277cc5 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -729,8 +729,7 @@ sub calcmsgid { my $uid = shift; my $data = shift; my $envelope = $data->{envelope}; - my $json = JSON::XS->new->allow_nonref->canonical; - my $coded = $json->encode([$envelope, $data->{'rfc822.size'}]); + my $coded = $json->encode([$envelope]); my $base = substr(sha1_hex($coded), 0, 9); my $msgid = "m$base"; From 1bee2373f8e6d976e5e838c834a0ee2cf1601bea Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 01:16:43 -0400 Subject: [PATCH 305/331] notchanged: correct error message --- JMAP/ImapDB.pm | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9277cc5..8cbcebc 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -992,7 +992,7 @@ sub update_messages { $updatemap{$ifolderid}{$uid} = $msgid; } else { - $notchanged{$msgid} = "No such message on server"; + $notchanged{$msgid} = {type => 'notFound', description => "No such message on server"}; } } @@ -1015,7 +1015,7 @@ sub update_messages { my $msgid = $updatemap{$ifolderid}{$uid}; my $action = $changes->{$msgid}; unless ($imapname and $uidvalidity) { - $notchanged{$msgid} = "No folder found"; + $notchanged{$msgid} = {type => 'notFound', description => "No folder found"}; next; } if (exists $action->{isUnread}) { @@ -1403,8 +1403,8 @@ sub update_mailboxes { $Self->begin(); - my @updated; - my %notupdated; + my @changed; + my %notchanged; my %namemap; # XXX - reorder the crap out of this if renaming multiple mailboxes due to deep rename foreach my $id (keys %$update) { @@ -1430,7 +1430,7 @@ sub update_mailboxes { $namemap{$oldname} = $imapname; - push @updated, $id; + push @changed, $id; } $Self->commit(); @@ -1440,9 +1440,9 @@ sub update_mailboxes { $Self->backend_cmd('rename_mailbox', $oldname, $imapname) if $oldname ne $imapname; } - $Self->sync_folders() if @updated; + $Self->sync_folders() if @changed; - return (\@updated, \%notupdated); + return (\@changed, \%notchanged); } sub destroy_mailboxes { @@ -1519,19 +1519,19 @@ sub update_calendar_events { $Self->begin(); my %todo; - my @updated; - my %notupdated; + my @changed; + my %notchanged; foreach my $uid (keys %$update) { my $calendar = $update->{$uid}; my ($resource) = $Self->dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); unless ($resource) { - $notupdated{$uid} = "No such event on server"; + $notchanged{$uid} = {type => 'notFound', description => "No such event on server"}; next; } $todo{$resource} = $calendar; - push @updated, $uid; + push @changed, $uid; } $Self->commit(); @@ -1540,7 +1540,7 @@ sub update_calendar_events { $Self->backend_cmd('update_event', $href, $todo{$href}); } - return (\@updated, \%notupdated); + return (\@changed, \%notchanged); } sub destroy_calendar_events { @@ -1628,13 +1628,13 @@ sub update_contact_groups { $Self->begin(); my %todo; - my @updated; + my @changed; my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { - $notchanged{$carduid} = "No such card on server"; + $notchanged{$carduid} = {type => 'notFound', description => "No such card on server"}; next; } my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); @@ -1646,7 +1646,7 @@ sub update_contact_groups { } $todo{$resource} = $card; - push @updated, $carduid; + push @changed, $carduid; } $Self->commit(); @@ -1655,7 +1655,7 @@ sub update_contact_groups { $Self->backend_cmd('update_card', $href, $todo{$href}); } - return (\@updated, \%notchanged); + return (\@changed, \%notchanged); } sub destroy_contact_groups { @@ -1741,13 +1741,13 @@ sub update_contacts { $Self->begin(); my %todo; - my @updated; + my @changed; my %notchanged; foreach my $carduid (keys %$changes) { my $contact = $changes->{$carduid}; my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { - $notchanged{$carduid} = "No such card on server"; + $notchanged{$carduid} = {type => 'notFound', description => "No such card on server"}; next; } my ($card) = Net::CardDAVTalk::VCard->new_fromstring($content); @@ -1768,7 +1768,7 @@ sub update_contacts { $card->VNotes($contact->{notes}) if exists $contact->{notes}; $todo{$resource} = $card; - push @updated, $carduid; + push @changed, $carduid; } $Self->commit(); @@ -1777,7 +1777,7 @@ sub update_contacts { $Self->backend_cmd('update_card', $href, $todo{$href}); } - return (\@updated, \%notchanged); + return (\@changed, \%notchanged); } sub destroy_contacts { From 6612b0adac6d9670f88f5b43b1675125a062fa7e Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 01:55:21 -0400 Subject: [PATCH 306/331] Start debugging --- JMAP/API.pm | 4 +++- JMAP/ImapDB.pm | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 6f4fe19..864a896 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -3,11 +3,13 @@ package JMAP::API; use JMAP::DB; -use JSON; use strict; use warnings; use Encode; use HTML::GenerateUtil qw(escape_html); +use JSON::XS; + +my $json = JSON::XS->new->utf8->canonical(); sub new { my $class = shift; diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 8cbcebc..96cbc1b 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -684,7 +684,8 @@ sub backfill { return unless @$data; - my $rest = 5000; + #DB::enable_profile(); + my $rest = 500; foreach my $row (@$data) { my $id = $row->{ifolderid}; my $label = $row->{label}; @@ -692,6 +693,8 @@ sub backfill { $rest -= $Self->do_folder($id, $label, $rest); last if $rest < 10; } + #DB::disable_profile(); + #exit 0; return 1; } From 97bc51ec4ba3b7299b5e2943d83622144c71d4fe Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 01:55:45 -0400 Subject: [PATCH 307/331] more not$foo error codes --- JMAP/ImapDB.pm | 16 ++++++++-------- bin/server.pl | 2 -- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 96cbc1b..c330787 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1107,7 +1107,7 @@ sub destroy_messages { $destroymap{$ifolderid}{$uid} = $msgid; } else { - $notdestroyed{$msgid} = "No such message on server"; + $notdestroyed{$msgid} = {type => 'notFound', description => "No such message on server"}; } } @@ -1123,7 +1123,7 @@ sub destroy_messages { my $imapname = $foldermap{$ifolderid}[1]; my $uidvalidity = $foldermap{$ifolderid}[2]; unless ($imapname) { - $notdestroyed{$_} = "No folder" for values %{$destroymap{$ifolderid}}; + $notdestroyed{$_} = {type => 'notFound', description => "No folder"} for values %{$destroymap{$ifolderid}}; } my $uids = [sort keys %{$destroymap{$ifolderid}}]; $Self->backend_cmd('imap_move', $imapname, $uidvalidity, $uids, undef); # no destination folder @@ -1493,7 +1493,7 @@ sub create_calendar_events { my $calendar = $new->{$cid}; my ($href) = $Self->dbh->selectrow_array("SELECT href FROM icalendars WHERE icalendarid = ?", {}, $calendar->{calendarId}); unless ($href) { - $notcreated{$cid} = "No such calendar on server"; + $notcreated{$cid} = {type => 'notFound', description => "No such calendar on server"}; next; } my $uid = new_uuid_string(); @@ -1560,7 +1560,7 @@ sub destroy_calendar_events { foreach my $uid (@$destroy) { my ($resource) = $Self->dbh->selectrow_array("SELECT resource FROM ievents WHERE uid = ?", {}, $uid); unless ($resource) { - $notdestroyed{$uid} = "No such event on server"; + $notdestroyed{$uid} = {type => 'notFound', description => "No such event on server"}; next; } @@ -1594,7 +1594,7 @@ sub create_contact_groups { #my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks WHERE iaddressbookid = ?", {}, $contact->{addressbookId}); my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { - $notcreated{$cid} = "No such addressbook on server"; + $notcreated{$cid} = {type => 'notFound', description => "No such addressbook on server"}; next; } my ($card) = Net::CardDAVTalk::VCard->new(); @@ -1675,7 +1675,7 @@ sub destroy_contact_groups { foreach my $carduid (@$destroy) { my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { - $notdestroyed{$carduid} = "No such card on server"; + $notdestroyed{$carduid} = {type => 'notFound', description => "No such card on server"}; next; } $todo{$resource} = 1; @@ -1705,7 +1705,7 @@ sub create_contacts { my $contact = $new->{$cid}; my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks"); unless ($href) { - $notcreated{$cid} = "No such addressbook on server"; + $notcreated{$cid} = {type => 'notFound', description => "No such addressbook on server"}; next; } my ($card) = Net::CardDAVTalk::VCard->new(); @@ -1797,7 +1797,7 @@ sub destroy_contacts { foreach my $carduid (@$destroy) { my ($resource, $content) = $Self->dbh->selectrow_array("SELECT resource, content FROM icards WHERE uid = ?", {}, $carduid); unless ($resource) { - $notdestroyed{$carduid} = "No such card on server"; + $notdestroyed{$carduid} = {type => 'notFound', description => "No such card on server"}; next; } $todo{$resource} = 1; diff --git a/bin/server.pl b/bin/server.pl index cb4fe45..54965d5 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -139,8 +139,6 @@ sub read_idle_line { }); } -my $json = JSON::XS->new->allow_nonref->pretty->canonical; - my $httpd = AnyEvent::HTTPD->new (port => 9000); my %backend; From 5f84298eb1bacf02e3de8a4f86fa0e8e18b273d6 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 01:56:43 -0400 Subject: [PATCH 308/331] turn off some noise --- JMAP/DB.pm | 2 +- bin/apiendpoint.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index ced47fc..fd5cf7e 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -70,7 +70,7 @@ sub log { } else { my ($level, @items) = @_; - #return if $level eq 'debug'; + return if $level eq 'debug'; my $time = time() - $Self->{start}; warn "[$level $time]: @items\n"; } diff --git a/bin/apiendpoint.pl b/bin/apiendpoint.pl index 8a61eb7..4086d53 100644 --- a/bin/apiendpoint.pl +++ b/bin/apiendpoint.pl @@ -3,7 +3,7 @@ use lib '/home/jmap/jmap-perl'; package JMAP::Backend; -use Mail::IMAPTalk qw(:trace); +#use Mail::IMAPTalk qw(:trace); # stuff complains otherwise - twice for luck use IO::Socket::SSL; From f432587e02c86a0688e737ca6731196893d15da3 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 01:57:03 -0400 Subject: [PATCH 309/331] backfill every 10 seconds --- bin/server.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/server.pl b/bin/server.pl index 54965d5..c6e2ed9 100644 --- a/bin/server.pl +++ b/bin/server.pl @@ -538,7 +538,7 @@ sub prod_backfill { $idler{$accountid}{backfilling} = 1; my $timer; - $timer = AnyEvent->timer(after => 60, cb => sub { + $timer = AnyEvent->timer(after => 10, cb => sub { send_backend_request("$accountid:backfill", 'backfill', $accountid, sub { $timer = undef; prod_backfill($accountid, @_); From d69d3cbb6a124e687d20a311f89e1926f564e480 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 02:41:14 -0400 Subject: [PATCH 310/331] Gmail BODY searches --- JMAP/ImapDB.pm | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c330787..c0d85de 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -53,6 +53,7 @@ my %ROLE_MAP = ( '\\trash' => 'trash', '\\sent' => 'sent', '\\junk' => 'spam', + '\\spam' => 'spam', '\\archive' => 'archive', '\\drafts' => 'drafts', ); @@ -880,6 +881,27 @@ sub imap_search { my $Self = shift; my @search = @_; + if ($Self->{is_gmail}) { + if ($search[0] eq 'text') { + @search = ('x-gm-raw', $search[1]); + } + if ($search[0] eq 'from') { + @search = ('x-gm-raw', "from:$search[1]"); + } + if ($search[0] eq 'to') { + @search = ('x-gm-raw', "to:$search[1]"); + } + if ($search[0] eq 'cc') { + @search = ('x-gm-raw', "cc:$search[1]"); + } + if ($search[0] eq 'subject') { + @search = ('x-gm-raw', "subject:$search[1]"); + } + if ($search[0] eq 'body') { + @search = ('x-gm-raw', $search[1]); + } + } + $Self->begin(); my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); if ($Self->{is_gmail}) { From 1df8637bdbab163da127984030adb6b116598238 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 02:46:51 -0400 Subject: [PATCH 311/331] Default values on all states --- JMAP/API.pm | 1 + JMAP/DB.pm | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 864a896..29e6024 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -28,6 +28,7 @@ sub setid { sub idmap { my $Self = shift; my $key = shift; + return unless $key; my $val = exists $Self->{idmap}{$key} ? $Self->{idmap}{$key} : $key; return $val; } diff --git a/JMAP/DB.pm b/JMAP/DB.pm index fd5cf7e..e202a7d 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -993,15 +993,15 @@ CREATE TABLE IF NOT EXISTS account ( email TEXT, displayname TEXT, picture TEXT, - jdeletedmodseq INTEGER, - jhighestmodseq INTEGER, - jstateMailbox TEXT, - jstateThread TEXT, - jstateMessage TEXT, - jstateContact TEXT, - jstateContactGroup TEXT, - jstateCalendar TEXT, - jstateCalendarEvent TEXT, + jdeletedmodseq INTEGER NOT NULL DEFAULT 1, + jhighestmodseq INTEGER NOT NULL DEFAULT 1, + jstateMailbox TEXT NOT NULL DEFAULT 1, + jstateThread TEXT NOT NULL DEFAULT 1, + jstateMessage TEXT NOT NULL DEFAULT 1, + jstateContact TEXT NOT NULL DEFAULT 1, + jstateContactGroup TEXT NOT NULL DEFAULT 1, + jstateCalendar TEXT NOT NULL DEFAULT 1, + jstateCalendarEvent TEXT NOT NULL DEFAULT 1, mtime DATE ); EOF From bd3d262d765fa68220a76bae835ba6b066bd43cc Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 17:23:03 +1000 Subject: [PATCH 312/331] JMAP: use subject as part of the threading algorithm --- JMAP/API.pm | 2 +- JMAP/DB.pm | 1 + JMAP/ImapDB.pm | 25 +++++++++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/JMAP/API.pm b/JMAP/API.pm index 29e6024..59b6f89 100644 --- a/JMAP/API.pm +++ b/JMAP/API.pm @@ -383,7 +383,7 @@ sub _build_sort { size => 'msgsize', isflagged => 'isFlagged', isunread => 'isUnread', - subject => 'msgsubject', + subject => 'sortsubject', from => 'msgfrom', to => 'msgto', ); diff --git a/JMAP/DB.pm b/JMAP/DB.pm index e202a7d..0b9e2da 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -942,6 +942,7 @@ CREATE TABLE IF NOT EXISTS jmessages ( msgmessageid TEXT, msgdate INTEGER, msgsize INTEGER, + sortsubject TEXT, jcreated INTEGER, jmodseq INTEGER, mtime DATE, diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index c0d85de..b29db5a 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -739,11 +739,14 @@ sub calcmsgid { my $replyto = _trimh($envelope->{'In-Reply-To'}); my $messageid = _trimh($envelope->{'Message-ID'}); - my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?)", {}, $replyto, $messageid); + my $encsub = Encode::decode('MIME-Header', $envelope->{Subject}); + my $sortsub = _normalsubject($encsub); + my ($thrid) = $Self->dbh->selectrow_array("SELECT DISTINCT thrid FROM ithread WHERE messageid IN (?, ?) AND sortsubject = ?", {}, $replyto, $messageid, $sortsub); + # XXX - merging? subject-checking? We have a subject here $thrid ||= "t$base"; foreach my $id ($replyto, $messageid) { next if $id eq ''; - $Self->dbh->do("INSERT OR IGNORE INTO ithread (messageid, thrid) VALUES (?, ?)", {}, $id, $thrid); + $Self->dbh->do("INSERT OR IGNORE INTO ithread (messageid, thrid, sortsubject) VALUES (?, ?, ?)", {}, $id, $thrid, $sortsub); } return ($msgid, $thrid); @@ -1245,12 +1248,29 @@ sub apply_data { } } +sub _normalsubject { + my $sub = shift; + + # Re: and friends + $sub =~ s/^[ \t]*[A-Za-z0-9]+://g; + # [LISTNAME] and friends + $sub =~ s/^[ \t]*\\[[^]]+\\]//g; + # Australian security services and frenemies + $sub =~ s/[\\[(SEC|DLM)=[^]]+\\][ \t]*$//g; + # any old whitespace + $sub =~ s/[ \t\r\n]+//g; + + return $sub; +} + sub _envelopedata { my $data = shift; my $envelope = decode_json($data || "{}"); my $encsub = Encode::decode('MIME-Header', $envelope->{Subject}); + my $sortsub = _normalsubject($encsub); return ( msgsubject => $encsub, + sortsubject => $sortsub, msgfrom => $envelope->{From}, msgto => $envelope->{To}, msgcc => $envelope->{Cc}, @@ -1897,6 +1917,7 @@ EOF $dbh->do(< Date: Wed, 29 Jul 2015 17:31:44 +1000 Subject: [PATCH 313/331] strip two unused backfilling items --- JMAP/ImapDB.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b29db5a..583efa6 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -807,7 +807,6 @@ sub do_folder { my $didold = 0; if ($res->{backfill}) { my $new = $res->{backfill}[1]; - $Self->{backfilling} = 1; foreach my $uid (sort { $a <=> $b } keys %$new) { my ($msgid, $thrid, @labels); if ($Self->{is_gmail}) { @@ -821,7 +820,6 @@ sub do_folder { $didold++; $Self->new_record($ifolderid, $uid, $new->{$uid}{'flags'}, \@labels, $new->{$uid}{envelope}, str2time($new->{$uid}{internaldate}), $msgid, $thrid, $new->{$uid}{'rfc822.size'}); } - delete $Self->{backfilling}; } if ($res->{update}) { From 4679e79bcfcd5495d238f981f1c6f507fb862776 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 17:34:56 +1000 Subject: [PATCH 314/331] let the API be happier --- JMAP/ImapDB.pm | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 583efa6..b3a1e64 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -807,7 +807,16 @@ sub do_folder { my $didold = 0; if ($res->{backfill}) { my $new = $res->{backfill}[1]; + my $count = 0; foreach my $uid (sort { $a <=> $b } keys %$new) { + $count++; + # release the lock frequently so we don't starve the API + if ($count > 50) { + $Self->commit(); + $Self->begin(); + $Self->{t}{backfilling} = 1; + $count = 0; + } my ($msgid, $thrid, @labels); if ($Self->{is_gmail}) { ($msgid, $thrid) = ($new->{$uid}{"x-gm-msgid"}, $new->{$uid}{"x-gm-thrid"}); From 4d8939d99e539d7d2fa2b41c50722d641d7f224f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 17:42:03 +1000 Subject: [PATCH 315/331] JMAP: use search --- JMAP/Sync/Common.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index 6ccd19d..ca5f8c1 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -351,10 +351,10 @@ sub imap_count { return \%res; } - my $data = $imap->fetch($uids, "UID"); + my $data = $imap->search($uids); $Self->_unselect($imap); - $res{data} = [sort { $a <=> $b } keys %$data]; + $res{data} = $data; return \%res; } From 89457377c85620097da83a39ba1fba992eb0106f Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Wed, 29 Jul 2015 17:46:51 -0400 Subject: [PATCH 316/331] handle unselectable folders --- JMAP/ImapDB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index b3a1e64..30d17cc 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -179,6 +179,7 @@ sub sync_folders { $Self->begin(); foreach my $name (keys %$data) { my $status = $data->{$name}; + next unless ref($status) eq 'HASH'; $Self->dmaybeupdate('ifolders', { uidvalidity => $status->{uidvalidity}, uidnext => $status->{uidnext}, From 75a5c8c64be378c08e77783f66f92c76b0dea7fd Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 03:43:20 -0400 Subject: [PATCH 317/331] skip undefined subjects --- JMAP/ImapDB.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 30d17cc..9e168d2 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1258,6 +1258,7 @@ sub apply_data { sub _normalsubject { my $sub = shift; + return unless defined $sub; # Re: and friends $sub =~ s/^[ \t]*[A-Za-z0-9]+://g; From 1510c5cfb9e12e64d3671f92ad3208315410308b Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 04:25:33 -0400 Subject: [PATCH 318/331] new_card - fix transactional --- JMAP/ImapDB.pm | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 9e168d2..ca1fee6 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -1752,6 +1752,7 @@ sub create_contacts { my %createmap; my %notcreated; + my %todo; foreach my $cid (keys %$new) { my $contact = $new->{$cid}; my ($href) = $Self->dbh->selectrow_array("SELECT href FROM iaddressbooks"); @@ -1778,8 +1779,14 @@ sub create_contacts { $card->VBirthday($contact->{birthday}) if exists $contact->{birthday}; $card->VNotes($contact->{notes}) if exists $contact->{notes}; - $Self->backend_cmd('new_card', $href, $card); $createmap{$cid} = { id => $uid }; + $todo{$href} = $card; + } + + $Self->commit(); + + foreach my $href (sort keys %todo) { + $Self->backend_cmd('new_card', $href, $todo{$href}); } return (\%createmap, \%notcreated); From a683f8efc470ad167761359833b5e79f23e4bc14 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 04:44:10 -0400 Subject: [PATCH 319/331] spell starttls correctly for sending via standard SMTP --- JMAP/Sync/Standard.pm | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/JMAP/Sync/Standard.pm b/JMAP/Sync/Standard.pm index 8bf3793..13e196e 100644 --- a/JMAP/Sync/Standard.pm +++ b/JMAP/Sync/Standard.pm @@ -90,18 +90,19 @@ sub send_email { my $ssl; $ssl = 'ssl' if $Self->{auth}{smtpSSL} == 2; - $ssl = 'startls' if $Self->{auth}{smtpSSL} == 3; + $ssl = 'starttls' if $Self->{auth}{smtpSSL} == 3; my $email = Email::Simple->new($rfc822); - sendmail($email, { - from => $Self->{auth}{username}, - transport => Email::Sender::Transport::SMTPS->new({ + my $detail = { helo => $ENV{jmaphost}, host => $Self->{auth}{smtpHost}, port => $Self->{auth}{smtpPort}, ssl => $ssl, sasl_username => $Self->{auth}{username}, sasl_password => $Self->{auth}{password}, - }), + }; + sendmail($email, { + from => $Self->{auth}{username}, + transport => Email::Sender::Transport::SMTPS->new($detail), }); } From 8eb866be79fc85cebd713516739a42cf0a80eb38 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 18:02:26 -0400 Subject: [PATCH 320/331] tidy up delete handling --- JMAP/DB.pm | 4 ++-- JMAP/ImapDB.pm | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index 0b9e2da..afb57fd 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -442,7 +442,7 @@ sub change_message { my $Self = shift; my ($msgid, $data, $newids) = @_; - $Self->dmaybedirty('jmessages', $data, {msgid => $msgid}); + my $bump = $Self->dmaybedirty('jmessages', $data, {msgid => $msgid}); my $oldids = $Self->dbh->selectcol_arrayref("SELECT jmailboxid FROM jmessagemap WHERE msgid = ? AND active = 1", {}, $msgid); my %old = map { $_ => 1 } @$oldids; @@ -450,7 +450,7 @@ sub change_message { foreach my $jmailboxid (@$newids) { if (delete $old{$jmailboxid}) { # just bump the modseq - $Self->dmaybeupdate('jmailboxes', {jcountsmodseq => $data->{jmodseq}}, {jmailboxid => $jmailboxid}); + $Self->dmaybeupdate('jmailboxes', {jcountsmodseq => $data->{jmodseq}}, {jmailboxid => $jmailboxid}) if $bump; } else { $Self->add_message_to_mailbox($msgid, $jmailboxid); diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index ca1fee6..57e2c92 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -878,7 +878,7 @@ sub do_folder { my $res = $Self->backend_cmd('imap_count', $imapname, $uidvalidity, "$uidfirst:$to"); $Self->begin(); my $uids = $res->{data}; - my $data = $Self->dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ? AND uid >= ?", {}, $ifolderid, $uidfirst); + my $data = $Self->dbh->selectcol_arrayref("SELECT uid FROM imessages WHERE ifolderid = ? AND uid >= ? AND uid <= ?", {}, $ifolderid, $uidfirst, $to); my %exists = map { $_ => 1 } @$uids; foreach my $uid (@$data) { next if $exists{$uid}; @@ -1170,13 +1170,12 @@ sub deleted_record { my $Self = shift; my ($folder, $uid) = @_; - my ($msgid, $jmailboxid) = $Self->dbh->selectrow_array("SELECT msgid, jmailboxid FROM imessages JOIN ifolders USING (ifolderid) WHERE imessages.ifolderid = ? AND uid = ?", {}, $folder, $uid); + my ($msgid) = $Self->dbh->selectrow_array("SELECT msgid FROM imessages WHERE ifolderid = ? AND uid = ?", {}, $folder, $uid); return unless $msgid; $Self->ddelete('imessages', {ifolderid => $folder, uid => $uid}); - $Self->ddirty('jmessages', {}, {msgid => $msgid}); # bump modeseq - $Self->delete_message_from_mailbox($msgid, $jmailboxid); + $Self->delete_message($msgid); } sub new_record { From c91584738fcacc83dcdfab62923883c2913c61ff Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 18:19:00 -0400 Subject: [PATCH 321/331] Sync:UID --- JMAP/Sync/Common.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/Sync/Common.pm b/JMAP/Sync/Common.pm index ca5f8c1..316a70b 100644 --- a/JMAP/Sync/Common.pm +++ b/JMAP/Sync/Common.pm @@ -351,7 +351,7 @@ sub imap_count { return \%res; } - my $data = $imap->search($uids); + my $data = $imap->search('uid', $uids); $Self->_unselect($imap); $res{data} = $data; From d40723302b33ef02ab47e6da2e5a77215fec0b15 Mon Sep 17 00:00:00 2001 From: jmap Date: Wed, 29 Jul 2015 18:31:33 -0400 Subject: [PATCH 322/331] x-gm-labels only needs to be fetched once --- JMAP/ImapDB.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 57e2c92..6eb3ff3 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -770,7 +770,7 @@ sub do_folder { my @immutable = qw(internaldate envelope rfc822.size); my @mutable; if ($Self->{is_gmail}) { - push @immutable, qw(x-gm-msgid x-gm-thrid x-gm-labels); + push @immutable, qw(x-gm-msgid x-gm-thrid); push @mutable, qw(x-gm-labels); } From b9ae67c3be766bb0e61104b3c82f03f3af6cd0dd Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Tue, 4 Aug 2015 10:19:34 +1000 Subject: [PATCH 323/331] DB: add 'dget' and 'dgetone' APIs --- JMAP/DB.pm | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/JMAP/DB.pm b/JMAP/DB.pm index afb57fd..b7d2207 100644 --- a/JMAP/DB.pm +++ b/JMAP/DB.pm @@ -900,7 +900,8 @@ sub dnuke { my $modseq = $Self->dirty($table); my @lkeys = sort keys %$limit; - my $sql = "UPDATE $table SET active = 0, jmodseq = ? WHERE active = 1 AND " . join(' AND ', map { "$_ = ?" } @lkeys); + my $sql = "UPDATE $table SET active = 0, jmodseq = ? WHERE active = 1"; + $sql .= " AND " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; $Self->log('debug', $sql, _dbl($modseq), _dbl(map { $limit->{$_} } @lkeys)); @@ -912,13 +913,35 @@ sub ddelete { my ($table, $limit) = @_; my @lkeys = sort keys %$limit; - my $sql = "DELETE FROM $table WHERE " . join(' AND ', map { "$_ = ?" } @lkeys); + my $sql = "DELETE FROM $table"; + $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; $Self->log('debug', $sql, _dbl(map { $limit->{$_} } @lkeys)); $Self->dbh->do($sql, {}, map { $limit->{$_} } @lkeys); } +sub dget { + my $Self = shift; + my ($table, $limit) = @_; + + my @lkeys = sort keys %$limit; + my $sql = "SELECT * FROM $table"; + $sql .= " WHERE " . join(' AND ', map { "$_ = ?" } @lkeys) if @lkeys; + + $Self->log('debug', $sql, _dbl(map { $limit->{$_} } @lkeys)); + + my $data = $Self->dbh->selectall_arrayref($sql, {Slice => {}}, map { $limit->{$_} } @lkeys); + return $data; +} + +# selectrow_arrayref? Nah +sub dgetone { + my $Self = shift; + my $res = $Self->dget(@_); + return $res->[0]; +} + sub _initdb { my $Self = shift; my $dbh = shift; From 199f3749ad0de5853403a22530938c3ffca63157 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Mon, 17 Aug 2015 08:54:37 +1000 Subject: [PATCH 324/331] more by-name --- JMAP/ImapDB.pm | 126 +++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/JMAP/ImapDB.pm b/JMAP/ImapDB.pm index 6eb3ff3..0df7f16 100644 --- a/JMAP/ImapDB.pm +++ b/JMAP/ImapDB.pm @@ -65,16 +65,16 @@ sub setuser { $Self->begin(); - my $data = $Self->dbh->selectrow_arrayref("SELECT username FROM iserver"); - if ($data and $data->[0]) { + my $data = $Self->dgetone('iserver'); + if ($data) { $Self->dmaybeupdate('iserver', $args); } else { $Self->dinsert('iserver', $args); } - my $user = $Self->dbh->selectrow_arrayref("SELECT email FROM account"); - if ($user and $user->[0]) { + my $user = $Self->dgetone('account'); + if ($user) { $Self->dmaybeupdate('account', {email => $args->{username}}); } else { @@ -92,20 +92,20 @@ sub access_token { my $Self = shift; $Self->begin(); - my ($hostname, $username, $password) = $Self->dbh->selectrow_array("SELECT imapHost, username, password FROM iserver"); + my $data = $Self->dgetone('iserver'); $Self->commit(); - return [$hostname, $username, $password]; + return [$data->{imapHost}, $data->{username}, $data->{password}]; } sub access_data { my $Self = shift; $Self->begin(); - my $config = $Self->dbh->selectall_arrayref("SELECT * FROM iserver", {Slice => {}}); + my $data = $Self->dgetone('iserver'); $Self->commit(); - return $config->[0]; + return $data; } # synchronous backend for now @@ -142,15 +142,15 @@ sub sync_folders { $Self->begin(); - my $ifolders = $Self->dbh->selectall_arrayref("SELECT ifolderid, sep, uidvalidity, imapname, label FROM ifolders"); - my %ibylabel = map { $_->[4] => $_ } @$ifolders; + my $ifolders = $Self->dget('ifolders'); + my %ibylabel = map { $_->{label} => $_ } @$ifolders; my %seen; my %getstatus; foreach my $name (sort keys %$folders) { my $sep = $folders->{$name}[0]; my $label = $folders->{$name}[1]; - my $id = $ibylabel{$label}[0]; + my $id = $ibylabel{$label}{ifolderid}; if ($id) { $Self->dmaybeupdate('ifolders', {sep => $sep, imapname => $name}, {ifolderid => $id}); } @@ -158,16 +158,16 @@ sub sync_folders { $id = $Self->dinsert('ifolders', {sep => $sep, imapname => $name, label => $label}); } $seen{$id} = 1; - unless ($ibylabel{$label}[2]) { + unless ($ibylabel{$label}{uidvalidity}) { # no uidvalidity, we need to get status for this one $getstatus{$name} = $id; } } foreach my $folder (@$ifolders) { - my $id = $folder->[0]; + my $id = $folder->{ifolderid}; next if $seen{$id}; - $Self->dbh->do("DELETE FROM ifolders WHERE ifolderid = ?", {}, $id); + $Self->ddelete('ifolders', {ifolderid => $id}); } $Self->dmaybeupdate('iserver', {imapPrefix => $prefix, lastfoldersync => time()}); @@ -198,28 +198,28 @@ sub sync_folders { sub sync_jmailboxes { my $Self = shift; $Self->begin(); - my $ifolders = $Self->dbh->selectall_arrayref("SELECT ifolderid, sep, imapname, label, jmailboxid FROM ifolders"); - my $jmailboxes = $Self->dbh->selectall_arrayref("SELECT jmailboxid, name, parentId, role, active FROM jmailboxes"); + my $ifolders = $Self->dget('ifolders'); + my $jmailboxes = $Self->dget('jmailboxes'); my %jbyid; my %roletoid; my %byname; foreach my $mailbox (@$jmailboxes) { - $jbyid{$mailbox->[0]} = $mailbox; - $roletoid{$mailbox->[3]} = $mailbox->[0] if $mailbox->[3]; - $byname{$mailbox->[2]}{$mailbox->[1]} = $mailbox->[0]; + $jbyid{$mailbox->{jmailboxid}} = $mailbox; + $roletoid{$mailbox->{role}} = $mailbox->{jmailboxid} if $mailbox->{role}; + $byname{$mailbox->{parentId}}{$mailbox->{name}} = $mailbox->{jmailboxid}; } my %seen; foreach my $folder (@$ifolders) { - next if lc $folder->[3] eq "\\allmail"; # we don't show this folder - my $fname = $folder->[2]; + next if lc $folder->{label} eq "\\allmail"; # we don't show this folder + my $fname = $folder->{imapname}; # check for roles first - my @bits = split "[$folder->[1]]", $fname; + my @bits = split "[$folder->{sep}]", $fname; shift @bits if ($bits[0] eq 'INBOX' and $bits[1]); # really we should be stripping the actual prefix, if any shift @bits if $bits[0] eq '[Gmail]'; # we special case this GMail magic next unless @bits; # also skip the magic '[Gmail]' top-level - my $role = $ROLE_MAP{lc $folder->[3]}; + my $role = $ROLE_MAP{lc $folder->{label}}; my $id = 0; my $parentId = 0; my $name; @@ -259,7 +259,7 @@ sub sync_jmailboxes { $id = $roletoid{$role}; $Self->dmaybedirty('jmailboxes', {active => 1, %details}, {jmailboxid => $id}); } - elsif (not $folder->[4]) { + elsif (not $folder->{active}) { # reactivate! $Self->dmaybedirty('jmailboxes', {active => 1}, {jmailboxid => $id}); } @@ -327,8 +327,8 @@ sub sync_jmailboxes { } foreach my $mailbox (@$jmailboxes) { - my $id = $mailbox->[0]; - next unless $mailbox->[4]; + my $id = $mailbox->{jmailboxid}; + next unless $mailbox->{active}; next if $seen{$id}; $Self->dupdate('jmailboxes', {active => 0}, {jmailboxid => $id}); } @@ -345,13 +345,13 @@ sub sync_calendars { $Self->begin(); - my $icalendars = $Self->dbh->selectall_arrayref("SELECT icalendarid, href, name, isReadOnly, color, syncToken FROM icalendars"); - my %byhref = map { $_->[1] => $_ } @$icalendars; + my $icalendars = $Self->dget('icalendars'); + my %byhref = map { $_->{href} => $_ } @$icalendars; my %seen; my %todo; foreach my $calendar (@$calendars) { - my $id = $calendar->{href} ? $byhref{$calendar->{href}}[0] : 0; + my $id = $calendar->{href} ? $byhref{$calendar->{href}}{icalendarid} : 0; my $data = { isReadOnly => $calendar->{isReadOnly}, href => $calendar->{href}, @@ -361,7 +361,7 @@ sub sync_calendars { }; if ($id) { $Self->dmaybeupdate('icalendars', $data, {icalendarid => $id}); - my $token = $byhref{$calendar->{href}}[5]; + my $token = $byhref{$calendar->{href}}{syncToken}; if ($token eq $calendar->{syncToken}) { $seen{$id} = 1; next; @@ -375,7 +375,7 @@ sub sync_calendars { } foreach my $calendar (@$icalendars) { - my $id = $calendar->[0]; + my $id = $calendar->{icalendarid}; next if $seen{$id}; $Self->ddelete('icalendars', {icalendarid => $id}); } @@ -392,19 +392,19 @@ sub sync_calendars { sub sync_jcalendars { my $Self = shift; - my $icalendars = $Self->dbh->selectall_arrayref("SELECT icalendarid, name, color, jcalendarid FROM icalendars"); - my $jcalendars = $Self->dbh->selectall_arrayref("SELECT jcalendarid, name, color, active FROM jcalendars"); + my $icalendars = $Self->dget('icalendars'); + my $jcalendars = $Self->dget('jcalendars'); my %jbyid; foreach my $calendar (@$jcalendars) { - $jbyid{$calendar->[0]} = $calendar; + $jbyid{$calendar->{jcalendarid}} = $calendar; } my %seen; foreach my $calendar (@$icalendars) { my $data = { - name => $calendar->[1], - color => $calendar->[2], + name => $calendar->{name}, + color => $calendar->{color}, isVisible => 1, mayReadFreeBusy => 1, mayReadItems => 1, @@ -414,19 +414,19 @@ sub sync_jcalendars { mayDelete => 1, mayRename => 1, }; - if ($calendar->[3] && $jbyid{$calendar->[3]}) { - $Self->dmaybedirty('jcalendars', $data, {jcalendarid => $calendar->[3]}); - $seen{$calendar->[3]} = 1; + my $id = $calendar->{jcalendarid}; + if ($id && $jbyid{$id}) { + $Self->dmaybedirty('jcalendars', $data, {jcalendarid => $id}); } else { - my $id = $Self->dmake('jcalendars', $data); - $Self->dupdate('icalendars', {jcalendarid => $id}, {icalendarid => $calendar->[0]}); - $seen{$id} = 1; + $id = $Self->dmake('jcalendars', $data); + $Self->dupdate('icalendars', {jcalendarid => $id}, {icalendarid => $calendar->{icalendarid}}); } + $seen{$id} = 1; } foreach my $calendar (@$jcalendars) { - my $id = $calendar->[0]; + my $id = $calendar->{jcalendarid}; next if $seen{$id}; $Self->dnuke('jcalendars', {jcalendarid => $id}); $Self->dnuke('jevents', {jcalendarid => $id}); @@ -451,7 +451,7 @@ sub do_calendars { foreach my $id (keys %$cals) { my $href = $cals->{$id}; my ($jcalendarid) = $Self->dbh->selectrow_array("SELECT jcalendarid FROM icalendars WHERE icalendarid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT ieventid, resource, content, uid FROM ievents WHERE icalendarid = ?", {Slice => {}}, $id); + my $exists = $Self->dget('ievents', {icalendarid => $id}); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -497,7 +497,7 @@ sub sync_addressbooks { $Self->begin(); - my $iaddressbooks = $Self->dbh->selectall_arrayref("SELECT iaddressbookid, href, syncToken FROM iaddressbooks", {Slice => {}}); + my $iaddressbooks = $Self->dget('iaddressbooks'); my %byhref = map { $_->{href} => $_ } @$iaddressbooks; my %seen; @@ -528,7 +528,7 @@ sub sync_addressbooks { foreach my $addressbook (@$iaddressbooks) { my $id = $addressbook->{iaddressbookid}; next if $seen{$id}; - $Self->dbh->do("DELETE FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); + $Self->ddelete('iaddressbooks', {iaddressbookid => $id}); } $Self->sync_jaddressbooks(); @@ -543,8 +543,8 @@ sub sync_addressbooks { sub sync_jaddressbooks { my $Self = shift; - my $iaddressbooks = $Self->dbh->selectall_arrayref("SELECT iaddressbookid, name, jaddressbookid FROM iaddressbooks", {Slice => {}}); - my $jaddressbooks = $Self->dbh->selectall_arrayref("SELECT jaddressbookid, name, active FROM jaddressbooks", {Slice => {}}); + my $iaddressbooks = $Self->dget('iaddressbooks'); + my $jaddressbooks = $Self->dget('jaddressbooks'); my %jbyid; foreach my $addressbook (@$jaddressbooks) { @@ -605,7 +605,7 @@ sub do_addressbooks { foreach my $id (keys %$books) { my $href = $books->{$id}; my ($jaddressbookid) = $Self->dbh->selectrow_array("SELECT jaddressbookid FROM iaddressbooks WHERE iaddressbookid = ?", {}, $id); - my $exists = $Self->dbh->selectall_arrayref("SELECT icardid, resource, content, uid, kind FROM icards WHERE iaddressbookid = ?", {Slice => {}}, $id); + my $exists = $Self->dget('icards', {iaddressbookid => $id}); my %res = map { $_->{resource} => $_ } @$exists; foreach my $resource (keys %{$allparsed{$href}}) { @@ -646,8 +646,8 @@ sub do_addressbooks { sub labels { my $Self = shift; unless ($Self->{labels}) { - my $data = $Self->dbh->selectall_arrayref("SELECT label, ifolderid, jmailboxid, imapname FROM ifolders"); - $Self->{labels} = { map { $_->[0] => [$_->[1], $_->[2], $_->[3]] } @$data }; + my $data = $Self->dget('ifolders'); + $Self->{labels} = { map { $_->{label} => [$_->{ifolderid}, $_->{jmailboxid}, $_->{imapname}] } @$data }; } return $Self->{labels}; } @@ -656,7 +656,7 @@ sub sync_imap { my $Self = shift; $Self->begin(); - my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $data = $Self->dget('ifolders'); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } @@ -707,7 +707,7 @@ sub firstsync { $Self->sync_folders(); $Self->begin(); - my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $data = $Self->dget('ifolders'); $Self->commit(); if ($Self->{is_gmail}) { @@ -762,9 +762,13 @@ sub do_folder { Carp::confess("NO FOLDERID") unless $ifolderid; $Self->begin(); - my ($imapname, $uidfirst, $uidnext, $uidvalidity, $highestmodseq) = - $Self->dbh->selectrow_array("SELECT imapname, uidfirst, uidnext, uidvalidity, highestmodseq FROM ifolders WHERE ifolderid = ?", {}, $ifolderid); - die "NO SUCH FOLDER $ifolderid" unless $imapname; + my $data = $Self->dgetone('ifolders', {ifolderid => $ifolderid}); + die "NO SUCH FOLDER $ifolderid" unless $data; + my $imapname = $data->{imapname}; + my $uidfirst = $data->{uidfirst}; + my $uidvalidity = $data->{uidvalidity}; + my $uidnext = $data->{uidnext}; + my $highestmodseq = $data->{highestmodseq}; my %fetches; my @immutable = qw(internaldate envelope rfc822.size); @@ -914,7 +918,7 @@ sub imap_search { } $Self->begin(); - my $data = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $data = $Self->dget('ifolders'); if ($Self->{is_gmail}) { $data = [ grep { lc $_->{label} eq '\\allmail' or lc $_->{label} eq '\\trash' } @$data ]; } @@ -959,7 +963,7 @@ sub import_message { my %flags = @_; $Self->begin(); - my $folderdata = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $folderdata = $Self->dget('ifolders'); $Self->commit(); my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; @@ -1032,10 +1036,10 @@ sub update_messages { } } - my $folderdata = $Self->dbh->selectall_arrayref("SELECT * FROM ifolders", {Slice => {}}); + my $folderdata = $Self->dget('ifolders'); my %foldermap = map { $_->{ifolderid} => $_ } @$folderdata; my %jmailmap = map { $_->{jmailboxid} => $_ } grep { $_->{jmailboxid} } @$folderdata; - my $jmapdata = $Self->dbh->selectall_arrayref("SELECT * FROM jmailboxes", {Slice => {}}); + my $jmapdata = $Self->dget('jmailboxes'); my %jidmap = map { $_->{jmailboxid} => $_->{role} } @$jmapdata; my %jrolemap = map { $_->{role} => $_->{jmailboxid} } grep { $_->{role} } @$jmapdata; @@ -1144,7 +1148,7 @@ sub destroy_messages { } } - my $folderdata = $Self->dbh->selectall_arrayref("SELECT ifolderid, imapname, uidvalidity, label, jmailboxid FROM ifolders"); + my $folderdata = $Self->dget('ifolders'); my %foldermap = map { $_->[0] => $_ } @$folderdata; my %jmailmap = map { $_->[4] => $_ } grep { $_->[4] } @$folderdata; From d153c2dfe9a68dadb5e80fd76c5daf4f723dcd96 Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 16 Aug 2015 16:19:16 +1000 Subject: [PATCH 325/331] remove custom DAVTalk wrappers - upstream does this now --- JMAP/Sync/Gmail.pm | 8 ++++---- Net/GmailCalendars.pm | 11 ----------- Net/GmailContacts.pm | 11 ----------- 3 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 Net/GmailCalendars.pm delete mode 100644 Net/GmailContacts.pm diff --git a/JMAP/Sync/Gmail.pm b/JMAP/Sync/Gmail.pm index b9f349b..99f0398 100644 --- a/JMAP/Sync/Gmail.pm +++ b/JMAP/Sync/Gmail.pm @@ -11,8 +11,8 @@ use JSON::XS qw(decode_json); use Email::Simple; use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::GmailSMTP; -use Net::GmailCalendars; -use Net::GmailContacts; +use Net::CalDAVTalk; +use Net::CardDAVTalk; use OAuth2::Tiny; use IO::All; @@ -45,7 +45,7 @@ sub connect_calendars { return $Self->{calendars}; } - $Self->{calendars} = Net::GmailCalendars->new( + $Self->{calendars} = Net::CalDAVTalk->new( user => $Self->{auth}{username}, access_token => $Self->access_token(), url => $Self->{auth}{caldavURL}, @@ -63,7 +63,7 @@ sub connect_contacts { return $Self->{contacts}; } - $Self->{contacts} = Net::GmailContacts->new( + $Self->{contacts} = Net::CardDAVTalk->new( user => $Self->{auth}{username}, access_token => $Self->access_token(), url => $Self->{auth}{carddavURL}, diff --git a/Net/GmailCalendars.pm b/Net/GmailCalendars.pm deleted file mode 100644 index fc8fbfd..0000000 --- a/Net/GmailCalendars.pm +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/perl -cw - -package Net::GmailCalendars; -use base 'Net::CalDAVTalk'; - -sub auth_header { - my $Self = shift; - return "Bearer $Self->{access_token}"; -} - -1; diff --git a/Net/GmailContacts.pm b/Net/GmailContacts.pm deleted file mode 100644 index a9669b7..0000000 --- a/Net/GmailContacts.pm +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/perl -cw - -package Net::GmailContacts; -use base 'Net::CardDAVTalk'; - -sub auth_header { - my $Self = shift; - return "Bearer $Self->{access_token}"; -} - -1; From de879ad54de6215c2bc0687087b340f60febcf7a Mon Sep 17 00:00:00 2001 From: Bron Gondwana Date: Sun, 16 Aug 2015 16:25:17 +1000 Subject: [PATCH 326/331] remove Mail::IMAPTalk - CPAN version is OK now --- Mail/IMAPTalk.pm | 5073 ---------------------------------------------- 1 file changed, 5073 deletions(-) delete mode 100755 Mail/IMAPTalk.pm diff --git a/Mail/IMAPTalk.pm b/Mail/IMAPTalk.pm deleted file mode 100755 index bbb0be3..0000000 --- a/Mail/IMAPTalk.pm +++ /dev/null @@ -1,5073 +0,0 @@ -package Mail::IMAPTalk; - -=head1 NAME - -Mail::IMAPTalk - IMAP client interface with lots of features - -=head1 SYNOPSIS - - use Mail::IMAPTalk; - - $IMAP = Mail::IMAPTalk->new( - Server => $IMAPServer, - Username => 'foo', - Password => 'bar', - ) || die "Failed to connect/login to IMAP server"; - - # Append message to folder - open(my $F, 'rfc822msg.txt'); - $IMAP->append($FolderName, $F) || dir $@; - close($F); - - # Select folder and get first unseen message - $IMAP->select($FolderName) || die $@; - $MsgId = $IMAP->search('not', 'seen')->[0]; - - # Get message envelope and print some details - $MsgEV = $IMAP->fetch($MsgId, 'envelope')->{$MsgId}->{envelope}; - print "From: " . $MsgEv->{From}; - print "To: " . $MsgEv->{To}; - print "Subject: " . $MsgEv->{Subject}; - - # Get message body structure - $MsgBS = $IMAP->fetch($MsgId, 'bodystructure')->{$MsgId}->{bodystructure}; - - # Find imap part number of text part of message - $MsgTxtHash = Mail::IMAPTalk::find_message($MsgBS); - $MsgPart = $MsgTxtHash->{text}->{'IMAP-Partnum'}; - - # Retrieve message text body - $MsgTxt = $IMAP->fetch($MsgId, "body[$MsgPart]")->{$MsgId}->{body}; - - $IMAP->logout(); - -=head1 DESCRIPTION - -This module communicates with an IMAP server. Each IMAP server command -is mapped to a method of this object. - -Although other IMAP modules exist on CPAN, this has several advantages -over other modules. - -=over 4 - -=item * - -It parses the more complex IMAP structures like envelopes and body -structures into nice Perl data structures. - -=item * - -It correctly supports atoms, quoted strings and literals at any -point. Some parsers in other modules aren't fully IMAP compatiable -and may break at odd times with certain messages on some servers. - -=item * - -It allows large return values (eg. attachments on a message) -to be read directly into a file, rather than into memory. - -=item * - -It includes some helper functions to find the actual text/plain -or text/html part of a message out of a complex MIME structure. -It also can find a list of attachements, and CID links for HTML -messages with attached images. - -=item * - -It supports decoding of MIME headers to Perl utf-8 strings automatically, -so you don't have to deal with MIME encoded headers (enabled optionally). - -=back - -While the IMAP protocol does allow for asynchronous running of commands, this -module is designed to be used in a synchronous manner. That is, you issue a -command by calling a method, and the command will block until the appropriate -response is returned. The method will then return the parsed results from -the given command. - -=cut - -# Export {{{ -require Exporter; -@ISA = qw(Exporter); -%EXPORT_TAGS = ( - Default => [ qw(get_body_part find_message build_cid_map generate_cid) ] -); -Exporter::export_ok_tags('Default'); - -my $AlwaysTrace = 0; - -sub import { - # Test for special case if need UTF8 support - our $AlreadyLoadedEncode; - my $Class = shift(@_); - - my %Parameters = map { $_ => 1 } @_; - - if (delete($Parameters{':utf8support'})) { - if (!$AlreadyLoadedEncode) { - eval "use Encode qw(decode decode_utf8);"; - $AlreadyLoadedEncode = 1; - } - } - - if (delete($Parameters{':trace'})) { - $AlwaysTrace = 1; - } - - @_ = ($Class, keys(%Parameters)); - - goto &Exporter::import; -} - -our $VERSION = '2.01'; -# }}} - -# Use modules {{{ -use Fcntl qw(:DEFAULT); -use Socket; -use IO::Select; -use IO::Handle; -use IO::Socket; -use Digest; -use Data::Dumper; - -# Choose the best socket class to use (all of these are sub-classes of IO::Socket) -my $DefSocketClass; -BEGIN { - for (qw(IO::Socket::IP IO::Socket::INET6 IO::Socket::INET)) { - if (eval "use $_; 1;") { $DefSocketClass = $_; last; } - } -} - -# Use Time::HiRes if available to handle select restarts -eval 'use Time::HiRes qw(time);'; - -use strict; -use warnings; -# }}} - -=head1 CLASS OVERVIEW - -The object methods have been broken in several sections. - -=head2 Sections - -=over 4 - -=item CONSTANTS - -Lists the available constants the class uses. - -=item CONSTRUCTOR - -Explains all the options available when constructing a new instance of the -C class. - -=item CONNECTION CONTROL METHODS - -These are methods which control the overall IMAP connection object, such -as logging in and logging out, how results are parsed, how folder names and -message id's are treated, etc. - -=item IMAP FOLDER COMMAND METHODS - -These are methods to inspect, add, delete and rename IMAP folders on -the server. - -=item IMAP MESSAGE COMMAND METHODS - -These are methods to retrieve, delete, move and add messages to/from -IMAP folders. - -=item HELPER METHODS - -These are extra methods that users of this class might find useful. They -generally do extra parsing on returned structures to provide higher -level functionality. - -=item INTERNAL METHODS - -These are methods used internally by the C object to get work -done. They may be useful if you need to extend the class yourself. Note that -internal methods will always 'die' if they encounter any errors. - -=item INTERNAL SOCKET FUNCTIONS - -These are functions used internally by the C object -to read/write data to/from the IMAP connection socket. The class does -its own buffering so if you want to read/write to the IMAP socket, you -should use these functions. - -=item INTERNAL PARSING FUNCTIONS - -These are functions used to parse the results returned from the IMAP server -into Perl style data structures. - -=back - -=head2 Method results - -All methods return undef on failure. There are four main modes of failure: - -=over 4 - -=item 1. An error occurred reading/writing to a socket. Maybe the server -closed it, or you're not connected to any server. - -=item 2. An error occurred parsing the response of an IMAP command. This is -usually only a problem if your IMAP server returns invalid data. - -=item 3. An IMAP command didn't return an 'OK' response. - -=item 4. The socket read operation timed out waiting for a response from -the server. - -=back - -In each case, some readable form of error text is placed in $@, or you -can call the C method. For commands which return -responses (e.g. fetch, getacl, etc), the result is returned. See each -command for details of the response result. For commands -with no response but which succeed (e.g. setacl, rename, etc) the result -'ok' is generally returned. - -=head2 Method parameters - -All methods which send data to the IMAP server (e.g. C, C, -etc) have their arguments processed before they are sent. Arguments may be -specified in several ways: - -=over 4 - -=item B - -The value is first checked and quoted if required. Values containing -[\000\012\015] are turned into literals, values containing -[\000-\040\{\} \%\*\"] are quoted by surrounding with a "..." pair -(any " themselves are turned into \"). undef is turned into NIL - -=item B - -The contents of the file is sent as an IMAP literal. Note that -because IMAPTalk has to know the length of the file being sent, -this must be a true file reference that can be seeked and not -just some stream. The entire file will be sent regardless of the -current seek point. - -=item B - -The string/data in the referenced item should be sent as is, no quoting will -occur, and the data won't be sent as quoted or as a literal regardless -of the contents of the string/data. - -=item B - -Emits an opening bracket, and then each item in the array separated -by a space, and finally a closing bracket. Each item in the array -is processed by the same methods, so can be a scalar, file ref, -scalar ref, another array ref, etc. - -=item B - -The hash reference should contain only 1 item. The key is a text -string which specifies what to do with the value item of the hash. - -=over 4 - -=item * 'Literal' - -The string/data in the value is sent as an IMAP literal -regardless of the actual data in the string/data. - -=item * 'Quote' - -The string/data in the value is sent as an IMAP quoted string -regardless of the actual data in the string/data. - -=back - -Examples: - - # Password is automatically quoted to "nasty%*\"passwd" - $IMAP->login("joe", 'nasty%*"passwd'); - # Append $MsgTxt as string - $IMAP->append("inbox", { Literal => $MsgTxt }) - # Append MSGFILE contents as new message - $IMAP->append("inbox", \*MSGFILE ]) - -=back - -=cut - -=head1 CONSTANTS - -These constants relate to the standard 4 states that an IMAP connection can -be in. They are passed and returned from the C method. See RFC 3501 -for more details about IMAP connection states. - -=over 4 - -=item I - -Current not connected to any server. - -=item I - -Connected to a server, but not logged in. - -=item I - -Connected and logged into a server, but not current folder. - -=item I - -Connected, logged in and have 'select'ed a current folder. - -=back - -=cut - -# Constants for the possible states the connection can be in {{{ -# Object not connected -use constant Unconnected => 0; -# connected; not logged in -use constant Connected => 1; -# logged in; no mailbox selected -use constant Authenticated => 2; -# mailbox selected -use constant Selected => 3; - -# What a link break is on the network connection -use constant LB => "\015\012"; -use constant LBLEN => length(LB); - -# Regexps used to determine if header is MIME encoded (we remove . from -# especials because of dumb ANSI_X3.4-1968 encoding) -my $RFC2047Token = qr/[^\x00-\x1f\(\)\<\>\@\,\;\:\"\/\[\]\?\=\ ]+/; -my $NeedDecodeUTF8Regexp = qr/=\?$RFC2047Token\?$RFC2047Token\?[^\?]*\?=/; - -# Known untagged responses -my %UntaggedResponses = map { $_ => 1 } qw(exists expunge recent); - -# Default responses -my %RespDefaults = ('annotation' => 'hash', 'metadata' => 'hash', 'fetch' => 'hash', 'list' => 'array', 'lsub' => 'array', 'sort' => 'array', 'search' => 'array'); - -# }}} - -=head1 CONSTRUCTOR - -=over 4 - -=cut - -=item Inew(%Options)> - -Creates new Mail::IMAPTalk object. The following options are supported. - -=item B - -=over 4 - -=item B - -The hostname or IP address to connect to. This must be supplied unless -the B option is supplied. - -=item B - -The port number on the host to connect to. Defaults to 143 if not supplied -or 993 if not supplied and UseSSL is true. - -=item B - -If true, use an IO::Socket::SSL connection. All other SSL_* arguments -are passed to the IO::Socket::SSL constructor. - -=item B - -An existing socket to use as the connection to the IMAP server. If you -supply the B option, you should not supply a B or B -option. - -This is useful if you want to create an SSL socket connection using -IO::Socket::SSL and then pass in the connected socket to the new() call. - -It's also useful in conjunction with the C method -described below for reusing the same socket beyond the lifetime of the IMAPTalk -object. See a description in the section C method for -more information. - -You must have write flushing enabled for any -socket you pass in here so that commands will actually be sent, -and responses received, rather than just waiting and eventually -timing out. you can do this using the Perl C call and -$| ($AUTOFLUSH) variable as shown below. - - my $ofh = select($Socket); $| = 1; select ($ofh); - -=item B - -For historical reasons, when reading from a socket, the module -sets the socket to non-blocking and does a select(). If you're -using an SSL socket that doesn't work, so you have to set -UseBlocking to true to use blocking reads instead. - -=item B - -If you supply a C option, you can specify the IMAP state the -socket is currently in, namely one of 'Unconnected', 'Connected', -'Authenticated' or 'Selected'. This defaults to 'Connected' if not -supplied and the C option is supplied. - -=item B - -If supplied and true, and a socket is supplied via the C -option, checks that a greeting line is supplied by the server -and reads the greeting line. - -=item B - -For historical reasons, the special name "INBOX" is rewritten as -Inbox because it looks nicer on the way out, and back on the way -in. If you want to preserve the name INBOX on the outside, set -this flag to true. - -=back - -=item B - -=over 4 - -=item B - -The username to connect to the IMAP server as. If not supplied, no login -is attempted and the IMAP object is left in the B state. -If supplied, you must also supply the B option and a login -is attempted. If the login fails, the connection is closed and B -is returned. If you want to do something with a connection even if the -login fails, don't pass a B option, but instead use the B -method described below. - -=item B - -The password to use to login to the account. - -=back - -=item B - -=over 4 - -=item B - -Control whether message ids are message uids or not. This is 1 (on) by -default because generally that's how most people want to use it. This affects -most commands that require/use/return message ids (e.g. B, B, -B, etc) - -=item B - -If supplied, sets the root folder prefix. This is the same as calling -C with the value passed. If no value is supplied, -C is called with no value. See the C -method for more details. - -=item B - -If supplied, sets the folder name text string separator character. -Passed as the second parameter to the C method. - -=item B - -If supplied, passed along with RootFolder to the C -method. - -=back - -Examples: - - $imap = Mail::IMAPTalk->new( - Server => 'foo.com', - Port => 143, - Username => 'joebloggs', - Password => 'mypassword', - Separator => '.', - RootFolder => 'INBOX', - ) || die "Connection to foo.com failed. Reason: $@"; - - $imap = Mail::IMAPTalk->new( - Socket => $SSLSocket, - State => Mail::IMAPTalk::Authenticated, - Uid => 0 - ) || die "Could not query on existing socket. Reason: $@"; - -=cut -sub new { - my $Proto = shift; - my $Class = ref($Proto) || $Proto; - my %Args = @_; - - # Two main possible new() modes. Either connect to server - # or use existing socket passed - $Args{Server} || $Args{Socket} - || die "No 'Server' or 'Socket' specified"; - $Args{Server} && $Args{Socket} - && die "Can not specify 'Server' and 'Socket' simultaneously"; - - # Set ourself to empty to start with - my $Self = {}; - bless ($Self, $Class); - - # Empty buffer - $Self->{ReadBuf} = ''; - - # Create new socket to server - my $Socket; - if ($Args{Server}) { - - # Set starting state - $Self->state(Unconnected); - - my %SocketOpts; - my $DefaultPort = 143; - my $SocketClass = $DefSocketClass; - - if (my $SSLOpt = $Args{UseSSL}) { - $SSLOpt = $SSLOpt eq '1' ? '' : " qw($SSLOpt)"; - eval "use IO::Socket::SSL$SSLOpt; 1;" || return undef; - $SocketClass = "IO::Socket::SSL"; - $DefaultPort = 993; - $SocketOpts{$_} = $Args{$_} for grep { /^SSL_/ } keys %Args; - } - - $SocketOpts{PeerHost} = $Self->{Server} = $Args{Server} || die "No Server name given"; - $SocketOpts{PeerPort} = $Self->{Port} = $Args{Port} || $DefaultPort; - - $Socket = ${SocketClass}->new(%SocketOpts) || return undef; - - # Force flushing after every write to the socket - my $ofh = select($Socket); $| = 1; select ($ofh); - - # Set to connected state - $Self->state(Connected); - } - - # We have an existing socket - else { - # Copy socket - $Socket = $Args{Socket}; - delete $Args{Socket}; - - # Set state - $Self->state(exists $Args{State} ? $Args{State} : Connected); - } - - $Self->{Socket} = $Socket; - - # Save socket for later use and create IO::Select - $Self->{Select} = IO::Select->new(); - $Self->{Select}->add($Socket); - $Self->{LocalFD} = fileno($Socket); - $Self->{UseBlocking} = $Args{UseBlocking}; - $Self->{Pedantic} = $Args{Pedantic}; - $Self->{PreserveINBOX} = $Args{PreserveINBOX}; - - # Do this now, so we trace greeting line as well - $Self->set_tracing($AlwaysTrace); - - # Process greeting - if ($Args{Server} || $Args{ExpectGreeting}) { - $Self->{CmdId} = "*"; - my ($CompletionResp, $DataResp) = $Self->_parse_response(''); - return undef if $CompletionResp !~ /^ok/i; - } - - # Start counter when sending commands - $Self->{CmdId} = 1; - - # Set base modes - $Self->uid(exists($Args{Uid}) ? $Args{Uid} : 1); - $Self->parse_mode(Envelope => 1, BodyStructure => 1, Annotation => 1); - $Self->{CurrentFolder} = ''; - $Self->{CurrentFolderMode} = ''; - - # Login first if specified - if ($Args{Username}) { - # If login fails, just return undef - $Self->login(@Args{'Username', 'Password'}) || return undef; - } - - # Set root folder and separator (if supplied) - $Self->set_root_folder( - $Args{RootFolder}, $Args{Separator}, $Args{AltRootRegexp}); - - return $Self; -} - -=back -=cut - -=head1 CONNECTION CONTROL METHODS - -=over 4 -=cut - -=item I - -Attempt to login user specified username and password. - -Currently there is only plain text password login support. If someone can -give me a hand implementing others (like DIGEST-MD5, CRAM-MD5, etc) please -contact me (see details below). - -=cut -sub login { - my $Self = shift; - my ($User, $Pwd) = @_; - my $PwdArr = { 'Quote' => $Pwd }; - - # Clear cached capability responses and the like - delete $Self->{Cache}; - - # Call standard command. Return undef if login failed - $Self->_imap_cmd("login", 0, "", $User, $PwdArr) - || return undef; - - # Set to authenticated if successful - $Self->state(Authenticated); - - return 1; -} - -=item I - -Log out of IMAP server. This usually closes the servers connection as well. - -=cut -sub logout { - my $Self = shift; - # Callback to say we're switching folders - $Self->cb_switch_folder($Self->{CurrentFolder}, ''); - $Self->_imap_cmd('logout', 0, ''); - # Returns the socket, which we immediately discard to close - $Self->release_socket(1); - return 1; -} - -=item I - -Set/get the current IMAP connection state. Returned or passed value should be -one of the constants (Unconnected, Connected, Authenticated, Selected). - -=cut -sub state { - my $Self = shift; - $Self->{State} = $_[0] if defined $_[0]; - return (defined($Self->{State}) ? $Self->{State} : ''); -} - -=item I - -Get/set the UID status of all UID possible IMAP commands. -If set to 1, all commands that can take a UID are set to 'UID Mode', -where any ID sent to IMAPTalk is assumed to be a UID. - -=cut -sub uid { - $_[0]->{Uid} = $_[1]; - return 1; -} - -=item I - -This method returns the IMAP servers capability command results. -The result is a hash reference of (lc(Capability) => 1) key value pairs. -This means you can do things like: - - if ($IMAP->capability()->{quota}) { ... } - -to test if the server has the QUOTA capability. If you just want a list of -capabilities, use the Perl 'keys' function to get a list of keys from the -returned hash reference. - -=cut -sub capability { - my $Self = shift; - - # If we've already executed the capability command once, just return the results - return $Self->{Cache}->{capability} - if exists $Self->{Cache}->{capability}; - - # Otherwise execute capability command - my $Capability = $Self->_imap_cmd("capability", 0, "capability"); - - # Better be a hash-ref... - ($Capability && ref($Capability) eq 'HASH') || return {}; - - # Save for any future queries and return - return ($Self->{Cache}->{capability} = $Capability); -} - -=item I - -Returns the result of the IMAP servers namespace command. - -=cut -sub namespace { - my $Self = shift; - - # If we've already executed the capability command once, just return the results - return $Self->{Cache}->{namespace} - if exists $Self->{Cache}->{namespace}; - - $Self->_require_capability('namespace') || return undef; - - # Otherwise execute capability command - my $Namespace = $Self->_imap_cmd("namespace", 0, "namespace"); - - # Save for any future queries and return - return ($Self->{Cache}->{namespace} = $Namespace); -} - -=item I - -Perform the standard IMAP 'noop' command which does nothing. - -=cut -sub noop { - my $Self = shift; - return $Self->_imap_cmd("noop", 0, "", @_); -} - -=item I - -Enabled the given imap extension - -=cut -sub enable { - my $Self = shift; - my $Feature = shift; - - # If we've already executed the enable command once, just return the results - return $Self->{Cache}->{enable}->{$Feature} - if exists $Self->{Cache}->{enable}->{$Feature}; - - $Self->_require_capability($Feature) || return undef; - - my $Result = $Self->_imap_cmd("enable", 0, "enabled", $Feature); - $Self->{Cache}->{enable} = $Result; - - return $Result && $Result->{$Feature}; -} - -=item I - -Returns true if the current socket connection is still open (e.g. the socket -hasn't been closed this end or the other end due to a timeout). - -=cut -sub is_open { - my $Self = shift; - - $Self->_trace("A: is_open test\n") if $Self->{Trace}; - - while (1) { - - # Ensure no data was left in our own read buffer - if ($Self->{ReadLine}) { - $Self->_trace("A: unexpected data in read buffer - '" .$Self->{ReadLine}. "'\n") - if $Self->{Trace}; - die "IMAPTalk: Unexpected data in read buffer '" . $Self->{ReadLine} . "'"; - } - $Self->{ReadLine} = undef; - - # See if there's any data to read - local $Self->{Timeout} = 0; - - # If no sockets with data, must be blocked, so must be connected - my $Atom = eval { $Self->_next_atom(); }; - - # If a timeout, socket is still connected and open - if ($@ && ($@ =~ /timed out/)) { - $Self->_trace("A: is_open test received timeout, still open\n") - if $Self->{Trace}; - return 1; - } - - # Other error, assume it's closed - if ($@) { - $Self->_trace("A: is_open test received error - $@\n") - if $Self->{Trace}; - $Self->{Socket}->close() if $Self->{Socket}; - $Self->{Socket} = undef; - $Self->state(Unconnected); - return undef; - } - - # There was something, find what it was - $Atom = $Self->_remaining_line(); - - $Self->_trace("A: is_open test returned data - '$Atom'\n") - if $Self->{Trace}; - - $Atom || die "IMAPTalk: Unexpected response while checking connection - $Atom"; - - # If it's a bye, we're being closed - if ($Atom =~ /^bye/i) { - $Self->_trace("A: is_open test received 'bye' response\n") - if $Self->{Trace}; - $Self->{Socket}->close(); - $Self->{Socket} = undef; - $Self->state(Unconnected); - return undef; - } - - # Otherwise it was probably some sort of alert, - # check again - } - -} - -=item I - -Change the root folder prefix. Some IMAP servers require that all user -folders/mailboxes live under a root folder prefix (current versions of -B for example use 'INBOX' for personal folders and 'user' for other -users folders). If no value is specified, it sets it to ''. You might -want to use the B method to find out what roots are -available. - -Setting this affects all commands that take a folder argument. Basically -if the foldername begins with root folder prefix, it's left as is, -otherwise the root folder prefix and separator char are prefixed to the -folder name. - -The AltRootRegexp is a regexp that if the start of the folder name matches, -does not have $RootFolder preprended. You can use this to protect -other namespaces in your IMAP server. - -Examples: - - # This is what cyrus uses - $IMAP->set_root_folder('INBOX', '.', qr/^user/); - - # Selects 'Inbox' (because 'Inbox' eq 'inbox' case insensitive) - $IMAP->select('Inbox'); - # Selects 'INBOX.blah' - $IMAP->select('blah'); - # Selects 'INBOX.Inbox.fred' - #IMAP->select('Inbox.fred'); - # Selects 'user.john' (because 'user' is alt root) - #IMAP->select('user.john'); # Selects 'user.john' - -=cut -sub set_root_folder { - my ($Self, $RootFolder, $Separator, $AltRootRegexp) = @_; - - $RootFolder = '' if !defined($RootFolder); - $Separator = '.' if !defined($Separator); - - # Strip off the Separator, if the IMAP-Server already appended it - $RootFolder =~ s/\Q$Separator\E$//; - - $Self->{RootFolder} = $RootFolder; - $Self->{AltRootRegexp} = $AltRootRegexp; - $Self->{Separator} = $Separator; - - # We map canonical IMAP INBOX to nicer looking Inbox, - # but have to be careful if the root is INBOX as well - - # INBOX -> Inbox (done in _fix_folder_name) - # INBOX.blah -> blah - # INBOX.inbox -> INBOX.inbox - # INBOX.Inbox -> INBOX.Inbox - # INBOX.inbox.inbox -> INBOX.inbox.inbox - # INBOX.Inbox.blah -> Inbox.blah - # user.xyz -> user.xyz - - # RootFolderMatch - # If folder passed in doesn't match this, then prepend $RootFolder . $Separator - # eg prepend inbox. if folder !/^inbox(\.inbox)*$|^user$|^user\./ - - # UnrootFolderMatch - # If folder returned matches this, strip $RootFolder . $Separator - # eg strip inbox. if folder /^inbox\.(?!inbox(\.inbox)*)/ - - my ($RootFolderMatch, $UnrootFolderMatch); - if ($RootFolder) { - # Note the /i on the end to make this case-insensitive - $RootFolderMatch = qr/\Q${RootFolder}\E(?:\Q${Separator}${RootFolder}\E)*/i; - $UnrootFolderMatch = qr/^\Q${RootFolder}${Separator}\E(?!${RootFolderMatch}$)/; - - if ($AltRootRegexp) { - $RootFolderMatch = qr/^${RootFolderMatch}$|^(?:${AltRootRegexp}(?:\Q${Separator}\E|$))/; - } else { - $RootFolderMatch = qr/^${RootFolderMatch}$/; - } - } - - @$Self{qw(RootFolderMatch UnrootFolderMatch)} - = ($RootFolderMatch, $UnrootFolderMatch); - - return 1; -} - -=item I<_set_separator($Separator)> - -Checks if the given separator is the same as the one we used before. -If not, it calls set_root_folder to recreate the settings with the new -Separator. - -=cut -sub _set_separator { - my ($Self, $Separator) = @_; - - #Nothing to do, if we have the same Separator as before - return 1 if (defined($Separator) && ($Self->{Separator} eq $Separator)); - return $Self->set_root_folder($Self->{RootFolder}, $Separator, $Self->{AltRootRegexp}); -} - -=item I - -Sets the mode whether to read literals as file handles or scalars. - -You should pass a filehandle here that any literal will be read into. To -turn off literal reads into a file handle, pass a 0. - -Examples: - - # Read rfc822 text of message 3 into file - # (note that the file will have /r/n line terminators) - open(F, ">messagebody.txt"); - $IMAP->literal_handle_control(\*F); - $IMAP->fetch(3, 'rfc822'); - $IMAP->literal_handle_control(0); - -=cut -sub literal_handle_control { - my $Self = shift; - $Self->{LiteralControl} = $_[0] if defined $_[0]; - return $Self->{LiteralControl} ? 1 : 0; -} - -=item I - -Release IMAPTalk's ownership of the current socket it's using so it's not -disconnected on DESTROY. This returns the socket, and makes sure that the -IMAPTalk object doesn't hold a reference to it any more and the connection -state is set to "Unconnected". - -This means you can't call any methods on the IMAPTalk object any more. - -If the socket is being released and being closed, then $Close is set to true. - -=cut -sub release_socket { - my $Self = shift; - - # Remove from the select object - $Self->{Select}->remove($Self->{Socket}) if ref($Self->{Select}); - my $Socket = $Self->{Socket}; - - # Delete any knowledge of the socket in our instance - delete $Self->{Socket}; - delete $Self->{Select}; - - $Self->_trace("A: Release socket, fileno=" . fileno($Socket) . "\n") - if $Self->{Trace}; - - # Set into no connection state - $Self->state(Mail::IMAPTalk::Unconnected); - - return $Socket; -} - -=item I - -Returns a text string which describes the last error that occurred. - -=cut -sub get_last_error { - my $Self = shift; - return $Self->{LastError}; -} - -=item I - -Returns the last completion response to the tagged command. - -This is either the string "ok", "no" or "bad" (always lower case) - -=cut -sub get_last_completion_response { - my $Self = shift; - return $Self->{LastRespCode}; -} - -=item I - -Returns the extra response data generated by a previous call. This is -most often used after calling B