diff --git a/initdb/init_android.db b/initdb/init_android.db index 290fa76..f8ae136 100644 Binary files a/initdb/init_android.db and b/initdb/init_android.db differ diff --git a/initdb/init_android_db.sql b/initdb/init_android_db.sql index 4d2a472..277f7e5 100644 --- a/initdb/init_android_db.sql +++ b/initdb/init_android_db.sql @@ -6,21 +6,21 @@ CREATE TABLE attachments (sms_id INTEGER,content_url TEXT,offset INTEGER); CREATE TABLE canonical_addresses (_id INTEGER PRIMARY KEY,address TEXT); CREATE TABLE drm (_id INTEGER PRIMARY KEY,_data TEXT); CREATE TABLE part (_id INTEGER PRIMARY KEY,mid INTEGER,seq INTEGER DEFAULT 0,ct TEXT,name TEXT,chset INTEGER,cd TEXT,fn TEXT,cid TEXT,cl TEXT,ctt_s INTEGER,ctt_t TEXT,_data TEXT,text TEXT); -CREATE TABLE pdu (_id INTEGER PRIMARY KEY,thread_id INTEGER,date INTEGER,msg_box INTEGER,read INTEGER DEFAULT 0,m_id TEXT,sub TEXT,sub_cs INTEGER,ct_t TEXT,ct_l TEXT,exp INTEGER,m_cls TEXT,m_type INTEGER,v INTEGER,m_size INTEGER,pri INTEGER,rr INTEGER,rpt_a INTEGER,resp_st INTEGER,st INTEGER,tr_id TEXT,retr_st INTEGER,retr_txt TEXT,retr_txt_cs INTEGER,read_status INTEGER,ct_cls INTEGER,resp_txt TEXT,d_tm INTEGER,d_rpt INTEGER,locked INTEGER DEFAULT 0,seen INTEGER DEFAULT 0); +CREATE TABLE pdu (_id INTEGER PRIMARY KEY,thread_id INTEGER,date INTEGER,date_sent INTEGER,msg_box INTEGER,read INTEGER DEFAULT 0,m_id TEXT,sub TEXT,sub_cs INTEGER,ct_t TEXT,ct_l TEXT,exp INTEGER,m_cls TEXT,m_type INTEGER,v INTEGER,m_size INTEGER,pri INTEGER,rr INTEGER,rpt_a INTEGER,resp_st INTEGER,st INTEGER,tr_id TEXT,retr_st INTEGER,retr_txt TEXT,retr_txt_cs INTEGER,read_status INTEGER,ct_cls INTEGER,resp_txt TEXT,d_tm INTEGER,d_rpt INTEGER,locked INTEGER DEFAULT 0,seen INTEGER DEFAULT 0); CREATE TABLE pending_msgs (_id INTEGER PRIMARY KEY,proto_type INTEGER,msg_id INTEGER,msg_type INTEGER,err_type INTEGER,err_code INTEGER,retry_index INTEGER NOT NULL DEFAULT 0,due_time INTEGER,last_try INTEGER); CREATE TABLE rate (sent_time INTEGER); CREATE TABLE raw (_id INTEGER PRIMARY KEY,date INTEGER,reference_number INTEGER,count INTEGER,sequence INTEGER,destination_port INTEGER,address TEXT,pdu TEXT); -CREATE TABLE sms (_id INTEGER PRIMARY KEY,thread_id INTEGER,address TEXT,person INTEGER,date INTEGER,protocol INTEGER,read INTEGER DEFAULT 0,status INTEGER DEFAULT -1,type INTEGER,reply_path_present INTEGER,subject TEXT,body TEXT,service_center TEXT,locked INTEGER DEFAULT 0,error_code INTEGER DEFAULT 0,seen INTEGER DEFAULT 0); +CREATE TABLE sms (_id INTEGER PRIMARY KEY,thread_id INTEGER,address TEXT,person INTEGER,date INTEGER,date_sent INTEGER,protocol INTEGER,read INTEGER DEFAULT 0,status INTEGER DEFAULT -1,type INTEGER,reply_path_present INTEGER,subject TEXT,body TEXT,service_center TEXT,locked INTEGER DEFAULT 0,error_code INTEGER DEFAULT 0,seen INTEGER DEFAULT 0); CREATE TABLE sr_pending (reference_number INTEGER,action TEXT,data TEXT); CREATE TABLE threads (_id INTEGER PRIMARY KEY,date INTEGER DEFAULT 0,message_count INTEGER DEFAULT 0,recipient_ids TEXT,snippet TEXT,snippet_cs INTEGER DEFAULT 0,read INTEGER DEFAULT 1,type INTEGER DEFAULT 0,error INTEGER DEFAULT 0,has_attachment INTEGER DEFAULT 0); CREATE VIRTUAL TABLE words USING FTS3 (_id INTEGER PRIMARY KEY, index_text TEXT, source_id INTEGER, table_to_use INTEGER); -CREATE TRIGGER pdu_update_thread_on_insert AFTER INSERT ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; -CREATE TRIGGER sms_update_thread_on_insert AFTER INSERT ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; -CREATE TRIGGER pdu_update_thread_date_subject_on_update AFTER UPDATE OF date, sub, msg_box ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; -CREATE TRIGGER sms_update_thread_date_subject_on_update AFTER UPDATE OF date, body, type ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; +CREATE TRIGGER pdu_update_thread_on_insert AFTER INSERT ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; +CREATE TRIGGER sms_update_thread_on_insert AFTER INSERT ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; +CREATE TRIGGER pdu_update_thread_date_subject_on_update AFTER UPDATE OF date, date_sent, sub, msg_box ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; +CREATE TRIGGER sms_update_thread_date_subject_on_update AFTER UPDATE OF date, date_sent, body, type ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; CREATE TRIGGER pdu_update_thread_read_on_update AFTER UPDATE OF read ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; CREATE TRIGGER sms_update_thread_read_on_update AFTER UPDATE OF read ON sms BEGIN UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END; -CREATE TRIGGER pdu_update_thread_on_delete AFTER DELETE ON pdu BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000) WHERE threads._id = old.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = old.thread_id; UPDATE threads SET snippet = (SELECT snippet FROM (SELECT date * 1000 AS date, sub AS snippet, thread_id FROM pdu UNION SELECT date, body AS snippet, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; UPDATE threads SET snippet_cs = (SELECT snippet_cs FROM (SELECT date * 1000 AS date, sub_cs AS snippet_cs, thread_id FROM pdu UNION SELECT date, 0 AS snippet_cs, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; END; +CREATE TRIGGER pdu_update_thread_on_delete AFTER DELETE ON pdu BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000) WHERE threads._id = old.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = old.thread_id; UPDATE threads SET snippet = (SELECT snippet FROM (SELECT date * 1000 AS date, sub AS snippet, thread_id FROM pdu UNION SELECT date, body AS snippet, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; UPDATE threads SET snippet_cs = (SELECT snippet_cs FROM (SELECT date * 1000 AS date, sub_cs AS snippet_cs, thread_id FROM pdu UNION SELECT date, 0 AS snippet_cs, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; END; CREATE TRIGGER delete_obsolete_threads_pdu AFTER DELETE ON pdu BEGIN DELETE FROM threads WHERE _id = old.thread_id AND _id NOT IN (SELECT thread_id FROM sms UNION SELECT thread_id from pdu); END; CREATE TRIGGER delete_obsolete_threads_when_update_pdu AFTER UPDATE OF thread_id ON pdu WHEN old.thread_id != new.thread_id BEGIN DELETE FROM threads WHERE _id = old.thread_id AND _id NOT IN (SELECT thread_id FROM sms UNION SELECT thread_id from pdu); END; CREATE TRIGGER insert_mms_pending_on_insert AFTER INSERT ON pdu WHEN new.m_type=130 OR new.m_type=135 BEGIN INSERT INTO pending_msgs (proto_type, msg_id, msg_type, err_type, err_code, retry_index, due_time) VALUES (1, new._id, new.m_type,0,0,0,0);END; diff --git a/smstools/android.py b/smstools/android.py index ae22e9f..8b4e471 100644 --- a/smstools/android.py +++ b/smstools/android.py @@ -6,7 +6,7 @@ class Android: def parse(self, filepath): - """ Parse a sqlite file to Text[] """ + """Parse a sqlite file from android format to Text[] """ db = sqlite3.connect(filepath) cursor = db.cursor() @@ -15,23 +15,26 @@ def parse(self, filepath): db.close() return texts + def parse_cursor(self, cursor): + """Gets content of the android db.""" texts = [] query = cursor.execute( - 'SELECT address, date, type, body \ + 'SELECT address, date, date_sent, type, body \ FROM sms \ ORDER BY _id ASC;') for row in query: - txt = core.Text(num=row[0],date=long(row[1]),incoming=(row[2]==2),body=row[3]) + #txt = core.Text(num=row[0],date=long(row[1]),incoming=(row[2]==2),body=row[3]) + txt = core.Text(num=row[0],date=long(row[1]),date_sent=long(row[2]),incoming=(row[3]==2),body=row[4]) texts.append(txt) return texts def write(self, texts, outfilepath): - """ write a Text[] to sqlite file """ + """Write a Text[] to sqlite file into android format.""" print "Creating empty Android SQLITE db" conn = sqlite3.connect(outfilepath) - conn.executescript(INIT_DB_SQL) cursor = conn.cursor() + cursor.executescript(INIT_DB_SQL) self.write_cursor(texts, cursor) @@ -42,11 +45,12 @@ def write(self, texts, outfilepath): def write_cursor(self, texts, cursor): - + """.""" if (cursor.execute("SELECT Count() FROM sms").fetchone()[0] > 0): raise sms_exceptions.NonEmptyStartDBError("Output DB has existing messages!") #populate fast lookup table: + # associate a phone number/mail to its _id as following contactIdFromNumber[phoneNumber]=_id contactIdFromNumber = {} query = cursor.execute('SELECT _id,address FROM canonical_addresses;') for row in query: @@ -59,7 +63,6 @@ def write_cursor(self, texts, cursor): starttime = time.time() for txt in texts: - clean_number = core.cleanNumber(txt.num) #add a new canonical_addresses lookup entry and thread item if it doesn't exist @@ -79,11 +82,12 @@ def write_cursor(self, texts, cursor): print "thread_id = "+ str(thread_id) cursor.execute( "SELECT * FROM threads WHERE _id=?", [contact_id] ) print "updated thread: " + str(cursor.fetchone()) - print "adding entry to message db: " + str([txt.num,txt.date,txt.body,thread_id,txt.incoming+1]) + print "adding entry to message db: " + str([txt.num,txt.date,txt.date,txt.body,thread_id,txt.incoming+1]) ## TODO try using cur.execute('BEGIN TRANSACTION') and cur.execute('COMMIT') every 1000 for speedup #add message to sms table - cursor.execute( "INSERT INTO sms (address,'date',body,thread_id,read,type,seen) VALUES (?,?,?,?,1,?,1)", [txt.num,txt.date,txt.body,thread_id,txt.incoming+1]) + #cursor.execute( "INSERT INTO sms (address,'date',body,thread_id,read,type,seen) VALUES (?,?,?,?,1,?,1)", [txt.num,txt.date,txt.body,thread_id,txt.incoming+1]) + cursor.execute( "INSERT INTO sms (address,date,date_sent,body,thread_id,read,type,seen) VALUES (?,?,?,?,?,1,?,1)", [txt.num, txt.date, txt.date, txt.body, thread_id, txt.incoming+1]) #print status (with fancy speed calculation) recalculate_every = 100 @@ -102,6 +106,7 @@ def write_cursor(self, texts, cursor): print row +#for each threads._id=sms.thread_id, set threads.read = 0 if a sms is not read, 1 else INIT_DB_SQL = "\ BEGIN TRANSACTION;\ @@ -112,18 +117,18 @@ def write_cursor(self, texts, cursor): CREATE TABLE canonical_addresses (_id INTEGER PRIMARY KEY,address TEXT);\ CREATE TABLE drm (_id INTEGER PRIMARY KEY,_data TEXT);\ CREATE TABLE part (_id INTEGER PRIMARY KEY,mid INTEGER,seq INTEGER DEFAULT 0,ct TEXT,name TEXT,chset INTEGER,cd TEXT,fn TEXT,cid TEXT,cl TEXT,ctt_s INTEGER,ctt_t TEXT,_data TEXT,text TEXT);\ -CREATE TABLE pdu (_id INTEGER PRIMARY KEY,thread_id INTEGER,date INTEGER,msg_box INTEGER,read INTEGER DEFAULT 0,m_id TEXT,sub TEXT,sub_cs INTEGER,ct_t TEXT,ct_l TEXT,exp INTEGER,m_cls TEXT,m_type INTEGER,v INTEGER,m_size INTEGER,pri INTEGER,rr INTEGER,rpt_a INTEGER,resp_st INTEGER,st INTEGER,tr_id TEXT,retr_st INTEGER,retr_txt TEXT,retr_txt_cs INTEGER,read_status INTEGER,ct_cls INTEGER,resp_txt TEXT,d_tm INTEGER,d_rpt INTEGER,locked INTEGER DEFAULT 0,seen INTEGER DEFAULT 0);\ +CREATE TABLE pdu (_id INTEGER PRIMARY KEY,thread_id INTEGER,date INTEGER,date_sent INTEGER,msg_box INTEGER,read INTEGER DEFAULT 0,m_id TEXT,sub TEXT,sub_cs INTEGER,ct_t TEXT,ct_l TEXT,exp INTEGER,m_cls TEXT,m_type INTEGER,v INTEGER,m_size INTEGER,pri INTEGER,rr INTEGER,rpt_a INTEGER,resp_st INTEGER,st INTEGER,tr_id TEXT,retr_st INTEGER,retr_txt TEXT,retr_txt_cs INTEGER,read_status INTEGER,ct_cls INTEGER,resp_txt TEXT,d_tm INTEGER,d_rpt INTEGER,locked INTEGER DEFAULT 0,seen INTEGER DEFAULT 0);\ CREATE TABLE pending_msgs (_id INTEGER PRIMARY KEY,proto_type INTEGER,msg_id INTEGER,msg_type INTEGER,err_type INTEGER,err_code INTEGER,retry_index INTEGER NOT NULL DEFAULT 0,due_time INTEGER,last_try INTEGER);\ CREATE TABLE rate (sent_time INTEGER);\ CREATE TABLE raw (_id INTEGER PRIMARY KEY,date INTEGER,reference_number INTEGER,count INTEGER,sequence INTEGER,destination_port INTEGER,address TEXT,pdu TEXT);\ -CREATE TABLE sms (_id INTEGER PRIMARY KEY,thread_id INTEGER,address TEXT,person INTEGER,date INTEGER,protocol INTEGER,read INTEGER DEFAULT 0,status INTEGER DEFAULT -1,type INTEGER,reply_path_present INTEGER,subject TEXT,body TEXT,service_center TEXT,locked INTEGER DEFAULT 0,error_code INTEGER DEFAULT 0,seen INTEGER DEFAULT 0);\ +CREATE TABLE sms (_id INTEGER PRIMARY KEY,thread_id INTEGER,address TEXT,person INTEGER,date INTEGER,date_sent INTEGER,protocol INTEGER,read INTEGER DEFAULT 0,status INTEGER DEFAULT -1,type INTEGER,reply_path_present INTEGER,subject TEXT,body TEXT,service_center TEXT,locked INTEGER DEFAULT 0,error_code INTEGER DEFAULT 0,seen INTEGER DEFAULT 0);\ CREATE TABLE sr_pending (reference_number INTEGER,action TEXT,data TEXT);\ CREATE TABLE threads (_id INTEGER PRIMARY KEY,date INTEGER DEFAULT 0,message_count INTEGER DEFAULT 0,recipient_ids TEXT,snippet TEXT,snippet_cs INTEGER DEFAULT 0,read INTEGER DEFAULT 1,type INTEGER DEFAULT 0,error INTEGER DEFAULT 0,has_attachment INTEGER DEFAULT 0);\ CREATE VIRTUAL TABLE words USING FTS3 (_id INTEGER PRIMARY KEY, index_text TEXT, source_id INTEGER, table_to_use INTEGER);\ -CREATE TRIGGER pdu_update_thread_on_insert AFTER INSERT ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ -CREATE TRIGGER sms_update_thread_on_insert AFTER INSERT ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ -CREATE TRIGGER pdu_update_thread_date_subject_on_update AFTER UPDATE OF date, sub, msg_box ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ -CREATE TRIGGER sms_update_thread_date_subject_on_update AFTER UPDATE OF date, body, type ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ +CREATE TRIGGER pdu_update_thread_on_insert AFTER INSERT ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ +CREATE TRIGGER sms_update_thread_on_insert AFTER INSERT ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ +CREATE TRIGGER pdu_update_thread_date_subject_on_update AFTER UPDATE OF date, date_sent, sub, msg_box ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.sub, snippet_cs = new.sub_cs WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ +CREATE TRIGGER sms_update_thread_date_subject_on_update AFTER UPDATE OF date, date_sent, body, type ON sms BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000), date_sent = (strftime('%s','now') * 1000), snippet = new.body, snippet_cs = 0 WHERE threads._id = new.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = new.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = new.thread_id; UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ CREATE TRIGGER pdu_update_thread_read_on_update AFTER UPDATE OF read ON pdu WHEN new.m_type=132 OR new.m_type=130 OR new.m_type=128 BEGIN UPDATE threads SET read = CASE (SELECT COUNT(*) FROM pdu WHERE read = 0 AND thread_id = threads._id AND (m_type=132 OR m_type=130 OR m_type=128)) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ CREATE TRIGGER sms_update_thread_read_on_update AFTER UPDATE OF read ON sms BEGIN UPDATE threads SET read = CASE (SELECT COUNT(*) FROM sms WHERE read = 0 AND thread_id = threads._id) WHEN 0 THEN 1 ELSE 0 END WHERE threads._id = new.thread_id; END;\ CREATE TRIGGER pdu_update_thread_on_delete AFTER DELETE ON pdu BEGIN UPDATE threads SET date = (strftime('%s','now') * 1000) WHERE threads._id = old.thread_id; UPDATE threads SET message_count = (SELECT COUNT(sms._id) FROM sms LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND sms.type != 3) + (SELECT COUNT(pdu._id) FROM pdu LEFT JOIN threads ON threads._id = thread_id WHERE thread_id = old.thread_id AND (m_type=132 OR m_type=130 OR m_type=128) AND msg_box != 3) WHERE threads._id = old.thread_id; UPDATE threads SET snippet = (SELECT snippet FROM (SELECT date * 1000 AS date, sub AS snippet, thread_id FROM pdu UNION SELECT date, body AS snippet, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; UPDATE threads SET snippet_cs = (SELECT snippet_cs FROM (SELECT date * 1000 AS date, sub_cs AS snippet_cs, thread_id FROM pdu UNION SELECT date, 0 AS snippet_cs, thread_id FROM sms) WHERE thread_id = OLD.thread_id ORDER BY date DESC LIMIT 1) WHERE threads._id = OLD.thread_id; END;\ diff --git a/smstools/core.py b/smstools/core.py index 7b53500..8bda59a 100644 --- a/smstools/core.py +++ b/smstools/core.py @@ -21,7 +21,7 @@ class Text: def __init__( self, **kwargs): - requiredArgs = ['num', 'date', 'incoming', 'body'] + requiredArgs = ['num', 'date', 'date_sent', 'incoming', 'body'] noneDefaultArgs = ['chatroom', 'members'] for arg in requiredArgs: @@ -42,7 +42,7 @@ def __eq__(self, other): def getParser(filepath): extension = os.path.splitext(filepath)[1] if extension == ".csv": - return tabular.Tabular() + return csv.CSV() if extension == ".json": return jsoner.JSONer() elif extension == ".db" or extension == ".sqlite": @@ -67,6 +67,7 @@ def getParser(filepath): def cleanNumber(numb): + """Return phone number or mail address.""" if not numb: return False if '@' in numb: return numb #allow email addresses stripped = ''.join(ch for ch in numb if ch.isalnum()) @@ -117,14 +118,15 @@ def warning(string): def getTestTexts(): ENCODING_TEST_STRING = u'Δ, Й, ק, ‎ م, ๗, あ, 叶, 葉, and 말.' - return [ Text(num="8675309", date=1326497882355L, incoming=True, body="Yo, what's up boo?"), \ - Text(num="+1(555)565-6565", date=1330568484000L, incoming=False, body="Goodbye cruel testing."),\ - Text(num="+1(555)565-6565", date=random.getrandbits(43), incoming=False, body=ENCODING_TEST_STRING)] + date_sent3 = random.getrandbits(43) + return [ Text(num="8675309", date=1326497882355L, date_sent=1326497882355L, incoming=True, body="Yo, what's up boo?"), \ + Text(num="+1(555)565-6565", date=1330568484000L, date_sent=1330568484000L, incoming=False, body="Goodbye cruel testing."),\ + Text(num="+1(555)565-6565", date=date_sent3, date_sent=date_sent3, incoming=False, body=ENCODING_TEST_STRING)] def compareTexts(t1, t2, throw_errors=True, - required_attrs=['num', 'incoming', 'body', 'date']): + required_attrs=['num', 'incoming', 'body', 'date', 'date_sent']): mismatched = [] for att in required_attrs: if t1.__dict__[att] != t2.__dict__[att]: diff --git a/smstools/ios6.py b/smstools/ios6.py index eb49b6f..f2e7a57 100644 --- a/smstools/ios6.py +++ b/smstools/ios6.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import sqlite3, uuid, os import core, sms_exceptions @@ -6,7 +8,7 @@ class IOS6: """ iOS 6 sqlite reader and writer """ def parse(self, filepath): - """ Parse iOS 6 sqlite file to Text[] """ + """ Open iOS 6 sqlite file and parse it to Text[].""" db = sqlite3.connect(filepath) cursor = db.cursor() texts = self.parse_cursor(cursor) @@ -16,7 +18,7 @@ def parse(self, filepath): return texts def parse_cursor(self, cursor): - + """Parse iOS6 sqlite content to Text[].""" handles = {} query = cursor.execute( 'SELECT ROWID, id, country FROM handle') @@ -46,8 +48,9 @@ def parse_cursor(self, cursor): text = core.Text( num = number, date = long((row[1] + 978307200)*1000), + date_sent = long((row[1] + 978307200)*1000), incoming = row[2] == 0, - body = row[3], + body = row[3], #.encode('utf8') chatroom = row[4], members=(chats[row[4]] if row[4] else None)) texts.append(text) @@ -135,7 +138,7 @@ def write_cursor(self, texts, cursor): print "built messages table with %i entries" % len(texts) - +# ajout CREATE TABLE sqlite_sequence (name, seq) INIT_DB_SQL = "\ BEGIN TRANSACTION;\ CREATE TABLE attachment (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, guid TEXT UNIQUE NOT NULL, created_date INTEGER DEFAULT 0, start_date INTEGER DEFAULT 0, filename TEXT, uti TEXT, mime_type TEXT, transfer_state INTEGER DEFAULT 0, is_outgoing INTEGER DEFAULT 0);\ diff --git a/smstools/tests/core_tests.py b/smstools/tests/core_tests.py index f396d4e..ca2856c 100644 --- a/smstools/tests/core_tests.py +++ b/smstools/tests/core_tests.py @@ -14,6 +14,7 @@ def assertTextsEqual(self, t1, t2): required_attrs=['num', 'incoming', 'body']) self.assertEqual(long(t1.date/1000), long(t2.date/1000)) if 'date' in warns: warns.remove('date') + if 'date_sent' in warns: warns.remove('date_sent') if warns: core.warning("text differ with %s" % (warns)) def get_test_db_files(self, directory=REAL_TEST_FILES_DIR, for_parser=None):