Friday, March 5, 2010

Fast way to get IMAP folder size information

Code that connects to an IMAP store over SSL, and obtains folder/subfolder information such as folder size by adding up the RFC822.size of all the messages in the folder. This is normally a very slow operation, but some pre-fetch code has been added to speed up this operation. Also a Map comparator implementation to sort folders in order of size.

 


public static String getMailboxInfo(String mail, String mailHost)
throws javax.mail.NoSuchProviderException, MessagingException, IOException, GeneralSecurityException, Exception {
String html = "";
String pattern = "%";
Config conf = nsit.Config.getInstance();
String mpUser = conf.getOption("mp_user");
String mpPassword = conf.getOption("mp_password");
// setup mail server
MailSSLSocketFactory sf = new MailSSLSocketFactory();
sf.setTrustAllHosts(true);

// set system properties
Properties mailProps = System.getProperties();
mailProps.put("mail.imaps.ssl.socketFactory", sf);
mailProps.put("mail.store.protocol", "imaps");
mailProps.put("mail.imaps.host", mailHost);
mailProps.put("mail.imaps.port", "993");
// get session
//Session session = Session.getDefaultInstance(mailProps);
mailProps.setProperty("mail.imaps.sasl.authorizationid", mail);
Session session = Session.getInstance(mailProps, new SASLMasqueradeAuthenticator(mpUser, mpPassword));
session.setDebug(false);

// Get a Store object
Store store = null;
Folder rf = null;

store = session.getStore();
store.connect();

rf = store.getDefaultFolder();
//dumpFolder(rf, false, "");

map.clear();

if ((rf.getType() & Folder.HOLDS_FOLDERS) != 0) {
Folder[] f = rf.list(pattern);
for (int i = 0; i < f.length; i++) {
dumpFolder(f[i], true, " ");
}
store.close();
}

ValueComparator bvc = new ValueComparator(map);
TreeMap sorted_map = new TreeMap(bvc);
sorted_map.putAll(map);

Set set = sorted_map.entrySet();
Iterator i = set.iterator();
while (i.hasNext()) {
Map.Entry me = (Map.Entry) i.next();
FolderInfo fi = (FolderInfo) me.getValue();
int fn = fi.folderNmsgs;
Long fs = fi.folderSize;
System.out.println(me.getKey() + " : " + fn + " : " + fs);
html += "
";

}
html += "
Mailbox/FolderMessage CountSize/Usage
" + me.getKey() + "" + fn + "" + fs + " MB
";
return html;
}

private static class SASLMasqueradeAuthenticator extends Authenticator {

String authcid, authcpass;

public SASLMasqueradeAuthenticator(String adminusername, String adminpassword) {
authcid = adminusername;
authcpass = adminpassword;
}

protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(authcid, authcpass);
}
}

static void dumpFolder(Folder folder, boolean recurse, String tab)
throws Exception, MessagingException {

if (folder.getType() == 2) {
if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
if (recurse) {
Folder[] f = folder.list();
for (int i = 0; i < f.length; i++) {
dumpFolder(f[i], recurse, tab + " ");
}
}
}
return;
}
folder.open(Folder.READ_ONLY);
Message messages[] = folder.getMessages();

// Tell JavaMail to do some pre-fetching of the usual cruft.
FetchProfile fp = new FetchProfile();
fp.add(FetchProfile.Item.ENVELOPE);
folder.fetch(messages, fp);

System.out.println(tab + "Name: " + folder.getName());
System.out.println(tab + "Full Name: " + folder.getFullName());
System.out.println(tab + "URL: " + folder.getURLName());
boolean verbose = true;

if (verbose) {
if (!folder.isSubscribed()) {
System.out.println(tab + "Not Subscribed");
}

if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
if (folder.hasNewMessages()) {
System.out.println(tab + "Has New Messages");
}
System.out.println(tab + "Total Messages: " +
folder.getMessageCount());
System.out.println(tab + "New Messages: " +
folder.getNewMessageCount());
System.out.println(tab + "Unread Messages: " +
folder.getUnreadMessageCount());
}
if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
System.out.println(tab + "Is Directory");
}


long folderSize = 0;
for (Message message : messages) {
folderSize += message.getSize();
}

long folderSizeMB = (folderSize / (1024L * 1024L));
System.out.printf("Folder Size in MB: %d\n", folderSizeMB);
FolderInfo fi = new FolderInfo(folder.getMessageCount(), folderSizeMB);
map.put(folder.getFullName(), fi);

/*
* Demonstrate use of IMAP folder attributes
* returned by the IMAP LIST response.
*/
if (folder instanceof IMAPFolder) {
IMAPFolder f = (IMAPFolder) folder;
String[] attrs = f.getAttributes();
if (attrs != null && attrs.length > 0) {
System.out.println(tab + "IMAP Attributes:");
for (int i = 0; i < attrs.length; i++) {
System.out.println(tab + " " + attrs[i]);
}
}
}
}

System.out.println();

if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
if (recurse) {
Folder[] f = folder.list();
for (int i = 0; i < f.length; i++) {
dumpFolder(f[i], recurse, tab + " ");
}
}
}
}

public static void main(String[] args) {
}
}

class ValueComparator implements Comparator {

Map base;

public ValueComparator(Map base) {
this.base = base;
}

public int compare(Object a, Object b) {

FolderInfo fa = (FolderInfo) base.get(a);
FolderInfo fb = (FolderInfo) base.get(b);
Long fsa = fa.folderSize;
Long fsb = fb.folderSize;

if ((Long) fsa < (Long) fsb) {
return 1;
} else if ((Long) fsa == (Long) fsb) {
return 1;
} else {
return -1;
}
}
}

class FolderInfo {

int folderNmsgs;
Long folderSize;

public FolderInfo(int i, Long l) {
this.folderNmsgs = i;
this.folderSize = l;
}
}


Monday, February 22, 2010

Using QuotaAwareStore

Using the QuotaAwareStore interface to obtain quota information for an IMAP folder
 
Store store = null;
Folder rf = null;
store = session.getStore();
store.connect();

rf = store.getDefaultFolder();

if ((rf.getType() & Folder.HOLDS_FOLDERS) != 0) {
Folder[] f = rf.list(pattern);
for (int i = 0; i < f.length; i++) {
log.info(f[i].getName());

IMAPSSLStore store3 = (IMAPSSLStore) f[i].getStore();
QuotaAwareStore qas = (QuotaAwareStore) store3;
Quota[] test = qas.getQuota(f[i].getName());
log.info("test.length " + test.length);
log.info("test[0] " + test[0]);
log.info("quota root " + test[0].quotaRoot);
Quota.Resource[] resources = test[0].resources;
log.info("resources.length " + resources.length);
log.info("resources[0].name " + resources[0].name);
log.info("resources[0].limit " + resources[0].limit);
log.info("resources[0].usage " + resources[0].usage);
}
store.close();

Get size of IMAP folder using JavaMail

Obtaining size of IMAP folder by totalling RFC822 message sizes of msgs in a folder.

          if (verbose) {
if (!folder.isSubscribed()) {
System.out.println(tab + "Not Subscribed");
}

if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
if (folder.hasNewMessages()) {
System.out.println(tab + "Has New Messages");
}
System.out.println(tab + "Total Messages: " +
folder.getMessageCount());
System.out.println(tab + "New Messages: " +
folder.getNewMessageCount());
System.out.println(tab + "Unread Messages: " +
folder.getUnreadMessageCount());
}
if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
System.out.println(tab + "Is Directory");
}

if (folder.getName().length() != 0) { //root folder
if (folder.getMessageCount() != 0) {
long msgs_size = 0;
log.info("folder.getMessageCount() " + folder.getMessageCount());
folder.open(Folder.READ_ONLY);
Message[] messages = folder.getMessages();
log.info("messages.length " + messages.length);
log.info("msgs_size " + msgs_size);
//rjm 010810:scp scp slowness due to something in here? set debugging statements
//if (folder.getName().equals("Sent Messages")) {
for (int x = 0; x < messages.length; x++) {
IMAPMessage tmp = (IMAPMessage) messages[x];
if (tmp.getSize() != -1) {
log.info("tmp.getSize() " + tmp.getSize());
msgs_size = msgs_size + tmp.getSize();
log.info("msgs_size " + msgs_size);//size in bytes
log.info(msgs_size / (1024L * 1024L)); //converted to MB
}
}
//}
}
}

Monday, February 1, 2010

Multiple hop tunnel to chain port forwarding

I was trying to set up a db connection to from a vm on my laptop to a db server that was configured to only accept connections from machines behind its own subnet. I had trouble setting up a multiple hop tunnel for chaining port forwarding through my firewall machine on the same subnet as the db. My first attempt involved two port forwards, on localhost and on the firewall machine, which didn't work for me. This approach I found at http://www.derkeiler.com/Newsgroups/comp.security.ssh/2006-03/msg00267.html involved constructing an end to end connection to the db via the firewall machine


When you have to go through multiple hops, it's usually better to get an
end-to-end connection. In this case:

ssh -oproxycommand="ssh -qaxT firewall nc %h %p" -L 5432:localhost:5432 dbserver

If you have a copy of the snail book, section 11.4 (p444) has a discussion
of these two approaches.

The annoyance with the second approach is that it requires having netcat
("nc") or something equivalent on the intermediate host. I hope that
someday OpenSSH will have this feature built in, i.e. connecting an exec
channel to a remote TCP connection.


Edit: I found some documentation on 'ProxyCommand' here that seems relevant

ProxyCommand
Specifies the command to use to connect to the server. The com-
mand string extends to the end of the line, and is executed with
the user's shell. In the command string, `%h' will be substitut-
ed by the host name to connect and `%p' by the port. The command
can be basically anything, and should read from its standard in-
put and write to its standard output. It should eventually con-
nect an sshd(8) server running on some machine, or execute sshd
-i somewhere. Host key management will be done using the Host-
Name of the host being connected (defaulting to the name typed by
the user). Setting the command to ``none'' disables this option
entirely. Note that CheckHostIP is not available for connects
with a proxy command.

This directive is useful in conjunction with nc(1) and its proxy
support. For example, the following directive would connect via
an HTTP proxy at 192.0.2.0:

ProxyCommand /usr/bin/nc -X connect -x 192.0.2.0:8080 %h %p

Monday, January 11, 2010

Connect to IMAP using JavaMail over SSL

Connect to and get mailbox information from an imaps server using JavaMail. I'm using a custom SASLMasqueradeAuthenticator method, configuring JavaMail's Socket Fetcher class to use "MailSSLSocketFactory".


    public static void getMailboxInfo(String mail, String mailHost)
throws javax.mail.NoSuchProviderException, MessagingException, IOException, GeneralSecurityException {
Config conf = nsit.Config.getInstance();
String mpUser = "";
String mpPassword = "";
// setup mail server
MailSSLSocketFactory sf = new MailSSLSocketFactory();
sf.setTrustAllHosts(true);

// set system properties
Properties mailProps = System.getProperties();
mailProps.put("mail.imaps.ssl.socketFactory", sf);
mailProps.put("mail.store.protocol", "imaps");
mailProps.put("mail.imaps.host", mailHost);
mailProps.put("mail.imaps.port", "993");

// get session
//Session session = Session.getDefaultInstance(mailProps);
mailProps.setProperty("mail.imaps.sasl.authorizationid", mail);
Session session = Session.getInstance(mailProps, new SASLMasqueradeAuthenticator(mpUser, mpPassword));
session.setDebug(false);
Store srcStore;
srcStore = session.getStore();
srcStore.connect();
Folder src = srcStore.getDefaultFolder();
//log.info("src.getName() " + src.getName());
//log.info("src.getType() " + src.getType());
//log.info("src.HOLDS_FOLDERS " + src.HOLDS_FOLDERS);

if ((src.getType() & Folder.HOLDS_FOLDERS) != 0) {
Folder[] subfolders = src.list();
log.info("subfolders.length " + subfolders.length);
for (int i = 0; i < subfolders.length; i++) {
long msgs_size = 0;

if (subfolders[i].getMessageCount() != 0) {
log.info("subfolders[i].getMessageCount() " + subfolders[i].getMessageCount());
subfolders[i].open(Folder.READ_ONLY);
Message[] messages = subfolders[i].getMessages();

for (int x = 0; x < messages.length; x++) {
//log.info("msg_size " + messages[i].getSize() + " bytes long.");
if (messages[i].getSize() != -1) {
msgs_size += messages[i].getSize();
}
}
log.info("msgs_size " + (msgs_size));
log.info("msgs_size " + (msgs_size / (1024L * 1024L)));

subfolders[i].close(false);
}
} else {
System.out.println("Not default folder");
}
}

private static class SASLMasqueradeAuthenticator extends Authenticator {

String authcid, authcpass;

public SASLMasqueradeAuthenticator(String adminusername, String adminpassword) {
authcid = adminusername;
authcpass = adminpassword;
}

protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(authcid, authcpass);
}
}

Wednesday, August 26, 2009

Stored functions to BASE64 ENCODE and DECODE in MySQL

-- base64.sql - MySQL base64 encoding/decoding functions
-- Copyright (C) 2006 Ian Gulliver
--
-- This program is free software; you can redistribute it and/or modify
-- it under the terms of version 2 of the GNU General Public License as
-- published by the Free Software Foundation.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with this program; if not, write to the Free Software
-- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

delimiter |

DROP TABLE IF EXISTS base64_data |
CREATE TABLE base64_data (c CHAR(1) BINARY, val TINYINT) |
INSERT INTO base64_data VALUES
('A',0), ('B',1), ('C',2), ('D',3), ('E',4), ('F',5), ('G',6), ('H',7), ('I',8), ('J',9),
('K',10), ('L',11), ('M',12), ('N',13), ('O',14), ('P',15), ('Q',16), ('R',17), ('S',18), ('T',19),
('U',20), ('V',21), ('W',22), ('X',23), ('Y',24), ('Z',25), ('a',26), ('b',27), ('c',28), ('d',29),
('e',30), ('f',31), ('g',32), ('h',33), ('i',34), ('j',35), ('k',36), ('l',37), ('m',38), ('n',39),
('o',40), ('p',41), ('q',42), ('r',43), ('s',44), ('t',45), ('u',46), ('v',47), ('w',48), ('x',49),
('y',50), ('z',51), ('0',52), ('1',53), ('2',54), ('3',55), ('4',56), ('5',57), ('6',58), ('7',59),
('8',60), ('9',61), ('+',62), ('/',63), ('=',0) |


DROP FUNCTION IF EXISTS BASE64_DECODE |
CREATE FUNCTION BASE64_DECODE (input BLOB)
RETURNS BLOB
CONTAINS SQL
DETERMINISTIC
SQL SECURITY INVOKER
BEGIN
DECLARE ret BLOB DEFAULT '';
DECLARE done TINYINT DEFAULT 0;

IF input IS NULL THEN
RETURN NULL;
END IF;

each_block:
WHILE NOT done DO BEGIN
DECLARE accum_value BIGINT UNSIGNED DEFAULT 0;
DECLARE in_count TINYINT DEFAULT 0;
DECLARE out_count TINYINT DEFAULT 3;

each_input_char:
WHILE in_count < 4 DO BEGIN
DECLARE first_char CHAR(1);

IF LENGTH(input) = 0 THEN
RETURN ret;
END IF;

SET first_char = SUBSTRING(input,1,1);
SET input = SUBSTRING(input,2);

BEGIN
DECLARE tempval TINYINT UNSIGNED;
DECLARE error TINYINT DEFAULT 0;
DECLARE base64_getval CURSOR FOR SELECT val FROM base64_data WHERE c = first_char;
DECLARE CONTINUE HANDLER FOR SQLSTATE '02000' SET error = 1;

OPEN base64_getval;
FETCH base64_getval INTO tempval;
CLOSE base64_getval;

IF error THEN
ITERATE each_input_char;
END IF;

SET accum_value = (accum_value << 6) + tempval;
END;

SET in_count = in_count + 1;

IF first_char = '=' THEN
SET done = 1;
SET out_count = out_count - 1;
END IF;
END; END WHILE;

-- We've now accumulated 24 bits; deaccumulate into bytes

-- We have to work from the left, so use the third byte position and shift left
WHILE out_count > 0 DO BEGIN
SET ret = CONCAT(ret,CHAR((accum_value & 0xff0000) >> 16));
SET out_count = out_count - 1;
SET accum_value = (accum_value << 8) & 0xffffff;
END; END WHILE;

END; END WHILE;

RETURN ret;
END |

DROP FUNCTION IF EXISTS BASE64_ENCODE |
CREATE FUNCTION BASE64_ENCODE (input BLOB)
RETURNS BLOB
CONTAINS SQL
DETERMINISTIC
SQL SECURITY INVOKER
BEGIN
DECLARE ret BLOB DEFAULT '';
DECLARE done TINYINT DEFAULT 0;

IF input IS NULL THEN
RETURN NULL;
END IF;

each_block:
WHILE NOT done DO BEGIN
DECLARE accum_value BIGINT UNSIGNED DEFAULT 0;
DECLARE in_count TINYINT DEFAULT 0;
DECLARE out_count TINYINT;

each_input_char:
WHILE in_count < 3 DO BEGIN
DECLARE first_char CHAR(1);

IF LENGTH(input) = 0 THEN
SET done = 1;
SET accum_value = accum_value << (8 * (3 - in_count));
LEAVE each_input_char;
END IF;

SET first_char = SUBSTRING(input,1,1);
SET input = SUBSTRING(input,2);

SET accum_value = (accum_value << 8) + ASCII(first_char);

SET in_count = in_count + 1;
END; END WHILE;

-- We've now accumulated 24 bits; deaccumulate into base64 characters

-- We have to work from the left, so use the third byte position and shift left
CASE
WHEN in_count = 3 THEN SET out_count = 4;
WHEN in_count = 2 THEN SET out_count = 3;
WHEN in_count = 1 THEN SET out_count = 2;
ELSE RETURN ret;
END CASE;

WHILE out_count > 0 DO BEGIN
BEGIN
DECLARE out_char CHAR(1);
DECLARE base64_getval CURSOR FOR SELECT c FROM base64_data WHERE val = (accum_value >> 18);

OPEN base64_getval;
FETCH base64_getval INTO out_char;
CLOSE base64_getval;

SET ret = CONCAT(ret,out_char);
SET out_count = out_count - 1;
SET accum_value = accum_value << 6 & 0xffffff;
END;
END; END WHILE;

CASE
WHEN in_count = 2 THEN SET ret = CONCAT(ret,'=');
WHEN in_count = 1 THEN SET ret = CONCAT(ret,'==');
ELSE BEGIN END;
END CASE;

END; END WHILE;

RETURN ret;
END |

Thursday, July 2, 2009

Shibboleth Relational Database DataConnector

Example Shibboleth Relational Database DataConnector configuration for MySQL with column mapping of AES encrypted attribute.


xmlns="urn:mace:shibboleth:2.0:resolver:dc"
id="MyDatabase">

jdbcURL="jdbc:mysql://localhost:3306/handle"
jdbcUserName="test"
jdbcPassword="test" />


SELECT AES_DECRYPT(password, 'key') FROM handle WHERE username='$requestContext.principalName'
]]>