# HG changeset patch # User cli # Date 1250771479 -7200 # Node ID bb6990c0dd1adf7a56e5a2f23993a59386e509c9 # Parent 961a8a3acb9a8d18a75cc58eac6bc3c49e60d829 Merging fixes from sonews/1.0.3 diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/config/BackendConfig.java --- a/org/sonews/config/BackendConfig.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/config/BackendConfig.java Thu Aug 20 14:31:19 2009 +0200 @@ -62,6 +62,12 @@ String configValue = values.get(key); if(configValue == null) { + if(StorageManager.current() == null) + { + Log.msg("Warning: BackendConfig not available, using default.", false); + return defaultValue; + } + configValue = StorageManager.current().getConfigValue(key); if(configValue == null) { diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/config/Config.java --- a/org/sonews/config/Config.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/config/Config.java Thu Aug 20 14:31:19 2009 +0200 @@ -78,7 +78,7 @@ /** The config key for the filename of the logfile */ public static final String LOGFILE = "sonews.log"; - public static final String[] AVAILABLE_KEYS = { + public static final String[] AVAILABLE_KEYS = { ARTICLE_MAXSIZE, EVENTLOG, FEED_NEWSPERRUN, diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/daemon/command/ListCommand.java --- a/org/sonews/daemon/command/ListCommand.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/daemon/command/ListCommand.java Thu Aug 20 14:31:19 2009 +0200 @@ -57,15 +57,15 @@ { final String[] command = line.split(" "); - if (command.length >= 2) + if(command.length >= 2) { - if (command[1].equalsIgnoreCase("OVERVIEW.FMT")) + if(command[1].equalsIgnoreCase("OVERVIEW.FMT")) { conn.println("215 information follows"); conn.println("Subject:\nFrom:\nDate:\nMessage-ID:\nReferences:\nBytes:\nLines:\nXref"); conn.println("."); } - else if (command[1].equalsIgnoreCase("NEWSGROUPS")) + else if(command[1].equalsIgnoreCase("NEWSGROUPS")) { conn.println("215 information follows"); final List list = Channel.getAll(); @@ -75,12 +75,12 @@ } conn.println("."); } - else if (command[1].equalsIgnoreCase("SUBSCRIPTIONS")) + else if(command[1].equalsIgnoreCase("SUBSCRIPTIONS")) { conn.println("215 information follows"); conn.println("."); } - else if (command[1].equalsIgnoreCase("EXTENSIONS")) + else if(command[1].equalsIgnoreCase("EXTENSIONS")) { conn.println("202 Supported NNTP extensions."); conn.println("LISTGROUP"); @@ -88,6 +88,11 @@ conn.println("XPAT"); conn.println("."); } + else if(command[1].equalsIgnoreCase("ACTIVE")) + { + // TODO: Implement wildcards for LIST ACTIVE + printGroupInfo(conn); + } else { conn.println("500 unknown argument to LIST command"); @@ -95,26 +100,31 @@ } else { - final List groups = Channel.getAll(); - if(groups != null) + printGroupInfo(conn); + } + } + + private void printGroupInfo(NNTPConnection conn) + throws IOException, StorageBackendException + { + final List groups = Channel.getAll(); + if (groups != null) + { + conn.println("215 list of newsgroups follows"); + for (Channel g : groups) { - conn.println("215 list of newsgroups follows"); - for (Channel g : groups) + if (!g.isDeleted()) { - if(!g.isDeleted()) - { - String writeable = g.isWriteable() ? " y" : " n"; - // Indeed first the higher article number then the lower - conn.println(g.getName() + " " + g.getLastArticleNumber() + " " - + g.getFirstArticleNumber() + writeable); - } + String writeable = g.isWriteable() ? " y" : " n"; + // Indeed first the higher article number then the lower + conn.println(g.getName() + " " + g.getLastArticleNumber() + " " + g.getFirstArticleNumber() + writeable); } - conn.println("."); } - else - { - conn.println("500 server database malfunction"); - } + conn.println("."); + } + else + { + conn.println("500 server database malfunction"); } } diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/daemon/command/PostCommand.java --- a/org/sonews/daemon/command/PostCommand.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/daemon/command/PostCommand.java Thu Aug 20 14:31:19 2009 +0200 @@ -21,12 +21,8 @@ import java.io.IOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.nio.charset.Charset; -import java.nio.charset.IllegalCharsetNameException; -import java.nio.charset.UnsupportedCharsetException; import java.sql.SQLException; import java.util.Arrays; -import java.util.Locale; import javax.mail.MessagingException; import javax.mail.internet.AddressException; import javax.mail.internet.InternetHeaders; @@ -303,7 +299,7 @@ boolean success = false; String[] groupnames = article.getHeader(Headers.NEWSGROUPS)[0].split(","); for(String groupname : groupnames) - { + { Group group = StorageManager.current().getGroup(groupname); if(group != null && !group.isDeleted()) { @@ -311,7 +307,7 @@ { // Send to mailing list; the Dispatcher writes // statistics to database - Dispatcher.toList(article); + Dispatcher.toList(article, group.getName()); success = true; } else diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/mlgw/Dispatcher.java --- a/org/sonews/mlgw/Dispatcher.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/mlgw/Dispatcher.java Thu Aug 20 14:31:19 2009 +0200 @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.mail.Address; import javax.mail.Authenticator; import javax.mail.Message; @@ -29,6 +31,7 @@ import javax.mail.internet.InternetAddress; import org.sonews.config.Config; import org.sonews.storage.Article; +import org.sonews.storage.Group; import org.sonews.storage.Headers; import org.sonews.storage.StorageBackendException; import org.sonews.storage.StorageManager; @@ -36,7 +39,7 @@ import org.sonews.util.Stats; /** - * Dispatches messages from mailing list or newsserver or vice versa. + * Dispatches messages from mailing list to newsserver or vice versa. * @author Christian Lins * @since sonews/0.5.0 */ @@ -58,86 +61,160 @@ } } + + /** + * Chunks out the email address of the full List-Post header field. + * @param listPostValue + * @return The matching email address or null + */ + private static String chunkListPost(String listPostValue) + { + // listPostValue is of form "" + Pattern mailPattern = Pattern.compile("(\\w+[-|.])*\\w+@(\\w+.)+\\w+"); + Matcher mailMatcher = mailPattern.matcher(listPostValue); + if(mailMatcher.find()) + { + return listPostValue.substring(mailMatcher.start(), mailMatcher.end()); + } + else + { + return null; + } + } + + /** + * This method inspects the header of the given message, trying + * to find the most appropriate recipient. + * @param msg + * @param fallback If this is false only List-Post and X-List-Post headers + * are examined. + * @return null or fitting group name for the given message. + */ + private static List getGroupFor(final Message msg, final boolean fallback) + throws MessagingException, StorageBackendException + { + List groups = null; + + // Is there a List-Post header? + String[] listPost = msg.getHeader(Headers.LIST_POST); + InternetAddress listPostAddr; + + if(listPost == null || listPost.length == 0 || "".equals(listPost[0])) + { + // Is there a X-List-Post header? + listPost = msg.getHeader(Headers.X_LIST_POST); + } + + if(listPost != null && listPost.length > 0 + && !"".equals(listPost[0]) && chunkListPost(listPost[0]) != null) + { + // listPost[0] is of form "" + listPost[0] = chunkListPost(listPost[0]); + listPostAddr = new InternetAddress(listPost[0], false); + groups = StorageManager.current().getGroupsForList(listPostAddr); + } + else if(fallback) + { + Log.msg("Using fallback recipient discovery for: " + msg.getSubject(), true); + groups = new ArrayList(); + // Fallback to TO/CC/BCC addresses + Address[] to = msg.getAllRecipients(); + for(Address toa : to) // Address can have '<' '>' around + { + if(toa instanceof InternetAddress) + { + List g = StorageManager.current().getGroupsForList((InternetAddress) toa); + groups.addAll(g); + } + } + } + + return groups; + } /** * Posts a message that was received from a mailing list to the * appropriate newsgroup. + * If the message already exists in the storage, this message checks + * if it must be posted in an additional group. This can happen for + * crosspostings in different mailing lists. * @param msg */ public static boolean toGroup(final Message msg) { try { - Address[] to = msg.getAllRecipients(); // includes TO/CC/BCC - if(to == null || to.length <= 0) + // Create new Article object + Article article = new Article(msg); + boolean posted = false; + + // Check if this mail is already existing the storage + boolean updateReq = + StorageManager.current().isArticleExisting(article.getMessageID()); + + List newsgroups = getGroupFor(msg, !updateReq); + List oldgroups = new ArrayList(); + if(updateReq) { - to = msg.getReplyTo(); + // Check for duplicate entries of the same group + Article oldArticle = StorageManager.current().getArticle(article.getMessageID()); + List oldGroups = oldArticle.getGroups(); + for(Group oldGroup : oldGroups) + { + if(!newsgroups.contains(oldGroup.getName())) + { + oldgroups.add(oldGroup.getName()); + } + } } - if(to == null || to.length <= 0) + if(newsgroups.size() > 0) { - Log.msg("Skipping message because no recipient!", false); - return false; + newsgroups.addAll(oldgroups); + StringBuilder groups = new StringBuilder(); + for(int n = 0; n < newsgroups.size(); n++) + { + groups.append(newsgroups.get(n)); + if (n + 1 != newsgroups.size()) + { + groups.append(','); + } + } + Log.msg("Posting to group " + groups.toString(), true); + + article.setGroup(groups.toString()); + //article.removeHeader(Headers.REPLY_TO); + //article.removeHeader(Headers.TO); + + // Write article to database + if(updateReq) + { + Log.msg("Updating " + article.getMessageID() + " with additional groups", true); + StorageManager.current().delete(article.getMessageID()); + StorageManager.current().addArticle(article); + } + else + { + Log.msg("Gatewaying " + article.getMessageID() + " to " + + article.getHeader(Headers.NEWSGROUPS)[0], true); + StorageManager.current().addArticle(article); + Stats.getInstance().mailGatewayed( + article.getHeader(Headers.NEWSGROUPS)[0]); + } + posted = true; } else { - boolean posted = false; - List newsgroups = new ArrayList(); - - for (Address toa : to) // Address can have '<' '>' around + StringBuilder buf = new StringBuilder(); + for (Address toa : msg.getAllRecipients()) { - if (toa instanceof InternetAddress) - { - List groups = StorageManager.current() - .getGroupsForList((InternetAddress)toa); - newsgroups.addAll(groups); - } + buf.append(' '); + buf.append(toa.toString()); } - - if (newsgroups.size() > 0) - { - StringBuilder groups = new StringBuilder(); - for(int n = 0; n < newsgroups.size(); n++) - { - groups.append(newsgroups.get(n)); - if(n + 1 != newsgroups.size()) - { - groups.append(','); - } - } - Log.msg("Posting to group " + groups.toString(), true); - - // Create new Article object - Article article = new Article(msg); - article.setGroup(groups.toString()); - article.removeHeader(Headers.REPLY_TO); - article.removeHeader(Headers.TO); - - // Write article to database - if(!StorageManager.current().isArticleExisting(article.getMessageID())) - { - StorageManager.current().addArticle(article); - Stats.getInstance().mailGatewayed( - article.getHeader(Headers.NEWSGROUPS)[0]); - } - else - { - Log.msg("Article " + article.getMessageID() + " already existing.", true); - } - posted = true; - } - else - { - StringBuilder buf = new StringBuilder(); - for(Address toa : to) - { - buf.append(' '); - buf.append(toa.toString()); - } - Log.msg("No group for" + buf.toString(), false); - } - return posted; + buf.append(" " + article.getHeader(Headers.LIST_POST)[0]); + Log.msg("No group for" + buf.toString(), false); } + return posted; } catch(Exception ex) { @@ -148,56 +225,53 @@ /** * Mails a message received through NNTP to the appropriate mailing list. + * This method MAY be called several times by PostCommand for the same + * article. */ - public static void toList(Article article) + public static void toList(Article article, String group) throws IOException, MessagingException, StorageBackendException { // Get mailing lists for the group of this article - List listAddresses = new ArrayList(); - String[] groupnames = article.getHeader(Headers.NEWSGROUPS)[0].split(","); - - for(String groupname : groupnames) + List rcptAddresses = StorageManager.current().getListsForGroup(group); + + if(rcptAddresses == null || rcptAddresses.size() == 0) { - String listAddress = StorageManager.current().getListForGroup(groupname); - if(listAddress != null) - { - listAddresses.add(listAddress); - } + Log.msg("No ML-address for " + group + " found.", false); + return; } - for(String listAddress : listAddresses) + for(String rcptAddress : rcptAddresses) { // Compose message and send it via given SMTP-Host String smtpHost = Config.inst().get(Config.MLSEND_HOST, "localhost"); - int smtpPort = Config.inst().get(Config.MLSEND_PORT, 25); + int smtpPort = Config.inst().get(Config.MLSEND_PORT, 25); String smtpUser = Config.inst().get(Config.MLSEND_USER, "user"); - String smtpPw = Config.inst().get(Config.MLSEND_PASSWORD, "mysecret"); + String smtpPw = Config.inst().get(Config.MLSEND_PASSWORD, "mysecret"); String smtpFrom = Config.inst().get( - Config.MLSEND_ADDRESS, article.getHeader(Headers.FROM)[0]); + Config.MLSEND_ADDRESS, article.getHeader(Headers.FROM)[0]); // TODO: Make Article cloneable() - String group = article.getHeader(Headers.NEWSGROUPS)[0]; article.getMessageID(); // Make sure an ID is existing article.removeHeader(Headers.NEWSGROUPS); article.removeHeader(Headers.PATH); article.removeHeader(Headers.LINES); article.removeHeader(Headers.BYTES); - article.setHeader("To", listAddress); - article.setHeader("Reply-To", listAddress); + article.setHeader("To", rcptAddress); + //article.setHeader("Reply-To", listAddress); - if(Config.inst().get(Config.MLSEND_RW_SENDER, false)) + if (Config.inst().get(Config.MLSEND_RW_SENDER, false)) { rewriteSenderAddress(article); // Set the SENDER address } SMTPTransport smtpTransport = new SMTPTransport(smtpHost, smtpPort); - smtpTransport.send(article, smtpFrom, listAddress); + smtpTransport.send(article, smtpFrom, rcptAddress); smtpTransport.close(); Stats.getInstance().mailGatewayed(group); - Log.msg("MLGateway: Mail " + article.getHeader("Subject")[0] - + " was delivered to " + listAddress + ".", true); + Log.msg("MLGateway: Mail " + article.getHeader("Subject")[0] + + " was delivered to " + rcptAddress + ".", true); } } diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/mlgw/SMTPTransport.java --- a/org/sonews/mlgw/SMTPTransport.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/mlgw/SMTPTransport.java Thu Aug 20 14:31:19 2009 +0200 @@ -84,6 +84,10 @@ public void send(Article article, String mailFrom, String rcptTo) throws IOException { + assert(article != null); + assert(mailFrom != null); + assert(rcptTo != null); + this.out.println("MAIL FROM: " + mailFrom); this.out.flush(); String line = this.in.readLine(); diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/storage/Article.java --- a/org/sonews/storage/Article.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/storage/Article.java Thu Aug 20 14:31:19 2009 +0200 @@ -119,7 +119,7 @@ final Object content = msg.getContent(); if(content instanceof String) { - this.body = ((String)content).getBytes(); + this.body = ((String)content).getBytes(getBodyCharset()); } else if(content instanceof Multipart) // probably subclass MimeMultipart { diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/storage/Headers.java --- a/org/sonews/storage/Headers.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/storage/Headers.java Thu Aug 20 14:31:19 2009 +0200 @@ -33,6 +33,7 @@ public static final String DATE = "date"; public static final String FROM = "from"; public static final String LINES = "lines"; + public static final String LIST_POST = "list-post"; public static final String MESSAGE_ID = "message-id"; public static final String NEWSGROUPS = "newsgroups"; public static final String NNTP_POSTING_DATE = "nntp-posting-date"; @@ -45,6 +46,7 @@ public static final String SUPERSEDES = "subersedes"; public static final String TO = "to"; public static final String X_COMPLAINTS_TO = "x-complaints-to"; + public static final String X_LIST_POST = "x-list-post"; public static final String X_TRACE = "x-trace"; public static final String XREF = "xref"; diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/storage/Storage.java --- a/org/sonews/storage/Storage.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/storage/Storage.java Thu Aug 20 14:31:19 2009 +0200 @@ -31,6 +31,11 @@ public interface Storage { + /** + * Stores the given Article in the storage. + * @param art + * @throws StorageBackendException + */ void addArticle(Article art) throws StorageBackendException; @@ -93,7 +98,14 @@ int getLastArticleNumber(Group group) throws StorageBackendException; - String getListForGroup(String groupname) + /** + * Returns a list of email addresses that are related to the given + * groupname. In most cases the list may contain only one entry. + * @param groupname + * @return + * @throws StorageBackendException + */ + List getListsForGroup(String groupname) throws StorageBackendException; String getOldestArticle() diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/storage/StorageBackendException.java --- a/org/sonews/storage/StorageBackendException.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/storage/StorageBackendException.java Thu Aug 20 14:31:19 2009 +0200 @@ -31,4 +31,9 @@ super(cause); } + public StorageBackendException(String msg) + { + super(msg); + } + } diff -r 961a8a3acb9a -r bb6990c0dd1a org/sonews/storage/impl/JDBCDatabase.java --- a/org/sonews/storage/impl/JDBCDatabase.java Mon Aug 17 11:00:51 2009 +0200 +++ b/org/sonews/storage/impl/JDBCDatabase.java Thu Aug 20 14:31:19 2009 +0200 @@ -1142,28 +1142,27 @@ } @Override - public String getListForGroup(String group) + public List getListsForGroup(String group) throws StorageBackendException { - ResultSet rs = null; + ResultSet rs = null; + List lists = new ArrayList(); try { this.pstmtGetListForGroup.setString(1, group); rs = this.pstmtGetListForGroup.executeQuery(); - if (rs.next()) + + while(rs.next()) { - return rs.getString(1); + lists.add(rs.getString(1)); } - else - { - return null; - } + return lists; } catch(SQLException ex) { restartConnection(ex); - return getListForGroup(group); + return getListsForGroup(group); } finally { diff -r 961a8a3acb9a -r bb6990c0dd1a test/unit/MLGWTests.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/unit/MLGWTests.java Thu Aug 20 14:31:19 2009 +0200 @@ -0,0 +1,38 @@ +/* + * SONEWS News Server + * see AUTHORS for the list of contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package test.unit; + +import junit.textui.TestRunner; +import test.util.mlgw.DispatcherTest; + +/** + * Unit tests for package org.sonews.mlgw. + * @author Christian Lins + * @since sonews/1.0.3 + */ +public class MLGWTests +{ + + public static void main(String[] args) + { + System.out.println("DispatcherTest"); + TestRunner.run(DispatcherTest.class); + } + +} diff -r 961a8a3acb9a -r bb6990c0dd1a test/unit/util/ResourceTest.java --- a/test/unit/util/ResourceTest.java Mon Aug 17 11:00:51 2009 +0200 +++ b/test/unit/util/ResourceTest.java Thu Aug 20 14:31:19 2009 +0200 @@ -41,7 +41,7 @@ assertNull(url); // This file should exist - url = Resource.getAsURL("org/sonews/daemon/Main.class"); + url = Resource.getAsURL("org/sonews/Main.class"); assertNotNull(url); } @@ -55,7 +55,7 @@ stream = Resource.getAsStream("this is bullshit"); assertNull(stream); - stream = Resource.getAsStream("org/sonews/daemon/Main.class"); + stream = Resource.getAsStream("org/sonews/Main.class"); assertNotNull(stream); } @@ -69,10 +69,10 @@ str = Resource.getAsString("this is bullshit", true); assertNull(str); - str = Resource.getAsString("org/sonews/daemon/Main.class", true); + str = Resource.getAsString("org/sonews/Main.class", true); assertNotNull(str); - str = Resource.getAsString("org/sonews/daemon/Main.class", false); + str = Resource.getAsString("org/sonews/Main.class", false); assertNotNull(str); assertEquals(str.indexOf("\n"), -1); } diff -r 961a8a3acb9a -r bb6990c0dd1a test/util/mlgw/DispatcherTest.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/util/mlgw/DispatcherTest.java Thu Aug 20 14:31:19 2009 +0200 @@ -0,0 +1,71 @@ +/* + * SONEWS News Server + * see AUTHORS for the list of contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package test.util.mlgw; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import junit.framework.TestCase; +import org.sonews.mlgw.Dispatcher; + +/** + * Tests the methods of class org.sonews.mlgw.Dispatcher. + * @author Christian Lins + * @since sonews/1.0.3 + */ +public class DispatcherTest extends TestCase +{ + + public DispatcherTest() + { + super("DispatcherTest"); + } + + public void testChunkListPost() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException + { + Dispatcher disp = new Dispatcher(); + + Class clazz = disp.getClass(); + Method methChunkListPost = clazz.getDeclaredMethod("chunkListPost", String.class); + methChunkListPost.setAccessible(true); + + try + { + // disp.chunkListPost(null) + methChunkListPost.invoke(disp, null); + fail("Should have raised an IllegalArgumentException"); + } + catch(IllegalArgumentException ex){} + + // disp.chunkListPost("") + Object obj = methChunkListPost.invoke(disp, ""); + assertNull(obj); + + // disp.chunkListPost("listPostValue is of form ") + obj = methChunkListPost.invoke(disp, "listPostValue is of form "); + assertNotNull(obj); + assertEquals("dev@openoffice.org", (String)obj); + + // disp.chunkListPost("