Changeset 628

Timestamp:

04/01/11 16:49:04 (2 years ago)

Author:

philsmart

Message:

 

Location:

raptor-ica-embedded/trunk

Files:

• pom.xml (modified) (1 diff)
• src/main/config/data-access.xml (modified) (3 diffs)
• src/main/config/ica-core.xml (modified) (5 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/dao/BaseEventParser.java (moved) (moved from raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/dao/RawData.java) (5 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/AuthenticationInput.java (deleted)
• src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/UsageInput.java (modified) (2 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/file/LogFileParser.java (modified) (12 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/file/LogFileParserForcedIncrement.java (deleted)
• src/main/java/uk/ac/cardiff/raptor/raptorica/engine/DataAccessRegister.java (modified) (1 diff)
• src/main/java/uk/ac/cardiff/raptor/raptorica/engine/ICAEngine.java (modified) (5 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/model/EntryHandler.java (modified) (1 diff)
• src/main/java/uk/ac/cardiff/raptor/raptorica/model/PersistantEntryHandler.java (modified) (12 diffs)
• src/main/java/uk/ac/cardiff/raptor/raptorica/model/exception (added)
• src/main/java/uk/ac/cardiff/raptor/raptorica/model/exception/HeaderException.java (added)
• src/main/java/uk/ac/cardiff/raptor/raptorica/model/exception/ParserException.java (added)
• src/main/java/uk/ac/cardiff/raptor/raptorica/server/RunServer.java (modified) (1 diff)
• src/main/resources/package.xml (modified) (1 diff)

raptor-ica-embedded/trunk/pom.xml

@@ -275 +275 @@
 
                 <!-- includes the information-model -->
-                <dependency>
-                        <groupId>uk.ac.cardiff.raptor</groupId>
-                        <artifactId>raptor-serviceinterfaces</artifactId>
-                        <version>0.4-SNAPSHOT</version>
-                        <exclusions>
-                                <exclusion>
-                                        <artifactId>slf4j-api</artifactId>
-                                        <groupId>org.slf4j</groupId>
-                                </exclusion>
-                        </exclusions>
-                </dependency>
+
                 <dependency>
                         <groupId>postgresql</groupId>

raptor-ica-embedded/trunk/src/main/config/data-access.xml

@@ -23 +23 @@
         <!--  List all the data access beans, used by the parser -->
         <bean id="dataAccessRegister" class="uk.ac.cardiff.raptor.raptorica.engine.DataAccessRegister">
-                <property name="authenticationModules">
+                <property name="parsingModules">
                         <list><!--
                                 <bean id="shibbolethAccessLFP" class="uk.ac.cardiff.raptor.raptorica.dao.external.file.LogFileParser">
@@ -45 +45 @@
                                 -->
                                 <bean id="shibbolethAuditLFP" class="uk.ac.cardiff.raptor.raptorica.dao.external.file.LogFileParser">
-                                        <property name="entryHandler"><ref bean="memoryEntryHandler"></ref></property>
+                                        <property name="entryHandler"><ref bean="persistantEntryHandler"></ref></property>
                                         <property name="logfile">
-                                                <value>file:///Users/philsmart/Documents/DataSets/Logs/idp-audit-17th.log
+                                                <value>file:///Users/philsmart/Documents/DataSets/Logs/shib2comb-sorted-to-20110213.log
                                                 </value>
                                         </property>
@@ -53 +53 @@
                                                 <ref bean="shibIDPAuditFileFormat" />
                                         </property>
-                                        <property name="entryType">
+                                        <property name="eventType">
                                                 <value>uk.ac.cardiff.model.event.ShibbolethIdpAuthenticationEvent</value>
                                         </property>
+                                        <property name="printParsingPosition"><value>true</value></property>
 
                                         <!--  The inclusion list is an OR of field name, value pairs that are loaded by the parser

raptor-ica-embedded/trunk/src/main/config/ica-core.xml

@@ -22 +22 @@
 
         <!-- Polling trigger setup -->
-
-
-
         <bean id="ICAProcessQuickTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerBean">
                 <property name="jobDetail">
@@ -30 +27 @@
                 </property>
                 <property name="repeatInterval" value="30000" />
-                <!-- <property name="repeatCount" value="0" />-->
         </bean>
 
@@ -48 +44 @@
         <bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
                 <property name="triggers">
-                        <list><!--
-                                <ref local="ICAProcessTrigger" />
-                                -->
+                        <list>
                                 <ref local="ICAProcessQuickTrigger" />
                         </list>
@@ -86 +80 @@
                 <property name="eventReleaseEngine"><ref bean="eventReleaseEngine"></ref></property>
         </bean>
-        
+
         <bean id="eventReleaseEngine" class="uk.ac.cardiff.raptor.registry.EventReleaseEngine">
                 <property name="attributeFilterEngine"><ref bean="attributeFilterEngine"/></property>
@@ -93 +87 @@
 
         <bean id="attributeFilterEngine" class="uk.ac.cardiff.raptor.attribute.filtering.AttrributeFilterEngine"/>
-        
-        <bean id="serviceEndpointInterface" class="uk.ac.cardiff.raptor.remoting.client.sei.impl.CxfServiceEndpointInterface"/>
+
+        <bean id="serviceEndpointInterface" class="uk.ac.cardiff.raptor.remoting.client.sei.impl.CxfServiceEndpointClient"/>
 
         <import resource="dao-beans.xml"/>

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/dao/BaseEventParser.java

@@ -19 +19 @@
 package uk.ac.cardiff.raptor.raptorica.dao;
 
+import java.util.Set;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import uk.ac.cardiff.model.event.Event;
 import uk.ac.cardiff.raptor.raptorica.model.EntryHandler;
+import uk.ac.cardiff.raptor.raptorica.model.exception.ParserException;
+import uk.ac.cardiff.raptor.raptorica.model.filter.ExclusionList;
+import uk.ac.cardiff.raptor.raptorica.model.filter.InclusionList;
 
 
@@ -30 +36 @@
  *
  */
-public abstract class RawData {
+public abstract class BaseEventParser {
 
-    /* class logger */
-    private final Logger log = LoggerFactory.getLogger(RawData.class);
+    /** class logger */
+    private final Logger log = LoggerFactory.getLogger(BaseEventParser.class);
 
     /** The handler that is responsible for storing events */
-    private EntryHandler entryHandler;
+    protected EntryHandler entryHandler;
 
-    public abstract void parse() throws Exception;
+    /** A list of <fieldname,value> pairs that should be excluded during parsing */
+    private ExclusionList exclusionList;
 
-    public RawData() {
-        //setEntryHandler(new MemoryEntryHandler());
+    /** A list of <fieldname, value> pairs that should be included during parsing */
+    private InclusionList inclusionList;
+
+    /**
+     * Default constructor
+     */
+    public BaseEventParser() {
     }
 
+    /**
+     * The method that must be provided to parse events from a particular datasource
+     *
+     * @throws ParserException
+     */
+    public abstract void parse() throws ParserException;
+
+
+    /**
+     * Creates a new object from a class name. Could be removed to a helper class
+     *
+     * @param className the fully qualified class name
+     * @return the new object instantiated as the type <code>className</code>
+     */
     public Object createObject(String className) {
         Object object = null;
@@ -49 +75 @@
             Class classDefinition = Class.forName(className);
             object = classDefinition.newInstance();
-            // this.setBeanProperty(object, "applicationContext",
-            // this.getApplicationContext());
         } catch (InstantiationException e) {
             log.warn(e.getMessage());
@@ -62 +86 @@
 
     /**
-     * removes all entries from the entry handler
+     * Method to remove all entries from the entry handler
      */
     public void removeAllEntries() {
-        getEntryHandler().removeAllEntries();
+        entryHandler.removeAllEntries();
 
     }
 
+    /**
+     * Sets and initialises the entry handler
+     *
+     * @param entryHandler
+     */
     public void setEntryHandler(EntryHandler entryHandler) {
         this.entryHandler = entryHandler;
@@ -74 +103 @@
     }
 
+    /**
+     * Gets the entry handler
+     *
+     * @return
+     */
     public EntryHandler getEntryHandler() {
         return entryHandler;
     }
 
+
+    /**
+     * Sets the exclusion list
+     *
+     * @param exclusionList
+     */
+    public void setExclusionList(ExclusionList exclusionList) {
+        this.exclusionList = exclusionList;
+    }
+
+    /**
+     * Gets the exclusion list
+     *
+     * @return
+     */
+    public ExclusionList getExclusionList() {
+        return exclusionList;
+    }
+
+
+    /**
+     * Sets the inclusion list
+     *
+     * @param inclusionList
+     */
+    public void setInclusionList(InclusionList inclusionList) {
+        this.inclusionList = inclusionList;
+    }
+
+
+    /**
+     * Gets the inclusion list
+     *
+     * @return
+     */
+    public InclusionList getInclusionList() {
+        return inclusionList;
+    }
+
+
+    /**
+     * Gets all the entries currently stored by this entry handler
+     *
+     * @return
+     */
+    public Set<Event> getAuthentications() {
+        return entryHandler.getEntries();
+
+    }
+
+
+
 }

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/UsageInput.java

@@ -22 +22 @@
 import java.util.Set;
 
-import uk.ac.cardiff.raptor.raptorica.dao.RawData;
+import uk.ac.cardiff.raptor.raptorica.dao.BaseEventParser;
 
 /**
@@ -28 +28 @@
  *
  */
-public abstract class UsageInput extends RawData{
+public abstract class UsageInput extends BaseEventParser{
 
 

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/dao/external/file/LogFileParser.java

@@ -17 +17 @@
 
 import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.reflect.Method;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
-import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
 import java.util.TimeZone;
 import java.io.BufferedInputStream;
 
-import uk.ac.cardiff.raptor.raptorica.dao.external.AuthenticationInput;
+import uk.ac.cardiff.raptor.raptorica.dao.BaseEventParser;
 import uk.ac.cardiff.raptor.raptorica.dao.external.format.Header;
 import uk.ac.cardiff.raptor.raptorica.dao.external.format.LogFileFormat;
-import uk.ac.cardiff.raptor.raptorica.dao.internal.ICADataConnection;
-import uk.ac.cardiff.raptor.raptorica.model.PersistentParserSupport;
+import uk.ac.cardiff.raptor.raptorica.model.exception.HeaderException;
+import uk.ac.cardiff.raptor.raptorica.model.exception.ParserException;
 import uk.ac.cardiff.raptor.raptorica.model.filter.ExclusionEntry;
 import uk.ac.cardiff.raptor.raptorica.model.filter.InclusionEntry;
@@ -48 +44 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
 
 import uk.ac.cardiff.raptor.raptorica.runtimeutils.ReflectionHelper;
-import uk.ac.cardiff.model.*;
 import uk.ac.cardiff.model.event.Event;
 
 /**
+ * A log file parser, used to parse files on the local filesystem according to a given
+ * format and transformed into the given event type.
+ *
  * @author phil
  *
- *         This is the primary log file parser.
- *
  */
-public class LogFileParser extends AuthenticationInput {
-    static Logger log = LoggerFactory.getLogger(LogFileParser.class);
+public class LogFileParser extends BaseEventParser {
+
+    /** The class logger*/
+    private final Logger log = LoggerFactory.getLogger(LogFileParser.class);
+
+    /** The specification of the format of this log file */
     private LogFileFormat format;
+
+    /** The fully qualified path of the log file */
     private String logfile;
-    private String entryType;
-
+
+    /** The fully qualified name of the event type class */
+    private String eventType;
+
+    /** Whether to print out the current position of parsing in the <code>logFile</code> */
+    private boolean printParsingPosition;
+
+    /**
+     * Default constructor
+     */
     public LogFileParser() {
-
-    }
-
-    public void parse() throws Exception {
-        log.info("parsing: {} instance: {}",logfile, this);
-
-        // Must use URL, as java.io does not work in a webapp
-        URL logfileURL = new URL(logfile);
-        URLConnection logfileconnection = logfileURL.openConnection();
-        BufferedReader in = new BufferedReader(new InputStreamReader(logfileconnection.getInputStream()));
-        int lineCount = 0;
-        String inputLine;
-
-        int totalNoLines =count(logfileURL);
-
-        log.debug("Parsing file with {} lines",totalNoLines);
-        while ((inputLine = in.readLine()) != null) {
-            // log.debug(inputLine);
-            lineCount++;
-            double linePercentage = (((double)lineCount/(double)totalNoLines)*100);
-            if (linePercentage%25>=0 && linePercentage%25<=0.0001) log.debug("Complete {}%",linePercentage);
-
-            StrTokenizer tokenizer = new StrTokenizer(inputLine, format.getDelimeter());
-            tokenizer.setIgnoreEmptyTokens(false);
-            ArrayList<String> allvalues = new ArrayList();
-            while (tokenizer.hasNext()) {
-                Object next = tokenizer.next();
-                if (next instanceof String)
-                    allvalues.add((String) next);
-                else {
-                    log.error("input column was not a string");
-                }
-            }
-            /* first, dynamically construct the supplied entry class, which is subclass of Event */
-            Event authE = (Event) this.createObject(getEntryType());
-
-            /* now populate its fields */
-            for (Header header : format.getHeaders()) {
-
+        super();
+    }
+
+    @Override
+    public void parse() throws ParserException {
+        log.info("parsing: {} instance: {}", logfile, this);
+
+        try {
+            URL logfileURL = new URL(logfile);
+
+            URLConnection logfileconnection = logfileURL.openConnection();
+
+            BufferedReader in = new BufferedReader(new InputStreamReader(logfileconnection.getInputStream()));
+            int lineCount = 0;
+            String inputLine;
+            int totalNoLines = count(logfileURL);
+
+            log.debug("Parsing file with {} lines", totalNoLines);
+            while ((inputLine = in.readLine()) != null) {
+                lineCount++;
+
+                if (printParsingPosition) {
+                    printParsingPosition(lineCount, totalNoLines);
+                }
+
+                // TODO TOKENIZE METHOD, then regex
+                StrTokenizer tokenizer = new StrTokenizer(inputLine, format.getDelimeter());
+                tokenizer.setIgnoreEmptyTokens(false);
+                ArrayList<String> allvalues = new ArrayList<String>();
+                while (tokenizer.hasNext()) {
+                    Object next = tokenizer.next();
+                    if (next instanceof String)
+                        allvalues.add((String) next);
+                    else {
+                        log.error("input column was not a string");
+                    }
+                }
+
+                Event authE = (Event) this.createObject(eventType);
                 try {
-                    if (!(header.getFieldNo() >= allvalues.size())) {
-                        String value = allvalues.get(header.getFieldNo());
-                        // log.debug("Field: " + header.getFieldName() +
-                        // " value: " + value+" Type:"+ header.getType());
-                        switch (header.getType()) {
+                    populateField(allvalues, authE);
+                } catch (HeaderException e) {
+                    log.error("ERROR: trying to access field {}, when only {} fields in log file", e.getHeaderNo(), allvalues.size());
+                }
+
+                boolean shouldBeIncluded = isIncluded(authE);
+                boolean preventAdd = isExcluded(authE);
+
+                if (shouldBeIncluded && !preventAdd) {
+                    entryHandler.addEntry(authE);
+                }
+
+            }
+            in.close();
+
+            log.debug("done, walked {} lines", lineCount);
+            log.debug("LogFileParser currently has: {} entries, latestEntry: {}", getEntryHandler().getEntries().size(), getEntryHandler().getLatestEntryTime());
+            entryHandler.endTransaction();
+        } catch (MalformedURLException e1) {
+            throw new ParserException("Could not find the source file ["+logfile+"] for parsing",e1);
+        } catch (IOException e2) {
+            throw new ParserException("Could not read from the source file ["+logfile+"] during parsing",e2);
+        }
+
+        // System.exit(1);
+
+    }
+
+    /**
+     * Prints, as a percentage, the line currently being parsed.
+     *
+     * @param lineCount
+     * @param totalNoLines
+     */
+    private void printParsingPosition(int lineCount, int totalNoLines) {
+        double linePercentage = (((double) lineCount / (double) totalNoLines) * 100);
+        if (linePercentage % 25 >= 0 && linePercentage % 25 <= 0.0001)
+            log.debug("Complete {}%", linePercentage);
+    }
+
+    /**
+     * Populates attributes in the Event class <code>authE</code> (or subclasses thereof) from the headers defined in <code>format</code>
+     *
+     * @param allvalues
+     * @param authE
+     * @throws HeaderException
+     */
+    private void populateField(List<String> allvalues, Event authE) throws HeaderException {
+        for (Header header : format.getHeaders()) {
+            try {
+                if (!(header.getFieldNo() >= allvalues.size())) {
+                    String value = allvalues.get(header.getFieldNo());
+                    switch (header.getType()) {
                         case DATE:
-                            addDate(value, header.getDateTimeFormat(), header.getTimeZone(),header.getFieldName(), authE);
+                            addDate(value, header.getDateTimeFormat(), header.getTimeZone(), header.getFieldName(), authE);
                             break;
                         case STRING:
@@ -124 +181 @@
                             addStringList(value, header.getFieldName(), authE, header.getListDelimeter());
                             break;
-                        }
-
-                    } else {
-                        log.error("trying to access field " + header.getFieldNo() + ", when only " + allvalues.size() + " fields in log file \n --> " + inputLine);
                     }
-                } catch (ClassCastException e) {
-                    log.error("Header key is not an integer, needs to be an integer (possibly look at data-access.xml).");
-                }
-            }
-
-            /* check its IN the inclusion list */
-            boolean shouldBeIncluded = false;
-            if (getInclusionList()!=null){
-                for (InclusionEntry inclusion : getInclusionList().getInclusionEntries()){
-                    String fieldname = inclusion.getFieldName();
-                    Object valueFromEntry = getValueFromObject(fieldname, authE);
-                    if (valueFromEntry instanceof String) {
-                        String valueFromEntryString = (String)valueFromEntry;
-                        if (inclusion.filter(valueFromEntryString)){
-                            //log.debug("entry [{}] had match true",valueFromEntryString);
-                            shouldBeIncluded = true;
-                        }
+
+                } else {
+                    StringBuilder exceptionMessage = new StringBuilder();
+                    exceptionMessage.append("failed trying to access field ");
+                    exceptionMessage.append(header.getFieldNo());
+                    throw new HeaderException(exceptionMessage.toString(), header.getFieldNo());
+                }
+            } catch (ClassCastException e) {
+                throw new HeaderException("Header key is not an integer, needs to be an integer (error could exist in data-access.xml)", header.getFieldNo(), e);
+
+            }
+        }
+    }
+
+    /**
+     * Determines if the event should not be stored, as determined by a <fieldname, value> match on the input <code>authE</code> event with those in the
+     * <code>exclusionList</code>.
+     *
+     * @param authE
+     * @return true if any <fieldname,value> pairs in the <code>autheE</code> event are found in the exclusion list, false otherwise
+     */
+    private boolean isExcluded(Event authE) {
+        if (getExclusionList() != null) {
+            for (ExclusionEntry exclusion : getExclusionList().getExclusionEntries()) {
+                String fieldname = exclusion.getFieldName();
+                Object valueFromEntry = getValueFromObject(fieldname, authE);
+                if (valueFromEntry instanceof String) {
+                    String valueFromEntryString = (String) valueFromEntry;
+                    if (exclusion.filter(valueFromEntryString)) {
+                        return true;
                     }
                 }
             }
-            else{
-                //no inclusion list, assume all
-                shouldBeIncluded = true;
-            }
-
-            /* now check its not in the exclusion list */
-            boolean preventAdd = false;
-            if (getExclusionList() != null) {
-                for (ExclusionEntry exclusion : getExclusionList().getExclusionEntries()){
-                    String fieldname = exclusion.getFieldName();
-                    Object valueFromEntry = getValueFromObject(fieldname, authE);
-                    if (valueFromEntry instanceof String) {
-                        String valueFromEntryString = (String)valueFromEntry;
-                        if (exclusion.filter(valueFromEntryString)){
-                            preventAdd=true;
-                        }
+        }
+        return false;
+    }
+
+    /**
+     * Determines if the event should stored, as determined by a <fieldname, value> match on the input <code>authE</code> event with those in the
+     * <code>exclusionList</code>.
+     *
+     * @param authE
+     * @return true if any <fieldname,value> pairs in the <code>autheE</code> event are found in the inclusion list, false otherwise
+     */
+    private boolean isIncluded(Event authE) {
+        if (getInclusionList() != null) {
+            for (InclusionEntry inclusion : getInclusionList().getInclusionEntries()) {
+                String fieldname = inclusion.getFieldName();
+                Object valueFromEntry = getValueFromObject(fieldname, authE);
+                if (valueFromEntry instanceof String) {
+                    String valueFromEntryString = (String) valueFromEntry;
+                    if (inclusion.filter(valueFromEntryString)) {
+                        return true;
                     }
                 }
             }
-
-            if (shouldBeIncluded && !preventAdd) {
-                /*
-                 * do not add the object to the arrayList if is timestamp is
-                 * older than the current latest entry this should save heap
-                 * space during operation
-                 */
-
-                getEntryHandler().addEntry(authE);
-            }
-
-        }
-
-        in.close();
-
-        log.debug("done, walked " + lineCount + " lines");
-        // getEntryHandler().addEntries(entries.);
-        log.debug("Currently has: " + getEntryHandler().getEntries().size() + " entries, latestEntry: " + getEntryHandler().getLatestEntryTime());
-        getEntryHandler().endTransaction();
-        //System.exit(1);
-
-        /*
-         * this is important, after one complete transaction, we push the
-         * latestTime entry by 1 millisecond, this stops the last entry of this
-         * transaction (n) being equal to itself in the second (n+1) transaction
-         * and so on. HOWEVER, this behaviour will also prevent a valid new
-         * entry in the n+1 transaction from being stored if it was a unique
-         * entry but at the same time as the last entry of the nth transaction.
-         * WE may therefore loose one or possibly two (more likely one) entry
-         * from the n+1 transactions - revise this. The good thing about this
-         * logic, is that we only need to check timestamps to obtain
-         * psuedo-incremental updates (even though the entire file is read in
-         * again, only newer entries are saved)
-         */
-        // entryHandler.endTransaction();
-
-    }
-
-    /**
-     * Fast method for finding the number of lines in a logfile
+        } else {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Efficient method for finding the number of lines in a logfile
      *
      * @param logfileURL
@@ -215 +254 @@
     private int count(URL logfileURL) throws IOException {
 
-            URLConnection logfileconnection = logfileURL.openConnection();
-            InputStream is = new BufferedInputStream(logfileconnection.getInputStream());
-            byte[] c = new byte[1024];
-            int count = 0;
-            int readChars = 0;
-            while ((readChars = is.read(c)) != -1) {
-                for (int i = 0; i < readChars; ++i) {
-                    if (c[i] == '\n')
-                        ++count;
-                }
-            }
-            return count;
+        URLConnection logfileconnection = logfileURL.openConnection();
+        InputStream is = new BufferedInputStream(logfileconnection.getInputStream());
+        byte[] c = new byte[1024];
+        int count = 0;
+        int readChars = 0;
+        while ((readChars = is.read(c)) != -1) {
+            for (int i = 0; i < readChars; ++i) {
+                if (c[i] == '\n')
+                    ++count;
+            }
+        }
+        return count;
     }
 
@@ -237 +276 @@
         try {
             String[] splitValue = value.split(delimeter);
-            // for (String s : splitValue)log.debug(s);
             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName);
             setValueOnObject(fieldAsMethod, splitValue, object);
         } catch (Exception e) {
-            log.error("Illegal StringList Value: " + value + " for field:" + fieldName + " | " + e.getMessage());
-            // e.printStackTrace();
+            log.error("Illegal StringList Value: {} for field: {}", new Object[] { value, fieldName, e });
         }
     }
@@ -249 +286 @@
         try {
             /*
-             * first check the end of the date, if it has a Z it should have
-             * time zone information e.g. -0800 or +0500 but if it ends in just
-             * Z, there is not valid timezone information supplied, so remove
+             * first check the end of the date, if it has a Z it should have time zone information e.g. -0800 or +0500 but if it ends in just Z, there is not
+             * valid timezone information supplied, so remove
              */
+            // TODO check this
             if (value.endsWith("Z"))
                 value = value.substring(0, value.length() - 1);
@@ -262 +299 @@
             setValueOnObject(fieldAsMethod, dt, object);
         } catch (Exception e) {
-            log.error("Illegal Date Format For: " + value + " and format:" + format + " | " + e.getMessage());
-            // e.printStackTrace();
+            log.error("Illegal Date Value: {} for field: {}", new Object[] { value, fieldName, e });
         }
 
@@ -273 +309 @@
             setValueOnObject(fieldAsMethod, value, object);
         } catch (Exception e) {
-            log.error("Illegal String Value: " + value + " for field:" + fieldName + " | " + e.getMessage());
-            // e.printStackTrace();
+            log.error("Illegal String Value: {} for field: {}", new Object[] { value, fieldName, e });
         }
 
@@ -281 +316 @@
     private void addInteger(String value, String fieldName, Object object) {
         try {
-            Integer valueAsInt = new Integer(Integer.parseInt(value));
             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName);
             setValueOnObject(fieldAsMethod, value, object);
         } catch (Exception e) {
-            log.error("Illegal Integer Value: " + value + " for field:" + fieldName + " | " + e.getMessage());
-            // e.printStackTrace();
+            log.error("Illegal Integer Value: {} for field: {}", new Object[] { value, fieldName, e });
         }
 
@@ -293 +326 @@
     private Object getValueFromObject(String fieldname, Object object) {
         try {
-            Class id = object.getClass();
+            Class<? extends Object> id = object.getClass();
             String fieldAsMethod = ReflectionHelper.prepareMethodNameGet(fieldname);
             Method getter = id.getMethod(fieldAsMethod, new Class[] {});
-            // log.debug("Trying to Set :"+param)
             Object result = getter.invoke(object, new Object[] {});
             return result;
         } catch (Throwable e) {
-            log.error("Field name '" + fieldname + "' does not match internal model attribute");
-            e.printStackTrace();
-            // System.exit(1);
-
+            log.error("Field name '{}' does not match internal model attribute", fieldname, e);
         }
         return null;
@@ -310 +339 @@
     private void setValueOnObject(String fieldname, Object param, Object object) {
         try {
-            Class id = object.getClass();
+            Class<? extends Object> id = object.getClass();
             Method setter = id.getMethod(fieldname, new Class[] { param.getClass() });
-            // log.debug("Trying to Set :"+param)
             setter.invoke(object, new Object[] { param });
         } catch (Throwable e) {
-            log.error("Field name '" + fieldname + "' does not match internal model attribute");
-            e.printStackTrace();
-            // System.exit(1);
-
-        }
-    }
-
-    private void preParse() {
-
+            log.error("Field name '{}' does not match internal model attribute", fieldname, e);
+
+        }
     }
 
@@ -342 +364 @@
     }
 
-    public void setEntryType(String entryType) {
-        this.entryType = entryType;
-    }
-
-    public String getEntryType() {
-        return entryType;
-    }
-
-    // public static void main(String args[]) throws IOException{
-    // LogFileParser lfp = new LogFileParser();
-    // lfp.setLogfile("/tmp/idp-access.log");
-    // lfp.setFormat(new LogFileFormat());
-    // lfp.parse();
-    // }
+    /**
+     * @param printParsingPosition
+     *            the printParsingPosition to set
+     */
+    public void setPrintParsingPosition(boolean printParsingPosition) {
+        this.printParsingPosition = printParsingPosition;
+    }
+
+    /**
+     * @return the printParsingPosition
+     */
+    public boolean isPrintParsingPosition() {
+        return printParsingPosition;
+    }
+
+    /**
+     * @param eventType the eventType to set
+     */
+    public void setEventType(String eventType) {
+        this.eventType = eventType;
+    }
+
+    /**
+     * @return the eventType
+     */
+    public String getEventType() {
+        return eventType;
+    }
 
 }

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/engine/DataAccessRegister.java

@@ -24 +24 @@
 import org.apache.log4j.Logger;
 
-import uk.ac.cardiff.raptor.raptorica.dao.external.AuthenticationInput;
+import uk.ac.cardiff.raptor.raptorica.dao.BaseEventParser;
 
 /**
  * @author philsmart
  *
- * Provides a register for parsing engines to plugin to. Also manages such engines.
+ * Provides a register for parsing modules to attach to.
  */
 public class DataAccessRegister {
-        static Logger log = Logger.getLogger(DataAccessRegister.class);
 
-        private List<AuthenticationInput> authenticationModules;
+        /** Class logger */
+        private final Logger log = Logger.getLogger(DataAccessRegister.class);
 
+        /** List of parsing modules */
+        private List<BaseEventParser> parsingModules;
+
+        /**
+         * Default constructor that sets an empty list of parsing modules by default.
+         */
         public DataAccessRegister(){
-                setAuthenticationModules(new ArrayList<AuthenticationInput>());
+            setParsingModules(new ArrayList<BaseEventParser>());
         }
 
-        public void setAuthenticationModules(List<AuthenticationInput> authenticationModules) {
-                for (AuthenticationInput authI : authenticationModules)log.info("Registering: "+authI.getClass());
-                this.authenticationModules = authenticationModules;
+        /**
+         * Sets and prints the name of all parsing modules passed as an input parameter
+         *
+         * @param parsingModules
+         */
+        public void setParsingModules(List<BaseEventParser> parsingModules) {
+                for (BaseEventParser parser : parsingModules)
+                    log.info("Registering: "+parser.getClass());
+                this.parsingModules = parsingModules;
         }
 
-        public List<AuthenticationInput> getAuthenticationModules() {
-                return authenticationModules;
+        /**
+         * Gets the list of parsing modules currently registered
+         *
+         * @return list of parsing modules, all subclasses of the {@link BaseEventParser} class
+         */
+        public List<BaseEventParser> getParsingModules() {
+                return parsingModules;
         }
 

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/engine/ICAEngine.java

@@ -20 +20 @@
 
 import java.util.ArrayList;
-import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Set;
 
 import uk.ac.cardiff.raptor.exceptions.ReleaseFailureException;
-import uk.ac.cardiff.raptor.raptorica.dao.external.AuthenticationInput;
-import uk.ac.cardiff.raptor.registry.EventReleaseEngine;
+import uk.ac.cardiff.raptor.raptorica.dao.BaseEventParser;
 import uk.ac.cardiff.raptor.remoting.client.EventReleaseClient;
 
@@ -37 +34 @@
 /**
  * @author philsmart
- * 
+ *
  *         Responsible for ALL low level capture operations
  */
@@ -54 +51 @@
 
         public void capturePerform() throws Exception {
-                for (AuthenticationInput authI : getDataAccessRegister().getAuthenticationModules()) {
-                        log.info("Capturing from {}", authI);
-                        authI.parse();
+                for (BaseEventParser parser : getDataAccessRegister().getParsingModules()) {
+                        log.info("Capturing from {}", parser);
+                        parser.parse();
                 }
         }
@@ -69 +66 @@
         private void retrieveTransactionFinished() {
                 log.debug("Retrieve Transaction Finished, entries are being removed from the ICA...");
-                for (AuthenticationInput authI : getDataAccessRegister().getAuthenticationModules()) {
-                        authI.removeAllEntries();
+                for (BaseEventParser parser : getDataAccessRegister().getParsingModules()) {
+                    parser.removeAllEntries();
                 }
-                log.debug("Retrieve Transaction Finished, entries are being removed form the ICA...done");
+                log.debug("Retrieve Transaction Finished, entries are being removed from the ICA...done");
 
         }
@@ -79 +76 @@
          * Converts all information from all modules into a single list that is sent
          * to the release engine
-         * 
+         *
          * @return
          */
         public boolean release() {
-                List<Event> eventsToSend = new ArrayList<Event>();              
-                for (AuthenticationInput authI : getDataAccessRegister().getAuthenticationModules()){
-                        eventsToSend.addAll(authI.getAuthentications());
+                List<Event> eventsToSend = new ArrayList<Event>();
+                for (BaseEventParser parser : getDataAccessRegister().getParsingModules()){
+                        eventsToSend.addAll(parser.getAuthentications());
                 }
-                
+
                 boolean success = false;
                 try {

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/model/EntryHandler.java

@@ -19 +19 @@
 package uk.ac.cardiff.raptor.raptorica.model;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Set;
 

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/model/PersistantEntryHandler.java

@@ -36 +36 @@
 /**
  * @author philsmart
- * 
+ *
  */
 public class PersistantEntryHandler implements EntryHandler {
 
         /* class logger */
-        static Logger log = LoggerFactory.getLogger(PersistantEntryHandler.class);
+        private final Logger log = LoggerFactory.getLogger(PersistantEntryHandler.class);
 
         /* information about entries, e.g. last entry */
@@ -64 +64 @@
                 log.info("Persistant entry handler [{}] initialising", this);
                 Integer rowCount = (Integer) dataConnection.runQueryUnique("select count(*) from Event", null);
-                log.info("Persistent data store has {} entries", rowCount);
+                log.info("Persistent data store has {} events", rowCount);
                 entryInformation = (EntryMetadata) dataConnection.runQueryUnique("from EntryMetadata", null);
                 log.debug("Have saved entryInformaiton: " + entryInformation);
@@ -71 +71 @@
                 log.debug("Entry Information " + entryInformation.getLatestEqualEntries());
                 // convert to set from list, maybe expensive
-                List<Event> entriesAsList = dataConnection.runQuery("from Event", null);
-                entries = new LinkedHashSet<Event>(entriesAsList);
+                //List<Event> entriesAsList = dataConnection.runQuery("from Event", null);
+                //entries = new LinkedHashSet<Event>(entriesAsList);
+                entries = new LinkedHashSet<Event>();
                 log.info("Persistant entry handler [{}] started", this);
         }
@@ -78 +79 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#addEntries(java.util.List)
          */
@@ -138 +139 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#endTransaction()
          */
@@ -152 +153 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#getEntries()
          */
@@ -162 +163 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#isAfter(uk.ac.cardiff.model.Entry)
          */
@@ -174 +175 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see
          * main.uk.ac.cf.model.EntryHandler#isEqualTime(uk.ac.cardiff.model.Entry)
@@ -187 +188 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see
          * main.uk.ac.cf.model.EntryHandler#isNewerOrEqual(uk.ac.cardiff.model.Entry
@@ -203 +204 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#removeAllEntries()
          */
@@ -233 +234 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see main.uk.ac.cf.model.EntryHandler#getLatestEntryTime()
          */
@@ -243 +244 @@
         /*
          * (non-Javadoc)
-         * 
+         *
          * @see
          * main.uk.ac.cf.model.EntryHandler#setLatestEntryTime(org.joda.time.DateTime

raptor-ica-embedded/trunk/src/main/java/uk/ac/cardiff/raptor/raptorica/server/RunServer.java

@@ +1 @@
+/**
+ * Copyright (C) 2010 Cardiff University, Wales <smartp@cf.ac.uk>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package uk.ac.cardiff.raptor.raptorica.server;
 

raptor-ica-embedded/trunk/src/main/resources/package.xml

@@ -1 +1 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright (C) 2010 Cardiff University, Wales <smartp@cf.ac.uk>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+            http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
 <assembly>
     <id>package</id>