Changeset 1490


Ignore:
Timestamp:
04/15/13 11:59:59 (7 years ago)
Author:
philsmart
Message:
 
Location:
raptor-parse/trunk
Files:
8 added
4 edited

Legend:

Unmodified
Added
Removed
  • raptor-parse/trunk/pom.xml

    r1452 r1490  
    8787            <groupId>uk.ac.cardiff.raptor</groupId> 
    8888            <artifactId>raptor-information-model</artifactId> 
    89             <version>1.1.2</version> 
     89            <version>1.1.3-SNAPSHOT</version> 
    9090            <scope>provided</scope> 
    9191        </dependency> 
     
    110110        </dependency> 
    111111 
     112        <!--  Jersey client for remoting e.g. Rest --> 
     113        <dependency> 
     114            <groupId>com.sun.jersey</groupId> 
     115            <artifactId>jersey-client</artifactId> 
     116            <version>1.16</version> 
     117        </dependency> 
     118 
     119        <dependency> 
     120            <groupId>org.codehaus.jackson</groupId> 
     121            <artifactId>jackson-mapper-asl</artifactId> 
     122            <version>1.9.11</version> 
     123        </dependency> 
    112124    </dependencies> 
    113125 
  • raptor-parse/trunk/src/main/java/uk/ac/cardiff/raptor/parse/BaseEventParser.java

    r1359 r1490  
    2121 
    2222import java.io.File; 
     23import java.io.UnsupportedEncodingException; 
     24import java.lang.reflect.Method; 
     25import java.net.URLDecoder; 
    2326import java.util.List; 
    24  
     27import java.util.TimeZone; 
     28 
     29import org.joda.time.DateTime; 
     30import org.joda.time.DateTimeZone; 
     31import org.joda.time.format.DateTimeFormat; 
     32import org.joda.time.format.DateTimeFormatter; 
    2533import org.slf4j.Logger; 
    2634import org.slf4j.LoggerFactory; 
    2735 
    2836import uk.ac.cardiff.model.event.Event; 
    29 import uk.ac.cardiff.raptor.parse.filter.ExclusionList; 
    30 import uk.ac.cardiff.raptor.parse.filter.InclusionList; 
     37import uk.ac.cardiff.raptor.parse.runtimeutils.ReflectionHelper; 
    3138import uk.ac.cardiff.raptor.store.IncrementalEventHandler; 
    3239 
    3340/** 
    34  * This is really not a base event parser, its specific to log file event parsing. 
     41 * The raptor base event parser that all concrete event parsers should extend. 
    3542 *  
    3643 */ 
     
    4047    private final Logger log = LoggerFactory.getLogger(BaseEventParser.class); 
    4148 
     49    /** 
     50     * Whether this event parser is enabled and should be used. Defaults to true. 
     51     */ 
     52    private boolean enabled = true; 
     53 
     54    /** The fully qualified name of the event type class. */ 
     55    protected String eventType; 
     56 
     57    /** A human readable name of the types of events this parser handles. */ 
     58    private String eventTypeFriendlyName; 
     59 
    4260    /** The handler that is responsible for storing events. */ 
    4361    protected IncrementalEventHandler eventHandler; 
    4462 
    45     /** A list of <fieldname,value> pairs that should be excluded during parsing. */ 
    46     private ExclusionList exclusionList; 
    47  
    48     /** 
    49      * A list of <fieldname, value> pairs that should be included during parsing 
    50      */ 
    51     private InclusionList inclusionList; 
    52  
    53     /** The fully qualified name of the event type class. */ 
    54     protected String eventType; 
    55  
    56     /** A human readable name of the types of events this parser handles. */ 
    57     private String eventTypeFriendlyName; 
    58  
    59     /** 
    60      * Whether this event parser is enabled and should be used. Defaults to true. 
    61      */ 
    62     private boolean enabled = true; 
    63  
    64     /** 
    65      * Default constructor 
    66      */ 
    67     public BaseEventParser() { 
    68     } 
    69  
    7063    /** 
    7164     * The method that must be provided to parse events from the <code>bytes</code> parameter 
     
    9285 
    9386    /** 
    94      * Method to reset this event parser back to its initial state.This involves removing all entries from the 
    95      * <code>IncrementalEntryHandler</code> and reseting all references to the last event(s) parsed. 
    96      */ 
    97     public void reset() { 
    98         eventHandler.reset(); 
    99     } 
    100  
    101     /** 
    10287     * Creates a new object from a class name. Could be removed to a helper class 
    10388     *  
     
    10590     * @return the new object instantiated as the type <code>className</code> 
    10691     */ 
    107     public Object createObject(String className) { 
     92    protected final Object createObject(String className) { 
    10893        Object object = null; 
    10994        try { 
     
    121106 
    122107    /** 
    123      * Method to remove all entries from the event handler 
    124      */ 
    125     public void removeAllEntries() { 
    126         eventHandler.removeAllEvents(); 
    127  
     108     * Method to reset this event parser back to its initial state.This involves removing all entries from the 
     109     * <code>IncrementalEntryHandler</code> and reseting all references to the last event(s) parsed. 
     110     */ 
     111    public void reset() { 
     112        eventHandler.reset(); 
     113    } 
     114 
     115    /** 
     116     * Decodes a URL (which must use the http(s) protocol) based on the ISO-8859-1 ISO-Latin character set. Keeps 
     117     * decoding (max 5 times) until the literal 'http(s)://' is obtained, as some URL's are double encoded. 
     118     *  
     119     * @param value the URL to be decoded. Must be a URL using the http protocol. 
     120     * @return the <code>value</code> URL decoded 
     121     */ 
     122    protected final String decode(String value) { 
     123        try { 
     124            int maxTries = 0; 
     125            String decodedURL = value; 
     126            while (true) { 
     127                decodedURL = URLDecoder.decode(decodedURL, "ISO-8859-1"); 
     128                if (decodedURL.contains("http://") || decodedURL.contains("https://")) { 
     129                    return decodedURL; 
     130                } 
     131                if (maxTries == 5) { 
     132                    log.warn("Maximum tries have been hit to decode the URL [{}]", value); 
     133                    break; 
     134                } 
     135                maxTries++; 
     136            } 
     137 
     138        } catch (UnsupportedEncodingException e) { 
     139            return value; 
     140        } 
     141        return value; 
     142    } 
     143 
     144    /** 
     145     * Converts the <code>value</code> into a list delimited by the <code> delimeter</code> input, and adds the list of 
     146     * string values to the <code>fieldName</code> of the <code>object</code> using Reflection. 
     147     *  
     148     * @param value the value to add to the <code>object</code> 
     149     * @param fieldName the name of the field that the value is added to 
     150     * @param delimeter the delimeter used to split the <code>value</code> into a list 
     151     * @param object the object with which to set the <code>value</code> on 
     152     */ 
     153    protected final void addStringList(String value, String fieldName, Object object, String delimeter) { 
     154        try { 
     155            String[] splitValue = value.split(delimeter); 
     156            String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
     157            setValueOnObject(fieldAsMethod, splitValue, object); 
     158        } catch (Exception e) { 
     159            log.error("Illegal StringList Value: {} for field: {}", new Object[] {value, fieldName, e}); 
     160        } 
     161    } 
     162 
     163    /** 
     164     * Converts the <code>value</code> into a date using the given <code>format</code> and <code>timezone</code> 
     165     * parameters, and adds the date to the <code>fieldName</code> of the <code>object</code> using Reflection. 
     166     *  
     167     * @param value the value to add to the <code>object</code> 
     168     * @param format the date format used to parse the <code>value</code> into a string 
     169     * @param timezone sets the timezone on the newly created date 
     170     * @param fieldName the name of the field that the value is added to 
     171     * @param object the object with which to set the <code>value</code> on 
     172     */ 
     173    protected final void addDate(String value, String format, String timezone, String fieldName, Object object) { 
     174        try { 
     175            DateTimeFormatter dtf = DateTimeFormat.forPattern(format); 
     176            dtf = dtf.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone))); 
     177            DateTime dt = dtf.parseDateTime(value.substring(0, value.length())); 
     178            String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
     179            setValueOnObject(fieldAsMethod, dt, object); 
     180        } catch (Exception e) { 
     181            log.error("Illegal Date Value: {} for field: {}", new Object[] {value, fieldName, e}); 
     182        } 
     183 
     184    } 
     185 
     186    /** 
     187     * Adds the string <code>value</code> to the <code>fieldName</code> of the <code>object</code> using Reflection. 
     188     *  
     189     * @param value the value to add to the <code>object</code> 
     190     * @param fieldName the name of the field that the value is added to 
     191     * @param object the object with which to set the <code>value</code> on 
     192     */ 
     193    protected final void addString(String value, String fieldName, Object object) { 
     194        try { 
     195            String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
     196            setValueOnObject(fieldAsMethod, value, object); 
     197        } catch (Exception e) { 
     198            log.error("Illegal String Value: {} for field: {}", new Object[] {value, fieldName, e}); 
     199        } 
     200 
     201    } 
     202 
     203    /** 
     204     * Converts the <code>value</code> into an Integer, and adds the Integer to the <code>fieldName</code> of the 
     205     * <code>object</code> using Reflection. 
     206     *  
     207     * @param value the value to add to the <code>object</code> 
     208     * @param fieldName the name of the field that the value is added to 
     209     * @param object the object with which to set the <code>value</code> on 
     210     */ 
     211    protected final void addInteger(String value, String fieldName, Object object) { 
     212        try { 
     213            String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
     214            int valueAsInt = Integer.parseInt(value); 
     215            setValueOnObject(fieldAsMethod, valueAsInt, object); 
     216        } catch (Exception e) { 
     217            log.error("Illegal Integer Value: {} for field: {}", new Object[] {value, fieldName, e}); 
     218        } 
     219 
     220    } 
     221 
     222    /** 
     223     * Sets the <code>param</code> object on the field <code>fieldname</code> of the object <code>object</code> 
     224     *  
     225     * @param fieldname the name of the field from which to set the value <code>param</code> 
     226     * @param param the object to set on the <code>fieldname</code> of the <code>object</code> 
     227     * @param object the object with which to set the object <code>param</code> 
     228     */ 
     229    protected final void setValueOnObject(String fieldname, Object param, Object object) { 
     230        try { 
     231            Class<? extends Object> id = object.getClass(); 
     232            Method setter = id.getMethod(fieldname, new Class[] {param.getClass()}); 
     233            setter.invoke(object, new Object[] {param}); 
     234        } catch (Throwable e) { 
     235            log.error("Field name '{}' does not match internal model attribute for type", new Object[] {fieldname, 
     236                    object.getClass()}, e); 
     237 
     238        } 
     239    } 
     240 
     241    /** 
     242     * @param enabled the enabled to set 
     243     */ 
     244    public void setEnabled(boolean enabled) { 
     245        this.enabled = enabled; 
     246    } 
     247 
     248    /** 
     249     * @return the enabled 
     250     */ 
     251    public boolean isEnabled() { 
     252        return enabled; 
     253    } 
     254 
     255    /** 
     256     * @param eventTypeFriendlyName the eventTypeFriendlyName to set 
     257     */ 
     258    public void setEventTypeFriendlyName(String eventTypeFriendlyName) { 
     259        this.eventTypeFriendlyName = eventTypeFriendlyName; 
     260    } 
     261 
     262    /** 
     263     * @return the eventTypeFriendlyName 
     264     */ 
     265    public String getEventTypeFriendlyName() { 
     266        return eventTypeFriendlyName; 
     267    } 
     268 
     269    /** 
     270     * @param eventType the eventType to set 
     271     */ 
     272    public void setEventType(String eventType) { 
     273        this.eventType = eventType; 
     274    } 
     275 
     276    /** 
     277     * @return the eventType 
     278     */ 
     279    public String getEventType() { 
     280        return eventType; 
    128281    } 
    129282 
     
    148301 
    149302    /** 
    150      * Sets the exclusion list 
    151      *  
    152      * @param exclusionList 
    153      */ 
    154     public void setExclusionList(ExclusionList exclusionList) { 
    155         this.exclusionList = exclusionList; 
    156     } 
    157  
    158     /** 
    159      * Gets the exclusion list 
    160      *  
    161      * @return 
    162      */ 
    163     public ExclusionList getExclusionList() { 
    164         return exclusionList; 
    165     } 
    166  
    167     /** 
    168      * Sets the inclusion list 
    169      *  
    170      * @param inclusionList 
    171      */ 
    172     public void setInclusionList(InclusionList inclusionList) { 
    173         this.inclusionList = inclusionList; 
    174     } 
    175  
    176     /** 
    177      * Gets the inclusion list 
    178      *  
    179      * @return 
    180      */ 
    181     public InclusionList getInclusionList() { 
    182         return inclusionList; 
    183     } 
    184  
    185     /** 
    186303     * Gets all the entries currently stored by this entry handler 
    187304     *  
     
    193310    } 
    194311 
    195     /** 
    196      * @param eventType the eventType to set 
    197      */ 
    198     public void setEventType(String eventType) { 
    199         this.eventType = eventType; 
    200     } 
    201  
    202     /** 
    203      * @return the eventType 
    204      */ 
    205     public String getEventType() { 
    206         return eventType; 
    207     } 
    208  
    209     /** 
    210      * @param eventTypeFriendlyName the eventTypeFriendlyName to set 
    211      */ 
    212     public void setEventTypeFriendlyName(String eventTypeFriendlyName) { 
    213         this.eventTypeFriendlyName = eventTypeFriendlyName; 
    214     } 
    215  
    216     /** 
    217      * @return the eventTypeFriendlyName 
    218      */ 
    219     public String getEventTypeFriendlyName() { 
    220         return eventTypeFriendlyName; 
    221     } 
    222  
    223     /** 
    224      * @param enabled the enabled to set 
    225      */ 
    226     public void setEnabled(boolean enabled) { 
    227         this.enabled = enabled; 
    228     } 
    229  
    230     /** 
    231      * @return the enabled 
    232      */ 
    233     public boolean isEnabled() { 
    234         return enabled; 
    235     } 
    236  
    237312} 
  • raptor-parse/trunk/src/main/java/uk/ac/cardiff/raptor/parse/external/file/LogFileParser.java

    r1370 r1490  
    2424import java.io.InputStream; 
    2525import java.io.InputStreamReader; 
    26 import java.io.UnsupportedEncodingException; 
    2726import java.lang.reflect.Method; 
    2827import java.net.MalformedURLException; 
    2928import java.net.URL; 
    3029import java.net.URLConnection; 
    31 import java.net.URLDecoder; 
    3230import java.util.ArrayList; 
    3331import java.util.List; 
    3432import java.util.Set; 
    35 import java.util.TimeZone; 
    3633import java.util.regex.Matcher; 
    3734import java.util.regex.Pattern; 
     
    3936import org.apache.commons.lang.text.StrTokenizer; 
    4037import org.joda.time.DateTime; 
    41 import org.joda.time.DateTimeZone; 
    42 import org.joda.time.format.DateTimeFormat; 
    43 import org.joda.time.format.DateTimeFormatter; 
    4438import org.slf4j.Logger; 
    4539import org.slf4j.LoggerFactory; 
    4640 
    4741import uk.ac.cardiff.model.event.Event; 
    48 import uk.ac.cardiff.raptor.parse.BaseEventParser; 
     42import uk.ac.cardiff.raptor.parse.BaseEventFileParser; 
    4943import uk.ac.cardiff.raptor.parse.ParserException; 
    5044import uk.ac.cardiff.raptor.parse.external.file.format.Header; 
     
    6256 *  
    6357 */ 
    64 public class LogFileParser extends BaseEventParser { 
     58public class LogFileParser extends BaseEventFileParser { 
    6559 
    6660    /** The class logger */ 
     
    125119    } 
    126120 
     121    @Override 
    127122    public void parse(File logfile) throws ParserException { 
    128123        try { 
     
    309304 
    310305    /** 
    311      * Decodes a URL (which must use the http(s) protocol) based on the ISO-8859-1 ISO-Latin character set. Keeps 
    312      * decoding (max 5 times) until the literal 'http(s)://' is obtained, as some URL's are double encoded. 
    313      *  
    314      * @param value the URL to be decoded. Must be a URL using the http protocol. 
    315      * @return the <code>value</code> URL decoded 
    316      */ 
    317     private String decode(String value) { 
    318         try { 
    319             int maxTries = 0; 
    320             String decodedURL = value; 
    321             while (true) { 
    322                 decodedURL = URLDecoder.decode(decodedURL, "ISO-8859-1"); 
    323                 if (decodedURL.contains("http://") || decodedURL.contains("https://")) { 
    324                     return decodedURL; 
    325                 } 
    326                 if (maxTries == 5) { 
    327                     log.warn("Maximum tries have been hit to decode the URL [{}]", value); 
    328                     break; 
    329                 } 
    330                 maxTries++; 
    331             } 
    332  
    333         } catch (UnsupportedEncodingException e) { 
    334             return value; 
    335         } 
    336         return value; 
    337     } 
    338  
    339     /** 
    340306     * Replace substrings from <code>value</code> with the given strings in <code>header</code> that match the regular 
    341307     * expression(s) in the regexReplaceAll from the <code>header</code> 
     
    414380 
    415381    /** 
    416      * Determines if the event should stored, as determined by a <fieldname, value> match on the input 
     382     * Determines if the event should be stored, as determined by a <fieldname, value> match on the input 
    417383     * <code>authE</code> event with those in the <code>exclusionList</code>. 
    418384     *  
     
    483449 
    484450    /** 
    485      * Converts the <code>value</code> into a list delimited by the <code> delimeter</code> input, and adds the list of 
    486      * string values to the <code>fieldName</code> of the <code>object</code> using Reflection. 
    487      *  
    488      * @param value the value to add to the <code>object</code> 
    489      * @param fieldName the name of the field that the value is added to 
    490      * @param delimeter the delimeter used to split the <code>value</code> into a list 
    491      * @param object the object with which to set the <code>value</code> on 
    492      */ 
    493     private void addStringList(String value, String fieldName, Object object, String delimeter) { 
    494         try { 
    495             String[] splitValue = value.split(delimeter); 
    496             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
    497             setValueOnObject(fieldAsMethod, splitValue, object); 
    498         } catch (Exception e) { 
    499             log.error("Illegal StringList Value: {} for field: {}", new Object[] {value, fieldName, e}); 
    500         } 
    501     } 
    502  
    503     /** 
    504      * Converts the <code>value</code> into a date using the given <code>format</code> and <code>timezone</code> 
    505      * parameters, and adds the date to the <code>fieldName</code> of the <code>object</code> using Reflection. 
    506      *  
    507      * @param value the value to add to the <code>object</code> 
    508      * @param format the date format used to parse the <code>value</code> into a string 
    509      * @param timezone sets the timezone on the newly created date 
    510      * @param fieldName the name of the field that the value is added to 
    511      * @param object the object with which to set the <code>value</code> on 
    512      */ 
    513     private void addDate(String value, String format, String timezone, String fieldName, Object object) { 
    514         try { 
    515             DateTimeFormatter dtf = DateTimeFormat.forPattern(format); 
    516             dtf = dtf.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone))); 
    517             DateTime dt = dtf.parseDateTime(value.substring(0, value.length())); 
    518             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
    519             setValueOnObject(fieldAsMethod, dt, object); 
    520         } catch (Exception e) { 
    521             log.error("Illegal Date Value: {} for field: {}", new Object[] {value, fieldName, e}); 
    522         } 
    523  
    524     } 
    525  
    526     /** 
    527      * Adds the string <code>value</code> to the <code>fieldName</code> of the <code>object</code> using Reflection. 
    528      *  
    529      * @param value the value to add to the <code>object</code> 
    530      * @param fieldName the name of the field that the value is added to 
    531      * @param object the object with which to set the <code>value</code> on 
    532      */ 
    533     private void addString(String value, String fieldName, Object object) { 
    534         try { 
    535             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
    536             setValueOnObject(fieldAsMethod, value, object); 
    537         } catch (Exception e) { 
    538             log.error("Illegal String Value: {} for field: {}", new Object[] {value, fieldName, e}); 
    539         } 
    540  
    541     } 
    542  
    543     /** 
    544      * Converts the <code>value</code> into an Integer, and adds the Integer to the <code>fieldName</code> of the 
    545      * <code>object</code> using Reflection. 
    546      *  
    547      * @param value the value to add to the <code>object</code> 
    548      * @param fieldName the name of the field that the value is added to 
    549      * @param object the object with which to set the <code>value</code> on 
    550      */ 
    551     private void addInteger(String value, String fieldName, Object object) { 
    552         try { 
    553             String fieldAsMethod = ReflectionHelper.prepareMethodNameSet(fieldName); 
    554             setValueOnObject(fieldAsMethod, value, object); 
    555         } catch (Exception e) { 
    556             log.error("Illegal Integer Value: {} for field: {}", new Object[] {value, fieldName, e}); 
    557         } 
    558  
    559     } 
    560  
    561     /** 
    562451     * Returns the value of the <code>fieldname</code> parameter from the <code>object</code> Object 
    563452     *  
     
    579468    } 
    580469 
    581     /** 
    582      * Sets the <code>param</code> object on the field <code>fieldname</code> of the object <code>object</code> 
    583      *  
    584      * @param fieldname the name of the field from which to set the value <code>param</code> 
    585      * @param param the object to set on the <code>fieldname</code> of the <code>object</code> 
    586      * @param object the object with which to set the object <code>param</code> 
    587      */ 
    588     private void setValueOnObject(String fieldname, Object param, Object object) { 
    589         try { 
    590             Class<? extends Object> id = object.getClass(); 
    591             Method setter = id.getMethod(fieldname, new Class[] {param.getClass()}); 
    592             setter.invoke(object, new Object[] {param}); 
    593         } catch (Throwable e) { 
    594             log.error("Field name '{}' does not match internal model attribute", fieldname, e); 
    595  
    596         } 
    597     } 
    598  
    599470    public void setLogfile(String logfile) { 
    600471        this.logfile = logfile; 
  • raptor-parse/trunk/src/main/java/uk/ac/cardiff/raptor/parse/external/file/format/Header.java

    r1359 r1490  
    4444    private String fieldName; 
    4545 
    46     /** The index in the currently tokenized line this header matches to. */ 
     46    /** 
     47     * The name of the source fieldname to extract the value from, this is used by some parsing types. 
     48     */ 
     49    private String sourceFieldName; 
     50 
     51    /** The index in the currently tokenized line this header matches to. This is used by some parsing types. */ 
    4752    private int fieldNo; 
    4853 
     
    291296    } 
    292297 
     298    /** 
     299     * @return Returns the sourceFieldName. 
     300     */ 
     301    public String getSourceFieldName() { 
     302        return sourceFieldName; 
     303    } 
     304 
     305    /** 
     306     * @param sourceFieldName The sourceFieldName to set. 
     307     */ 
     308    public void setSourceFieldName(String sourceFieldName) { 
     309        this.sourceFieldName = sourceFieldName; 
     310    } 
     311 
    293312} 
Note: See TracChangeset for help on using the changeset viewer.