expression-lexer.rkt

#lang racket

(require "parse-util.rkt")
(require parser-tools/lex)
(require parser-tools/lex-sre)
(require (prefix-in : parser-tools/lex-sre))

(define expression-lexer 
 (lexer
   [#\space (expression-lexer input-port)]
   ["or" "or"]
   ["either"  "either"]
   ["any" "any"]
   ["instead" "instead"]
   ["and" "and"]
   ["both" "both"]
   ["also" "also"]
   ["including" "including"]
   ["not" "not"]
   ["never" "never"]
   ["neither" "neither"]
   ["no" "no"]
   ["don't" "don't"]
   ["isn't" "isn't"]
   ["concatenate" "concatenate"]
   ["concat" "concat"]
   ["combine" "combine"]
   ["transform" "transform"]
   ["append" "append"]
   ["merge" "merge"]
   ["mix" "mix"]
   ["encode" "encode"]
   ["quotient" "quotient"]
   ["remainder" "remainder"]
   ["modulo" "modulo"]
   ["mod" "mod"]
   [#\%  "%"]
   ["abs" "abs"]
   ["absolute" "absolute"]
   ["magnitude" "magnitude"]
   ["positive" "positive"]
   ["ceiling" "ceiling"]
   ["round" "round"]
   ["up" "up"]
   ["floor" "floor"]
   ["down" "down"]
   ["truncate" "truncate"]
   ["trunc" "trunc"]
   ["drop" "drop"]
   ["shorten" "shorten"]
   ["abbreviate" "abbreviate"]
   ["cut" "cut"]
   ["sign" "sign"]
   ["positive" "positive"]
   ["or" "or"]
   ["negative" "negative"]
   ["if" "if"]
   ["whether" "whether"]
   ["when" "when"]
   ["unless" "unless"]
   ["case" "case"]
   ["in" "in"]
   ["then" "then"]
   ["do" "do"]
   ["subsequently" "subsequently"]
   ["else" "else"]
   ["otherwise" "otherwise"]
   ["substring" "substring"]
   ["extract" "extract"]
   ["characters" "characters"]
   ["string" "string"]
   ["left" "left"]
   ["right" "right"]
   ["mid" "mid"]
   ["indexof" "indexof"]
   ["index" "index"]
   ["of" "of"]
   ["find" "find"]
   ["search" "search"]
   ["length" "length"]
   ["len" "len"]
   [#\+  "+"]
   ["plus" "plus"]
   [#\-  "-"]
   ["minus" "minus"]
   ["takeaway" "takeaway"]
   ["subtract" "subtract"]
   ["deduct" "deduct"]
   [#\/  "/"]
   ["divide" "divide"]
   ["by" "by"]
   [#\*  "*"] 
   ["multiply" "multiply"]
   ["times" "times"]
   ["exponent" "exponent"]
   ["exp" "exp"]
   ["power" "power"]
   ["logarithm" "logarithm"]
   ["log" "log"]
   ["ln" "ln"]
   ["natural" "natural"]
   ["sqrt" "sqrt"]
   ["square" "square"]
   ["root" "root"]
   ["lower" "lower"]
   ["to" "to"]
   ["case" "case"]
   ["convert" "convert"]
   ["upper" "upper"]
   ["trim" "trim"]
   ["remove" "remove"]
   ["trailing" "trailing"]
   ["spaces" "spaces"]
   ["pare" "pare"]
   ["cut" "cut"]
   ["replace" "replace"]
   ["overlay" "overlay"]
   ["change" "change"]
   ["matches" "matches"]
   ["contains" "contains"]
   ["includes" "includes"]
   ["has" "has"]
   ["appears" "appears"]
   ["in" "in"]
   ["now" "now"]
   ["current" "current"]
   ["time" "time"]
   ["today" "today"]
   ["retrieve" "retrieve"]
   ["pull" "pull"]
   ["out" "out"]
   ["grab" "grab"]
   ["fetch" "fetch"]
   ["obtain" "obtain"]
   ["get" "get"]
   [#\>  ">"]
   ["greater" "greater"]
   ["than" "than"]
   ["equal" "equal"]
   ["is" "is"]
   ["more" "more"]
   ["larger" "larger"]
   ["higher" "higher"]
   ["bigger" "bigger"]
   ["older" "older"]
    [#\<  "<"]
    ["less" "less"]
    ["smaller" "smaller"]
    ["lower" "lower"]
    ["younger" "younger"]
    [">="  ">="]
    ["<="  "<="]
    ["!="  "!="]
    ["<>"  "<>"]
    [#\=  "="]
    ["=="  "=="]
    ["equals" "equals"]
    ["same" "same"]
    ["as" "as"]
    ["("  "("]
    [")"  ")"]
    ["," ","]
    [#\' "'"]
    [#\" "\""]
    [(seq (:or (char-range #\a #\z) (char-range #\A #\Z)) (:* (:or (char-range #\a #\z) (char-range #\A #\Z) (char-range #\0 #\9) #\_))) lexeme]
    [(seq (:or #\' #\") (:* (:or (char-range #\a #\z) (char-range #\A #\Z) (char-range #\0 #\9) #\_ #\ )) (:or #\' #\")) (trim-first-last-chars lexeme)]
    [(seq (char-range #\0 #\9) (char-range #\0 #\9) (:or #\/ #\-) (char-range #\0 #\9) (char-range #\0 #\9)  (:or #\/ #\-) (char-range #\0 #\9) (char-range #\0 #\9)(char-range #\0 #\9) (char-range #\0 #\9) ) lexeme]
    [(seq (:? (:or #\- #\+)) (:+ (char-range #\0 #\9)) (:? (seq #\. (:+ (char-range #\0 #\9))))) lexeme]
    [(seq (:? (:or #\- #\+)) (seq #\. (:+ (char-range #\0 #\9)))) lexeme]
   ))

(define (lex in)
  (let ((exp (expression-lexer in)))
    (if (eq? 'eof exp)
        '()
        (cons exp (lex in)))))

(define (test1)
  (lex  (open-input-string "a2_3*b12=2.3 19/09/1994 12-12-2000")))

(define (test2)
  (lex (open-input-string "((min_servers*price_per_server) * 100)")))

(define (test3)
  (lex (open-input-string "funny same as foo")))

(define (test4)
  (lex (open-input-string "funny 'same as' foo")))

(define (test6)
  (lex (open-input-string "funny \"same as\" foo")))

(define (test5)
  (lex (open-input-string "foo*.03")))

(define (test7)
  (lex (open-input-string "customer_tier = 'Tier 0' or customer_tier = 'Tier 1' and MRR is greater than 2000")))

(define (test8)
  (lex (open-input-string "sum (if 'Start Date' > 12-3-2015 then 'Start Date' else 0) by organization")))

(define (test9)
  (lex (open-input-string "sum if Start_Date 2.3 >12/03/2015 then 'Start Date' else 0 by organization")))

(provide lex)