parser.rl

/*
 * THIS C FILE GENERATED BY RAGEL - DO NOT EDIT!
 * Ragel State Machine Compiler: http://www.complang.org/ragel/
 * Source: http://www.complang.org/ragel/ragel-6.5.tar.gz
 */

%%{
	machine parser;
	access cmd->;
	
	action set_year {
		cmd->timestamp.tm_year = (((*(fpc - 3) - '0') * 1000) + ((*(fpc - 2) - '0') * 100) + ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0')) - 1900;
	}
	
	action set_month {
		cmd->timestamp.tm_mon = (((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0')) - 1;
	}
	
	action set_month_jan {
		cmd->timestamp.tm_mon = 0;
	}
	
	action set_month_feb {
		cmd->timestamp.tm_mon = 1;
	}
	
	action set_month_mar {
		cmd->timestamp.tm_mon = 2;
	}
	
	action set_month_apr {
		cmd->timestamp.tm_mon = 3;
	}
	
	action set_month_may {
		cmd->timestamp.tm_mon = 4;
	}
	
	action set_month_jun {
		cmd->timestamp.tm_mon = 5;
	}
	
	action set_month_jul {
		cmd->timestamp.tm_mon = 6;
	}
	
	action set_month_aug {
		cmd->timestamp.tm_mon = 7;
	}
	
	action set_month_sep {
		cmd->timestamp.tm_mon = 8;
	}
	
	action set_month_oct {
		cmd->timestamp.tm_mon = 9;
	}
	
	action set_month_nov {
		cmd->timestamp.tm_mon = 10;
	}
	
	action set_month_dec {
		cmd->timestamp.tm_mon = 11;
	}
	
	action set_day {
		cmd->timestamp.tm_mday = ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0');
	}
	
	action set_hour {
		cmd->timestamp.tm_hour = ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0');
	}
	
	action set_minute {
		cmd->timestamp.tm_min = ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0');
	}
	
	action set_second {
		cmd->timestamp.tm_sec = ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0');
	}
	
	action set_timezoneoffset {
		int hours = ((*(fpc - 3) - '0') * 10) + (*(fpc - 2) - '0');
		int minutes = ((*(fpc - 1) - '0') * 10) + (*(fpc - 0) - '0');
		cmd->timestamp.tm_gmtoff = ((hours % 12) * 3600) + ((minutes % 60) * 60);
		if (*(fpc - 4) == '-') {
			cmd->timestamp.tm_gmtoff = -cmd->timestamp.tm_gmtoff;
		}
	}
	
	action begin_timestamp {
		timestamp = fpc;
		cmd->timestamp.tm_gmtoff = 0;
	}
	
	action end_timestamp {
		cmd->time = timegm(&cmd->timestamp);
		if ((cmd->time < cmd->start_time) || ((cmd->time > cmd->stop_time))) {
			cmd->time = 0;
		}
	}
	
	action set_httpd_access_timestamp {
		flag = 1; // match from start of timestamp to the end of text
	}
	
	action set_httpd_error_timestamp {
		flag = 1; // match from start of timestamp to the end of text
	}
	
	action set_nmp2_timestamp {
		flag = 0; // match from start of text to the end of text
	}
	
	action begin_text {
		text = fpc;
	}
	
	action end_text {
		if (cmd->time) {
			int len = fpc + 1 - timestamp;
			//fwrite(timestamp, 1, len, stdout);
			if (match(cmd->pattern, flag ? timestamp : text)) {
				if (cmd->format) {
					if ((cmd->limit == 0) || ((cmd->marker >= cmd->skip) && (cmd->marker < cmd->skip + cmd->limit))) {
						switch (cmd->format) {
						case TEXTFORMAT:
							if (fwrite(timestamp, 1, len, stdout) != len) {
								fprintf(stderr, "logfile: could not write stdout\n");
								result = 1;
								fbreak;
							}
							break;
						case XMLFORMAT:
							{
								char ch, *line = timestamp;
								int i, c = 0, n = len - 1;
								printf("\t<string>");
								while (c < n) {
									for (i = c; i < n; i++) {
										if (line[i] == '<' || line[i] == '>' || line[i] == '&') {
											break;
										}
									}
									ch = line[i];
									line[i] = '\0';
									printf("%s", &line[c]);
									if (ch == '<') {
										printf("&lt;");
									}
									else if (ch == '>') {
										printf("&gt;");
									}
									else if (ch == '&') {
										printf("&amp;");
									}
									line[i] = ch;
									c = i + 1;
								}
								printf("</string>\n");
								break;
							}
						}
					}
				}
				else {
// 					if (cmd->idx_file) {
// 						if (fwrite(&cmd->offset, cmd->offset_size, 1, cmd->idx_file) != 1) {
// 							fprintf(stderr, "logfile: could not write idx file\n");
// 							result = 1;
// 							fbreak;
// 						}
// 					}
// 					if (cmd->log_file) {
// 						if (fwrite(timestamp, 1, len, cmd->log_file) != len) {
// 							fprintf(stderr, "logfile: could not write log file\n");
// 							result = 1;
// 							fbreak;
// 						}
// 					}
					if (rec) {
						cmd->index = (cmd->time - cmd->start_time) / cmd->value;
						rec[cmd->index].count++;
					}
				}
				cmd->marker++;
				cmd->offset += len;
				stat->totalBytes += len;
				stat->totalLength++;
			}
			stat->totalSearchBytes += len;
			stat->totalSearchLength++;
			cmd->time = 0; // reset
		}
	}
	
	year = ( digit digit digit digit ) @set_year;
	
	month = ( digit digit ) @set_month;
	
	month_jan = ( 'Jan' ) @set_month_jan;
	
	month_feb = ( 'Feb' ) @set_month_feb;
	
	month_mar = ( 'Mar' ) @set_month_mar;
	
	month_apr = ( 'Apr' ) @set_month_apr;
	
	month_may = ( 'May' ) @set_month_may;
	
	month_jun = ( 'Jun' ) @set_month_jun;
	
	month_jul = ( 'Jul' ) @set_month_jul;
	
	month_aug = ( 'Aug' ) @set_month_aug;
	
	month_sep = ( 'Sep' ) @set_month_sep;
	
	month_oct = ( 'Oct' ) @set_month_oct;
	
	month_nov = ( 'Nov' ) @set_month_nov;
	
	month_dec = ( 'Dec' ) @set_month_dec;
	
	month_all = ( month_jan | month_feb | month_mar | month_apr | month_may | month_jun | 
				  month_jul | month_aug | month_sep | month_oct | month_nov | month_dec );
	
	day = ( digit digit ) @set_day;
	
	hour = ( digit digit ) @set_hour;
	
	minute = ( digit digit ) @set_minute;
	
	second = ( digit digit ) @set_second;
	
	timezoneoffset = ( ( '+' | '-' ) digit digit digit digit ) @set_timezoneoffset;
	
	ipv4 = ( digit+ '.' digit+ '.' digit+ '.' digit+ );
	
	ip = ( ipv4 ); # Note: there is currently no support for ipv6
	
	httpd_access_timestamp = ( ip ' - ' [^ ]+ ' [' day '/' month_all '/' year ':' hour ':' minute ':' second ' ' timezoneoffset ']' ) @set_httpd_access_timestamp;
	
	httpd_error_timestamp = ( '[' [A-Z][a-z][a-z] ' ' month_all ' ' day ' ' hour ':' minute ':' second ' ' year ']' ) @set_httpd_error_timestamp;
	
	nmp2_timestamp = ( year '-' month '-' day [ \t] hour ':' minute ':' second ) @set_nmp2_timestamp;
	
	# Important Notice: log entries in httpd error_log without a timestamp - it is possible - are skipped
	timestamp = ( httpd_access_timestamp | httpd_error_timestamp | nmp2_timestamp ) >begin_timestamp @end_timestamp;
	
	line = timestamp [ \t] ( any* '\n' ) >begin_text @end_text;
	
	main := line*;
}%%

%% write data noerror nofinal;

int parser_read(Statistics *stat, Record *rec, Command *cmd, const char *name)
{
	char *text;
	char *timestamp;
	gzFile *file;
	int flag;
	int have = 0;
	int result = 0;
	
	cmd->time = 0; // reset
	
	if ((file = gzopen(name, "rb"))) {
	
		%% write init;
		
		while (1) {
			char *p, *pe, *data = cmd->buffer + have;
			int len, space = cmd->buffer_size - have;
			/* fprintf(stderr, "logfile: space: %i\n", space); */
			
			if (space == 0) { 
				fprintf(stderr, "logfile: buffer out of space\n");
				result = 1;
				break;
			}
			
			len = gzread(file, data, space); // fread(data, 1, space, stdin);
			/* fprintf(stderr, "logfile: len: %i\n", len); */
			if (len == 0) {
				/* fprintf(stderr, "logfile: end of file\n"); */
				break;
			}
			
			/* Find the last newline by searching backwards. This is where 
			 * we will stop processing on this iteration. */
			p = cmd->buffer;
			pe = cmd->buffer + have + len - 1;
			while (*pe != '\n' && pe >= cmd->buffer) {
				pe--;
			}
			pe += 1;
			
			/* fprintf(stderr, "logfile: running on: %i\n", pe - p); */
			
			%% write exec;
			
			/* Have we reached or exceeded the limit. */
			if (cmd->limit && (cmd->marker >= cmd->skip + cmd->limit)) {
				have = 0; /* We need a controlled terminate. */
				break;
			}
			
			/* How much is still in the buffer. */
			have = data + len - pe;
			if (have > 0) {
				memmove(cmd->buffer, pe, have);
			}
			
			/* fprintf(stderr, "logfile: have: %i\n", have); */
			
			if (len < space) {
				/* fprintf(stderr, "logfile: end of file\n"); */
				break;
			}
		}
		
		if (have > 0) {
			fprintf(stderr, "logfile: input not newline terminated\n");
			result = 1;
		}
		gzclose(file);
	}
	
	return result;
}

int parser_test(Command *cmd, const char *name)
{
	int head, tail;
	time_t head_time, tail_time;
	struct tm head_timestamp, tail_timestamp;
	
	if (sscanf(name, cmd->name_format/*"%4d%2d%2d%2d%2d%2d-%4d%2d%2d%2d%2d%2d."*/, 
			&head_timestamp.tm_year, &head_timestamp.tm_mon, &head_timestamp.tm_mday, 
			&head_timestamp.tm_hour, &head_timestamp.tm_min, &head_timestamp.tm_sec, 
			&tail_timestamp.tm_year, &tail_timestamp.tm_mon, &tail_timestamp.tm_mday, 
			&tail_timestamp.tm_hour, &tail_timestamp.tm_min, &tail_timestamp.tm_sec) == 12) {
		
		head_timestamp.tm_year -= 1900;
		head_timestamp.tm_mon -= 1;
		head_timestamp.tm_isdst = 0;
		head_timestamp.tm_gmtoff = 0;
		head_time = timegm(&head_timestamp);
		
		tail_timestamp.tm_year -= 1900;
		tail_timestamp.tm_mon -= 1;
		tail_timestamp.tm_isdst = 0;
		tail_timestamp.tm_gmtoff = 0;
		tail_time = timegm(&tail_timestamp);
		
		head = head_time >= cmd->start_time && head_time <= cmd->stop_time;
		tail = tail_time >= cmd->start_time && tail_time <= cmd->stop_time;
		
		return head | tail;
	}
	else {
		int len = strlen(cmd->name);
		if (len) {
			if (cmd->name[len - 1] == '*') {
				return strncmp(name, cmd->name, len - 1) == 0; // partial match OK in certain cases
			}
			else {
				return strcmp(name, cmd->name) == 0; // exact match required to enforce name_format order
			}
		}
	}
	
	return 0;
}

int parser(Command *cmd)
{
	struct dirent **dir;
	int nentries, entry;
	int result = 0;
	
	if (chdir(cmd->dir) == 0) {
		if (cmd->start_time && cmd->stop_time) {
			if (cmd->start_time < cmd->stop_time) {
				cmd->value = ((cmd->stop_time - cmd->start_time) / 86400) > 1 ? 86400 : 3600;
				cmd->count = ((cmd->stop_time - cmd->start_time) / cmd->value) + 1;
				Record *rec = malloc(cmd->count * sizeof(Record));
				if (rec) {
					for (cmd->index = 0; cmd->index < cmd->count; cmd->index++) {
						reset_record(&rec[cmd->index]);
					}
					cmd->timestamp.tm_isdst = 0;
					cmd->timestamp.tm_gmtoff = 0;
					nentries = scandir(".", &dir, 0, alphasort);
					if (nentries >= 0) {
						Statistics stat;
						stat.totalBytes = 0;
						stat.totalLength = 0;
						stat.totalSearchBytes = 0;
						stat.totalSearchLength = 0;
						if (cmd->format) {
							// nothing
						}
						else {
							print_begin_xml();
							print_begin_files_xml();
						}
						start_clock(&stat);
						for (entry = 0; entry < nentries; entry++) {
							if (strcmp(dir[entry]->d_name, ".") == 0) {
								goto free_dir_entry;
							}
							if (strcmp(dir[entry]->d_name, "..") == 0) {
								goto free_dir_entry;
							}
							if (cmd->limit && (cmd->marker >= cmd->skip + cmd->limit)) {
								goto free_dir_entry;
							}
							if (parser_test(cmd, dir[entry]->d_name)) {
								if (cmd->format) {
									// nothing
								}
								else {
									print_file_xml(dir[entry]->d_name, cmd->marker);
								}
								/* fprintf(stderr, "logfile: %s\n", dir[entry]->d_name); */
								if (result = parser_read(&stat, rec, cmd, dir[entry]->d_name)) {
									break;
								}
							}
						free_dir_entry:
							free(dir[entry]);
						}
						stop_clock(&stat);
						if (cmd->format) {
							// nothing
						}
						else {
							print_end_files_xml();
							print_records_xml(rec, cmd);
							print_statistics_xml(&stat);
							print_end_xml();
						}
						free(dir);
					}
					free(rec);
				}
				else {
					fprintf(stderr, "logfile: could not allocate record (%d)\n", cmd->count);
					result = 1;
				}
			}
			else {
				fprintf(stderr, "logfile: start (%ld) is later than stop (%ld)\n", (long int)cmd->start_time, (long int)cmd->stop_time);
				result = 1;
			}
		}
		else {
			fprintf(stderr, "logfile: start (%ld) stop (%ld) not specified\n", (long int)cmd->start_time, (long int)cmd->stop_time);
			result = 1;
		}
	}
	else {
		fprintf(stderr, "logfile: could not change directory (%s)\n", cmd->dir);
		result = 1;
	}
	
	return result;
}