#!/usr/local/bin/perl5
#
# $Id: raw-data.pl,v 1.14 2009/02/01 04:49:12 winter Exp $
use Data::Dumper;
use HTML::Entities;
use Date::Format;
use Cfg;
sub
strip_html {
$_ = decode_entities(shift);
$_ =~ s/\s*\<[^\>]+\>\s*//g;
$_ =~ s/[\.\!\?\,]+//g; # errant punctuation
return $_;
}
sub
extract_number {
$_ = strip_html(shift);
if (/=.*[[:digit:]]+/) {
(undef, $_) = split(/=/)
}
if (/([[:digit:]]+)/) {
$i = int($1);
if ($i > 0 && $i < 2001) {
return int($1);
}
}
# if (/one/) {
# return 1;
# }
return undef;
}
sub
detect_comments {
$_ = shift;
$_ =~ s/Posted Via AR15\.Com Mobile//g;
$_ =~ s/
//g; # hard breaks
$_ =~ s/
//g; # hard returns
$_ =~ s/\s*<\/*(a|b|font|i|span|div|img)[^\>]*>\s*//g; # Tags
$_ =~ s/:\)//g; # emoticons
$_ =~ s/\[.*\]//g; # emoticons
$_ =~ s/[\.!?,]+//g; # errant punctuation
$_ =~ s/\s+$//g; # trailing whitespace
$_ =~ s/^\s+//g; # leading whitespace
$_ =~ s/one/1/g;
return grep(/[^[:digit:]]+/, $_) ? 1 : 0;
}
while (<>) {
chop;
($page, $post, $pdate, $edate, $poster, $plevel, $message) = split(/\t/);
next if ($post == 0); # Skip the OP.
$number = extract_number($message);
$comments = detect_comments($message);
$edited = $edate != 0 ? 1 : 0;
$hash{$poster}++;
push @posts,
{
poster => $poster,
page => int($page),
post => int($post),
level => $plevel,
post_date => int($pdate),
edit_date => int($edate),
edited => $edited,
comments => $comments,
message => $message,
number => $number
};
}
foreach $post ( @posts ) {
$post->{multiple} = ($hash{$post->{poster}} > 1) || 0;
}
print <<__EOF__;
${title} - Raw Data
Data for last whole page only.
Update checks every 10 minutes.
Last update: \@DATE\@
Last page: \@PAGE\@
See the report here.
__EOF__
$i = 1;
foreach $post ( @posts ) {
$c = ($i++ % 2) ? "l" : "d";
$n = $post->{number} || " ";
$rm = encode_entities($post->{message}) || " ";
$m = $post->{message} || " ";
($pd = time2str("%c", $post->{post_date})) =~ s/\s/ /g;
($ed = $post->{edited} ? time2str("%c", $post->{edit_date}) : " ") =~ s/\s/ /g;
($l = $post->{level}) =~ s/\s/ /g;
print <<__EOF__;
| $post->{page} |
$post->{post} |
$pd |
$ed |
$post->{edited} |
$post->{comments} |
$post->{multiple} |
$post->{poster} |
$l |
$n |
$m |
$rm |
__EOF__
}
print <<__EOF__;
__EOF__