#!/usr/bin/env perl
use strict;
use warnings;
use MyLibrary::Process;

if (@ARGV != 2) {
	die "Usage: ./my_tfidf.pl <.sent file> <prefix>\n";
}
open(my $in, "<", "$ARGV[0]") or die "can't open $ARGV[0]!";
open(OUT, ">", "$ARGV[1].tf") or die "can't open $ARGV[1].tf!";

my @lines = <$in>;
my $num=0;
#print "$num\n";

my %idf_hash;
loop:foreach my $sent (@lines){
	#next loop if $sent =~ /^\</;
	#next loop if $sent =~ /http/;
	#next loop if $sent =~ /\*\//;
	#next loop if $sent =~ /\</;
	#next loop if $sent =~ /@/;
	#next loop if $sent =~ /\*/;
	#next loop if $sent =~ /--/; 
	#next loop if $sent =~ /\{/; #next loop if $sent =~ /\}/; #next loop if $sent =~ /^\s*return/;
	next loop if skip($sent);
	chomp($sent);
	$num++;
	
	#$sent =~ s/[0-9]+|\@/ /g;
#	$sent =~ s/=|>|\?|!|:|\"|;|\/|\^|&|\~|\'|\]|\[|\,/ /g;
	$sent = cleanSentence($sent);
	my @tmp = split /\s+/, $sent;
	nextword:foreach my $word (@tmp) {
		$word =~ tr/A-Z/a-z/;
#		$word =~ s/\./ /g;
#		$word =~ s/[0-9]+//g;
#		$word =~ s/#|\(|\)//g;
		#$word =~ s/=|>|\?|!|:|\"|;|\/|\^|&|\~|\'|\]|\[//g;
		next nextword if $word eq '';
		if(exists $idf_hash{$word}) {
			my $count = $idf_hash{$word};
			$count++;
			$idf_hash{$word} = $count;
		} else {
			$idf_hash{$word} = 1;
		}
	}
}
foreach my $key (keys %idf_hash) {
	#my $ratio = $num/$idf_hash{$key};
	#$ratio++; #in case not in the corpus
	#my $idf = log($ratio)/log(10);
	my $idf = 1 - $idf_hash{$key}/$num;
	print OUT "$key: $idf \n";
}
