April 20, 2005: Cloud Nine

#!/usr/bin/perl -w
use strict; 

# Where I put the Text::Ngrams Module:
use lib qw(/home/alex/perllib); 

# lowest font size, in px
my $base_size = 10; 

# where are our text entries?
my $dir_to_search = '/home/alex/www/alex/ephemeris'; 

# what's the site URL?
my $site_url = 'http://skazat.com'; 

# what's the most words you want back? (before we filter for common words)
my $word_limit = 1250; 


# No more to really change; 
#---------------------------------------------------------------------#

use File::Find; 
use Text::Ngrams; 

my @files; 
my %count = (); 
my %results; 





# A list of words, we won't take into account, since either: 
# (1 They're part of the HTML of the page, 
# (2 They're locally weird
# (3 they're very common. 

my @banned_words = qw(

p strong img href em value param width border height com net org align style
images target blank skazat src solid black NAME PARAM http alt And s px www I
jpg gif table tr td id indexOf FFFFFF index serif row url br san meta size 
header blockquote bgcolor html text li  ol font quot ul nbsp hspace vspace cd
cccccc bold arial mailto span hr valign

currents verdana cgi am gt didn don wasn ssi th nav div

the of to and a in is it you that he was for on are with 
as I his they be at one have this from or had by hot word 
but what some we can out other were all there when up use 
your how said an each she which do their time if will way 
about many then them write would like so these her long make 
thing see him two has look more day could go come did number
 sound no most people my over know water than call first who 
may down side been now find any new work part take get place
 made live where after back little only round man year came 
show every good me give our under open seem together next 
white children begin got walk example ease paper group always 
music those both mark often letter until mile river car feet 
care second book carry took science eat room friend began idea
 fish mountain stop once base hear horse cut sure watch color 
face wood main enough plain girl usual young ready above ever 
red list though feel talk bird soon body dog family direct 
pose leave song measure door product black short numeral class 
wind question happen complete ship area half rock order fire 
south problem piece told knew pass since top whole king space 
heard best hour better true . during hundred five remember step
 early hold west ground interest reach fast verb sing listen 
six table travel less morning gentle woman captain practice 
separate difficult doctor please protect noon whose locate ring 
character insect caught period indicate radio spoke atom human 
history effect electric expect crop modern element hit student 
corner party supply bone rail imagine provide agree thus 
capital won't chair danger fruit rich thick soldier process 
operate guess necessary sharp wing create neighbor wash bat 
rather crowd corn compare poem string bell depend meat rub 
tube famous dollar stream fear sight thin triangle planet hurry
 chief colony clock mine tie enter major fresh search send 
yellow gun allow print dead spot desert suit current lift rose
 continue block chart hat sell success company subtract event 
particular deal swim term opposite wife shoe shoulder spread 
arrange camp invent cotton born determine quart nine name very
through just form sentence great think say help low line differ turn
cause much mean before move right boy old too same tell does set three
want air well also play small end put home read hand port large spell
add even land here must big high such follow act why ask men change went
light kind off need house picture try us again animal point mother world
near build self earth father head stand own page should country found
answer school grow study still learn plant cover food sun four between
state keep eye never last let thought city tree cross farm hard start
might story saw far sea draw left late run don't while press close night
real life few north ten simple several vowel toward war lay against
pattern slow center love person money serve appear road map rain rule
govern pull cold notice voice unit power town fine certain fly fall lead
cry dark machine note wait plan figure star box noun field rest correct
able pound done beauty drive stood contain front teach week final gave
green oh quick develop ocean warm free minute strong special mind behind
clear tail produce fact street inch multiply nothing course stay wheel
full force blue object decide surface deep moon island foot system busy
test record boat common gold possible plane stead dry wonder laugh
thousand ago ran check game shape equate hot miss brought heat snow tire
bring yes distant fill east paint language among truck noise level
chance gather shop stretch throw shine property column molecule select
wrong gray repeat require broad prepare salt nose plural anger claim
continent oxygen sugar death pretty skill women season solution magnet
silver thank branch match suffix especially fig afraid huge sister steel
discuss forward similar guide experience score apple bought led pitch
coat mass card band rope slip win dream evening condition feed tool
]total basic smell valley nor double seat arrive master track parent
shore division sheet substance favor connect post spend chord fat glad
original share station dad bread charge proper bar offer segment slave
duck instant market degree populate chick dear enemy reply drink occur
support speech nature range steam motion path liquid log meant quotient
teeth shell neck
); 

# make a simple lookup table...
my %banned_words = (); 
$banned_words{lc($_)} = 1 foreach @banned_words; 


my $ng = Text::Ngrams->new(type => 'word', windowsize => 1, limit => $word_limit);

# walk the directory structure, make note of the plain text (HTML) files...
find(\&wanted, $dir_to_search);
sub wanted {
	if(-r $File::Find::name && -f _ && -s _){ 
 		push(@files, $File::Find::name); 
	}
}
    
my $count = $ng->process_files(@files);

my @results = split("\n", $ng->to_string()); 

# Take note of what words were found, and how many 

foreach(@results){ 

	next if $_ =~ /\<NUMBER\>/;	    # We don't care about numbers.
	next unless $_ =~ m/\t/; 	    # a tab usually delimits 
								    # the word from the number
								
	my ($word, $count) = split("\t"); 
	
	next if length($word) <= 1; 	# has to be a word of 
									# more than one letter...
	next if its_banned($word); 	    # can't be a banned word...

	# make sure everything is lowercased...
	if(exists($results{lc($word)})){ 
		$results{$word} += $count;  
	}else{ 
		$results{lc($word)} = $count;  
	}
}


# Find the smallest amount, and set as, "0", 
# adjust all other numbers to relate to that
# baseline.

%results = massage_for_size(\%results); 

# Make out our links. 
my $str; 
foreach my $w(sort keys %results){ 
	$str .= wb_link($w, $results{$w});
}

# print it out!
print $str; 




sub massage_for_size { 

	my $results = shift; 
	
	my @numbers = values %$results; 
	
	@numbers = sort { $a <=> $b} @numbers; 
	my $smallest = $numbers[0];	
	foreach(keys %$results){ 
		$results{$_} -= $smallest; 
	}

	return %$results; 
}




sub wb_link { 

	my ($word, $count) = @_; 
	return '<span style="font-size:'. ( (int($count/3))  + $base_size ) .'px"><a href="http://www.google.com/search?q=site:'.$site_url.'%20' . $word . '">'. $word . '</a> </span> ' . "\n"; 

}




sub its_banned { 

	my $word = shift; 
	return 1 if exists $banned_words{lc($word)}; 

}


__END__

=cut

Copyright (c) 2005 alex Simoni 

This program is provided "as is" without expressed or implied warranty. This is free software; you can redistribute it and/or modify it under the same terms as Perl itself.

=cut