# FILE: search.pl
# DESCRIPTION: Keyword and New Message searching
#-------------------------------------------------------------------------------
# DISCUS COPYRIGHT NOTICE
#
# Discus is copyright (c) 2002 by DiscusWare, LLC, all rights reserved.
# The use of Discus is governed by the Discus License Agreement which is
# available from the Discus WWW site at:
#    http://www.discusware.com/discus/license
#
# Pursuant to the Discus License Agreement, this copyright notice may not be
# removed or altered in any way.
#-------------------------------------------------------------------------------

use strict;
use vars qw($GLOBAL_OPTIONS $DCONF $PARAMS);

###
### search_control
###
### Controls search engine functions
###

sub search_control {
	my $FORMref = defined $_[0] ? $_[0] : parse_form($ENV{'QUERY_STRING'}, $ENV{'CONTENT_LENGTH'});
	my $cookie_str = $_[1];
	dreq("template", "fcn-logs");
	my $sparam = search_form_prepare($FORMref, $cookie_str);
	my ($s_out, $sparam2, $tree) = search_perform($sparam);
	$sparam = $sparam2 if defined $sparam2;
	search_output($s_out, $sparam, $FORMref, $tree) if defined $s_out;
	search_first($FORMref, $sparam);
}

###
### search_perform
###
### Performs a search on the posting logs and/or keyword searching logs,
### returning the results in appropriate reference
###

sub search_perform {
	my ($sparam) = @_;
	return undef if ! defined $sparam->{thing};
	my $s = undef;
	my $t = undef;
	return ( { cache => $sparam->{thing}->{scache} }) if $sparam->{thing}->{scache};
	return search_keyword_cache($sparam) if $sparam->{thing}->{cache};
	($s, $t) = search_keyword($sparam) if $sparam->{thing}->{search};
	($s, $t) = search_logs($sparam) if $sparam->{thing}->{'log'};
	($s, $t) = search_tree($sparam) if $sparam->{thing}->{tree};
	return ($s, undef, $t);
}

###
### search_output
###
### Outputs the results of a keyword or new message search
###

sub search_output {
	my ($o, $sparam, $FORMref, $tree) = @_;
	search_output_array($o, $sparam, $FORMref, $tree) if ref $o eq "ARRAY";
	search_output_tree($o, $sparam, $FORMref, $tree) if ref $o eq "HASH";
}

###
### search_tree_info
###
### Gets information needed from the tree file for search results
###

sub search_tree_info {
	my ($topic, $tree_in) = @_;
	my $tree = defined $tree_in ? $tree_in : read_tree($topic, { no_lock => 1, no_unlock => 1 });
	my $u = {};
	foreach my $t (@{ $tree }) {
		$u->{$t->{page}} = $t;
	}
	$u->{secure} = -e "$DCONF->{message_dir}/$topic" ? 0 : 1;
	return $u;
}

###
### search_tree_build_parents
###
### Goes backward to build up the parents of a particular page
###

sub search_tree_build_parents {
	my ($page, $tree) = @_;
	my $t = $tree->{$page};
	my @p = ($t);
	until ($t->{parent} == $t->{page} || $t->{parent} == 0) {
		push @p, $tree->{ $t->{parent} };
		$t = $tree->{$t->{parent}};
	}
	@p = reverse @p;
	return \@p;
}

###
### search_output_array
###
### Keyword search output
###

sub search_output_array {
	my ($arr, $sparam, $FORMref, $tree) = @_;
	my $subst = {};
	my $results_per_page = defined $GLOBAL_OPTIONS->{search_results_per_page} ? $GLOBAL_OPTIONS->{search_results_per_page} : 15;
	$results_per_page = $FORMref->{rpp} if ($FORMref->{rpp} > 0 && $GLOBAL_OPTIONS->{search_results_per_page_chooser});
	my $s = $FORMref->{start} == 0 ? 1 : $FORMref->{start};
	my @arr = @{ $arr };
	$subst->{general}->{matches} = scalar(@arr);
	if (scalar(@arr) == 0 && $sparam->{query}->{tree} == 0) {
		my $l = search_stop_word_file();
		my @q = ();
		foreach my $k (keys(%{ $l })) { push @q, quotemeta($k); }
		my $x = join("|", @q);
		my $sstr = $sparam->{query_string};
		$sstr =~ s/(^|\b)($x)(\b|$)//gio;
		$subst->{general}->{too_general} = 1 if $sstr !~ /\S/;
	}
	my $maxscore = $arr[0]->{score} == 0 ? 1 : $arr[0]->{score};
	$subst->{general}->{cache} = $arr[0]->{cache} ? $arr[0]->{cache} : $sparam->{cache};
	$subst->{general}->{query_string} = $sparam->{query_string};
	$subst->{general}->{perpage} = $results_per_page;
	$subst->{general}->{lower} = int($s);
	$subst->{general}->{lower} = 1 if $subst->{general}->{lower} < 1;
	$subst->{general}->{start} = $s;
	$subst->{general}->{prev} = $s-$results_per_page;
	$subst->{general}->{next} = $s+$results_per_page;
	$subst->{general}->{prev} = 1 if $subst->{general}->{prev} <= 1;
	$subst->{general}->{upper} = $subst->{general}->{lower} + $results_per_page - 1;
	$subst->{general}->{upper} = $subst->{general}->{matches} if $subst->{general}->{matches} < $subst->{general}->{upper};
	my @arout = ();
	my $ctr = $subst->{general}->{lower};
	my @u = ();
	if ($GLOBAL_OPTIONS->{search_results_page_limit} ne "0") {
		@u = splice (@arr, ($subst->{general}->{lower} - 1), (1+$subst->{general}->{upper}-$subst->{general}->{lower}));
	} else {
		@u = @arr;
		$subst->{general}->{lower} = 1;
		$subst->{general}->{start} = 1;
		$subst->{general}->{upper} = $subst->{general}->{matches};
		$subst->{general}->{perpage} = $subst->{general}->{matches};
		$subst->{general}->{prev} = 0;
		$subst->{general}->{next} = $subst->{general}->{matches}+1;
	}
	my $tinfo = {};
	foreach my $r (@u) {
		next if $r->{topic} == 0;
		my $i = {};
		$i->{number} = $ctr++;
		$tinfo->{$r->{topic}} = search_tree_info($r->{topic}, $tree->{$r->{topic}}) if ! defined $tinfo->{$r->{topic}};
		$i->{parents} = search_tree_build_parents($r->{page}, $tinfo->{$r->{topic}});
		$i->{topic} = $r->{topic};
		$i->{page} = $r->{page};
		$i->{secure} = $tinfo->{$r->{topic}}->{secure};
		if ($r->{url} ne "" && $r->{param} =~ m|^LINK:(.*?)$|) {
			$i->{url} = $r->{url};
			$i->{target} = $1;
		}
		my @j = ();
		if ($r->{match}) {
			foreach my $q (@{ $r->{match} }) {
				my $i2 = {};
				$i2->{postindex} = $q->{postindex};
				$i2->{'time'} = $q->{'time'};
				$i2->{text} = search_keyword_result_line($q->{text}, $sparam);
				push @j, $i2;
			}
			$i->{recent} = (sort { $b->{'time'} <=> $a->{'time'} } @{$r->{match}})[0]->{'time'};
			$i->{matchnum} = scalar(@{ $r->{match} });
			$i->{last_poster} = $r->{last_poster};
		} elsif ($r->{lastmod}) {
			$i->{lastmod} = $r->{lastmod};
			$i->{post_count} = 0 + scalar(split(/,/, $r->{post_list}));
			$i->{last_poster} = $r->{last_poster};
		} elsif ($r->{'time'}) {
			$i->{recent} = $r->{'time'};
		}
		$i->{matches} = \@j;
		$i->{score} = int(100*$r->{score}/$maxscore);
		push @arout, $i;
	}
	$subst->{array} = \@arout;
	screen_out("srchres1", $subst, $sparam->{stuff}->{cookie_str});
}

###
### search_keyword_result_line
###
### Finds the words from your query that matched and presents an output string
###

sub search_keyword_result_line {
	my ($text, $o) = @_;
	my @r = search_prepare_query($o, 1);
	my @q = @{ $r[0] };
	push @q, @{ $r[2] };
	my $str = join("|", @q);
	$str = join("", "(", $str, ")");
	$str = join($str, '(^|\b)', '(\b|$)') if $o->{search_opt}->{whole};
	my $k = $o->{search_opt}->{sensitive} ? "" : "(?i)";
	my $u = ""; my $c = 0;
	while ($text =~ /$k(^|\b)(.{0,20})($str)(.{0,20})(\b|$)/) {
		$text = $'; $u .= join("", $&, " ... ");
		$c++; last if $c >= 2;
	}
	$u =~ s/$k$str/\%open$&\%close/g;
	$u =~ s/\s+\.\.\.\s+$//;
	return $u;
}

###
### search_log_create_cache
###
### Creates a cache file for a new message search or other search whose
### results are displayed in a message tree
###

sub search_log_create_cache {
	my ($s, $tt, $uid) = @_;
	return undef if ref $s ne "ARRAY";
	return undef if ref $tt ne "ARRAY";
	return undef if scalar(@{ $tt }) <= 0;
	return undef if scalar(@{ $s }) <= 0;
	if (! -d "$DCONF->{admin_dir}/msg_index/searches") {
		unlink "$DCONF->{admin_dir}/msg_index/searches" if -f "$DCONF->{admin_dir}/msg_index/searches";
		mkdir "$DCONF->{admin_dir}/msg_index/searches", oct($DCONF->{perms0777}) || error_message("Directory Error", "Could not create 'searches' subdirectory.  This is probably a permissions problem.  Make sure that the 'msg_index' subdirectory is world writable.");
		chmod(oct($DCONF->{perms0777}), "$DCONF->{admin_dir}/msg_index/searches");
	}
	my $cfile = join("", time, $$, $ENV{REMOTE_ADDR}); $cfile =~ s/\D//g;
	$cfile++ while -r "$DCONF->{admin_dir}/msg_index/searches/$cfile.txt";
	my $file = join("/", $DCONF->{admin_dir}, "msg_index", "searches", "$cfile.txt");
	my @store_keys = ('topic', 'parent', 'page', 'name', 'lastmod', 'param', 'properties', 'level');
	open (FILE, "> $file");
	print FILE "$uid\n";
	foreach my $t (@{ $tt }) {
		my @l = ();
Z1:		foreach my $z (@store_keys) {
			push @l, join("=>", $z, escape($t->{$z}));
		}
		print FILE join("\t", "T", @l), "\n";
	}
	foreach my $t (@{ $s }) {
		my @l = ();
Z:		foreach my $z (keys %{$t}) {
			next Z if $z eq "post_list";
			next Z if $z eq "name" && $t->{datatype} eq "mp";
			next Z if $z eq "linked_text" && $t->{datatype} eq "s";
			next Z if $z eq "unlinked_text" && $t->{datatype} eq "s";
			push @l, join("=>", $z, escape($t->{$z})) if $t->{$z} ne "";
		}
		print FILE join("\t", "S$t->{topic}", @l), "\n";
	}
	close (FILE);
	chmod(oct($DCONF->{perms0666}), "$file");
	$PARAMS->{file_access}->{$file}->{write}++;
	$PARAMS->{files_written}++;
	$PARAMS->{file_access}->{$file}->{read} += 0;
	return $cfile;
}

###
### search_output_tree
###
### Outputs search results in a tree view
###

sub search_output_tree {
	my ($hash_in, $sparam, $FORMref, $trees) = @_;
	my $subst = {};
	if (ref $sparam->{stuff} eq 'HASH' && ref $sparam->{stuff}->{result} eq 'ARRAY') {
		$subst->{general}->{your_last_search_time} = 0 + ($sparam->{stuff}->{result})->[0]->{stime};
		$subst->{general}->{your_last_search_time} = 0 if $subst->{general}->{your_last_search_time} != $sparam->{cutoff};
	}
	$subst->{general}->{result_posts} = 0;
	$subst->{general}->{result_pages} = 0;
	$subst->{general}->{result_topics} = 0;
	$subst->{general}->{new_window} = 1 if $sparam->{preferences} =~ m|g|;
	$subst->{general}->{link_to_page} = 1;
	if ($hash_in->{cache} == 0) {
		my @s = ();
		my @tt = ();
		my $tr = read_tree(undef, { no_lock => 1, no_unlock => 1, zero_ok => 1});
		foreach my $tt (reverse @{ $tr }) {
			my $topic = $tt->{topic};
			if (defined $hash_in->{$topic}) {
				$subst->{general}->{result_topics}++;
				$subst->{secure}->{$topic} = -e "$DCONF->{admin_dir}/secure/$topic";
				unshift @tt, $tt;
				my $st = defined $trees->{$topic} ? $trees->{$topic} : read_tree($topic, { no_lock => 1, no_unlock => 1, zero_ok => 1});
				my $p = {};
				foreach my $z (reverse @{ $st }) {
					if (defined $hash_in->{$topic}->{$z->{page}} || defined $p->{$z->{page}}) {
						my %i = %{ $z };
						$i{datatype} = "s";
						if (! defined $p->{$z->{parent}}) {
							if (!($sparam->{query_string} eq "" && $z->{param} =~ /Archive/ && $GLOBAL_OPTIONS->{skip_archives_nm} == 1)) {
								$i{last_of_parent} = 1;
								$p->{$z->{parent}} = 1;
							}
						}
						$i{expanded} = 1;
						if (ref $hash_in->{$topic}->{$z->{page}} eq "ARRAY") {
							$subst->{general}->{result_pages}++;
							my $c = 0;
							my %k = map { $_, $c++ } split(/,/, $i{post_list});
							my @r = sort { $k{$b->{'postindex'}} <=> $k{$a->{'postindex'}} } @{ $hash_in->{$topic}->{$z->{page}} };
							my $fl = 1;
							$fl = 0 if $p->{$z->{page}};
							foreach my $r (@r) {
								$subst->{general}->{result_posts}++;
								my %l = %{ $z };
								$l{datatype} = "mp";
								$l{is_last_post} = $fl;
								$l{postnum} = $r->{postindex};
								$l{linked_text} = $r->{poststr};
								$l{posttime} = $r->{'time'};
								$l{unlinked_text} = $r->{firstchars};
								$l{last_of_parent} = $i{last_of_parent};
								$fl = 0;
								unshift @s, \%l;
							}
						}
						unshift @s, \%i;
					}
				}
			}
		}
		if ($GLOBAL_OPTIONS->{tree_search_no_cache} == 1 || $subst->{general}->{result_posts} <= $GLOBAL_OPTIONS->{tree_search_expand_all_point} ) {
			foreach my $t (@tt) {
				$t->{expanded} = 1;
			}
			foreach my $ss (@s) {
				next if ($sparam->{query_string} eq "" && $GLOBAL_OPTIONS->{skip_archives_nm} == 1 && $ss->{param} =~ m|Archive|);
				push @{ $subst->{subtopic_tree} }, $ss;
			}
			$subst->{general}->{expand_url} = '#" onClick="return false;" e="';
			$subst->{general}->{expand_url_X} = 1;
		} elsif ($subst->{general}->{result_posts} <= $GLOBAL_OPTIONS->{tree_search_cutoff_point}) {
			if (scalar (@tt)) {
				$tt[0]->{expanded} = 1;
				$subst->{general}->{topic_shown} = $tt[0]->{topic};
			}
			foreach my $ss (@s) {
				next if ($sparam->{query_string} eq "" && $GLOBAL_OPTIONS->{skip_archives_nm} == 1 && $ss->{param} =~ m|Archive|);
				push @{ $subst->{subtopic_tree} }, $ss if $ss->{topic} == $subst->{general}->{topic_shown};
			}
			$subst->{general}->{cache} = search_log_create_cache(\@s, \@tt, $sparam->{stuff}->{uid});
			$subst->{general}->{expand_url} = join("/", $DCONF->{script_url}, "search.$DCONF->{cgi_extension}");
		} else {
			my @p = ();
			$subst->{subtopic_tree} = \@p;
			$subst->{general}->{cache} = search_log_create_cache(\@s, \@tt, $sparam->{stuff}->{uid});
			$subst->{general}->{expand_url} = join("/", $DCONF->{script_url}, "search.$DCONF->{cgi_extension}");
		}
		$subst->{topic_tree} = \@tt;
	} else {
		my $cfile = $hash_in->{cache};
		$cfile =~ s/\D//g;
		my $file = join("/", $DCONF->{admin_dir}, "msg_index", "searches", "$cfile.txt");
		my @s = ();
		my @tt = ();
		my $x = {};
		my $selt = $sparam->{stuff}->{st};
		$selt = 0 if $sparam->{stuff}->{x} eq "c";
		my $tseen = {};
		my $rpage = {};
		if (open (FILE, "< $file")) {
			$PARAMS->{file_access}->{$file}->{write} += 0;
			$PARAMS->{files_read}++;
			$PARAMS->{file_access}->{$file}->{read}++;
			my $uid = <FILE>; chomp $uid;
			if ($DCONF->{pro} && $GLOBAL_OPTIONS->{search_verify_uid} && $sparam->{stuff}->{uid} ne $uid) {
				close(FILE);
				error_message(read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_TITLE}, read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_EXPL}, 0, 1);
			}
WFILE:		while (my $f = <FILE>) {
				chomp $f;
				my ($type, $other) = split(/\t/, $f, 2);
				$type = substr($type, 0, 1);
				my @o = split(/\t/, $other);
				my $i = {};
				foreach my $o (@o) {
					if ($o =~ m|^(\w+)=>(.*)|) {
						$i->{$1} = unescape($2);
					}
				}
				if ($type eq "S") {
					push @s, $i if $i->{topic} == $selt;
					if ($i->{datatype} eq "mp") {
						$subst->{general}->{result_posts}++;
						$x->{$i->{page}} = 1;
					}
				} else {
					$tseen->{$i->{topic}} = 1;
					$i->{expanded} = ($selt == $i->{topic});
					$subst->{secure}->{$i->{topic}} = -e "$DCONF->{admin_dir}/secure/$i->{topic}";
					push @tt, $i;
				}
			}
			close (FILE);
			$subst->{general}->{result_topics} = scalar keys %{$tseen};
		}
		if (scalar(@tt) == 0) {
			error_message(read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_TITLE}, read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_EXPL}, 0, 1);
		}
		$subst->{general}->{topic_shown} = $selt;
		foreach my $ss (@s) {
			next if ($sparam->{query_string} eq "" && $GLOBAL_OPTIONS->{skip_archives_nm} == 1 && $ss->{param} =~ m|Archive|);
			push @{ $subst->{subtopic_tree} }, $ss if $ss->{topic} == $subst->{general}->{topic_shown};
		}
		$subst->{topic_tree} = \@tt;
		$subst->{general}->{cache} = $cfile;
		$subst->{general}->{result_pages} = scalar(keys(%{ $x }));
		$subst->{general}->{expand_url} = join("/", $DCONF->{script_url}, "search.$DCONF->{cgi_extension}");
	}
	$DCONF->{html_url} =~ m%^(https?://.*?)(/|$)%; my $html_server = $1; my $html_after = join("", $2, $');
	$DCONF->{message_url} =~ m%^(https?://.*?)(/|$)%; my $message_server = $1; my $message_after = join("", $2, $');
	$DCONF->{script_url} =~ m%^(https?://.*?)(/|$)%; my $script_server = $1; my $script_after = join("", $2, $');
	if ($script_server eq $html_server) {
		$subst->{general}->{icon_url} = join("/", $html_after, $DCONF->{icon_dir});
		$subst->{general}->{script_url} = $script_after;
	} else {
		$subst->{general}->{icon_url} = join("/", $DCONF->{html_url}, $DCONF->{icon_dir});
		$subst->{general}->{script_url} = $DCONF->{script_url};
	}
	$subst->{general}->{message_url} = $message_server eq $script_server ? $message_after : $DCONF->{message_url};
	$subst->{general}->{screen} = 1;
	screen_out("treeview", $subst, $sparam->{stuff}->{cookie_str});
}

###
### search_keyword_cache
###
### Spits out results from a keyword search cache
###

sub search_keyword_cache {
	my ($sparam, $dbh) = @_;
	dreq("maintain");
	cleaning_remove_old_files("$DCONF->{admin_dir}/msg_index/searches", '^\d+\.txt$', 2, 0);
	my $cachefile = $sparam->{thing}->{cache};
	$cachefile =~ s/\D//g;
	my $u = readfile("$DCONF->{admin_dir}/msg_index/searches/$cachefile.txt", "search_keyword_cache", { no_lock => 1, no_unlock => 1, create => 1, zero_ok => 1 });
	if (! defined $u || ref $u ne "ARRAY" || scalar (@{ $u }) == 0) {
		error_message(read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_TITLE}, read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_EXPL}, 0, 1);
	}
	my $k = shift (@{ $u });
	my ($uid, $qs) = split(/\//, trim(unescape($k)), 2);
	if ($DCONF->{pro} && $GLOBAL_OPTIONS->{search_verify_uid} && $sparam->{stuff}->{uid} ne $uid) {
		error_message(read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_TITLE}, read_language()->{SEARCH_ENGINE_CACHE_EXPIRED_EXPL}, 0, 1);
	}
	my @x = ();
	foreach my $x (@{ $u }) {
		push @x, search_keyword_cache_line_to_hash($x);
	}
	return (\@x, { uid => $uid, query_string => $qs, cache => $cachefile });
}

###
### search_keyword_create_cache
###
### Creates a keyword search cache
###

sub search_keyword_create_cache {
	my ($s, $sparam) = @_;
	return undef if ref $s ne "ARRAY";
	return undef if scalar(@{ $s }) <= 0;
	if (! -d "$DCONF->{admin_dir}/msg_index/searches") {
		unlink "$DCONF->{admin_dir}/msg_index/searches" if -f "$DCONF->{admin_dir}/msg_index/searches";
		mkdir "$DCONF->{admin_dir}/msg_index/searches", oct($DCONF->{perms0777}) || error_message("Directory Error", "Could not create 'searches' subdirectory.  This is probably a permissions problem.  Make sure that the 'msg_index' subdirectory is world writable.");
		chmod(oct($DCONF->{perms0777}), "$DCONF->{admin_dir}/msg_index/searches");
	}
	my @u = ();
	my $u = join("/", $sparam->{stuff}->{uid}, escape($sparam->{query_string}));
	push @u, join("", $u, "\n");
	foreach my $_s (@{ $s }) {
		push @u, search_keyword_cache_hash_to_line($_s);
	}
	my $cfile = join("", time, $$, $ENV{REMOTE_ADDR}); $cfile =~ s/\D//g;
	$cfile++ while -r "$DCONF->{admin_dir}/msg_index/searches/$cfile.txt";
	writefile("$DCONF->{admin_dir}/msg_index/searches/$cfile.txt", \@u, "search_keyword_create_cache", { create => 1, no_lock => 1, no_unlock => 1 });
	chmod(oct($DCONF->{perms0666}), "$DCONF->{admin_dir}/msg_index/searches/$cfile.txt");
	return $cfile;
}

###
### search_keyword_cache_hash_to_line
###
### Converts a keyword cache hash to a line
###

sub search_keyword_cache_hash_to_line {
	my ($hash) = @_;
	my $u = "";
	if ($hash->{match}) {
		my @z = ();
		foreach my $z (@{ $hash->{match} }) {
			push @z, join("=", $z->{postindex}, $z->{'time'}, trim($z->{text}));
		}
		$u = join("\t", "m", @z);
	} elsif ($hash->{lastmod}) {
		$u = join("\t", "t", $hash->{lastmod}, $hash->{post_list});
	}
	my $l = join("\t", $hash->{score}, $hash->{topic}, $hash->{page}, $u);
	$l .= "\n";
	return $l;
}

###
### search_keyword_cache_line_to_hash
###
### Converts a keyword cache line to a hash
###

sub search_keyword_cache_line_to_hash {
	my ($line) = @_;
	chomp $line;
	my $h = {};
	my @u = split(/\t/, $line);
	($h->{score}, $h->{topic}, $h->{page}, $h->{type}) = splice @u, 0, 4;
	if ($h->{type} eq "m") {
		my @m = ();
		while (my $m = shift @u) {
			my $u = {};
			($u->{postindex}, $u->{'time'}, $u->{text}) = split(/=/, $m, 3);
			push @m, $u;
		}
		$h->{match} = \@m;
	} elsif ($h->{type} eq "t") {
		($h->{lastmod}, $h->{post_list}) = splice @u, 0, 2;
	}
	return $h;
}

###
### search_keyword
###
### Searches the keyword search log or database
###

sub search_keyword {
	my ($sparam, $dbh) = @_;
	my ($s, $trees);
	if ($GLOBAL_OPTIONS->{database} && $DCONF->{pro}) {
		dreq("sql-logs-PRO");
		($s, $trees) = search_keyword_database($sparam);
	} else {
		($s, $trees) = search_keyword_files($sparam);
	}
	my $cfile = search_keyword_create_cache($s, $sparam);
	$s->[0]->{cache} = $cfile if $cfile;
	return ($s, $trees);
}

###
### search_logs
###
### Searches the posting log
###

sub search_logs {
	my ($sparam, $dbh) = @_;
	return ({}, undef) if ! defined $sparam->{where};
	my $searches = {};
	$searches->{bottom_cutoff} = $sparam->{cutoff};
	$searches->{top_cutoff} = time;
	$searches->{poster_match} = $sparam->{query_string};
	$searches->{query_info} = $sparam;
	$searches->{id} = $sparam->{id};
	my $result = {};
	my $trees = {};
	my $resultsnum = 0;
	my $resultcache = undef;
	dreq("fcn-logs");
	foreach my $ht ( @{ log_read_file($sparam->{where}, undef, $searches) } ) {
		if ($ht->{topic} != $resultsnum) {
			$resultsnum = $ht->{topic};
			$resultcache = undef;
		}
		my ($page, $resultcache0, $treecache0) = search_post_locater($ht->{postindex}, $resultcache, $trees->{$ht->{topic}}, $ht->{topic});
		$resultcache = $resultcache0 if ! defined $resultcache;
		$trees->{$ht->{topic}} = $treecache0 if defined $treecache0;
		next if $page eq "";
		$ht->{page} = $page; $ht->{where} = join("/", $ht->{topic}, $ht->{page});
		push @{ $result->{$ht->{topic}}->{$ht->{page}} }, $ht;
	}
	return ($result, $trees);
}

###
### search_post_locater
###
### Locates a post within a given tree file
###

sub search_post_locater {
	my ($postnum, $result_cache, $treecache, $topic) = @_;
	if (defined $PARAMS->{postloc_db_sth}) {
		return sql_post_locater($topic, $postnum);
	}
	return $result_cache->{$postnum} if $result_cache->{$postnum};
	return undef if ref $result_cache eq 'HASH' && scalar keys %{$result_cache};
	$treecache = read_tree($topic, { no_lock => 1, no_unlock => 1 }) if ! defined $treecache;
	my $result = {};
	foreach my $line (@{ $treecache }) {
		foreach my $post (split(/,/, $line->{post_list})) {
			$result->{$post} = $line->{page};
		}
	}
	return ($result->{$postnum}, $result, $treecache);
}

###
### search_tree
###
### Searches the tree file for subject lines
###

sub search_tree {
	my ($o, $dbh) = @_;
	my @r = search_prepare_query($o);
	my @req = @{ $r[0] };
	my @neg = @{ $r[1] };
	my @opt = @{ $r[2] };
	if (scalar(@req) == 0 && scalar(@opt) == 0) {
		my @l = ();
		return \@l;
	}
	my @k = ();
	my $tr = {};
	foreach my $topic (keys(%{ $o->{where} })) {
		my $tree = read_tree($topic, { no_lock => 1, no_unlock => 1 });
		next if ref $tree ne "ARRAY";
		$tr->{$topic} = $tree;
		my $cutoff_field = $o->{cutoff} > 0 ? 'lastmod' : undef;
		my $cutoff_time = $o->{cutoff} > 0 ? $o->{cutoff} : undef;
		my @u = search_does_it_match(\@req, \@neg, \@opt, 'name', $o->{search_opt}->{sensitive}, $cutoff_field, $cutoff_time, @{ $tree });
		push @k, @u if (scalar(@u) && ref $u[0] eq "HASH");
	}
	@k = sort { $b->{score} <=> $a->{score} } @k;
	return (\@k, $tr) if scalar(@k) == 0;
	$k[0]->{cache} = search_keyword_create_cache(\@k, $o);
	return (\@k, $tr);
}

###
### search_prepare_query
###
### Creates a search query (required, excluded, optional) from a
### query string
###

sub search_prepare_query {
	my ($o, $flag, $no_stop) = @_;
	my $sstr = $o->{query_string};
	my @req = (); my @opt = (); my @neg = ();
	my $quot = join("|", quotemeta('"'), quotemeta('&#34;'), quotemeta('&quot;'));
	my $plus = join("|", quotemeta('+'), quotemeta('&#43;'));
	my $minus = join("|", quotemeta('-'), quotemeta('&#45;'));
	while ($sstr =~ m%(?:$plus)(?:$quot)(.*?)(?:$quot)%) { push @req, trim($1); $sstr = join("", $`, $'); }
	while ($sstr =~ m%(?:$minus)(?:$quot)(.*?)(?:$quot)%) { push @neg, trim($1); $sstr = join("", $`, $'); }
	while ($sstr =~ m%(?:$quot)(.*?)(?:$quot)%) { push @opt, trim($1); $sstr = join("", $`, $'); }
	while ($sstr =~ m%(?:$plus)(\S+)%) { push @req, trim($1); $sstr = join("", $`, $'); }
	while ($sstr =~ m%(?:$minus)(\S+)%) { push @neg, trim($1); $sstr = join("", $`, $'); }
	while ($sstr =~ m%(\S+)%) { push @opt, trim($1); $sstr = join("", $`, $'); }
	if (! $no_stop) {
		@req = grep (/\S/, map { search_keyword_is_stop_word($_) } @req);
		@opt = grep (/\S/, map { search_keyword_is_stop_word($_) } @opt);
		@neg = grep (/\S/, map { search_keyword_is_stop_word($_) } @neg);
	}
	@req = map { quotemeta($_) } @req;
	@neg = map { quotemeta($_) } @neg;
	@opt = map { quotemeta($_) } @opt;
	return (\@req, \@neg, \@opt) if $flag;
	if ($o->{search_opt}->{whole}) {
		@req = map { join($_, '(^|\b)', '(\b|$)') } @req;
		@neg = map { join($_, '(^|\b)', '(\b|$)') } @neg;
		@opt = map { join($_, '(^|\b)', '(\b|$)') } @opt;
	}
	if ($o->{search_opt}->{and}) {
		push @req, @opt;
		@opt = ();
	}
	return (\@req, \@neg, \@opt);
}

###
### search_keyword_is_stop_word
###
### Sees if a particular word is a 'stop word'
###

sub search_keyword_is_stop_word {
	my ($word) = @_;
	my $swf = search_stop_word_file();
	my $regexp = join("|", map { quotemeta($_) } keys %{$swf});
	my @newword = ();
	foreach my $comp (split(/\s+/, $word)) {
		push @newword, $comp if $comp !~ /^($regexp)$/i;
	}
	return join(" ", @newword);
}

###
### search_keyword_files
###
### Searches the keyword search logs
###

sub search_keyword_files {
	my ($o) = @_;
	my @r = search_prepare_query($o);
	my @req = @{ $r[0] };
	my @neg = @{ $r[1] };
	my @opt = @{ $r[2] };
	if (scalar(@req) == 0 && scalar(@opt) == 0) {
		my @l = ();
		return \@l;
	}
	my $k = {};
	my $results = {};
	my $trees = {};
	my $found_count = 0;
T:	foreach my $topic (keys(%{ $o->{where} })) {
		my $file = join("/", $DCONF->{admin_dir}, "msg_index", "$topic-search.txt");
		$PARAMS->{files_read}++;
		$PARAMS->{file_access}->{$file}->{read} += 1;
		$PARAMS->{file_access}->{$file}->{write} += 0;
		performance_string("< search_keyword_files $file") if $GLOBAL_OPTIONS->{performance_monitoring};
		open (SFILE, "< $file");
F:		while (<SFILE>) {
			my $x = search_line_to_hash($_);
			next F if ($o->{cutoff} && $x->{'time'} < $o->{cutoff});
			my $d = search_does_it_match(\@req, \@neg, \@opt, 'text', $o->{search_opt}->{sensitive}, $x);
			next F if ! $d;
			my ($page, $resultcache, $treecache) = search_post_locater($d->{postindex}, $results->{$topic}, $trees->{$topic}, $topic);
			$trees->{$topic} = $treecache if defined $treecache;
			$results->{$topic} = $resultcache if defined $resultcache;
			next F if $page == 0;
			$d->{page} = $page; $d->{topic} = $topic;
			if ($d) {
				$found_count += 1 if ! defined $k->{$topic}->{$page};
				push @{ $k->{$topic}->{$page} }, $d;
				if ($found_count >= $GLOBAL_OPTIONS->{search_link_limit_value} && $GLOBAL_OPTIONS->{search_link_limit} == 1 && $GLOBAL_OPTIONS->{search_link_limit_value} > 0) {
					last T;
				}
			}
		}
		close (SFILE);
	}
	my @k = ();
	foreach my $topic (keys(%{ $k })) {
		my $l = $k->{$topic};
		foreach my $page (keys(%{ $l })) {
			my $u = {};
			$u->{topic} = $topic;
			$u->{page} = $page;
			foreach my $m (sort { $b->{score} <=> $a->{score} } @{ $l->{$page} }) {
				push @{ $u->{match} }, $m;
				$u->{score} += $m->{score};
			}
			push @k, $u;
		}
	}
	@k = sort { $b->{score} <=> $a->{score} } @k;
	return (\@k, $trees);
}

###
### search_does_it_match
###
### Checks a line, or series of lines, to see if they match
###

sub search_does_it_match {
	my ($req, $neg, $opt, $k, $sens) = splice @_, 0, 5;
	my @pd = @_;
	my $cutoff_field = undef;
	my $cutoff_time = undef;
	if (ref $pd[0] ne "HASH") {
		($cutoff_field, $cutoff_time) = splice @pd, 0, 2;
	}
	my $opt_re = join("|", @{ $opt });
	my $neg_re = join("|", @{ $neg });
	$opt_re = join("", "(", $opt_re, ")") if $opt_re ne "";
	$neg_re = join("", "(", $neg_re, ")") if $neg_re ne "";
	$opt_re = join("", "(?i)", $opt_re) if (! $sens && $opt_re);
	$neg_re = join("", "(?i)", $neg_re) if (! $sens && $neg_re);
	my @o = ();
	my @r = @{ $req };
	my $r = scalar(@r);
	my @p = @{ $opt }; push @p, @r;
	if (! $sens) {
		foreach my $p (@p) {
			$p = join("", "(?i)", $p);
		}
		foreach my $p (@r) {
			$p = join("", "(?i)", $p);
		}
	}
	while (! defined $pd[0] && scalar(@pd) > 0) {
		shift @pd;
	}
	my $l = 0;
O:	while (my $i = shift @pd) {
		next if ref $i ne "HASH";
		$l++;
		my $u = $i->{$k};
		next if ($neg_re ne "" && $u =~ /$neg_re/o);
		if ($cutoff_field ne "" && ! $i->{islink}) {
			next if $i->{$cutoff_field} < $cutoff_time;
		}
		if ($r > 0) {
I:			foreach my $rr (@r) {
				next O if $u !~ /$rr/;

			}
		} else {
			next if $u !~ /$opt_re/o;
		}
		my $score = 0;
		my $matches = 0;
		foreach my $rr (@p) {
			if ($u =~ /$rr/) {
				$score += length($rr);
				while ($u =~ /$rr/g) {
					$matches += (length($rr) / length($u));
				}
			}
		}
		$i->{score} = ($score * (1.5 ** $matches));
		push @o, $i;
	}
	return @o if $l != 1;
	return $o[0];
}

###
### search_form_prepare
###
### Configures the search engine for a search about to be performed based on
### the input from the searching form.  Note that if you have your own form
### that you want to use for searching, you can write your own subroutine to
### turn data from that form into the internal searching language used here
### and then pass it to the searching processor engine.
###

sub search_form_prepare {
	my ($FORMref, $cookie_str) = @_;
	my $o = {};
	my ($cache, $ckstr, $result, $privcache);
	my $silent = undef;
	if ($FORMref->{nmsm} == 2) {
		$silent = $FORMref->{silent} == 1 ? 1 : 0;
	}
	$FORMref->{COOKIE}->{user} = "" if $FORMref->{nmsm} == 2;
	$FORMref->{password} = $FORMref->{passwd} if ! defined $FORMref->{password};
	if ($DCONF->{pro}) {
		dreq("authwrap-PRO");
		($cache, $ckstr, $result, $privcache) = user_utility_login_check($FORMref, undef, $silent);
		if ($FORMref->{nmsm} == 2) {
			if (! defined $result || ref $result ne "ARRAY" || scalar(@{ $result }) == 0) {
				error_message(read_language()->{BPAUTHERROR}, read_language()->{ERROR_AUTHENTICATION_WRONG_USERNAME_AND_PW}, 0, 1);
			}
		}
		$o->{favorites} = $cache->{favorites};
		$o->{preferences} = $cache->{preferences};
		$cookie_str .= $ckstr if $ckstr;
	} elsif ($FORMref->{nmsm} == 2) {
		dreq("authpass");
		my $arg = {};
		$arg->{new_message} = 1 if ($silent == 0 && defined $silent);
		$arg->{silent} = 1 if $silent == 1;
		$result = check_password($FORMref->{username}, $FORMref->{password}, $arg, $FORMref->{COOKIE});
		if (! defined $result || ref $result ne "ARRAY" || scalar(@{ $result }) == 0) {
			error_message(read_language()->{BPAUTHERROR}, read_language()->{ERROR_AUTHENTICATION_WRONG_USERNAME_AND_PW}, 0, 1);
		}
	}
	$o->{stuff}->{cookie_str} = $cookie_str;
	$o->{stuff}->{result} = $result;
	$o->{stuff}->{privcache} = $privcache;
	$o->{stuff}->{uid} = $FORMref->{COOKIE}->{uid};
	$o->{stuff}->{x} = $FORMref->{x};
	$o->{stuff}->{st} = $FORMref->{st};
	return $o if ($FORMref->{stype} eq "" && $FORMref->{method} eq "" && $FORMref->{query} eq "" && $FORMref->{cache} eq "" && $FORMref->{scache} eq "");
	if ($FORMref->{stype} eq "") {
		if ($FORMref->{method} ne "") {
			$o->{thing}->{log} = 1 if $FORMref->{method} eq "check";
			$o->{thing}->{log} = 1 if $FORMref->{method} eq "last";
		}
		if ($FORMref->{query} ne "") {
			$o->{thing}->{tree} = 1 if $FORMref->{lookin} == 1;
			$o->{thing}->{log} = 1 if $FORMref->{lookin} == 2;
			$o->{thing}->{search} = 1 if $FORMref->{lookin} == 3;
		}
	}
	$o->{thing}->{log} if ($FORMref->{nmsm} && $FORMref->{stype} !~ m|s|);
	if ($FORMref->{stype} =~ m|s|) {
		$o->{thing}->{tree} = 1 if $FORMref->{slookin} == 1;
		$o->{thing}->{log} = 1 if $FORMref->{slookin} == 2;
		$o->{thing}->{search} = 1 if $FORMref->{slookin} == 3;
	} elsif ($FORMref->{stype} =~ m|n|) {
		$o->{thing}->{log} = 1;
	}
	if ($FORMref->{query} || $FORMref->{stype} =~ m|s|) {
		$o->{query_string} = defined $FORMref->{squery} ? $FORMref->{squery} : $FORMref->{query};
	}
	if ($FORMref->{method} || $FORMref->{nmsm}) {
		if (ref $result eq "ARRAY" && scalar(@{ $result }) && ($FORMref->{nmsm} == 2 || $FORMref->{method} eq "check")) {
			$o->{cutoff} = 0 + $result->[0]->{stime};
			$o->{cutoff} = time - (7*24*60*60) if $o->{cutoff} == 0;
		} else {
			$o->{cutoff} = time - 60*$FORMref->{units}*($FORMref->{nnumber}+$FORMref->{number});
		}
	}
	if ($FORMref->{sopts}) {
		$o->{search_opt}->{and} = 1 if $FORMref->{sopts} == 1;
	}
	if ($FORMref->{smethod}) {
		$o->{search_opt}->{whole} = 1 if $FORMref->{smethod} == 1;
	}
	if ($FORMref->{scase}) {
		$o->{search_opt}->{sensitive} = 1 if $FORMref->{scase} == 1;
	}
	if ($FORMref->{cache}) {
		$o->{thing}->{cache} = $FORMref->{cache};
	}
	if ($FORMref->{scache}) {
		$o->{thing}->{scache} = $FORMref->{scache};
	}
	$FORMref->{where} = $cache->{favorites}	if ($FORMref->{where} eq "all" && $FORMref->{method} && $cache->{preferences} =~ m|h| && $cache->{favorites});
	my $topics = board_topics();
	my @t = grep($_->{type} == 1, @{$topics});
	my %t = map { $_->{number}, 1 } @t;
	if ($FORMref->{where} =~ m|^[\d\,]+$|) {
		my @u = split(/,/, $FORMref->{where});
		foreach my $u (@u) {
			next if ! $t{$u};
			if ($DCONF->{pro}) {
				$o->{where}->{$u} = 1 if $cache->{topic_auth}->{$u};
			} else {
				$o->{where}->{$u} = 1;
			}
		}
	} elsif ($FORMref->{where} eq "all") {
		foreach my $uu (@t) {
			my $u = $uu->{number};
			if ($DCONF->{pro}) {
				$o->{where}->{$u} = 1 if $cache->{topic_auth}->{$u};
			} else {
				$o->{where}->{$u} = 1;
			}
		}
	}
	$o->{query_string} = char_convert($o->{query_string});
	return $o;
}

###
### search_first
###
### First interface for searching
###

sub search_first {
	my ($FORMref, $sparam) = @_;
	my $subst = {};
	my $cookie_str = "";
	my $tree = readfile("$DCONF->{admin_dir}/tree.txt", "search_first", { zero_ok => 1, no_lock => 1, no_unlock => 1});
	if ($DCONF->{pro}) {
		my $topics = board_topics(undef, undef, $tree, 0, 1);
		($subst->{topics}, $cookie_str) = topic_list_simple_prune($topics, $FORMref, $sparam->{stuff}->{cookie_str}, $sparam->{stuff}->{result}, $sparam->{stuff}->{privcache});
		$subst->{general}->{favorites} = $sparam->{favorites};
		$subst->{general}->{preferences} = $sparam->{preferences};
	} else {
		$subst->{topics} = board_topics(undef, undef, $tree, 0, 1);
	}
	my @t = @{ $subst->{topics} };
	@t = grep ($_->{type} == 1, @t);
	$subst->{topics} = \@t;
	$subst->{fill}->{username} = $FORMref->{COOKIE}->{user};
	$subst->{fill}->{password} = $FORMref->{COOKIE}->{rpwd};
	screen_out("search", $subst, $cookie_str);
}

1;
