#!/usr/bin/env perl
# find_duplicate_links
# This Perl script scans html files in gnu.org (especially in proprietary/*)
# to detect duplicated items.
# Send any comment, correction, optimization, etc. to <felicien@gnu.org>
#
# Copyright 2017 (C) Felicien PILLOT <felicien@gnu.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#### Modules
use Cwd;               # get current directory
use Getopt::Long;      # parse command line options
use HTML::TreeBuilder; # parse HTML file into a tree
use strict;            # be strict
use warnings;          # warn

#### Global variables (as less as possible)
my (%allItems,         # Hash containing every <a> item from html page
    %dupItems,         # Hash containing only duplicated <a> items
    $cron_output,      # Integer specifying the type of output
    $verbose);         # Integer specifying verbosity

#### Subroutines (sorted alphabetically)
sub check_element
{
    # Declare local variables
    my ($a,          # HTML::Element: contains a <a> tag
	$class,      # String: contains the class of the <a>
	$file,       # String: contains the currently processed file
	$text1,      # String: contains the whole text from the parent <p>
	$text2,      # String: idem as text1
	$url);       # String containing the href (url) of the <a>    
    # Set variables related to the currently processed element (<a>)
    $a = $_[0];
    $file = $_[1];
    $url = $a->attr ('href');
    $class = $a->attr ('class');

    # Check if there is a duplicated link
    if (grep (/^$url/, keys %allItems)) {
	if (($class // '') eq "not-a-duplicate" ||
	    $allItems{$url}->attr ('class') eq "not-a-duplicate") {
	    print "$url is $class\n" if ($verbose > 0);
	} else {
	    my $outp = ($class // '');
	    print "$outp\n";
	    print "$url found twice\n" if ($verbose > 0);
	    # Get context (text from <p>)
	    $text1 = $a->parent ()->as_text ();
	    $text2 = $allItems{$url}->parent ()->as_text ();
	    # Remove beginning spaces
	    $text1 =~ s/^\s+//;
	    $text2 =~ s/^\s+//;
	    # Store it in %dupItems
	    $dupItems{$file}->{$url} = [$text1, $text2];
	}
    }
    # Store every external urls in %allItems
    $allItems{$url} = $a if ($url =~ /^http/);
}

# Print top and bottom texts, for mailing the output through the cron job
sub cron_output
{
    print "*** This is an automatic email adressed to webmasters\@gnu.org\n"
	. "*** It will be sent twice a month, according to #1198654 RT ticket\n"
	. "*** about duplicate links found in www.gnu.org/proprietary/*\n"
	. "*** The crontab from which this mail is sent is located at\n"
	. "*** /var/spool/cron/crontabs/felicien\n"
	. "*** For any information, contribution, comment, etc., contact "
	. "<felicien\@gnu.org>\n"
	. "Here is the output of the Perl script 'find_duplicate_links':\n\n";
    if (output ()) {
	print "Please check these occurrences, if they are true duplications,\n"
	    . "remove the redundant one or make a reference to the first item"
	    . " ;\nif they are two different items with the same link, set a"
	    . " class\nattribute containing 'not-a-duplicate' in the <a> tag:\n"
	    . "<a class=\"not-a-duplicate\" href=\"...\">\n";
    }
}

# Print help information
sub help
{
    print "help ()\n" if ($verbose > 1);
    print "Usage: find_duplicate_links.sh [OPTIONS] [PATH]\n"
	. "OPTIONS:\n"
	. "  -c, --cron   \t\t\tFormat the output accordingly to a 'cron sent'"
	. " email\n  -h, --help   \t\t\tDisplay this help message then exit\n"
	. "  -p, --pattern=FILE_PATTERN\tSet FILE_PATTERN as explained below\n"
	. "      --verbose\t\t\tIncrease debug & information display\n"
	. "      --version\t\t\tDisplay version then exit\n"
	. "FILE_PATTERN is a part (without extension) of one or more filenames"
	. " from\nthe current directory. Don't worry, the script won't edit "
	. "translated files.\nFor example: the FILE_PATTERN \"malware\" will "
	. "check malware-adobe.html,\nmalware-amazon.html, malware-apple.html "
	. "but not malware-adobe.fr.html.\n";
    exit;
}

# Generate HTML::Tree from file, then check each <a> element
sub main_sub
{
    print "main_sub ()\n" if ($verbose > 1);
    # Declare local variables
    my ($dir,        # String: contains the current directory
	$pattern,    # String: given by user -- see help() for details
	$tree);      # HTML::Tree: contains all elements from the html page
    # Get variables
    if ($ARGV[0]) {
	if ($ARGV[0] =~ /^\//) {
	    $dir = $ARGV[0];
	} else {
	    $dir = getcwd ()."/".$ARGV[0];
	}
    } else {
	$dir = getcwd ();
    }
    $pattern = $_[1] // "";
    print "Directory: $dir\n" if ($verbose > 0);
    # Parse files
    opendir (DIR, $dir) or die $!;
    while (readdir(DIR)) {
        next unless (/^$pattern[^\.]*\.html/);
	print "Selected file: $_\n" if ($verbose > 0);
	# Reset some variables
	%allItems = ();
	$tree = HTML::TreeBuilder->new_from_file ($dir."/".$_);
	# Walk through each <a> tag
	foreach $a ($tree->find ('a')) {
	    check_element ($a, $_);
	}
    }
    closedir(DIR);
    
    # Display results
    $cron_output ? cron_output () : output ();
    print "The script ended successfully\n" if ($verbose > 0);
    exit 1;
}

# Display output for terminal stdout
sub output {
    # Declare local variables
    my ($file,       # String: filename contained in %dupItems
	$url);       # String: url (href) contained in $dupItems{$file}
    # Walk through %dupItems
    if (%dupItems) {
	foreach $file (keys %dupItems) {
	    foreach $url (keys %{$dupItems{$file}}) {
		# Display results for each duplication
		print "In $file, these items point to the same link:\n"
		    . "$url\n"
		    . "* $dupItems{$file}{$url}[0]\n"
		    . "* $dupItems{$file}{$url}[1]\n\n";
	    }
	}
	return 1;
    } else {
	print "No duplicated link has been found.\n";
	return 0;
    }
}

# Print version information
sub version
{
    print "version ()\n" if ($verbose > 1);
    print "find_duplicate_links 0.7\n"
	. "Copyright (C) 2017 Felicien Pillot <felicien\@gnu.org>\n"
	. "License GPLv3+: GNU GPL version 3 or later "
	. "<http://gnu.org/licenses/gpl.html>\n"
	. "This is free software: you are free to change and"
	. "redistribute it.\n"
	. "There is NO WARRANTY, to the extent permitted by law.\n";
    exit 1;
}

### Main

$verbose = 0;
$cron_output = 0;
# Parse the command line arguments
GetOptions ("cron"        => \$cron_output,
	    "help"        => \&help,
	    "pattern=s"   => \&main_sub,
	    "verbose+"    => \$verbose,
	    "version"     => \&version);
# Even if no --pattern has been given, try the main loop
main_sub ();