#!/usr/bin/perl

# Copyright 2007 Gérald Sédrati-Dinet
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

use strict;
use warnings;

use LWP::UserAgent;
use XML::Twig;
use Unicode::String qw(utf8 latin1);
use File::Basename;
use Getopt::Long;

# Some files will be save under tree hierarchy based on executable location
use FindBin qw($Bin);
use lib "$Bin";

# Default values
my $output_dir = "$Bin/xml/mps";
my $debug = 0;
my $this_mp;

# Parse command line options
Getopt::Long::Configure("bundling");
GetOptions(
  'mp|m=s'         => \$this_mp,
  'output|o=s'     => \$output_dir,
  'debug|d'        => \$debug,
  'help|h'         =>
    sub {
      print STDERR <<USAGE;
Usage : $0 [-hd] [-m <MemberOfParliamentWikiName>] [-o <output_dir]
Options can be:
  --mep, -m <MpWikiName>:            wiki name of the MP to processed
                                     default: unset => all MPs are processed
  --output, -o <output_dir>:         output directory of xml pages
                                     default: $Bin/xml/mps
  --debug, -d                        debug
                                     default: no debug
  --help, -h:                        print this message
USAGE
      exit 0;
    }
);

my %groups = (
  'Union pour un Mouvement Populaire' => 'UMP',
  'Socialiste' => 'PS',
  'Union pour la Démocratie Française' => 'UDF',
  'Député-e-s Communistes et Républicains' => 'PCF',
  'Députés n\'appartenant à aucun groupe' => '?',
);

my %no_groups = (
  'Gérard Charasse' => 'PRG',
  'Émile Zuccarelli' => 'PRG',
  'Noël Mamère' => 'Verts',
  'Martine Billard' => 'Verts',
  'Yves Cochet' => 'Verts',
  'Édouard Leveau' => 'CNIP (divers-droite)',
  'Véronique Besse' => 'MPF',
  'Joël Sarlot' => 'MPF',
  'Alfred Marie-Jeanne' => 'MIM (divers-gauche)',
  'Huguette Bello' => 'PCR',
  'Pierre-Christophe Baguet' => 'ex UDF',
  'Philippe Edmond-Mariette' => 'BPM (divers-gauche)',
);

# Create a user agent object
my $ua = LWP::UserAgent->new;
$ua->agent("$0 (gibus perl script to fetch information for wiki.ffii.fr)");
my $headers = HTTP::Headers->new('Accept-Language' => 'fr');

# Fetch list of departments
my $departments_url = 'http://www.assemblee-nationale.fr/12/qui/circonscriptions/';

# Create a request
my $departments_req = HTTP::Request->new(GET => $departments_url, $headers);

# Pass request to the user agent and get a response back
my $departments_res = $ua->request($departments_req);

# Check the outcome of the response
unless ($departments_res->is_success) {
  die "Error fetching list of departments: $departments_url: ", $departments_res->status_line, "\n";
}

# Parse each department
foreach my $department_line (split /\n/, $departments_res->content) {
  $department_line = latin1($department_line)->utf8;
  # Dirty hack to fix formating for Guadeloupe
  $department_line = '                      <b><font color="#860019">GUADELOUPE</font></b>&nbsp;<font size="-1" color="#333333"><a href="971.asp">971</a></font><br>
'
    if $department_line eq '                      GUADELOUPE</b></font>&nbsp;<font size="-1" color="#333333"><a href="971.asp">971</a></font><br>
';
  next unless $department_line =~ /<font color="#860019">([^<]+)<\/font><\/b>&nbsp;<font size="-1" color="#333333"><a href="([^"]+)">([^<]+)<\/a>/o;
  my ($department_name, $department_url, $department_number) = ($1, $2, $3);

  my $constituencies_html;
  my @constituencies;
  if ($department_url =~ /^\d\d[ab\d]\.asp$/) {
    # Fetch list of constituencies
    $department_url = "http://www.assemblee-nationale.fr/12/qui/circonscriptions/$department_url";
    my $department_req = HTTP::Request->new(GET => $department_url, $headers);
    my $department_res = $ua->request($department_req);
    unless ($department_res->is_success) {
      die "Error fetching list of constituencies: $department_url: ", $department_res->status_line, "\n";
    }

    $constituencies_html = latin1($department_res->content)->utf8;
    @constituencies = ($constituencies_html =~ /^c\d+ = "([^"]+)"/gom);
  }
  # Hack for DOM/TOM with one contituency by department
  else {
    $constituencies_html = "          <AREA SHAPE=\"CIRCLE\" COORDS=\"21,6,71\" HREF=\"$department_url\"  onMouseOver=\"affiche\(c1\)\">";
    push @constituencies, $department_name;
  }

  $constituencies_html =~ s/\015\012?/\n/go;
  # Parse each constituency
  foreach my $constituency_line (split /\n/, $constituencies_html) {
    next unless $constituency_line =~ /^\s+<AREA SHAPE="CIRCLE" COORDS="[^"]+" HREF="([^"]+)"\s+onMouseOver="affiche\(c(\d+)\)"/moi;
    my ($constituency_url, $constituency_index) = ($1, $2);
    my $constituency_name = $constituencies[$constituency_index-1];
    $constituency_name =~ s/^[^:]+:\s//o;
    $constituency_name =~ s/\.$//o;
    my @constituency_names = split /, /, $constituency_name;

    # Fetch deputy
    my $mp_url = "http://www.assemblee-nationale.fr$constituency_url";

    # Dirty hack to fix bugs on access by constituency not updated to new format
    $mp_url = 'http://www.assemblee-nationale.fr/12/tribun/fiches_id/664.asp'
      if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/97701.asp';
    $mp_url = 'http://www.assemblee-nationale.fr/12/tribun/fiches_id/1793.asp'
      if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/97801.asp';
    $mp_url = 'http://www.assemblee-nationale.fr/12/tribun/fiches_id/1375.asp'
      if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/97802.asp';
    $mp_url = 'http://www.assemblee-nationale.fr/12/tribun/fiches_id/690.asp'
      if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/97901.asp';
    $mp_url = 'http://www.assemblee-nationale.fr/12/tribun/fiches_id/266763.asp'
      if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/97902.asp';

    # Dirty hack to fix Jean de Gaulle's resignation (January 2nd 2007)
    next if $mp_url eq 'http://www.assemblee-nationale.fr/12/tribun/fiches_cir/07508.asp';

    my $mp_req = HTTP::Request->new(GET => $mp_url, $headers);
    my $mp_res = $ua->request($mp_req);
    unless ($mp_res->is_success) {
      die "Error fetching deputy: $mp_url: ", $mp_res->status_line, "\n";
    }
    my $mp_html = latin1($mp_res->content)->utf8;

    # Skip vacant posts
    next if $mp_html =~ /Si(?:&egrave;|è)ge vacant/o;

    # Extract infos
    # Name
    (my $mp_name) = ($mp_html =~ /<div id="corps"><h1 class="titre">([^>]+)<\/h1>/o);
    my $mp_wiki_name = wikify($mp_name);
    next if (defined $this_mp and $mp_wiki_name ne $this_mp);

    (my $mp_first_name = $mp_name) =~ s/ ([-'\wÀÂÄÉÈÊËÍÎÏÓÔÖÙÛÜÇàâäéèêëíîïóôöùûüç]+)$//o;
    my $mp_last_name = $1;
    # Hack for compound name with single quote (eg. Louis Giscard d'Estaing)
    if ($mp_last_name =~ /'/o and $mp_first_name =~ /^([-'\wÀÂÄÉÈÊËÍÎÏÓÔÖÙÛÜÇàâäéèêëíîïóôöùûüç]+) ([-'\wÀÂÄÉÈÊËÍÎÏÓÔÖÙÛÜÇàâäéèêëíîïóôöùûüç]+)$/o) {
      $mp_first_name = $1;
      $mp_last_name = "$2 $mp_last_name";
    }
    # Hack for compound name with particule (eg. Jean de Gaulle)
    if ($mp_first_name =~ /^([-'\wÀÂÄÉÈÊËÍÎÏÓÔÖÙÛÜÇàâäéèêëíîïóôöùûüç]+) (de)$/o) {
      $mp_first_name = $1;
      $mp_last_name = "$2 $mp_last_name";
    }

    # Gender
    (my $mp_gender) = ($mp_html =~ /<h1>Informations générales<\/h1><ul><li>(\S+) $mp_name<\/li>/);

    # Picture
    (my $mp_id) = ($mp_html =~ /<img src="\/12\/tribun\/photos\/(\d+)\.jpg" \/>/o);
    my $mp_pic_url = "http://www.assemblee-nationale.fr/12/tribun/photos/$mp_id.jpg";
    my $mp_pic_location = "$Bin/IMG/deputes/$mp_wiki_name.jpg";
    $ua->mirror($mp_pic_url, $mp_pic_location) unless $debug;

    $mp_url = "http://www.assemblee-nationale.fr/12/tribun/fiches_id/$mp_id.asp";

    # Birth
    my ($mp_birth_gender, $mp_birth_date, $mp_birth_place) = ($mp_html =~ /<li>(Né(?:e?)) le (.+?) à ([^>]+)<\/li>/o);
    my ($mp_birth_day, $mp_birth_month, $mp_birth_year) = ($mp_birth_date =~ /^(\d+) (\S+) (\d+)$/o);
    my ($mp_birth_city, $mp_birth_department) = ($mp_birth_place =~ /^(.+) \(([^)]+)\)$/o);

    # Contituency (get department name here to avoir upper-case only)
    ($department_name, my $constituency_nb) = ($mp_html =~ /<u>Circonscription d'élection<\/u> : <\/td><td>(.+?) \(([^)]+)\)<\/td>/o);

    # Political group
    my ($mp_group_prez, $mp_grp) = ($mp_html =~ /<tr><td class="objet"><u>(Président du g|G)roupe politique<\/u> : <\/td><td>([^<]+)<\/td><\/tr>/o);
    $mp_group_prez =~ s/(?: du g|G)$//o;
    my $mp_grp_abbrev = $mp_grp;
    my $ap = '';
    if ($mp_grp_abbrev =~ s/^Apparenté //o) {
      $ap = 'Ap. ';
    }
    $mp_grp =~ s/^Apparenté //o;
    die "Unknown group for <$department_url> <$constituency_url> <$mp_url>\n" unless exists $groups{$mp_grp_abbrev};
    $mp_grp_abbrev = $groups{$mp_grp_abbrev};
    $mp_grp_abbrev = $no_groups{$mp_name} if $mp_grp_abbrev eq '?' and exists $no_groups{$mp_name};
    $mp_grp_abbrev = $ap.$mp_grp_abbrev;
    
    # Profession
    (my $mp_prof) = ($mp_html =~ /<tr><td class="objet"><u>Profession<\/u> : <\/td><td>([^<]+)<\/td><\/tr>/o);

    # Web
    (my $mp_web) = ($mp_html =~ /<li><u>Site internet<\/u>[^<]+<a href="([^"]+)"/o);

    # Adresses
    (my $mp_addresses) = ($mp_html =~ /<li><u>Adresses<\/u> :\s+<ul>(.+?)<\/ul>/o);
    # Extract email if included in addresses
    my $mp_email;
    if ($mp_addresses =~ s/<br \/><A HREF="[^"]+">([^<]+)<\/A><\/li>//o) {
      $mp_email = $1;
    }
    # Mail can alternatively be enscripted before addresses
    elsif ($mp_html =~ /<u>Mél<\/u> : <script>document.write\('<A HREF="MAILTO:([^\\]+)\\x40'\.replace\(\/_\/g,''\) \+ 'asse_mblee-nat_ionale'\.replace\(\/_\/g,''\) \+ '\.fr">Assemblée nationale<\/A>'\)<\/script>/o) {
      ($mp_email = $1.'@assemblee-nationale.fr') =~ s/_//go;
    }
    else {
      warn "No email for $mp_name ($mp_url)\n" if $debug;
    }
    # Parse each address
    my @mp_address = parse_addresses($mp_addresses, $mp_url);

    # Mandates
    (my $mp_mandates) = ($mp_html =~ /<div class="mandats">(.+)<h1>Place dans l'hémicycle<\/h1>/os);
    # Bad hack to fix Jean De Gaulle (http://www.assemblee-nationale.fr/12/tribun/fiches_id/1437.asp)
    # who has no section "Place dans l'hémicycle"
    unless ($mp_mandates) {
      ($mp_mandates) = ($mp_html =~ /<div class="mandats">(.+)<\/div><div id="footer">/os);
    }
    die "No mandate for $mp_name ($mp_url)\n" unless $mp_mandates;
    $mp_mandates =~ s/\s+/ /go;
    # Parse each mandate
    my %mp_function;
    my @mp_mandate = parse_mandates_functions($mp_mandates,\%mp_function);

    # Build XML tree
    my $xml = XML::Twig->new(pretty_print => 'indented');
    $xml->set_encoding('UTF-8');
    my $root = XML::Twig::Elt->new('politician');
    $xml->set_root($root);
    $root->insert_new_elt('first_child', 'mps.dbxml');
    my $info = $root->insert_new_elt('last_child', 'infos');
    my $name = $info->insert_new_elt('last_child', 'name');
    $name->insert_new_elt('last_child', 'gender', $mp_gender);
    $name->insert_new_elt('last_child', 'first', $mp_first_name);
    $name->insert_new_elt('last_child', 'last', $mp_last_name);
    $name->insert_new_elt('last_child', 'wiki', $mp_wiki_name);
    $info->insert_new_elt('last_child', 'picture', basename($mp_pic_location));
    my $birth = $info->insert_new_elt('last_child', 'birth');
    $birth->insert_new_elt('last_child', 'gender', $mp_birth_gender);
    my $birth_date = $birth->insert_new_elt('last_child', 'date');
    $birth_date->insert_new_elt('last_child', 'day', $mp_birth_day);
    $birth_date->insert_new_elt('last_child', 'month', $mp_birth_month);
    $birth_date->insert_new_elt('last_child', 'year', $mp_birth_year);
    my $birth_place = $birth->insert_new_elt('last_child', 'place');
    $birth_place->insert_new_elt('last_child', 'city', $mp_birth_city);
    $birth_place->insert_new_elt('last_child', 'department', $mp_birth_department);
    my $constituency = $info->insert_new_elt('last_child', 'constituency');
    my $department = $constituency->insert_new_elt('last_child', 'department');
    $department->insert_new_elt('last_child', 'name', $department_name);
    $department->insert_new_elt('last_child', 'number', $department_number);
    $constituency->insert_new_elt('last_child', 'number', $constituency_nb);
    my $cantons = $constituency->insert_new_elt('last_child', 'cantons');
    map {$cantons->insert_new_elt('last_child', 'name', $_)} @constituency_names;
    my $group = $info->insert_new_elt('last_child', 'group');
    $group->set_att('role', $mp_group_prez) if $mp_group_prez;
    $group->set_att('status', 'Apparenté') if $ap;
    $group->insert_new_elt('last_child', 'name', $mp_grp);
    $group->insert_new_elt('last_child', 'abbreviation', $mp_grp_abbrev);
    $info->insert_new_elt('last_child', 'profession', $mp_prof) if $mp_prof;
    my $contact = $root->insert_new_elt('last_child', 'contact');
    $contact->insert_new_elt('last_child', web => { type => "Page sur le site de l'Assemblée nationale" }, $mp_url);
    $contact->insert_new_elt('last_child', 'web', $mp_web) if $mp_web;
    $contact->insert_new_elt('last_child', 'email', $mp_email) if $mp_email;
    foreach my $addr (@mp_address) {
      my $address = $contact->insert_new_elt('last_child', 'address');
      $address->insert_new_elt('last_child', 'street', $addr->{STREET}) if $addr->{STREET};
      $address->insert_new_elt('last_child', 'postcode', $addr->{POSTCODE}) if $addr->{POSTCODE};
      $address->insert_new_elt('last_child', 'city', $addr->{CITY}) if $addr->{CITY};
      if ($addr->{TEL}) {
        map {$address->insert_new_elt('last_child', 'phone', $_)} @{$addr->{TEL}};
      }
      if ($addr->{FAX}) {
        map {$address->insert_new_elt('last_child', 'fax', $_)} @{$addr->{FAX}};
      }
    }
    my $functions = $root->insert_new_elt('last_child', 'functions');
    foreach my $func (keys %mp_function) {
      foreach my $val (@{$mp_function{$func}}) {
        my $function = $functions->insert_new_elt('last_child', 'function');
        $function->insert_new_elt('last_child', 'type', $func);
        $function->insert_new_elt('last_child', 'label', $val->{VALUE});
        $function->insert_new_elt('last_child', 'role', $val->{ROLE});
        $function->insert_new_elt('last_child', 'mission', $val->{MISSION}) if $val->{MISSION};
      }
    }
    my $mandates = $root->insert_new_elt('last_child', 'mandates');
    foreach my $mand (@mp_mandate) {
      my $mandate = $mandates->insert_new_elt('last_child', 'mandate');
      $mandate->set_att('current', $mand->{CURRENT});
      $mandate->insert_new_elt('last_child', 'type', $mand->{TYPE});
      $mandate->insert_new_elt('last_child', 'institution', $mand->{INSTITUTION}) if $mand->{INSTITUTION};
      $mandate->insert_new_elt('last_child', 'role', $mand->{ROLE}) if $mand->{ROLE};
      my $election_date=$mandate->insert_new_elt('last_child', 'election_date') if $mand->{ELECTION_DAY};
      $election_date->insert_new_elt('last_child', 'day', $mand->{ELECTION_DAY}) if $mand->{ELECTION_DAY};
      $election_date->insert_new_elt('last_child', 'month', $mand->{ELECTION_MONTH}) if $mand->{ELECTION_MONTH};
      $election_date->insert_new_elt('last_child', 'year', $mand->{ELECTION_YEAR}) if $mand->{ELECTION_YEAR};
      my $begin_term=$mandate->insert_new_elt('last_child', 'begin_term') if $mand->{BEGIN_DAY};
      $begin_term->insert_new_elt('last_child', 'day', $mand->{BEGIN_DAY}) if $mand->{BEGIN_DAY};
      $begin_term->insert_new_elt('last_child', 'month', $mand->{BEGIN_MONTH}) if $mand->{BEGIN_MONTH};
      $begin_term->insert_new_elt('last_child', 'year', $mand->{BEGIN_YEAR}) if $mand->{BEGIN_YEAR};
      $begin_term->insert_new_elt('last_child', 'reason', $mand->{BEGIN_REASON}) if $mand->{BEGIN_REASON};
      my $end_term=$mandate->insert_new_elt('last_child', 'end_term') if $mand->{END_DAY};
      $end_term->insert_new_elt('last_child', 'day', $mand->{END_DAY}) if $mand->{END_DAY};
      $end_term->insert_new_elt('last_child', 'month', $mand->{END_MONTH}) if $mand->{END_MONTH};
      $end_term->insert_new_elt('last_child', 'year', $mand->{END_YEAR}) if $mand->{END_YEAR};
      $end_term->insert_new_elt('last_child', 'reason', $mand->{END_REASON}) if $mand->{END_REASON};
    }
    my $activities = $root->insert_new_elt('last_child', 'activities');
    $activities->insert_new_elt('last_child', 'questions', "http://questions.assemblee-nationale.fr/resultats-questions.asp?ResultMaxDocs=0&C1=QE&C2=QG&C3=QOSD&B1=Envoyer&T2=brevet*&AUT=$mp_last_name+$mp_first_name&SortField=NUM&SortOrder=DESC&NumLegislature=12&ResultCount=100");
    $activities->insert_new_elt('last_child', 'propositions', "http://recherche.assemblee-nationale.fr/resultats.asp?QueryText=brevet&Collection=site&theme=Propositions-de-loi-et-de-résolutions_12_topic&allsite=non&SortField=title&SortOrder=DESC&AUT=$mp_first_name+$mp_last_name&proposition=oui");
    $activities->insert_new_elt('last_child', 'reports', "http://recherche.assemblee-nationale.fr/resultats.asp?QueryText=brevet+et+$mp_first_name+$mp_last_name%3Cin%3E%+title&Collection=site&theme=Rapports_12_topic&allsite=non&SortField=title&SortOrder=DESC&AUT=$mp_first_name+$mp_last_name&rapport=oui");
    $activities->insert_new_elt('last_child', 'speeches', "http://recherche.assemblee-nationale.fr/resultats.asp?QueryText=brevet*+et+%28$mp_first_name+$mp_last_name+%3Cin%3E+a%29&Collection=site&allsite=non&theme=Int\%E9graux-des-d\%E9bats-en-s\%E9ance_12_topic&AUT=$mp_first_name+$mp_last_name&intervention=oui");

    my $xml_location = "$output_dir/$mp_wiki_name.xml";
    open XML, ">$xml_location" or die "Cannot write in file $xml_location: $!\n";
    $xml->flush(\*XML);
	  close XML or die "Error when closing $xml_location: $!\n";

    warn "name=<$mp_name> first=<$mp_first_name> last=<$mp_last_name> wiki=<$mp_wiki_name> department=<$department_name>\n\nmandate=[ ".join("]\nmandate=[ ", map({"type=!$_->{TYPE}! current=!$_->{CURRENT}! ".($_->{INSTITUTION}?"institution=!$_->{INSTITUTION}! ":'').($_->{ELECTION_DAY}?"election_day=!$_->{ELECTION_DAY}! ":'').($_->{ELECTION_MONTH}?"election_month=!$_->{ELECTION_MONTH}! ":'').($_->{ELECTION_YEAR}?"election_year=!$_->{ELECTION_YEAR}! ":'').($_->{BEGIN_DAY}?"begin_day=!$_->{BEGIN_DAY}! ":'').($_->{BEGIN_MONTH}?"begin_month=!$_->{BEGIN_MONTH}! ":'').($_->{BEGIN_YEAR}?"begin_year=!$_->{BEGIN_YEAR}! ":'').($_->{BEGIN_REASON}?"begin_reason=!$_->{BEGIN_REASON}! ":'').($_->{END_DAY}?"end_day=!$_->{END_DAY}! ":'').($_->{END_MONTH}?"end_month=!$_->{END_MONTH}! ":'').($_->{END_YEAR}?"end_year=!$_->{END_YEAR}! ":'').($_->{END_REASON}?"end_reason=!$_->{END_REASON}! ":'').($_->{ROLE}?"role=!$_->{ROLE}! ":'')} @mp_mandate))."]\n\nfunction=[".join("]\nfunction=[", map({"$_=> <".join("> <", map({"$_->{VALUE}".($_->{ROLE}?" ($_->{ROLE})":'').($_->{MISSION}?" ($_->{MISSION})":'')} @{$mp_function{$_}}))} keys %mp_function))."]\n" if $debug;
  }
}

sub parse_addresses {
  my $addresses = shift;
  my $url = shift;
  my @address;

  # Parse each address
  while ($addresses =~ /<li>(.+?)<\/li>/go) {
    my $addr_str = $1;
    my %addr;
    if ($addr_str =~ s/^?([^<]+)<br \/>(\d+) ([^<]+)<br \/>//o) {
      ($addr{STREET}, $addr{POSTCODE}, $addr{CITY}) = ($1, $2, $3);
    }

    # Clean trimming characters
    $addr{STREET} =~ s/[, ]+$//o if $addr{STREET};
    $addr{STREET} =~ s/^ +//o if $addr{STREET};

    # Parse reminder (telephone, fax)
    $addr_str =~ s/Autre télé(phone|copie)/Télé$1/go if $addr_str;
    while ($addr_str =~ /Téléphone : ([^<]+)<br \/>/go) {
      my $tel = $1;
      $tel =~ s/^\s+//go;
      $tel =~ s/\s+$//go;
      push @{$addr{TEL}}, $tel;
    }
    while ($addr_str =~ /Télécopie : ([^<]+)<br \/>/go) {
      my $fax = $1;
      $fax =~ s/^\s+//go;
      $fax =~ s/\s+$//go;
      push @{$addr{FAX}}, $fax;
    }

    push @address, \%addr;
  }
  return @address;
}

sub parse_mandates_functions {
    my $mandates = shift;
    my $function_hashref = shift;
    my @mandates_str = split /<a name="P\d+" id="P\d+"><\/a>/o, $mandates;
    shift @mandates_str;
    my @mandate;
    foreach my $mandates_str (@mandates_str) {
      $mandates_str =~ s/<h1>([^<]+)<\/h1>//o;
      my $type = $1;
      my $current = ($type =~ s/Ancien(?:ne)?s (m|f)(andats|onctions)/\u$1$2/o) ? 'false' : 'true';
      $type =~ s/ en cours$//o;
      my $deputy = ($mandates_str =~ s/<\/div>$//o) ? 'true' : 'false';
      $mandates_str =~ s/^<ul>(.+)<\/ul>$/<mandates>$1<\/mandates>/o;
      $mandates_str =~ s/<ul><\/ul>//go;
      my $mandates_xml = XML::Twig->new(pretty_print => 'record',output_encoding => 'UTF-8');
      $mandates_xml->parse("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n".$mandates_str);

      if ($deputy eq 'true') {
        foreach my $chunk ($mandates_xml->root->children('li')) {
          if ($chunk->first_child('ul')) {
            my $field = $chunk->first_child('u')->sprint(1);
            if ($field eq 'Commission') {
              foreach my $by_role ($chunk->first_child('ul')->children('li')) {
                my $value = $by_role->sprint(1);
                my ($role, $mission);
                if ($value =~ s/^(.+) au nom de la (commission[^:]+): //o) {
                  $role = $1;
                  $mission = $value;
                  ($value = ucfirst($2)) =~ s/ $//o;
                }
                elsif ($value =~ s/^(.+) de la (commission)/\u$2/o) {
                  $role = $1;
                }
                my %val_by_role = ('ROLE'=>$role,'VALUE'=>$value);
                $val_by_role{'MISSION'} = $mission  if $mission;
                push @{$function_hashref->{$field}}, \%val_by_role;
              }
            }
            else {
              foreach my $by_role ($chunk->children('ul')) {
                my $values = $by_role->first_child('li')->sprint(1);
                my $role;
                if ($values =~ s/(\S+) : - //o) {
                  $role = $1;
                }
                foreach my $value (split / - /, $values) {
                  my %val_by_role = ('ROLE'=>$role,'VALUE'=>$value);
                  push @{$function_hashref->{$field}}, \%val_by_role;
                }
              }
            }
          }
          else {
            create_mandate(\@mandate, $type, $current, undef, $chunk->sprint(1));
            delete $mandate[$#mandate]->{ROLE};
          }
        }
      }
      else {
        foreach my $mandate_xml ($mandates_xml->root->children('li')) {
          my $reelection = $mandate_xml->first_child('ul');
          if ($reelection) {
            my $institution = $mandate_xml->first_child('#TEXT');
            foreach my $this_mandate ($reelection->children('li')) {
              create_mandate(\@mandate, $type, $current, $institution->sprint(1), $this_mandate->sprint(1));
            }
          }
          else {
            create_mandate(\@mandate, $type, $current, undef, $mandate_xml->sprint(1));
          }
        }
      }
    }

    return @mandate;
}

sub parse_mandate_dates {
  my $ref_mandate = shift;
  my $str = shift;
  if ($str =~ s/(?:É|Réé)lue? le (\d\d)\/(\d\d)\/(\d\d\d\d)//o) {
    ($ref_mandate->{ELECTION_DAY}, $ref_mandate->{ELECTION_MONTH}, $ref_mandate->{ELECTION_YEAR}) =($1, $2, $3);
  }
  if ($str =~ s/(?:(?: - Mandat )?du|Date de début de mandat :) (\d\d)\/(\d\d)\/(\d\d\d\d)(?: \(([^)]+)\))?//o) {
    ($ref_mandate->{BEGIN_DAY}, $ref_mandate->{BEGIN_MONTH}, $ref_mandate->{BEGIN_YEAR}) =($1, $2, $3);
    $ref_mandate->{BEGIN_REASON} = $4 if $4;
  }
  if ($str =~ s/ au (\d\d)\/(\d\d)\/(\d\d\d\d)(?: \(([^)]+)\))?//o) {
    ($ref_mandate->{END_DAY}, $ref_mandate->{END_MONTH}, $ref_mandate->{END_YEAR}) =($1, $2, $3);
    $ref_mandate->{END_REASON} = $4 if $4;
  }
  $str =~ s/, $//o;
  $str =~ s/ - $//o;
  $ref_mandate->{ROLE} = $str if $str;
  if (exists $ref_mandate->{END_REASON} and not exists $ref_mandate->{BEGIN_REASON}) {
    $ref_mandate->{ROLE} = $ref_mandate->{END_REASON};
    delete $ref_mandate->{END_REASON};
  }
  if (exists $ref_mandate->{INSTITUTION} and not exists $ref_mandate->{ROLE}) {
    $ref_mandate->{ROLE} = $ref_mandate->{INSTITUTION};
    delete $ref_mandate->{INSTITUTION};
  }
}

sub create_mandate {
  my $ref_mandate = shift;
  my %mandate;
  my $str;
  ($mandate{TYPE}, $mandate{CURRENT}, $mandate{INSTITUTION}, $str) = @_;
  parse_mandate_dates(\%mandate, $str);
  push @{$ref_mandate}, \%mandate;
  return @$ref_mandate;
}

sub wikify {
  my $str = shift;
  $str = desaccent($str);
  $str =~ s/-//go;
  return $str;
}

sub desaccent {
  my $str = shift;
  my $mp_name = $str;
  $str =~ s/À/A/go;
  $str =~ s/Â/A/go;
  $str =~ s/Ä/A/go;
  $str =~ s/É/E/go;
  $str =~ s/È/E/go;
  $str =~ s/Ê/E/go;
  $str =~ s/Ë/E/go;
  $str =~ s/Í/I/go;
  $str =~ s/Î/I/go;
  $str =~ s/Ï/I/go;
  $str =~ s/Ó/O/go;
  $str =~ s/Ô/O/go;
  $str =~ s/Ö/O/go;
  $str =~ s/Ù/U/go;
  $str =~ s/Û/U/go;
  $str =~ s/Ü/U/go;
  $str =~ s/Ç/C/go;
  $str =~ s/à/a/go;
  $str =~ s/â/a/go;
  $str =~ s/ä/a/go;
  $str =~ s/é/e/go;
  $str =~ s/è/e/go;
  $str =~ s/ê/e/go;
  $str =~ s/ë/e/go;
  $str =~ s/í/i/go;
  $str =~ s/î/i/go;
  $str =~ s/ï/i/go;
  $str =~ s/ó/o/go;
  $str =~ s/ô/o/go;
  $str =~ s/ö/o/go;
  $str =~ s/ù/u/go;
  $str =~ s/û/u/go;
  $str =~ s/ü/u/go;
  $str =~ s/ç/c/go;
  $str =~ s/[ ']//go;
  return $str;
}