#!/usr/bin/perl
#
# DW::Worker::ContentImporter::LiveJournal::Entries
#
# Importer worker for LiveJournal-based sites entries.
#
# Authors:
#      Andrea Nall <anall@andreanall.com>
#      Mark Smith <mark@dreamwidth.org>
#
# Copyright (c) 2009-2011 by Dreamwidth Studios, LLC.
#
# This program is free software; you may redistribute it and/or modify it under
# the same terms as Perl itself.  For a copy of the license, please reference
# 'perldoc perlartistic' or 'perldoc perlgpl'.
#

package DW::Worker::ContentImporter::LiveJournal::Entries;
use strict;
use base 'DW::Worker::ContentImporter::LiveJournal';

use Carp qw/ croak confess /;
use Time::HiRes qw/ tv_interval gettimeofday /;
use DW::Task::SphinxCopier;
use DW::Worker::ContentImporter::Local::Entries;

sub work {

    # VITALLY IMPORTANT THAT THIS IS CLEARED BETWEEN JOBS
    %DW::Worker::ContentImporter::LiveJournal::MAPS = ();
    LJ::start_request();

    my ( $class, $job ) = @_;
    my $opts = $job->arg;
    my $data = $class->import_data( $opts->{userid}, $opts->{import_data_id} );

    return $class->decline($job) unless $class->enabled($data);

    eval { try_work( $class, $job, $opts, $data ); };
    if ( my $msg = $@ ) {
        $msg =~ s/\r?\n/ /gs;
        return $class->temp_fail( $data, 'lj_entries', $job, 'Failure running job: %s', $msg );
    }

    # FIXME: temporary hack to reclaim memory when we have imported entries
    exit 0;
}

sub try_work {
    my ( $class, $job, $opts, $data ) = @_;
    my $begin_time = [ gettimeofday() ];

    # we know that we can potentially take a while, so budget some hours for
    # the import job before someone else comes in to snag it
    $job->grabbed_until( time() + 3600 * 12 );
    $job->save;

    # failure wrappers for convenience
    my $fail      = sub { return $class->fail( $data, 'lj_entries', $job, @_ ); };
    my $ok        = sub { return $class->ok( $data, 'lj_entries', $job ); };
    my $temp_fail = sub { return $class->temp_fail( $data, 'lj_entries', $job, @_ ); };
    my $status    = sub { return $class->status( $data, 'lj_entries', {@_} ); };

    # logging sub
    my ( $logfile, $last_log_time );
    $logfile = $class->start_log(
        "lj_entries",
        userid         => $opts->{userid},
        import_data_id => $opts->{import_data_id}
    ) or return $temp_fail->('Internal server error creating log.');

    my $log = sub {
        $last_log_time ||= [ gettimeofday() ];

        my $fmt = "[%0.4fs %0.1fs] " . shift() . "\n";
        my $msg = sprintf( $fmt, tv_interval($last_log_time), tv_interval($begin_time), @_ );

        print $logfile $msg;
        $job->debug($msg);

        $last_log_time = [ gettimeofday() ];
    };

    # setup
    my $u = LJ::load_userid( $data->{userid} )
        or return $fail->( 'Unable to load target with id %d.', $data->{userid} );
    $log->( 'Import begun for %s(%d).', $u->user, $u->userid );
    $log->( 'memory usage is now %dMB', LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    # title munging
    my $title = sub {
        my $msg = sprintf( shift(), @_ );
        $msg = " $msg" if $msg;

        $0 = sprintf( 'content-importer [entries: %s(%d)%s]', $u->user, $u->id, $msg );
    };
    $title->();

    # load entry map
    my $entry_map = DW::Worker::ContentImporter::Local::Entries->get_entry_map($u) || {};
    $log->( 'Loaded entry map with %d entries.', scalar( keys %$entry_map ) );
    $log->( 'memory usage is now %dMB',          LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    # and xpost map
    my $xpost_map = $class->get_xpost_map( $u, $data ) || {};
    $log->( 'Loaded xpost map with %d entries.', scalar( keys %$xpost_map ) );
    $log->( 'memory usage is now %dMB',          LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    # get the itemid of the most recent entry (just so we know how many entries there have
    # been in the life of this account)
    $log->('Fetching the most recent entry.');
    my $last = $class->call_xmlrpc(
        $data,
        'getevents',
        {
            ver         => 1,
            selecttype  => 'lastn',
            howmany     => 50,
            lineendings => 'unix',
        }
    );

    my $xmlrpc_fail = 'XMLRPC failure: ' . ( $last ? $last->{faultString} : '[unknown]' );
    $xmlrpc_fail .= " (community: $data->{usejournal})" if $data->{usejournal};
    return $temp_fail->($xmlrpc_fail) if !$last || $last->{fault};

    # we weren't able to get any data. Maybe weren't able to connect to remote server?
    # we want to try again
    return $temp_fail->('Failed to fetch the most recent entry.')
        unless ref $last->{events} eq 'ARRAY';

    # look for the latest non-DOOO entry
    # first entry may be a sticky entry (can't tell via props)
    #   so we guess by taking the max non-DOOO entry on this list
    my $maxid;
    foreach my $event ( @{ $last->{events} } ) {
        if ( $event->{props}->{opt_backdated} ) {
            $log->( 'Skipping %d -- backdated', $event->{itemid} );
        }
        else {
            $maxid = $event->{itemid} > $maxid ? $event->{itemid} : $maxid;
        }
    }
    $log->( 'Discovered that the maximum jitemid on the remote is %d.', $maxid ) if $maxid;

    # we got entries but weren't able to find a single *non*-DOOO entry.
    # No idea what the actual latest entry is. Give up now, tell the user to fix it
    unless ($maxid) {
        $log->( 'Got %d entries but all were DOOO.', scalar @{ $last->{events} } );
        return $fail->(
"We weren't able to find your latest entry because you have several entries that are dated out of order (DOOO, or backdated). "
                . "Please edit all the entries on %s that are DOOO and set their date to one that's earlier than your latest non-DOOO entry. "
                . "Then try your import again.",
            $data->{hostname}
        );
    }

    my ( $duplicates_map, %has ) = $class->find_prunable_items(
        $u, $entry_map, $xpost_map,
        hostname       => $data->{hostname},
        username       => $data->{username},
        log_sub        => $log,
        force_reimport => $data->{options}->{lj_entries_remap_icon}
    );
    $title->('post-prune');

    # this is a useful helper sub we use
    my $count         = 0;
    my $process_entry = sub {
        my $evt  = $_[0];
        my $user = $data->{usejournal} // $data->{username};

        # Key for entry_map. We know the username and the site, so we set this to
        # something that is dependable.
        $evt->{key} =
            $data->{hostname} . '/' . $user . '/' . ( $evt->{itemid} * 256 + $evt->{anum} );

        # We also need to calculate the remote URL for ESN purposes.
        my $ext_u = DW::External::User->new(
            user => $user,
            site => $data->{hostname}
        );

        # Although I don't think this is likely to ever croak,
        # I'm being extra-cautious and using an eval here to
        # make sure this can't interrupt the import process.
        $evt->{url} = eval { $ext_u->site->entry_url( $ext_u, %$evt ) };

        $count++;
        $log->(
            '    %d %s %s; mapped = %d (import_source) || %d (xpost).',
            $evt->{itemid}, $evt->{url}, $evt->{logtime},
            $entry_map->{ $evt->{key} },
            $xpost_map->{ $evt->{itemid} }
        );

        # always set the picture_keyword property though, in case they're a paid
        # user come back to fix their keywords.  this forcefully overwrites their
        # local picture keyword
        if ( my $jitemid = $entry_map->{ $evt->{key} } ) {
            my $entry = LJ::Entry->new( $u, jitemid => $jitemid );
            my $kw    = $evt->{props}->{picture_keyword};
            if ( $u->userpic_have_mapid ) {
                $entry->set_prop( picture_mapid => $u->get_mapid_from_keyword( $kw, create => 1 ) );
            }
            else {
                $entry->set_prop( picture_keyword => $kw );
            }
        }

        # now try to skip it if we already have it
        return
               if $entry_map->{ $evt->{key} }
            || $xpost_map->{ $evt->{itemid} }
            || $has{ $evt->{itemid} };

        # try to detect suspiciously similar entries
        # but first clean subject to make sure it matches what we actually inserted into the db
        # separate condition from above so that we can log that we guessed these are similar
        $evt->{subject} = $class->remap_lj_user( $data, $evt->{subject} || "" );
        my $dupecheck_key = $evt->{eventtime} . '-' . $evt->{subject};
        if ( $duplicates_map->{$dupecheck_key} ) {
            $log->(
                "Skipping entry on %s. Remote entry jitemid %d similar to local jitemid %d.",
                $evt->{eventtime}, $evt->{itemid}, $duplicates_map->{$dupecheck_key}
            );
            $has{ $evt->{itemid} } = 1;

            return;
        }

        # clean up event for LJ and remap friend groups
        my @item_errors;
        my $allowmask = $evt->{allowmask};
        my $newmask   = $class->remap_groupmask( $data, $allowmask );

        # if we are unable to determine a good groupmask, then fall back to making
        # the entry private and mark the error.
        if ( $allowmask != 1 && $newmask == 1 ) {
            $newmask = 0;
            push @item_errors, "Could not determine groups to post to.";
        }
        $evt->{allowmask} = $newmask;

        # now try to determine if we need to post this as a user
        my $posteru;
        if ( $data->{usejournal} ) {
            my ( $posterid, $fid ) = $class->get_remapped_userids( $data, $evt->{poster}, $log );

            unless ($posterid) {

                # FIXME: need a better way of totally dying...
                push @item_errors,
                    "Unable to map poster from LJ user '$evt->{poster}' to local user.";
                $status->(
                    remote_url => $evt->{url},
                    errors     => \@item_errors,
                );
                return;
            }

            $posteru = LJ::load_userid($posterid);
        }

        # we just link polls to the original site
        # FIXME: this URL should be from some method and not manually constructed
        my $event = $evt->{event};
        $event =~
            s!<.+?-poll-(\d+?)>![<a href="http://www.$data->{hostname}/poll/?id=$1">Poll #$1</a>]!g;

        if ( $event =~ m/<.+?-embed-.+?>/ ) {
            $event =~ s/<.+?-embed-.+?>//g;

            push @item_errors,
                "Entry contained an embed tag, please manually re-add the embedded content.";
        }

        if ( $event =~ m/<.+?-template-.+?>/ ) {
            $event =~ s/<.+?-template-.+?>//g;

            push @item_errors,
                "Entry contained a template tag, please manually re-add the templated content.";
        }

        $evt->{event} = $class->remap_lj_user( $data, $event );

        # actually post it
        my ( $ok, $res ) =
            DW::Worker::ContentImporter::Local::Entries->post_event( $data, $entry_map, $u,
            $posteru, $evt, \@item_errors );

        # we don't need this text anymore, so nuke it to try to save memory
        delete $evt->{event};
        delete $evt->{subject};

        # now record any errors that happened
        $status->(
            remote_url => $evt->{url},
            post_res   => $res,
            errors     => \@item_errors,
        ) if @item_errors;

    };

    # helper to load some events -- if this function does not return a true value,
    # the caller must exit (errors were encountered).
    my $fetch_events = sub {

        # let them know we're still working
        $job->grabbed_until( time() + 3600 );
        $job->save;

        # clear "request" cache of db handles to force revalidation, whenever fetching a block
        # of entries, to avoid possibly using one that was idle too long without testing it.
        $LJ::DBIRole->clear_req_cache();

        $log->( 'Fetching %d items.', scalar @_ );
        $title->( 'getevents - %d to %d', $_[0], $_[-1] );

        # try to get it from the remote server
        my $hash = $class->call_xmlrpc(
            $data,
            'getevents',
            {
                ver         => 1,
                itemids     => join( ',', @_ ) . ",",
                selecttype  => 'multiple',
                lineendings => 'unix',
            }
        );

        # if we get an error, then we have to abort the import
        my $xmlrpc_fail = 'XMLRPC failure: ' . ( $hash ? $hash->{faultString} : '[unknown]' );
        $xmlrpc_fail .= " (community: $data->{usejournal})" if $data->{usejournal};
        if ( !$hash || $hash->{fault} ) {
            $temp_fail->($xmlrpc_fail);
            return 0;
        }

        # good, import this event
        $process_entry->($_) foreach @{ $hash->{events} || [] };
        return 1;
    };

    # now get the actual events
    my @toload;
    foreach my $jid ( 0 .. $maxid ) {
        push @toload, $jid
            unless exists $has{$jid} && $has{$jid};
        if ( scalar @toload == 100 ) {
            return unless $fetch_events->(@toload);
            @toload = ();
        }
    }
    if ( scalar @toload > 0 ) {
        return unless $fetch_events->(@toload);
    }

    # mark the comments mode as ready to schedule
    my $dbh = LJ::get_db_writer();
    $dbh->do(
        q{UPDATE import_items SET status = 'ready'
          WHERE userid = ? AND item IN ('lj_comments')
          AND import_data_id = ? AND status = 'init'},
        undef, $u->id, $opts->{import_data_id}
    );

    # Kick off a indexing job for this user
    if (@LJ::SPHINX_SEARCHD) {
        DW::TaskQueue->dispatch(
            DW::Task::SphinxCopier->new( { userid => $u->id, source => "importen" } ) );
    }

    return $ok->();
}

sub find_prunable_items {
    my ( $class, $u, $entry_map, $xpost_map, %opts ) = @_;

    my $log            = $opts{log_sub} || sub { warn @_ };
    my $force_reimport = $opts{force_reimport};
    my $hostname       = $opts{hostname};
    my $username       = $opts{username};

    # this is an optimization.  since we never do an edit event (only post!) we will
    # never get changes anyway.  so let's remove from the list of things to sync any
    # post that we already know about.  (not that we really care, but it's much nicer
    # on people we're pulling from.)
    my %has;    # jitemids we have
    foreach my $url ( keys %$entry_map ) {

        # but first, let's skip anything that isn't from the server we are importing
        # from.  this assumes URLs never have other hostnames, so if someone were to
        # register testlivejournal.com and do an import, they will have trouble
        # importing.  if they want to do that to befunge this logic, more power to them.
        $url =~ s/-/_/g;    # makes \b work below
        next
            unless $url =~ /\Q$hostname\E/
            && $url =~ /\b$username\b/;

        unless ( $url =~ m!/(\d+)(?:\.html)?$! ) {
            $log->( 'URL %s not of expected format in prune.', $url );
            next;
        }

        # if we want a paid user or someone to get their picture keyword updated,
        # skip them here.
        next if $force_reimport;

        # yes, we can has this jitemid from the remote side
        $has{ $1 >> 8 } = 1;
    }
    $log->( 'Identified %d items we already know about (first pass).', scalar( keys %has ) );
    $log->( 'memory usage is now %dMB', LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    # this is another optimization.  we know crossposted entries can be removed from
    # the list of things we will import, as we generated them to begin with.
    foreach my $itemid ( keys %$xpost_map ) {
        $has{$itemid} = 1;
    }

    $log->( 'Identified %d items we already know about (second pass).', scalar( keys %has ) );
    $log->( 'memory usage is now %dMB', LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    my $duplicates_map =
        DW::Worker::ContentImporter::Local::Entries->get_duplicates_map( $u, \%has ) || {};
    $log->( 'Found %d items to check for duplicates.', scalar( keys %$duplicates_map ) );
    $log->( 'memory usage is now %dMB', LJ::gtop()->proc_mem($$)->resident / 1024 / 1024 );

    return ( $duplicates_map, %has );
}

1;