mourningdove/cgi-bin/DW/Logic/LogItems.pm
2026-05-24 01:03:05 +00:00

668 lines
24 KiB
Perl

#!/usr/bin/perl
#
# DW::Logic::LogItems
#
# Contains logic used to calculate what items should be showed on the reading page
# and other related functions. Functions related to loading large numbers of entries
# in a complicated fashion should be in here. General purpose entry functionality
# should be in LJ::Entry, etc.
#
# Authors:
# Mark Smith <mark@dreamwidth.org>
#
# Copyright (c) 2009 by Dreamwidth Studios, LLC.
#
# This program is free software; you may redistribute it and/or modify it under
# the same terms as Perl itself. For a copy of the license, please reference
# 'perldoc perlartistic' or 'perldoc perlgpl'.
#
package DW::Logic::LogItems;
use strict;
use Carp qw/ confess /;
# name: $u->watch_items
# des: Return watch list items for a given user, filter, and period.
# args: hash of items, key/values:
# - remote
# - itemshow
# - skip
# - friends (opt) friends rows loaded via [func[LJ::get_friends]]
# - friends_u (opt) u objects of all friends loaded
# - idsbycluster (opt) hashref to set clusterid key to [ [ journalid, itemid ]+ ]
# - dateformat: either "S2" for S2 code, or anything else for S1
# - friendsoffriends: load friends of friends, not just friends
# - security: (public|access) or a group number
# - showtypes: /[PICNYF]/
# - events_date: date to load events for ($u must have friendspage_per_day)
# - content_filter: object of type DW::User::ContentFilters::Filter
# returns: Array of item hashrefs containing the same elements
sub watch_items {
my ( $u, %args ) = @_;
$u = LJ::want_user($u) or confess 'invalid user object';
# bail very, very early for accounts that are too big for a reading page
return () if $LJ::FORCE_EMPTY_SUBSCRIPTIONS{ $u->id };
# not sure where best to do this, so we're doing it here: don't allow
# content filters for community reading pages.
delete $args{content_filter} unless $u->is_individual;
# we only allow viewing protected content on your own reading page, if you
# are viewing someone else's reading page, we assume you're logged out
my $remote = LJ::want_user( delete $args{remote} ) || LJ::get_remote();
$remote = undef if $remote && $remote->id != $u->id;
# prepare some variables for later... many variables
my @items = ();
my $itemshow = $args{itemshow} + 0;
my $skip = $args{skip} + 0;
$skip = 0 if $skip < 0;
my $getitems = $itemshow + $skip;
# friendspage per day is allowed only for journals with the special cap 'friendspage_per_day'
my $events_date = $args{events_date};
$events_date = '' unless $remote && $u->can_use_daily_readpage;
my $filter = $args{content_filter};
my $max_age = $LJ::MAX_FRIENDS_VIEW_AGE || 3600 * 24 * 14; # 2 week default.
my $lastmax = $LJ::EndOfTime - ( $events_date || ( time() - $max_age ) );
my $lastmax_cutoff = 0
; # if nonzero, never search for entries with rlogtime higher than this (set when cache in use)
# given a hash of friends rows, strip out rows with invalid journaltype
my $filter_journaltypes = sub {
my ( $friends, $friends_u, $memcache_only, $valid_types ) = @_;
return unless $friends && $friends_u;
$valid_types ||= uc $args{showtypes} if defined $args{showtypes};
# make (F)eeds an alias for s(Y)ndicated
$valid_types =~ s/F/Y/g if defined $valid_types;
# load u objects for all the given
LJ::load_userids_multiple( [ map { $_, \$friends_u->{$_} } keys %$friends ],
[$remote], $memcache_only );
# delete u objects based on 'showtypes'
foreach my $fid ( keys %$friends_u ) {
my $fu = $friends_u->{$fid};
if ( !$fu
|| !$fu->is_visible
|| $valid_types && index( uc($valid_types), $fu->{journaltype} ) == -1 )
{
delete $friends_u->{$fid};
delete $friends->{$fid};
}
}
# all args passed by reference
return;
};
my @friends_buffer = ();
my $fr_loaded = 0; # flag: have we loaded friends?
# normal friends mode
my $get_next_friend = sub {
# return one if we already have some loaded.
return $friends_buffer[0] if @friends_buffer;
return undef if $fr_loaded;
# get all friends for this user and groupmask
my $friends = $u->watch_list( community_okay => 1 );
my %friends_u;
# strip out people who aren't in the filter, if we have one
if ($filter) {
foreach my $fid ( keys %$friends ) {
delete $friends->{$fid}
unless $filter->contains_userid($fid);
}
}
# strip out rows with invalid journal types
$filter_journaltypes->( $friends, \%friends_u );
# get update times for all the friendids
my $tu_opts = {};
my $fcount = scalar keys %$friends;
if ( $LJ::SLOPPY_FRIENDS_THRESHOLD && $fcount > $LJ::SLOPPY_FRIENDS_THRESHOLD ) {
$tu_opts->{memcache_only} = 1;
}
my $times =
$events_date
? LJ::get_times_multi( $tu_opts, keys %$friends )
: { updated => LJ::get_timeupdate_multi( $tu_opts, keys %$friends ) };
my $timeupdate = $times->{updated};
# now push a properly formatted @friends_buffer row
foreach my $fid ( keys %$timeupdate ) {
my $fu = $friends_u{$fid};
my $rupdate = $LJ::EndOfTime - ( $timeupdate->{$fid} || 0 );
my $clusterid = $fu->{'clusterid'};
push @friends_buffer, [ $fid, $rupdate, $clusterid, $friends->{$fid}, $fu ];
}
@friends_buffer =
sort { $a->[1] <=> $b->[1] }
grep {
( $timeupdate->{ $_->[0] } || 0 ) >= $lastmax and # reverse index
(
$events_date
? $times->{created}->{ $_->[0] } < $events_date
: 1
)
} @friends_buffer;
# note that we've already loaded the friends
$fr_loaded = 1;
# return one if we just found some, else we're all
# out and there's nobody else to load.
return @friends_buffer ? $friends_buffer[0] : undef;
};
# memcached friends of friends mode
$get_next_friend = sub {
# return one if we already have some loaded.
return $friends_buffer[0] if @friends_buffer;
return undef if $fr_loaded;
# get journal's friends
my $friends = $u->watch_list or return undef;
my %friends_u;
# fill %allfriends with all friendids and cut $friends
# down to only include those that match $filter
my %allfriends = ();
foreach my $fid ( keys %$friends ) {
$allfriends{$fid}++;
# if the person is in an active filter, allow them, else delete them
next unless $filter && !$filter->contains_userid($fid);
delete $friends->{$fid};
}
# strip out invalid friend journaltypes
$filter_journaltypes->( $friends, \%friends_u, "memcache_only", "P" );
# get update times for all the friendids
my $f_tu = LJ::get_timeupdate_multi( { 'memcache_only' => 1 }, keys %$friends );
# get friends of friends
my $ffct = 0;
my %ffriends = ();
foreach my $fid ( sort { $f_tu->{$b} <=> $f_tu->{$a} } keys %$friends ) {
last if $ffct > 50;
my $fu = $friends_u{$fid};
my $ff = $fu->watch_list( community_okay => 1, memcache_only => 1 );
my $ct = 0;
while ( my $ffid = each %$ff ) {
last if $ct > 100;
next if $allfriends{$ffid} || $ffid == $u->id;
$ffriends{$ffid} = $ff->{$ffid};
$ct++;
}
$ffct++;
}
# strip out invalid friendsfriends journaltypes
my %ffriends_u;
$filter_journaltypes->( \%ffriends, \%ffriends_u, "memcache_only" );
# get update times for all the friendids
my $ff_tu = LJ::get_timeupdate_multi( { 'memcache_only' => 1 }, keys %ffriends );
# build friends buffer
foreach my $ffid ( sort { $ff_tu->{$b} <=> $ff_tu->{$a} } keys %$ff_tu ) {
my $rupdate = $LJ::EndOfTime - $ff_tu->{$ffid};
my $clusterid = $ffriends_u{$ffid}->{'clusterid'};
# since this is ff mode, we'll force colors to ffffff on 000000
$ffriends{$ffid}->{'fgcolor'} = "#000000";
$ffriends{$ffid}->{'bgcolor'} = "#ffffff";
push @friends_buffer,
[ $ffid, $rupdate, $clusterid, $ffriends{$ffid}, $ffriends_u{$ffid} ];
}
@friends_buffer = sort { $a->[1] <=> $b->[1] } @friends_buffer;
# note that we've already loaded the friends
$fr_loaded = 1;
# return one if we just found some fine, else we're all
# out and there's nobody else to load.
return @friends_buffer ? $friends_buffer[0] : undef;
}
if $args{friendsoffriends} && @LJ::MEMCACHE_SERVERS;
# friends of friends disabled w/o memcache
confess 'friends of friends mode requires memcache'
if $args{friendsoffriends} && !@LJ::MEMCACHE_SERVERS;
my $loop = 1;
my $itemsleft = $getitems; # even though we got a bunch, potentially, they could be old
my $fr;
while ( $loop && ( $fr = $get_next_friend->() ) ) {
shift @friends_buffer;
# load the next recent updating friend's recent items
my $friendid = $fr->[0];
$args{friends}->{$friendid} = $fr->[3]; # friends row
$args{friends_u}->{$friendid} = $fr->[4]; # friend u object
my @newitems = LJ::get_log2_recent_user(
{
clusterid => $fr->[2],
userid => $friendid,
remote => $remote,
itemshow => $itemsleft,
filter => $filter,
notafter => $lastmax,
dateformat => $args{dateformat},
update => $LJ::EndOfTime - $fr->[1], # reverse back to normal
events_date => $events_date,
security => $args{security},
}
);
# stamp each with clusterid if from cluster, so ljviews and other
# callers will know which items are old (no/0 clusterid) and which
# are new
$_->{clusterid} = $fr->[2] foreach @newitems;
if (@newitems) {
push @items, @newitems;
$itemsleft--;
my $evtime = sub { LJ::mysqldate_to_time( $_[0]->{eventtime} ) };
# sort all the total items by rlogtime (recent at beginning).
# if there's an in-second tie, the "newer" post is determined by
# the later eventtime (if known/different from rlogtime), then by
# the higher jitemid, which means nothing if the posts aren't in
# the same journal, but means everything if they are (which happens
# almost never for a human, but all the time for RSS feeds)
@items = sort {
$a->{rlogtime} <=> $b->{rlogtime}
|| $evtime->($b) <=> $evtime->($a)
|| $b->{jitemid} <=> $a->{jitemid}
} @items;
# cut the list down to what we need.
@items = splice( @items, 0, $getitems ) if ( @items > $getitems );
}
if ( @items == $getitems ) {
$lastmax = $items[-1]->{'rlogtime'};
$lastmax = $lastmax_cutoff if $lastmax_cutoff && $lastmax > $lastmax_cutoff;
# stop looping if we know the next friend's newest entry
# is greater (older) than the oldest one we've already
# loaded.
my $nextfr = $get_next_friend->();
$loop = 0 if ( $nextfr && $nextfr->[1] > $lastmax );
}
}
# remove skipped ones
splice( @items, 0, $skip ) if $skip;
# get items
foreach (@items) {
$args{owners}->{ $_->{'ownerid'} } = 1;
}
# return the itemids grouped by clusters, if callers wants it.
if ( ref $args{idsbycluster} eq "HASH" ) {
foreach (@items) {
push @{ $args{idsbycluster}->{ $_->{'clusterid'} } },
[ $_->{'ownerid'}, $_->{'itemid'} ];
}
}
return @items;
}
*LJ::User::watch_items = \&watch_items;
*DW::User::watch_items = \&watch_items;
# name: $u->recent_items
# des: Returns journal entries for a given account.
# takes hash of options as arguments
# -- err: scalar ref to return error code/msg in
# -- remote: remote user's $u
# -- clusterid: clusterid of userid
# -- tagids: arrayref of tagids to return entries with
# -- security: (public|access|private) or a group number
# -- clustersource: if value 'slave', uses replicated databases
# -- order: if 'logtime', sorts by logtime, not eventtime
# -- friendsview: if true, sorts by logtime, not eventtime
# -- notafter: upper bound inclusive for rlogtime/revttime (depending on sort mode),
# defaults to no limit
# -- skip: items to skip
# -- itemshow: items to show
# -- viewall: if set, no security is used.
# -- dateformat: if "S2", uses S2's 'alldatepart' format.
# -- itemids: optional arrayref onto which itemids should be pushed
# -- posterid: optional, return (community) posts made by this poster only
# returns: array of hashrefs containing keys:
# -- itemid (the jitemid)
# -- posterid
# -- security
# -- alldatepart (in S1 or S2 fmt, depending on 'dateformat' req key)
# -- system_alldatepart (same as above, but for the system time)
# -- ownerid (if in 'friendsview' mode)
# -- rlogtime (if in 'friendsview' mode)
sub recent_items {
my ( $u, %args ) = @_;
$u = LJ::want_user($u) or confess 'invalid user object';
my $userid = $u->id;
my @items = (); # what we'll return
my $err = $args{err};
my $remote = LJ::want_user( delete $args{remote} );
my $remoteid = $remote ? $remote->id : 0;
my $max_hints = $LJ::MAX_SCROLLBACK_LASTN; # temporary
my $sort_key = "revttime";
my $clusterid = $args{'clusterid'} + 0;
my @sources = ("cluster$clusterid");
if ( my $ab = $LJ::CLUSTER_PAIR_ACTIVE{$clusterid} ) {
@sources = ("cluster${clusterid}${ab}");
}
unshift @sources, ( "cluster${clusterid}lite", "cluster${clusterid}slave" )
if $args{'clustersource'} eq "slave";
my $logdb = LJ::get_dbh(@sources);
# community/friend views need to post by log time, not event time
$sort_key = "rlogtime" if ( $args{'order'} eq "logtime"
|| $args{'friendsview'} );
# 'notafter':
# the friends view doesn't want to load things that it knows it
# won't be able to use. if this argument is zero or undefined,
# then we'll load everything less than or equal to 1 second from
# the end of time. we don't include the last end of time second
# because that's what backdated entries are set to. (so for one
# second at the end of time we'll have a flashback of all those
# backdated entries... but then the world explodes and everybody
# with 32 bit time_t structs dies)
#
# Unless we are not on a friends view, then want to use the actual end of time.
my $notafter = $args{notafter} ? $args{notafter} + 0 : 0;
$notafter ||= $args{friendsview} ? $LJ::EndOfTime - 1 : $LJ::EndOfTime;
my $skip = $args{skip} ? $args{skip} + 0 : 0;
my $itemshow = $args{itemshow} ? $args{itemshow} + 0 : 0;
$itemshow = $max_hints if $itemshow > $max_hints;
my $maxskip = $max_hints - $itemshow;
if ( $skip < 0 ) { $skip = 0; }
if ( $skip > $maxskip ) { $skip = $maxskip; }
my $itemload = $itemshow + $skip;
my $mask = 0;
if ( $remote && ( $remote->is_person || $remote->is_identity ) && $remoteid != $userid ) {
# if this is a community we're viewing, fake the mask to select on, as communities
# no longer have masks to users
if ( $u->is_community ) {
$mask = $remote->member_of($u) ? 1 : 0;
}
else {
$mask = $u->trustmask($remote);
}
}
# decide what level of security the remote user can see
my $secwhere = "";
if ( $userid == $remoteid
|| ( $remote && $remote->can_manage($u) )
|| $args{'viewall'} )
{
# no extra where restrictions... user can see all their own stuff
# community administrators can also see everything in their comms
# alternatively, if 'viewall' opt flag is set, security is off.
}
elsif ($mask) {
# can see public or things with them in the mask
$secwhere = "AND (security='public' OR (security='usemask' AND allowmask & $mask != 0))";
}
else {
# not a friend? only see public.
$secwhere = "AND security='public' ";
}
# because LJ::get_friend_items needs rlogtime for sorting.
my $extra_sql = '';
if ( $args{'friendsview'} ) {
$extra_sql .= "journalid AS 'ownerid', rlogtime, ";
}
# if we need to get by tag, get an itemid list now
my $jitemidwhere = '';
if ( ref $args{tagids} eq 'ARRAY' && @{ $args{tagids} } ) {
my $jitemids;
# $args{tagmode} = $args{getargs}->{mode} eq 'and' ? 'and' : 'or';
if ( $args{tagmode} eq 'and' ) {
my $limit = $LJ::TAG_INTERSECTION;
die "\$LJ::TAG_INTERSECTION not set!"
unless $limit && $limit > 0;
my $need = scalar @{ $args{tagids} };
$#{ $args{tagids} } = $limit - 1 if $need > $limit;
my $in = join( ',', map { $_ + 0 } @{ $args{tagids} } );
my $sth = $logdb->prepare(
"SELECT jitemid, kwid FROM logtagsrecent WHERE journalid = ? AND kwid IN ($in)");
$sth->execute($userid);
die $logdb->errstr if $logdb->err;
my %mix;
while ( my $row = $sth->fetchrow_arrayref ) {
my ($jitemid) = @$row;
$mix{$jitemid}++;
}
foreach my $jitemid ( keys %mix ) {
delete $mix{$jitemid} if $mix{$jitemid} < $need;
}
$jitemids = [ keys %mix ];
}
else { # mode: 'or'
# select jitemids uniquely
my $in = join( ',', map { $_ + 0 } @{ $args{tagids} } );
$jitemids = $logdb->selectcol_arrayref(
qq{
SELECT DISTINCT jitemid FROM logtagsrecent WHERE journalid = ? AND kwid IN ($in)
}, undef, $userid
);
die $logdb->errstr if $logdb->err;
}
# set $jitemidwhere iff we have jitemids
if (@$jitemids) {
$jitemidwhere = " AND jitemid IN (" . join( ',', map { $_ + 0 } @$jitemids ) . ")";
}
else {
# no items, so show no entries
return ();
}
}
# if we need to filter by security, build up the where clause for that too
my $securitywhere = '';
if ( $args{'security'} ) {
my $security = $args{'security'};
if ( ( $security eq "public" ) || ( $security eq "private" ) ) {
$securitywhere = " AND security = \"$security\"";
}
elsif ( $security eq "access" ) {
$securitywhere = " AND security = \"usemask\" AND allowmask = 1";
}
elsif ( $security =~ /^\d+$/ ) {
$securitywhere =
" AND security = \"usemask\" AND (allowmask & " . ( 1 << $security ) . ")";
}
}
my $sql;
my $dateformat = "%a %W %b %M %y %Y %c %m %e %d %D %p %i %l %h %k %H";
if ( $args{'dateformat'} eq "S2" ) {
$dateformat = "%Y %m %d %H %i %s %w"; # yyyy mm dd hh mm ss day_of_week
}
my ( $sql_limit, $sql_select ) = ( '', '' );
if ( $args{'ymd'} ) {
my ( $year, $month, $day );
if ( $args{'ymd'} =~ m!^(\d\d\d\d)/(\d\d)/(\d\d)\b! ) {
( $year, $month, $day ) = ( $1, $2, $3 );
# check
if ( $year !~ /^\d+$/ ) { $$err = "Corrupt or non-existant year."; return (); }
if ( $month !~ /^\d+$/ ) { $$err = "Corrupt or non-existant month."; return (); }
if ( $day !~ /^\d+$/ ) { $$err = "Corrupt or non-existant day."; return (); }
if ( $month < 1 || $month > 12 || int($month) != $month ) {
$$err = "Invalid month.";
return ();
}
if ( $year < 1970 || $year > 2038 || int($year) != $year ) {
$$err = "Invalid year: $year";
return ();
}
if ( $day < 1 || $day > 31 || int($day) != $day ) { $$err = "Invalid day."; return (); }
if ( $day > LJ::days_in_month( $month, $year ) ) {
$$err = "That month doesn't have that many days.";
return ();
}
}
else {
$$err = "wrong date: " . $args{'ymd'};
return ();
}
$sql_limit = "LIMIT 2000";
$sql_select = "AND year=$year AND month=$month AND day=$day";
$extra_sql .= "allowmask, ";
}
else {
$sql_limit = "LIMIT $skip,$itemshow";
$sql_select = "AND $sort_key <= $notafter";
}
my $posterwhere;
if ( $args{posterid} && $args{posterid} =~ /^(\d+)$/ ) {
$posterwhere = " AND posterid=$1";
}
else {
$posterwhere = "";
}
$sql = qq{
SELECT jitemid AS 'itemid', posterid, security, $extra_sql
DATE_FORMAT(eventtime, "$dateformat") AS 'alldatepart', anum,
DATE_FORMAT(logtime, "$dateformat") AS 'system_alldatepart',
allowmask, eventtime, logtime
FROM log2 USE INDEX ($sort_key)
WHERE journalid=$userid $sql_select $secwhere $jitemidwhere $securitywhere $posterwhere
ORDER BY journalid, $sort_key
$sql_limit
};
unless ($logdb) {
$$err = "nodb" if ref $err eq "SCALAR";
return ();
}
my $sth = $logdb->prepare($sql);
$sth->execute;
if ( $logdb->err ) { die $logdb->errstr; }
# keep track of the last alldatepart, and a per-minute buffer
my $last_time;
my @buf;
my $flush = sub {
return unless @buf;
push @items, sort { $b->{itemid} <=> $a->{itemid} } @buf;
@buf = ();
};
while ( my $li = $sth->fetchrow_hashref ) {
push @{ $args{'itemids'} }, $li->{'itemid'};
my $sortdate = {
rlogtime => 'system_alldatepart',
revttime => 'alldatepart'
}->{$sort_key};
$flush->()
unless defined $last_time
&& $li->{$sortdate} eq $last_time;
push @buf, $li;
$last_time = $li->{$sortdate};
# construct an LJ::Entry singleton
my $entry = LJ::Entry->new( $userid, jitemid => $li->{itemid} );
$entry->absorb_row(%$li);
}
$flush->();
return @items;
}
*LJ::User::recent_items = \&recent_items;
*DW::User::recent_items = \&recent_items;
# name: $u->active_entries
# des: Returns 10 last active entries for an account
# returns: array of itemids
sub active_entries {
my ($u) = @_;
my $uid = $u->userid;
# check memcache first
my $activeentries = LJ::MemCache::get( [ $uid, "activeentries:$uid" ] );
return @$activeentries if $activeentries;
# get latest 10 entries that were commented on - we will see whether $remote can view them later
# disregard screened and deleted comments when ordering
# NOTE: we have to force the index because MySQL's optimizer gets this wrong. we know that our
# data is going to be near the top.
my $entryids = $u->selectcol_arrayref(
q{SELECT DISTINCT nodeid FROM talk2 FORCE INDEX (PRIMARY)
WHERE journalid = ? AND state NOT IN ('D', 'S')
ORDER BY jtalkid DESC LIMIT 10},
undef, $u->id
);
die $u->errstr if $u->err;
return unless $entryids && @$entryids;
# memcache this data in the form: activeentries:journalid
LJ::MemCache::set( [ $uid, "activeentries:$uid" ], \@$entryids );
# return. we check whether the user viewing is allowed to view these entries later
return @$entryids;
}
*LJ::User::active_entries = \&active_entries;
*DW::User::active_entries = \&active_entries;
1;