360 lines
12 KiB
Prolog
Executable file
360 lines
12 KiB
Prolog
Executable file
#!/usr/bin/perl
|
|
# This code was forked from the LiveJournal project owned and operated
|
|
# by Live Journal, Inc. The code has been modified and expanded by
|
|
# Dreamwidth Studios, LLC. These files were originally licensed under
|
|
# the terms of the license supplied by Live Journal, Inc, which can
|
|
# currently be found at:
|
|
#
|
|
# http://code.livejournal.org/trac/livejournal/browser/trunk/LICENSE-LiveJournal.txt
|
|
#
|
|
# In accordance with the original license, this code and all its
|
|
# modifications are provided under the GNU General Public License.
|
|
# A copy of that license can be found in the LICENSE file included as
|
|
# part of this distribution.
|
|
|
|
use strict;
|
|
|
|
BEGIN {
|
|
require "$ENV{LJHOME}/cgi-bin/ljlib.pl";
|
|
}
|
|
use LJ::User;
|
|
use LJ::Userpic;
|
|
use DW::BlobStore;
|
|
|
|
use Getopt::Long;
|
|
use IPC::Open3;
|
|
use Digest::MD5;
|
|
use Log::Log4perl;
|
|
|
|
# this script is a migrater that will move userpics from an old storage method
|
|
# into whatever blobstore method is defined in the site config.
|
|
|
|
# the basic theory is that we iterate over all clusters, find all userpics that
|
|
# aren't in either mogile or blobstore right now, and put them in blobstore
|
|
|
|
# determine
|
|
my ( $one, $besteffort, $dryrun, $user, $verify, $verbose, $clusters, $purge );
|
|
my $rv = GetOptions(
|
|
"best-effort" => \$besteffort,
|
|
"one" => \$one,
|
|
"dry-run" => \$dryrun,
|
|
"user=s" => \$user,
|
|
"verify" => \$verify,
|
|
"verbose" => \$verbose,
|
|
"purge-old" => \$purge,
|
|
"clusters=s" => \$clusters,
|
|
);
|
|
unless ($rv) {
|
|
die <<ERRMSG;
|
|
This script supports the following command line arguments:
|
|
|
|
--clusters=X[-Y]
|
|
Only handle clusters in this range. You can specify a
|
|
single number, or a range of two numbers with a dash.
|
|
|
|
--user=username
|
|
Only move this particular user.
|
|
|
|
--one
|
|
Only move one user. (But it moves all their pictures.)
|
|
This is used for testing.
|
|
|
|
--verify
|
|
If specified, this option will reload the userpic from
|
|
BlobStore and make sure it's been stored successfully.
|
|
|
|
--dry-run
|
|
If on, do not update the database. This mode will put
|
|
the userpic in BlobStore and let you examine the image
|
|
and make sure everything is okay. It will not update the
|
|
userpic2 table, though.
|
|
|
|
--best-effort
|
|
Normally, if a problem is encountered (null userpic, md5
|
|
mismatch, connection failure, etc) the script will die to
|
|
make sure everything goes well. With this flag, we don't
|
|
die and instead just print to standard error.
|
|
|
|
--purge-old
|
|
Sometimes we run into data that is for users that have since
|
|
moved to a different cluster. Normally we ignore it, but
|
|
with this option, we'll clean that data up as we find it.
|
|
|
|
--verbose
|
|
Be very chatty.
|
|
ERRMSG
|
|
}
|
|
|
|
# make sure ljconfig is setup right (or so we hope)
|
|
die "Please define \@LJ::BLOBSTORES in your site config\n"
|
|
unless @LJ::BLOBSTORES && scalar @LJ::BLOBSTORES;
|
|
|
|
# setup stderr if we're in best effort mode
|
|
if ($besteffort) {
|
|
my $oldfd = select(STDERR);
|
|
$| = 1;
|
|
select($oldfd);
|
|
}
|
|
|
|
# use a custom log4perl config, to make sure we're getting
|
|
# DEBUG messages from Blobstore if we've requested verbose
|
|
# output, but not under normal use
|
|
|
|
my $conf = 'log4perl.rootLogger=' . ( $verbose ? 'DEBUG' : 'ERROR' );
|
|
$conf .= q{, STDERR
|
|
|
|
log4perl.appender.STDERR=Log::Log4perl::Appender::Screen
|
|
log4perl.appender.STDERR.stderr=1
|
|
log4perl.appender.STDERR.layout=Log::Log4perl::Layout::SimpleLayout
|
|
};
|
|
Log::Log4perl::init( \$conf );
|
|
|
|
# operation modes
|
|
if ($user) {
|
|
|
|
# move a single user
|
|
my $u = LJ::load_user($user);
|
|
die "No such user: $user\n" unless $u;
|
|
handle_userid( $u->{userid}, $u->{clusterid} );
|
|
|
|
}
|
|
else {
|
|
# parse the clusters
|
|
my @clusters;
|
|
if ($clusters) {
|
|
if ( $clusters =~ /^(\d+)(?:-(\d+))?$/ ) {
|
|
my ( $min, $max ) = map { $_ + 0 } ( $1, $2 || $1 );
|
|
push @clusters, $_ foreach $min .. $max;
|
|
}
|
|
else {
|
|
die "Error: --clusters argument not of right format.\n";
|
|
}
|
|
}
|
|
else {
|
|
@clusters = @LJ::CLUSTERS;
|
|
}
|
|
|
|
# now iterate over the clusters to pick
|
|
my $ctotal = scalar(@clusters);
|
|
my $ccount = 0;
|
|
foreach my $cid ( sort { $a <=> $b } @clusters ) {
|
|
|
|
# status report
|
|
$ccount++;
|
|
print "\nChecking cluster $cid...\n\n";
|
|
|
|
# get a handle
|
|
my $dbcm = get_db_handle($cid);
|
|
|
|
# get all userids
|
|
print "Getting userids...\n";
|
|
my $limit = $one ? 'LIMIT 1' : '';
|
|
my $userids = $dbcm->selectcol_arrayref(
|
|
"SELECT DISTINCT userid FROM userpic2 WHERE (location <> 'mogile' AND location <> 'blobstore') OR location IS NULL $limit"
|
|
);
|
|
my $total = scalar(@$userids);
|
|
|
|
# iterate over userids
|
|
my $count = 0;
|
|
print "Beginning iteration over userids...\n";
|
|
foreach my $userid (@$userids) {
|
|
|
|
# move this userpic
|
|
my $extra = sprintf( "[%6.2f%%, $ccount of $ctotal] ", ( ++$count / $total * 100 ) );
|
|
handle_userid( $userid, $cid, $extra );
|
|
}
|
|
|
|
# don't hit up more clusters
|
|
last if $one;
|
|
}
|
|
}
|
|
print "\n";
|
|
|
|
print "Updater terminating.\n";
|
|
|
|
#############################################################################
|
|
### helper subs down here
|
|
|
|
# take a userid and move their pictures. returns 0 on error, 1 on successful
|
|
# move of a user's pictures, and 2 meaning the user isn't ready for moving
|
|
# (dversion < 7, etc)
|
|
sub handle_userid {
|
|
my ( $userid, $cid, $extra ) = @_;
|
|
|
|
# load user to move and do some sanity checks
|
|
my $u = LJ::load_userid($userid);
|
|
unless ($u) {
|
|
LJ::end_request();
|
|
LJ::start_request();
|
|
$u = LJ::load_userid($userid);
|
|
}
|
|
die "ERROR: Unable to load userid $userid\n"
|
|
unless $u;
|
|
|
|
# if they're expunged, they might have data somewhere if they were
|
|
# copy-moved from A to B, then expunged on B. now we're on A and
|
|
# need to delete it ourselves (if purge-old is on)
|
|
if ( $u->{clusterid} == 0 && $u->is_expunged ) {
|
|
return unless $purge;
|
|
|
|
# if we get here, the user has indicated they want data purged, get handle
|
|
my $to_purge_dbcm = get_db_handle($cid);
|
|
my $ct = $to_purge_dbcm->do( "DELETE FROM userpic2 WHERE userid = ?", undef, $u->{userid} );
|
|
print "\tnotice: purged $ct old rows.\n\n"
|
|
if $verbose;
|
|
return;
|
|
}
|
|
|
|
# get a handle
|
|
my $dbcm = get_db_handle( $u->{clusterid} );
|
|
|
|
# print that we're doing this user
|
|
print "$extra$u->{user}($u->{userid})\n";
|
|
|
|
# if a user has been moved to another cluster, but the source data from
|
|
# userpic2 wasn't deleted, we need to ignore the user or purge their data
|
|
if ( $u->{clusterid} != $cid ) {
|
|
return unless $purge;
|
|
|
|
# verify they have some rows on the new side
|
|
my $count = $dbcm->selectrow_array( "SELECT COUNT(*) FROM userpic2 WHERE userid = ?",
|
|
undef, $u->{userid} );
|
|
return unless $count;
|
|
|
|
# if we get here, the user has indicated they want data purged, get handle
|
|
my $to_purge_dbcm = get_db_handle($cid);
|
|
|
|
# delete the old data
|
|
if ($dryrun) {
|
|
print "\tnotice: need to delete userpic2 rows.\n\n"
|
|
if $verbose;
|
|
}
|
|
else {
|
|
my $ct =
|
|
$to_purge_dbcm->do( "DELETE FROM userpic2 WHERE userid = ?", undef, $u->{userid} );
|
|
print "\tnotice: purged $ct old rows.\n\n"
|
|
if $verbose;
|
|
}
|
|
|
|
# nothing else to do here
|
|
return;
|
|
}
|
|
|
|
# get all their photos that aren't in mogile or blobstore already
|
|
my $picids = $dbcm->selectall_arrayref(
|
|
"SELECT picid, md5base64, fmt, location FROM userpic2 WHERE userid = ? AND ( (location <> 'mogile' AND location <> 'blobstore') OR location IS NULL )",
|
|
undef, $u->{userid}
|
|
);
|
|
return unless @$picids;
|
|
|
|
# now we have a userid and picids, get the photos from the blob server
|
|
foreach my $row (@$picids) {
|
|
my ( $picid, $md5, $fmt, $loc ) = @$row;
|
|
print "\tstarting move for picid $picid\n"
|
|
if $verbose;
|
|
|
|
my $format = { G => 'gif', J => 'jpg', P => 'png' }->{$fmt};
|
|
|
|
my $data;
|
|
|
|
# no target? then it's in the database
|
|
unless ( defined $loc ) {
|
|
|
|
($data) = $dbcm->selectrow_array(
|
|
'SELECT imagedata FROM userpicblob2 WHERE userid = ? AND picid = ?',
|
|
undef, $u->{userid}, $picid );
|
|
|
|
}
|
|
|
|
# get length
|
|
my $len = length($data);
|
|
if ( $besteffort && !$len ) {
|
|
print STDERR "empty_userpic userid=$u->{userid} picid=$picid\n";
|
|
print "\twarning: empty userpic.\n\n"
|
|
if $verbose;
|
|
next;
|
|
}
|
|
die "Error: data from location=$loc empty ($u->{user}, 'userpic', $format, $picid)\n"
|
|
unless $len;
|
|
|
|
# verify the md5 of this picture with what's in the database
|
|
my $blobmd5 = Digest::MD5::md5_base64($data);
|
|
if ( $besteffort && ( $md5 ne $blobmd5 ) ) {
|
|
print STDERR
|
|
"md5_mismatch userid=$u->{userid} picid=$picid dbmd5=$md5 blobmd5=$blobmd5\n";
|
|
print "\twarning: md5 mismatch; database=$md5, blobserver=$blobmd5\n\n"
|
|
if $verbose;
|
|
next;
|
|
}
|
|
die "\tError: data from blobserver md5 mismatch: database=$md5, blobserver=$blobmd5\n"
|
|
unless $md5 eq $blobmd5;
|
|
print "\tverified md5; database=$md5, blobserver=$blobmd5\n"
|
|
if $verbose;
|
|
|
|
# get filehandle to blobstore and put the file there
|
|
print "\tdata length = $len bytes, uploading to BlobStore...\n"
|
|
if $verbose;
|
|
my $storage_key = LJ::Userpic->storage_key( $u->userid, $picid );
|
|
my $bstore = DW::BlobStore->store( userpics => $storage_key, \$data );
|
|
if ( $besteffort && !$bstore ) {
|
|
print STDERR "store_failed userid=$u->{userid} picid=$picid\n";
|
|
print "\twarning: failed in call to store\n\n"
|
|
if $verbose;
|
|
next;
|
|
}
|
|
die "Unable to store file in BlobStore\n"
|
|
unless $bstore;
|
|
|
|
# extra verification
|
|
if ($verify) {
|
|
my $data2 = DW::BlobStore->retrieve( userpics => $storage_key );
|
|
my $eq = ( $data2 && $$data2 eq $data ) ? 1 : 0;
|
|
if ( $besteffort && !$eq ) {
|
|
print STDERR "verify_failed userid=$u->{userid} picid=$picid\n";
|
|
print "\twarning: verify failed; picture not updated\n\n"
|
|
if $verbose;
|
|
next;
|
|
}
|
|
die "\tERROR: picture NOT stored successfully, content mismatch\n"
|
|
unless $eq;
|
|
print "\tverified length = " . length($$data2) . " bytes...\n"
|
|
if $verbose;
|
|
}
|
|
|
|
# done moving this picture
|
|
unless ($dryrun) {
|
|
print "\tupdating database for this picture...\n"
|
|
if $verbose;
|
|
$dbcm->do( "UPDATE userpic2 SET location = 'blobstore' WHERE userid = ? AND picid = ?",
|
|
undef, $u->{userid}, $picid );
|
|
}
|
|
|
|
# get the paths so the user can verify if they want
|
|
if ($verbose) {
|
|
|
|
# Log4perl will print the blobstore path when in
|
|
# debug mode, no need to calculate it again here
|
|
print "\tverify site url: $LJ::SITEROOT/userpic/$picid/$u->{userid}\n";
|
|
print "\tpicture update complete.\n\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
# a sub to get a cluster handle and set it up for our use
|
|
sub get_db_handle {
|
|
my $cid = shift;
|
|
|
|
my $dbcm = LJ::get_cluster_master( { raw => 1 }, $cid );
|
|
unless ($dbcm) {
|
|
print STDERR "handle_unavailable clusterid=$cid\n";
|
|
die "ERROR: unable to get raw handle to cluster $cid\n";
|
|
}
|
|
eval {
|
|
$dbcm->do("SET wait_timeout = 28800");
|
|
die $dbcm->errstr if $dbcm->err;
|
|
};
|
|
die "Couldn't set wait_timeout on $cid: $@\n" if $@;
|
|
$dbcm->{'RaiseError'} = 1;
|
|
|
|
return $dbcm;
|
|
}
|