5 =head1 NAME $RCSfile: mspb.pl,v $
7 MultiSite PlayBack: This script updates the CCDB database by playing back
8 multisite transcations.
16 Andrew DeFaria <Andrew@ClearSCM.com>
24 Fri Mar 11 19:09:52 PST 2011
28 $Date: 2011/05/05 18:39:56 $
34 Usage mspb.pl: [-u|sage] [-ve|rbose] [-deb|ug] [-vo|b <vob>]
37 -u|sage: Displays usage
40 -deb|ug: Output debug messages
42 -vo|b <vob>: Vob to process (Default: All vobs)
46 This script updates the CCDB database with Clearcase UCM meta data by playing
47 back multisite transactions.
49 If no parameters are specified then mspb attempts to replay all transactions
50 from all vobs listed in CCDB. To add a new vob use -vob. Epoch numbers are kept
51 in CCDB to keep track of the last oplog operation that had been played back for
54 Note that only certain transactions are played back, those that correspond to
55 actions important to the metadata kept in CCDB. Also, if a transaction fails,
56 i.e. the add or deletion of a record fails, then the error is silently ignored.
57 This allows you to playback transactions without worry that replaying an
58 already played transcation will cause the data to become out of sync. This is
59 much like multisite's syncreplica itself.
69 use lib "$FindBin::Bin/lib", "$FindBin::Bin/../lib";
73 use Clearcase::Element;
74 use Clearcase::UCM::Activity;
82 my $VERSION = '$Revision: 1.2 $';
83 ($VERSION) = ($VERSION =~ /\$Revision: (.*) /);
85 my (%opts, %totals, $log);
101 # Special casing op_time. For some odd reason we have more than one value
102 # on a single line. One case of this is where the keyword is "op_time".
103 # We've seen lines like this:
105 # op_time= 2008-10-15T18:48:39Z create_time= 2008-10-15T18:48:39Z
107 # So by now $value = '2008-10-15T18:48:39Z create_time= 2008-10-15T18:48:39Z'
108 # so we'll split it based on ' '.
109 if ($key eq 'op_time') {
111 ($value, $_) = split / /, $value;
114 $record{$key} = $value;
116 # Now parse $create time
123 # Some values are wrapped in quotes
124 if ($value =~ /(\'|\")(.*)(\'|\")/) {
128 # If a key occurs multiple times then make its value an array
130 if (ref $record{$key} eq 'ARRAY') {
131 push @{$record{$key}}, $value;
133 $record{$key} = [ $record{$key}, $value];
136 $record{$key} = $value;
144 sub GetSubmittedDate ($$) {
145 my ($activity, $pvob) = @_;
147 $pvob = Clearcase::vobtag $pvob;
149 my $cmd = "describe -fmt \"%Na\" activity:$activity\@$pvob";
151 my ($status, @output) = $Clearcase::CC->execute ($cmd);
155 foreach (split / /, $output[0]) {
157 if ($1 =~ /submit_time/i) {
158 my ($year, $mon, $mday, $hour, $min, $sec) = ymdhms $2;
159 return "$year-$mon-$mday $hour:$min:$sec";
169 sub AddActivity ($%) {
170 my ($pvob, %oplog) = @_;
172 my ($cmd, $err, $status, $msg, @output, %existingRec);
174 $totals{'Activities processed'}++;
178 # Add an activity (if not already existing)
179 %existingRec = $ccdb->GetActivity ($oplog{name_p}, $pvob);
181 unless (%existingRec) {
182 my $submitted = GetSubmittedDate $oplog{name_p}, $pvob;
184 my $cmd = "describe -fmt \"%[owner]p\" activity:$oplog{name_p}\@"
185 . Clearcase::vobtag $pvob;
187 my ($status, @output) = $Clearcase::CC->execute ($cmd);
189 ($err, $msg) = $ccdb->AddActivity ({
190 oid => $oplog{activity_oid},
191 name => $oplog{name_p},
194 submitted => $submitted,
200 $totals{'Activities added'}++;
203 # Add a stream (if not already existing)
204 $cmd = "lsactivity -fmt \"%[stream]p\" $oplog{name_p}\@"
205 . Clearcase::vobtag $pvob;
207 ($status, @output) = $Clearcase::CC->execute ($cmd);
210 # There are times when an activity is subsequently deleted. Since we are
211 # playing back Multisite transactions we will see the mkactivity first, then
212 # later see a corresponding rmactivity. Since Multisite has already played
213 # all of these transactions to Clearcase itself the activity is gone - and
214 # therefore the lsactivity will fail to find the stream. If that's the case
215 # then just return. If not then issue a warning.
216 unless ($output[0] =~ /activity not found/) {
217 $log->warn ("Can't find stream for activity:$oplog{name_p}\@"
218 . Clearcase::vobtag $pvob
222 return (1, "Unable to execute command: $cmd (Status: $status)\n"
223 . join ("\n", @output))
228 %existingRec = $ccdb->GetStream ($output[0], $pvob);
230 unless (%existingRec) {
231 ($err, $msg) = $ccdb->AddStream ({
237 $log->warn ("Unable to add stream:$output[0]\@"
238 . Clearcase::vobtag $pvob
239 . " (Error: $err)\n$msg");
244 $totals{'Streams added'}++;
249 # Link them (if not already linked)
250 %existingRec = $ccdb->GetStreamActivityXref (
256 unless (%existingRec) {
257 ($err, $msg) = $ccdb->AddStreamActivityXref ({
258 stream => $output[0],
259 activity => $oplog{name_p},
264 $log->warn ("Unable to add stream_activity_xref:$output[0]\@"
265 . Clearcase::vobtag $pvob
266 . " activity:$oplog{name_p} (Error: $err)\n$msg");
271 $totals{'Stream/Activity Xrefs added'}++;
278 my ($pvob, %oplog) = @_;
282 $totals{'Streams processed'}++;
284 # Add a stream (if not already existing)
285 my %existingRec = $ccdb->GetStream ($oplog{name_p}, $pvob);
287 unless (%existingRec) {
288 my $pvobTag = Clearcase::vobtag $pvob;
289 my $cmd = "lsstream -fmt \"%[project]p\" $oplog{name_p}\@$pvobTag";
291 my ($status, @output) = $Clearcase::CC->execute ($cmd);
294 $log->err ("Unable to execute command: $cmd (Status: $status)"
295 . join ("\n", @output));
296 return ($status, join ("\n", @output));
299 ($err, $msg) = $ccdb->AddStream ({
300 oid => $oplog{activity_oid},
301 name => $oplog{name_p},
302 project => $output[0],
307 $totals{'Streams added'}++;
314 sub ProcessActivity ($$%) {
315 my ($operation, $pvob, %oplog) = @_;
317 # Many operations in Multisite's oplog have an op of mkactivity but are
318 # actually operations on other objects based on actype_oid. The following are
329 # We only handle activity and stream here
332 if ($oplog{actype_oid}) {
333 $actype = $Clearcase::CC->oid2name ($oplog{actype_oid}, $pvob);
335 if ($operation eq 'rmactivity' and $oplog{comment}) {
336 $actype = 'activity';
344 if ($operation eq 'mkactivity') {
345 if ($actype eq 'activity') {
346 AddActivity $pvob, %oplog;
347 } elsif ($actype eq 'stream') {
348 AddStream $pvob, %oplog;
350 } elsif ($operation eq 'rmactivity') {
351 if ($actype eq 'activity') {
352 # For rmactivity there's nothing but the comment to go on to get the
353 # activity's name. The comment must be of a format of "Destroyed activity
354 # "<activity_name>@<pvob>"." complete with nested double quotes.
355 my ($activity, $pvob);
357 # Note: There are rmactivity's that lack the comment! Nothing we can do
358 # with these except to ignore them!
360 unless $oplog{comment};
362 # Note: <pvob> is a vob tag of the variety of the client. So, for example,
363 # it can be a Windows style pvob (e.g. \\pvob) even though we are running
364 # on a Linux machine where the pvob needs to be /vob/pvob!
365 if ($oplog{comment} =~ /Destroyed activity \"activity:(\S+)\@(\S+)\"/) {
367 $pvob = Clearcase::vobname ($2);
371 unless ($activity or $pvob);
373 $totals{'Activities processed'}++;
374 $totals{'Activities deleted'}++;
376 return $ccdb->DeleteActivity ($activity, $pvob);
377 } elsif ($actype eq 'stream') {
378 # Note: I have yet to see an rmactivity stream with even an actype_oid!
379 $totals{'Streams processed'}++;
380 $totals{'Streams deleted'}++;
382 return $ccdb->DeleteStreamOID ($oplog{activity_oid});
389 sub ProcessBaseline ($$%) {
390 my ($operation, $pvob, %oplog) = @_;
392 my ($cmd, $err, $status, $msg, @output, %existingRec);
394 my $pvobTag = Clearcase::vobtag $pvob;
396 $totals{'Baselines processed'}++;
398 if ($operation eq 'mkcheckpoint') {
401 # Add an activity (if not already existing)
402 %existingRec = $ccdb->GetBaseline ($oplog{name_p}, $pvob);
404 unless (%existingRec) {
405 ($err, $msg) = $ccdb->AddBaseline ({
406 oid => $oplog{checkpoint_oid},
407 name => $oplog{name_p},
414 $totals{'Baselines added'}++;
417 # Add a stream_baseline_xref entry
418 $cmd = "lsbl -fmt \"%[bl_stream]p\" $oplog{name_p}\@$pvobTag";
420 ($status, @output) = $Clearcase::CC->execute ($cmd);
423 $log->err ("Unable to execute command: $cmd (Status: $status)"
424 . join ("\n", @output));
428 ($err, $msg) = $ccdb->AddStreamBaselineXref ({
429 stream => $output[0],
430 baseline => $oplog{name_p},
437 $totals{'Stream/Baseline Xrefs added'}++;
440 unless $oplog{activity_oid};
442 # Loop through activities
443 my @activities = ref $oplog{activity_oid} eq 'ARRAY'
444 ? @{$oplog{activity_oid}}
445 : ($oplog{activity_oid});
447 foreach (@activities) {
448 my $activity = $Clearcase::CC->oid2name ($_, $pvob);
450 # I think $activity will be blank if after this mkcheckpoint somebody
451 # did an rmactivity...
455 # Check to see if the activity exists
458 %existingRec = $ccdb->GetActivity ($activity, $pvob);
460 unless (%existingRec) {
461 ($err, $msg) = $ccdb->AddActivity ({
470 # Link them (if not already linked)
471 %existingRec = $ccdb->GetBaselineActivityXref (
472 $oplog{name_p}, $activity, $pvob
475 unless (%existingRec) {
476 ($err, $msg) = $ccdb->AddBaselineActivityXref ({
477 baseline => $oplog{name_p},
478 activity => $activity,
483 $log->warn ("Unable to add baseline_activity_xref:$output[0]\@"
484 . "$pvobTag baseline:$oplog{name_p} activity:$_ (Error:"
490 $totals{'Baseline/Activity Xrefs added'}++;
493 } elsif ($operation eq 'rmcheckpoint') {
494 $totals{'Baselines deleted'}++;
496 return $ccdb->DeleteBaselineOID ($oplog{checkpoint_oid});
502 sub ProcessElement ($$%) {
503 my ($operation, $vob, %oplog) = @_;
506 unless $oplog{version_oid};
508 my $elementVersion = $Clearcase::CC->oid2name ($oplog{version_oid}, $vob);
509 my ($element, $version) = split /$Clearcase::SFX/, $elementVersion;
511 # Remove VOBTAG_PREFIX from $element
512 $element = '/' . Clearcase::vobname $element;
514 my $cmd = "describe -fmt \"%[activity]Xp\" oid:$oplog{version_oid}\@"
515 . Clearcase::vobtag $vob;
517 my ($status, @output) = $Clearcase::CC->execute ($cmd);
520 $log->err ("Unable to execute command: $cmd (Status: $status)"
521 . join ("\n", @output));
525 # If this operation is not attached to an activity then we're not interested.
529 my ($activity, $pvob) = split /\@/, $output[0];
531 # Remove leading "activity:"
532 $activity = substr $activity, 9;
535 $pvob = Clearcase::vobname $pvob;
537 my ($err, $msg, %existingRec);
539 if ($operation eq 'checkin'
540 or $operation eq 'checkout') {
541 %existingRec = $ccdb->GetChangeset ($activity, $element, $version, $pvob);
543 unless (%existingRec) {
544 my $create_time = $oplog{create_time};
546 # Create time from Multisite are of the format: 2008-10-15T18:48:39Z
547 $create_time =~ s/T/ /;
548 $create_time =~ s/Z//;
550 ($err, $msg) = $ccdb->AddChangeset ({
551 activity => $activity,
555 created => $create_time,
559 $log->err ("Unable to AddChangeset ($activity, $element, $version, "
565 # Update Activity's submitted field (if this create time gt submitted)
566 my %activity = $ccdb->GetActivity ($activity, $pvob);
569 $activity{submitted} ||= $create_time;
571 if ($create_time ge $activity{submitted}) {
572 $activity{submitted} = $create_time;
574 my ($err, $msg) = $ccdb->UpdateActivity (
580 $log->err ("Unable to update activity: $activity pvob: $pvob - "
581 . " submitted: $create_time")
586 $totals{'Changesets added'}++;
588 } elsif ($operation eq 'uncheckout'
589 or $operation eq 'rmver') {
590 %existingRec = $ccdb->GetChangeset ($activity, $element, $version, $pvob);
593 ($err, $msg) = $ccdb->DeleteChangeset (
601 $log->err ("Unable to DeleteChangeset ($activity, $element, $version, "
607 $totals{'Changesets deleted'}++;
609 } elsif ($operation eq 'rmelem') {
610 %existingRec = $ccdb->GetChangeset ($activity, $element, $version, $pvob);
613 ($err, $msg) = $ccdb->DeleteElementAll ($element);
616 $log->err ("Unable to DeleteElementAll ($element)\n$msg");
621 $totals{'Elements removed'}++;
628 sub ProcessRename ($%) {
629 my ($vob, %oplog) = @_;
632 unless $oplog{comment};
636 # Parse comment to find what got renamed and the from and to names
637 if ($oplog{comment} =~ /Changed name of (.+?) from \"(\S+)\" to \"(\S+)\"/) {
642 # Only interested in these objects
644 unless $object =~ /activity/i or
645 $object =~ /baseline/i or
646 $object =~ /stream/i;
652 my $method = 'Update' . ucfirst $object;
654 my ($err, $str) = $ccdb->$method ($from, $vob, \%update);
657 $log->err ("Unable to rename $object from $from -> $to (pvob:$vob");
663 if ($object eq 'activity') {
664 $totals{'Activities renamed'}++;
665 } elsif ($object eq 'baseline') {
666 $totals{'Baselines renamed'}++;
667 } elsif ($object eq 'stream') {
668 $totals{'Streams renamed'}++;
674 sub ProcessOperation ($$%) {
675 my ($operation, $vob, %oplog) = @_;
677 # For now let's only process the activity opcodes... We'll add more later.
678 my @interestingOpcodes = (
695 unless InArray $operation, @interestingOpcodes;
697 if ($operation eq 'mkactivity'
698 or $operation eq 'rmactivity') {
699 return ProcessActivity ($operation, $vob, %oplog);
700 } elsif ($operation eq 'mkcheckpoint'
701 or $operation eq 'rmcheckpoint') {
702 return ProcessBaseline ($operation, $vob, %oplog);
703 } elsif ($operation eq 'checkin'
704 or $operation eq 'checkout'
705 or $operation eq 'rmelem'
706 or $operation eq 'rmver'
707 or $operation eq 'uncheckout') {
708 return ProcessElement ($operation, $vob, %oplog);
709 } elsif ($operation eq 'rename') {
710 return ProcessRename ($vob, %oplog);
714 sub ProcessOplog (%) {
717 # Start dumpoplog off at the appropriate oplog number
718 my $cmd = 'multitool dumpoplog -long -invob '
719 . Clearcase::vobtag ($vob{name})
720 . " -from $vob{epoch}";
723 open my $oplog, "$cmd|"
724 or error "Cannot execute $cmd", 1;
729 # Look for the next oplog entry
737 } elsif (!$inRecord) {
741 my ($operation, $status, @output);
749 ProcessOperation $operation, $vob{name}, ParseOplog $oplog;
751 # Update vob's last_oplog
752 my ($err, $msg) = $ccdb->UpdateVob ($vob{name}, \%vob);
754 $log->err ("Unable to update vob:$vob{name}\'s epoch to "
769 my %vob = $ccdb->GetVob ($name);
771 $log->msg ("Processing vob:$name ($vob{type})");
774 my $vob = Clearcase::Vob->new (Clearcase::vobtag $name);
776 ($err, $msg) = $ccdb->AddVob ({
778 type => $vob->vob_registry_attributes !~ /ucmvob/ ? 'base' : 'ucm',
782 $log->err ("Unable to add vob $name (Error: $err)\n$msg");
784 $totals{'Vobs added'}++;
787 %vob = $ccdb->GetVob ($name);
794 $totals{Errors} = $log->errors;
795 $totals{Warnings} = $log->warnings;
797 Stats \%totals, $log;
803 my $startTime = time;
807 'verbose' => sub { set_verbose },
808 'usage' => sub { Usage },
810 ) or Usage "Unknown option";
814 $SIG{__DIE__} = $SIG{INT} = $SIG{ABRT} = $SIG{QUIT} = $SIG{USR2} = 'EndProcess';
819 push @vobs, $opts{vob};
822 my (@ucmvobs, @basevobs);
824 push @ucmvobs, $$_{name}
825 foreach ($ccdb->FindVob ('*', 'ucm'));
828 push @basevobs, $$_{name}
829 foreach ($ccdb->FindVob ('*', 'base'));
831 push @vobs, $_ foreach (sort @ucmvobs);
832 push @vobs, $_ foreach (sort @basevobs);
836 $log->msg ('1 vob to process');
838 $log->msg (scalar @vobs . ' vobs to process');
844 $totals{'Vobs processed'}++;
847 display_duration $startTime, $log;
849 $totals{Errors} = $log->errors;
850 $totals{Warnings} = $log->warnings;
852 Stats \%totals, $log;