5 =head1 NAME $RCSfile: update.pl,v $
7 Updates the CCDB database
15 Andrew DeFaria <Andrew@ClearSCM.com>
23 Fri Mar 11 19:09:52 PST 2011
27 $Date: 2011/05/05 18:37:05 $
33 Usage update.pl: [-u|sage] [-ve|rbose] [-deb|ug]
38 [[-p|vob <pvob> -a|ctivity <activity>]|
39 [-p|vob <pvob> -b|aseline <baseline>]|
40 [-p|vob <pvob> -s|tream <stream>]]]
47 -u|sage: Displays usage
50 -deb|ug: Output debug messages
52 -vo|b <vob>: Vob to process
54 -p|vob <pvob>: PVOB to operate on
55 -a|ctivity <activity>: Activity to process
56 -b|aseline <baseline>: Baseline to process
57 -s|tream <stream>: Stream to process
59 -o|plog [<vob>]: Process oplog (Default: All vobs)
65 This script updates the CCDB database with Clearcase UCM meta data. It operates
70 In this mode, indicated by specifying either no options or a -pvob and
71 optionally one of -activity, -baseline or -stream, update.pl will query
72 Clearcase and gather all metadata for the specified option.
74 You can run update.pl with no paramters to process all pvobs in the current
75 registry region of you can specify a -pvob to process. This is generally how
76 the script is run. Note you can parallelize update.pl by running it multiple
77 times each with its own -pvob. In this case the script will log activity to
80 Or you can run "fix ups" to add individual activities, baselines or streams by
81 specifying -activity (or -baseline/-stream) and its -pvob. Note however that
82 the object is not validated (In such cases we don't check that say activity and
83 pvob are valid - we just add them to the database).
85 Additionally you can use -vob to add a vob to CCDB. This should be a relatively
86 infrequent operation and it is necessary to add vobs that -oplog will process.
88 =head2 Check Change Sets mode
90 Even with this script initially popullating CCDB and with the appropriate
91 triggers set to fire to keep CCDB up to date, and even with -oplog mode to apply
92 changes from other sites the CCDB may still become out of sync with Clearcase.
93 This is due to the fact that orphaned files can effect change set membership in
94 UCM and Clearcase does not call any triggers or otherwise notify you of the
95 problem. To illustrate, if the user is running under UCM and checks out a
96 directory, makes an element and checks it in, but then cancels the checkout of
97 the directory, Clearcase is forced to orphan the file by placing it in
98 lost+found. A warning is issued to the user, however no triggers are called.
100 Investigating the change set we see that the elements that were orphaned are
101 indicated in the change set but their paths have been altered to indicate that
102 the elements are in lost+found! One would think that Clearcase would fire the
103 chactivity trigger but it seems that trigger is only fired when elements change
104 from one activity to another. In this case the elements are changing, but the
105 activity is the same activity. To me this is a bug and Clearcase should fire the
106 chactivity trigger with CLEARCASE_ACTIVITY == CLEARCASE_TO_ACTIVITY. If this
107 were the case we could handle this situation with triggers.
109 Check change set mode instead goes through all of the changesets in CCDB and
110 verifies that the changeset in CCDB matches the changeset as listed by
111 lsactivity -long. If not it updates it. This is an intense activity that will
112 be time consuming but I can see no other way to fix up this problem.
122 use lib "$FindBin::Bin/lib", "$FindBin::Bin/../lib";
128 use Clearcase::UCM::Activity;
129 use Clearcase::Element;
135 my $VERSION = '$Revision: 1.4 $';
136 ($VERSION) = ($VERSION =~ /\$Revision: (.*) /);
138 my (%opts, %totals, $log);
140 my $ccdb = CCDB->new;
143 sub ProcessFolder ($$);
146 my ($activity, $pvob) = @_;
148 $pvob = Clearcase::vobtag $pvob;
150 my $cmd = "lsact -fmt \"%[versions]CQp\" $activity\@$pvob";
152 my ($status, @output) = $Clearcase::CC->execute ($cmd);
154 $log->err ("Unable to execute $cmd\n" . join ("\n", @output), $status)
157 # Need to split up change set. It's presented to us as quoted and space
158 # separated however the change set elements themselves can have spaces in
161 # "/vob/foo/file name with spaces@@/main/1", "/vob/foo/file name2@@/main/2"
163 # So we'll split on '", ""'! Note that this will leave us with the first
164 # element with a leading '"' and the last element with a trailing '"' which
165 # we will have to handle.
167 # Additionally we will call collapseOverExtendedViewPathname to normalize
168 # the over extended pathnames to element hashes.
171 @output = split /\", \"/, $output[0]
175 # Skip any cleartool warnings. We are getting warnings of the form:
176 # "A version in the change set of activity "63332.4" is currently
177 # unavailable". Probably some sort of subtle corruption that we can ignore.
178 # (It should be fixed but we aren't going to be doing that here!)
179 next if /cleartool: Warning/;
181 # Strip any remaining '"'s
184 # Remove vob prefix but keep the leading "/"
185 $_ = '/' . Clearcase::vobname $_;
187 my %element = Clearcase::Element::collapseOverExtendedVersionPathname $_;
189 push @changeset, \%element;
195 sub baselineActivities (%) {
198 my $pvobTag = Clearcase::vobtag $baseline{pvob};
200 my $cmd = "lsbl -fmt \"%[activities]p\" $baseline{name}\@$pvobTag";
202 my ($status, @output) = $Clearcase::CC->execute ($cmd);
204 $log->err ("Unable to execute $cmd\n" . join ("\n", @output), $status)
209 return split / /, $output[0];
210 } # baselineActivities
215 my %pvob = $ccdb->GetVob ($pvob);
219 my ($err, $msg) = $ccdb->AddVob ({
225 $log->err ("Unable to add pvob:$pvob\n$msg");
227 $totals{'Pvobs added'}++;
229 $log->msg ("Added pvob:$pvob");
235 sub UpdateFolder ($$) {
236 my ($folder, $pvob) = @_;
238 my %folder = $ccdb->GetFolder ($folder, $pvob);
242 my ($err, $msg) = $ccdb->AddFolder ({
248 $log->err ("Unable to add folder:$folder\n$msg");
250 $totals{'Folders added'}++;
252 $log->msg ("Added folder:$folder");
258 sub UpdateSubfolder ($$$) {
259 my ($parent, $subfolder, $pvob) = @_;
261 my %subfolder = $ccdb->GetSubfolder ($parent, $subfolder, $pvob);
263 return if %subfolder;
265 my ($err, $msg) = $ccdb->AddSubfolder ({
267 subfolder => $subfolder,
272 $log->err ("Unable to add subfolder:$parent/$subfolder\n$msg");
274 $totals{'Subfolders added'}++;
276 $log->msg ("Added subfolder:$parent/$subfolder");
282 sub UpdateProject ($$$) {
283 my ($project, $folder, $pvob) = @_;
285 my %project = $ccdb->GetProject ($project, $folder, $pvob);
289 my ($err, $msg) = $ccdb->AddProject ({
296 $log->err ("Unable to add project:$project folder:$folder pvob:$pvob\n$msg");
298 $totals{'Projects added'}++;
300 $log->msg ("Added Project:$project");
306 sub UpdateStream ($$) {
307 my ($name, $pvob) = @_;
309 my %stream = $ccdb->GetStream ($name, $pvob);
313 # Determine the integration stream for this stream's project. First get
314 # project for the stream.
315 my $pvobTag = Clearcase::vobtag ($pvob);
317 my $cmd = "lsstream -fmt \"%[project]p\" $name\@$pvobTag";
319 my ($status, @output) = $Clearcase::CC->execute ($cmd);
322 $log->err ("Unable to execute $cmd\n" . join ("\n", @output));
327 # Now get the intergration stream for this project
328 $cmd = "lsproject -fmt \"%[istream]p\" $output[0]\@$pvobTag";
330 ($status, @output) = $Clearcase::CC->execute ($cmd);
333 $log->err ("Unable to execute $cmd\n" . join ("\n", @output));
338 my $type = 'integration'
339 if $name eq $output[0];
341 my ($err, $msg) = $ccdb->AddStream ({
348 $log->err ( "Unable to add stream:$name\n$msg");
350 $log->msg ("Added stream:$name");
351 $totals{'Streams added'}++;
355 sub UpdateChangeset ($$$) {
356 my ($activity, $pvob, $element) = @_;
359 name => '/' . Clearcase::vobname $element->pname,
360 version => $element->version,
363 my %changeset = $ccdb->GetChangeset (
365 '/' . Clearcase::vobname $element->pname,
370 return if %changeset;
372 my ($err, $msg) = $ccdb->AddChangeset ({
373 activity => $activity,
374 element => $element{name},
375 version => $element{version},
381 $log->err ("Unable to add changeset activity:$activity "
382 . "element:$element{name}$Clearcase::SFX$element{version}\n$msg");
384 $totals{'Changesets added'}++;
386 $log->msg ("Linked activity:$activity -> element:$element{name}");
392 sub UpdateActivity ($$) {
393 my ($name, $pvob) = @_;
395 my %activity = $ccdb->GetActivity ($name, $pvob);
399 my ($err, $msg) = $ccdb->AddActivity ({
405 $log->err ("Unable to add activity:$name\n$msg");
407 $totals{'Activities added'}++;
409 $log->msg ("Added activity $name");
415 sub UpdateBaselineActivityXref (%) {
418 $log->msg ("Processing Baseline Activities for $baseline{name}");
420 my %baselineActivityXref = (
421 baseline => $baseline{name},
422 pvob => $baseline{pvob},
425 foreach (baselineActivities %baseline) {
428 # Often activities in a baseline have not yet been added so add them here.
429 # (Not sure why this is the case...)
431 my %existingRec = $ccdb->GetActivity ($_, $baseline{pvob});
433 UpdateActivity $_, $baseline{pvob}
436 $baselineActivityXref{activity} = $_;
438 %existingRec = $ccdb->GetBaselineActivityXref (
439 $baselineActivityXref{baseline},
440 $baselineActivityXref{activity},
441 $baselineActivityXref{pvob}
444 unless (%existingRec) {
445 ($err, $msg) = $ccdb->AddBaselineActivityXref (\%baselineActivityXref);
448 $log->err ("Unable to add baseline:$baselineActivityXref{name}"
449 . " activity: $baselineActivityXref{activity}\n"
453 $totals{'Baseline Activity Xrefs added'}++;
458 $log->msg ("Processed Baseline Activities for $baseline{name}");
461 } # UpdateBaselineActivityXref
463 sub UpdateBaseline ($$) {
464 my ($name, $pvob) = @_;
466 my %baseline = $ccdb->GetBaseline ($name, $pvob);
470 my ($err, $msg) = $ccdb->AddBaseline ({
476 $log->err ("Unable to add baseline:$name\n$msg");
478 $totals{'Baselines added'}++;
480 $log->msg ("Added baseline:$name");
482 my %baseline = $ccdb->GetBaseline ($name, $pvob);
484 UpdateBaselineActivityXref (%baseline);
490 sub UpdateStreamActivityXref ($$$) {
491 my ($stream, $activity, $pvob) = @_;
493 my %streamActivityXref = $ccdb->GetStreamActivityXref (
499 return if %streamActivityXref;
501 my ($err, $msg) = $ccdb->AddStreamActivityXref ({
503 activity => $activity,
508 $log->err ("Unable to add stream_activity_xref stream:$stream "
509 . "activity:$activity\n$msg");
512 $totals{'Stream Activity Xrefs added'}++;
514 $log->msg ("Linked stream:$stream -> activity:$activity");
518 } # UpdateStreamActivityXref
520 sub ProcessElements ($$) {
521 my ($name, $pvob) = @_;
523 $log->msg ("Finding changeset for activity:$name");
525 my $activity = Clearcase::UCM::Activity->new ($name, $pvob);
527 foreach ($activity->changeset) {
530 # Remove vob prefix but keep the leading "/"
531 my $elementName = '/' . Clearcase::vobname $element->pname;
534 "Processing element:$elementName"
539 UpdateChangeset $name, $pvob, $element;
542 $log->msg ("Processed changeset for activity:$name");
547 sub ProcessActivities ($$) {
548 my ($stream, $pvob) = @_;
550 $log->msg ("Finding activities in stream:$stream");
552 my $pvobTag = Clearcase::vobtag ($pvob);
554 my $cmd = "lsstream -fmt \"%[activities]p\" $stream\@$pvobTag";
556 my ($status, @output) = $Clearcase::CC->execute ($cmd);
559 $log->err ("Unable to execute $cmd\n" . join ("\n", @output), $status);
566 foreach (sort split / /, $output[0]) {
567 next if /^DEFAULT.*NO_CHECKIN/;
569 UpdateActivity ($_, $pvob);
571 $totals{'Activities processed'}++;
573 UpdateStreamActivityXref $stream, $_, $pvob;
575 ProcessElements $_, $pvob;
578 $log->msg ("Processed activities in stream:$stream");
581 } # ProcessActivities
583 sub ProcessBaselines ($$) {
584 my ($stream, $pvob) = @_;
586 $log->msg ("Finding baselines in stream:$stream");
588 my $pvobTag = Clearcase::vobtag ($pvob);
590 my $cmd = "lsbl -stream $stream\@$pvobTag -short";
592 my ($status, @baselines) = $Clearcase::CC->execute ($cmd);
595 $log->err ("Unable to execute $cmd\n" . join ("\n", @baselines));
600 foreach (sort @baselines) {
601 UpdateBaseline ($_, $pvob);
603 $totals{'Baselines processed'}++;
606 $log->msg ("Processed baselines in stream:$stream");
611 sub ProcessStream ($$) {
612 my ($name, $pvob) = @_;
614 $totals{'Streams processed'}++;
616 UpdateStream $name, $pvob;
618 ProcessActivities $name, $pvob;
619 ProcessBaselines $name, $pvob;
624 sub ProcessProject ($$$) {
625 my ($project, $folder, $pvob) = @_;
627 my $pvobTag = Clearcase::vobtag $pvob;
629 $log->msg ("Processing project:$project\@$pvobTag");
631 UpdateProject ($project, $folder, $pvob);
633 my $cmd = "lsstream -short -in $project\@$pvobTag";
635 my ($status, @output) = $Clearcase::CC->execute ($cmd);
638 $log->err ("Unable to execute $cmd\n" . join ("\n", @output));
644 ProcessStream $_, $pvob;
650 sub ProcessFolder ($$) {
651 my ($folder, $pvob) = @_;
653 my $pvobTag = Clearcase::vobtag $pvob;
655 $log->msg ("Processing folder:$folder\@$pvobTag");
657 UpdateFolder ($folder, $pvob);
659 my $cmd = "lsfolder -fmt \"%[contains_folders]p\" $folder\@$pvobTag";
661 my ($status, @output) = $Clearcase::CC->execute ($cmd);
664 $log->err ("Unable to execute command $cmd (Status: $status)\n"
665 . join ("\n", @output), 1);
672 foreach (split / /, $output[0]) {
673 ProcessFolder $_, $pvob;
675 UpdateSubfolder ($folder, $_, $pvob);
678 $cmd = "lsfolder -fmt \"%[contains_projects]p\" $folder\@$pvobTag";
680 ($status, @output) = $Clearcase::CC->execute ($cmd);
683 $log->err ("Unable to execute command $cmd (Status: $status)\n"
684 . join ("\n", @output), 1);
691 foreach (split / /, $output[0]) {
692 ProcessProject $_, $folder, $pvob;
698 sub ProcessPvob ($) {
701 $log->msg ("Processing pvob:$pvobName");
703 UpdatePvob $pvobName;
705 ProcessFolder ('RootFolder', $pvobName);
709 $log->msg ("Finding streams in pvob:$pvobName");
711 my $pvob = Clearcase::vobtag ($pvobName);
713 my $cmd = "lsstream -invob $pvob -short";
714 my ($status, @streams) = $Clearcase::CC->execute ($cmd);
716 $log->err ("Unable to execute $cmd\n" . join ("\n", @streams), $status)
723 foreach (sort @streams) {
726 $totals{'Streams processed'}++;
728 ProcessStream $stream{name}, $stream{pvob};
731 $totals{'Pvobs processed'}++;
733 $log->msg ("Finished processing pvob:$pvobName");
743 my %existingRec = $ccdb->GetVob ($name);
745 unless (%existingRec) {
746 my $vob = Clearcase::Vob->new (Clearcase::vobtag $name);
748 # If vob doesn't exist then $vob is just an empty shell. Check to see if
749 # another field is present to make sure the vob really exists. A vob should
750 # always have a region, for example.
754 my $vobRegistryAttributes = $vob->vob_registry_attributes;
756 my $type = ($vobRegistryAttributes and
757 $vobRegistryAttributes =~ /ucmvob/) ? 'ucm' : 'base';
759 ($err, $msg) = $ccdb->AddVob ({
765 $log->err ("Unable to add vob $name (Error: $err)\n$msg");
767 $totals{'Vobs added'}++;
777 my $startTime = time;
781 'verbose' => sub { set_verbose },
782 'usage' => sub { Usage },
789 ) or Usage "Unknown option";
793 $nbrOpts++ if $opts{pvob};
794 $nbrOpts++ if $opts{activity};
795 $nbrOpts++ if $opts{baseline};
796 $nbrOpts++ if $opts{stream};
797 $nbrOpts++ if $opts{vob};
799 Usage "Cannot specify -checkchangeset and any other options"
800 if $opts{checkchangeset} and $nbrOpts != 0;
802 Usage "Cannot specify -vob and any other options"
803 if $opts{vob} and ($nbrOpts != 1 or $opts{checkchangeset});
805 my $me = $FindBin::Script;
808 if ($opts{activity} and $opts{pvob} and
809 ($opts{baseline} or $opts{stream})) {
810 Usage "If -activity is specified then -pvob should be the only other "
813 } elsif ($opts{baseline} and $opts{pvob} and
814 ($opts{activity} or $opts{stream})) {
815 Usage "If -baseline is specified then -pvob should be the only other "
818 } elsif ($opts{stream} and $opts{pvob} and
819 ($opts{activity} or $opts{baseline})) {
820 Usage "If -stream is specified then -pvob should be the only other option";
822 } elsif ($opts{pvob}) {
825 $nbrOpts++ if $opts{activity};
826 $nbrOpts++ if $opts{baseline};
827 $nbrOpts++ if $opts{stream};
829 if ($nbrOpts != 0 and $nbrOpts > 1) {
830 Usage "If -pvob is specified then it must be used alone or in "
831 . "conjunction\nwith only one of -activity, -baseline or -stream "
832 . "must be specified\n";
837 if ($opts{activity} and $opts{pvob}) {
840 $log->msg ("$FindBin::Script V$VERSION");
842 UpdateActivity ($opts{activity}, $opts{pvob});
843 } elsif ($opts{baseline} and $opts{pvob}) {
846 $log->msg ("$FindBin::Script V$VERSION");
848 UpdateBaseline ($opts{baseline}, $opts{pvob});
849 } elsif ($opts{stream} and $opts{pvob}) {
852 $log->msg ("$FindBin::Script V$VERSION");
854 UpdateStream ($opts{stream}, $opts{pvob});
855 } elsif ($opts{pvob}) {
856 $log = Logger->new (name => "$me.$opts{pvob}");
858 $log->msg ("$FindBin::Script V$VERSION");
860 ProcessPvob $opts{pvob};
861 } elsif ($opts{checkchangeset}) {
862 error "The -checkchangeset option is not implemented yet", 1;
863 } elsif ($opts{vob}) {
866 $log->msg ("$FindBin::Script V$VERSION");
868 ProcessVob $opts{vob};
872 my $UCM = Clearcase::UCM->new;
874 $log->msg ("$FindBin::Script V$VERSION");
877 foreach ($UCM->pvobs);
880 display_duration $startTime, $log;
882 $totals{Errors} = $log->errors;
884 Stats \%totals, $log;