1 ###############################################################################
3 # Package: NaturalDocs::SourceDB
5 ###############################################################################
7 # SourceDB is an experimental package meant to unify the tracking of various elements in the source code.
11 # - All extension packages must call <RegisterExtension()> before they can be used.
14 # Architecture: The Idea
16 # For quite a while Natural Docs only needed <SymbolTable>. However, 1.3 introduced the <ClassHierarchy> package
17 # which duplicated some of its functionality to track classes and parent references. 1.4 now needs <ImageReferenceTable>,
18 # so this package was an attempt to isolate the common functionality so the wheel doesn't have to keep being rewritten as
19 # the scope of Natural Docs expands.
21 # SourceDB is designed around <Extensions> and items. The purposefully vague "items" are anything in the source code
22 # that we need to track the definitions of. Extensions are the packages to track them, only they're derived from
23 # <NaturalDocs::SourceDB::Extension> and registered with this package instead of being free standing and duplicating
24 # functionality such as watched files.
26 # The architecture on this package isn't comprehensive yet. As more extensions are added or previously made free standing
27 # packages are migrated to it it will expand to encompass them. However, it's still experimental so this concept may
28 # eventually be abandoned for something better instead.
31 # Architecture: Assumptions
33 # SourceDB is built around certain assumptions.
37 # SourceDB assumes that only the first item per file with a particular item string is relevant. For example, if two functions
38 # have the exact same name, there's no way to link to the second one either in HTML or internally so it doesn't matter for
39 # our purposes. Likewise, if two references are exactly the same they go to the same target, so it doesn't matter whether
40 # there's one or two or a thousand. All that matters is that at least one reference exists in this file because you only need
41 # to determine whether the entire file gets rebuilt. If two items are different in some meaningful way, they should generate
42 # different item strings.
44 # Watched file parsing:
46 # SourceDB assumes the parse method is that the information that was stored from Natural Docs' previous run is loaded, a
47 # file is watched, that file is reparsed, and then <AnalyzeWatchedFileChanges()> is called. When the file is reparsed all
48 # items within it are added the same as if the file was never parsed before.
50 # If there's a new item this time around, that's fine no matter what. However, a changed item wouldn't normally be
51 # recorded because the previous run's definition is seen as the first one and subsequent ones are ignored. Also, deleted
52 # items would normally not be recorded either because we're only adding.
54 # The watched file method fixes this because everything is also added to a second, clean database specifically for the
55 # watched file. Because it starts clean, it always gets the first definition from the current parse which can then be
56 # compared to the original by <AnalyzeWatchedFileChanges()>. Because it starts clean you can also compare it to the
57 # main database to see if anything was deleted, because it would appear in the main database but not the watched one.
59 # This means that functions like <ChangeDefinition()> and <DeleteDefinition()> should only be called by
60 # <AnalyzeWatchedFileChanges()>. Externally only <AddDefinition()> should be called. <DeleteItem()> is okay to be
61 # called externally because entire items aren't managed by the watched file database, only definitions.
64 ###############################################################################
66 # This file is part of Natural Docs, which is Copyright © 2003-2010 Greg Valure
67 # Natural Docs is licensed under version 3 of the GNU Affero General Public License (AGPL)
68 # Refer to License.txt for the complete details
74 use NaturalDocs::SourceDB::Extension;
75 use NaturalDocs::SourceDB::Item;
76 use NaturalDocs::SourceDB::ItemDefinition;
77 use NaturalDocs::SourceDB::File;
78 use NaturalDocs::SourceDB::WatchedFileDefinitions;
81 package NaturalDocs::SourceDB;
84 ###############################################################################
91 # A unique identifier for each <NaturalDocs::SourceDB> extension as given out by <RegisterExtension()>.
96 ###############################################################################
103 # An array of <NaturalDocs::SourceDB::Extension>-derived extensions, as added with <RegisterExtension()>. The indexes
104 # are the <ExtensionIDs> and the values are package references.
109 # array: extensionUsesDefinitionObjects
111 # An array where the indexes are <ExtensionIDs> and the values are whether that extension uses its own definition class
112 # derived from <NaturalDocs::SourceDB::ItemDefinition> or it just tracks their existence.
114 my @extensionUsesDefinitionObjects;
121 # The array of source items. The <ExtensionIDs> are the indexes, and the values are hashrefs mapping the item
122 # string to <NaturalDocs::SourceDB::Item>-derived objects. Hashrefs may be undef.
130 # A hashref mapping source <FileNames> to <NaturalDocs::SourceDB::Files>.
136 # object: watchedFile
138 # When a file is being watched for changes, will be a <NaturalDocs::SourceDB::File> for that file. Is undef otherwise.
140 # When the file is parsed, items are added to both this and the version in <files>. Thus afterwards we can compare the two to
141 # see if any were deleted since the last time Natural Docs was run, because they would be in the <files> version but not this
148 # string: watchedFileName
150 # When a file is being watched for changes, will be the <FileName> of the file being watched. Is undef otherwise.
156 # object: watchedFileDefinitions
158 # When a file is being watched for changes, will be a <NaturalDocs::SourceDB::WatchedFileDefinitions> object. Is undef
161 # When the file is parsed, items are added to both this and the version in <items>. Since only the first definition is kept, this
162 # will always have the definition info from the file whereas the version in <items> will have the first definition as of the last time
163 # Natural Docs was run. Thus they can be compared to see if the definitions of items that existed the last time around have
166 my $watchedFileDefinitions;
170 ###############################################################################
171 # Group: Extension Functions
175 # Function: RegisterExtension
177 # Registers a <NaturalDocs::SourceDB::Extension>-derived package and returns a unique <ExtensionID> for it. All extensions
178 # must call this before they can be used.
180 # Registration Order:
182 # The order in which extensions register is important. Whenever possible, items are added in the order their extensions
183 # registered. However, items are changed and deleted in the reverse order. Take advantage of this to minimize
184 # churn between extensions that are dependent on each other.
186 # For example, when symbols are added or deleted they may cause references to be retargeted and thus their files need to
187 # be rebuilt. However, adding or deleting references never causes the symbols' files to be rebuilt. So it makes sense that
188 # symbols should be created before references, and that references should be deleted before symbols.
192 # extension - The package or object of the extension. Must be derived from <NaturalDocs::SourceDB::Extension>.
193 # usesDefinitionObjects - Whether the extension uses its own class derived from <NaturalDocs::SourceDB::ItemDefinition>
194 # or simply tracks each definitions existence.
198 # An <ExtensionID> unique to the extension. This should be saved because it's required in functions such as <AddItem()>.
200 sub RegisterExtension #(package extension, bool usesDefinitionObjects) => ExtensionID
202 my ($self, $extension, $usesDefinitionObjects) = @_;
204 push @extensions, $extension;
205 push @extensionUsesDefinitionObjects, $usesDefinitionObjects;
207 return scalar @extensions - 1;
213 ###############################################################################
214 # Group: File Functions
220 # Loads the data of the source database and all the extensions. Will call <NaturalDocs::SourceDB::Extension->Load()> for
221 # all of them, unless there's a situation where all the source files are going to be reparsed anyway in which case it's not needed.
227 # No point loading if RebuildData is set.
228 if (!NaturalDocs::Settings->RebuildData())
230 # If any load fails, stop loading the rest and just reparse all the source files.
233 for (my $extension = 0; $extension < scalar @extensions && $success; $extension++)
235 $success = $extensions[$extension]->Load();
239 { NaturalDocs::Project->ReparseEverything(); };
247 # Saves the data of the source database and all its extensions. Will call <NaturalDocs::SourceDB::Extension->Save()> for all
254 for (my $extension = scalar @extensions - 1; $extension >= 0; $extension--)
256 $extensions[$extension]->Save();
262 # Function: PurgeDeletedSourceFiles
264 # Removes all data associated with deleted source files.
266 sub PurgeDeletedSourceFiles
270 my $filesToPurge = NaturalDocs::Project->FilesToPurge();
272 # Extension is the outermost loop because we want the extensions added last to have their definitions removed first to cause
273 # the least amount of churn between interdependent extensions.
274 for (my $extension = scalar @extensions - 1; $extension >= 0; $extension--)
276 foreach my $file (keys %$filesToPurge)
278 if (exists $files{$file})
280 my @items = $files{$file}->ListItems($extension);
282 foreach my $item (@items)
284 $self->DeleteDefinition($extension, $item, $file);
295 ###############################################################################
296 # Group: Item Functions
302 # Adds the passed item to the database. This will not work if the item string already exists. The item added should *not*
303 # already have definitions attached. Only use this to add blank items and then call <AddDefinition()> instead.
307 # extension - An <ExtensionID>.
308 # itemString - The string serving as the item identifier.
309 # item - An object derived from <NaturalDocs::SourceDB::Item>.
313 # Whether the item was added, that is, whether it was the first time this item was added.
315 sub AddItem #(ExtensionID extension, string itemString, NaturalDocs::SourceDB::Item item) => bool
317 my ($self, $extension, $itemString, $item) = @_;
319 if (!defined $items[$extension])
320 { $items[$extension] = { }; };
322 if (!exists $items[$extension]->{$itemString})
324 if ($item->HasDefinitions())
325 { die "Tried to add an item to SourceDB that already had definitions."; };
327 $items[$extension]->{$itemString} = $item;
338 # Returns the <NaturalDocs::SourceDB::Item>-derived object for the passed <ExtensionID> and item string, or undef if there
341 sub GetItem #(ExtensionID extension, string itemString) => bool
343 my ($self, $extensionID, $itemString) = @_;
345 if (defined $items[$extensionID])
346 { return $items[$extensionID]->{$itemString}; }
353 # Function: DeleteItem
355 # Deletes the record of the passed <ExtensionID> and item string. Do *not* delete items that still have definitions. Use
356 # <DeleteDefinition()> first.
360 # extension - The <ExtensionID>.
361 # itemString - The item's identifying string.
365 # Whether it was successful, meaning whether an entry existed for it.
367 sub DeleteItem #(ExtensionID extension, string itemString) => bool
369 my ($self, $extension, $itemString) = @_;
371 if (defined $items[$extension] && exists $items[$extension]->{$itemString})
373 if ($items[$extension]->{$itemString}->HasDefinitions())
374 { die "Tried to delete an item from SourceDB that still has definitions."; };
376 delete $items[$extension]->{$itemString};
387 # Returns whether there is an item defined for the passed <ExtensionID> and item string.
389 sub HasItem #(ExtensionID extension, string itemString) => bool
391 my ($self, $extension, $itemString) = @_;
393 if (defined $items[$extension])
394 { return (exists $items[$extension]->{$itemString}); }
401 # Function: GetAllItemsHashRef
403 # Returns a hashref of all the items defined for an extension. *Do not change the contents.* The keys are the item strings and
404 # the values are <NaturalDocs::SourceDB::Items> or derived classes.
406 sub GetAllItemsHashRef #(ExtensionID extension) => hashref
408 my ($self, $extension) = @_;
409 return $items[$extension];
414 ###############################################################################
415 # Group: Definition Functions
419 # Function: AddDefinition
421 # Adds a definition to an item. Assumes the item was already created with <AddItem()>. If there's already a definition for this
422 # file in the item, the new definition will be ignored.
426 # extension - The <ExtensionID>.
427 # itemString - The item string.
428 # file - The <FileName> the definition is in.
429 # definition - If you're using a custom <NaturalDocs::SourceDB::ItemDefinition> class, you must include an object for it here.
430 # Otherwise this parameter is ignored.
434 # Whether the definition was added, which is to say, whether this was the first definition for the passed <FileName>.
436 sub AddDefinition #(ExtensionID extension, string itemString, FileName file, optional NaturalDocs::SourceDB::ItemDefinition definition) => bool
438 my ($self, $extension, $itemString, $file, $definition) = @_;
443 my $item = $self->GetItem($extension, $itemString);
446 { die "Tried to add a definition to an undefined item in SourceDB."; };
448 if (!$extensionUsesDefinitionObjects[$extension])
449 { $definition = 1; };
451 my $result = $item->AddDefinition($file, $definition);
456 if (!exists $files{$file})
457 { $files{$file} = NaturalDocs::SourceDB::File->New(); };
459 $files{$file}->AddItem($extension, $itemString);
464 if ($self->WatchingFileForChanges())
466 $watchedFile->AddItem($extension, $itemString);
468 if ($extensionUsesDefinitionObjects[$extension])
469 { $watchedFileDefinitions->AddDefinition($extension, $itemString, $definition); };
478 # Function: ChangeDefinition
480 # Changes the definition of an item. This function is only used for extensions that use custom
481 # <NaturalDocs::SourceDB::ItemDefinition>-derived classes.
485 # extension - The <ExtensionID>.
486 # itemString - The item string.
487 # file - The <FileName> the definition is in.
488 # definition - The definition, which must be an object derived from <NaturalDocs::SourceDB::ItemDefinition>.
490 sub ChangeDefinition #(ExtensionID extension, string itemString, FileName file, NaturalDocs::SourceDB::ItemDefinition definition)
492 my ($self, $extension, $itemString, $file, $definition) = @_;
494 my $item = $self->GetItem($extension, $itemString);
497 { die "Tried to change the definition of an undefined item in SourceDB."; };
499 if (!$extensionUsesDefinitionObjects[$extension])
500 { die "Tried to change the definition of an item in an extension that doesn't use definition objects in SourceDB."; };
502 if (!$item->HasDefinition($file))
503 { die "Tried to change a definition that doesn't exist in SourceDB."; };
505 $item->ChangeDefinition($file, $definition);
506 $extensions[$extension]->OnChangedDefinition($itemString, $file);
511 # Function: GetDefinition
513 # If the extension uses custom <NaturalDocs::SourceDB::ItemDefinition> classes, returns it for the passed definition or undef
514 # if it doesn't exist. Otherwise returns whether it exists.
516 sub GetDefinition #(ExtensionID extension, string itemString, FileName file) => NaturalDocs::SourceDB::ItemDefinition or bool
518 my ($self, $extension, $itemString, $file) = @_;
520 my $item = $self->GetItem($extension, $itemString);
525 return $item->GetDefinition($file);
530 # Function: DeleteDefinition
532 # Removes the definition for the passed item. Returns whether it was successful, meaning whether a definition existed for that
535 sub DeleteDefinition #(ExtensionID extension, string itemString, FileName file) => bool
537 my ($self, $extension, $itemString, $file) = @_;
539 my $item = $self->GetItem($extension, $itemString);
544 my $result = $item->DeleteDefinition($file);
548 $files{$file}->DeleteItem($extension, $itemString);
549 $extensions[$extension]->OnDeletedDefinition($itemString, $file, !$item->HasDefinitions());
557 # Function: HasDefinitions
559 # Returns whether there are any definitions for this item.
561 sub HasDefinitions #(ExtensionID extension, string itemString) => bool
563 my ($self, $extension, $itemString) = @_;
565 my $item = $self->GetItem($extension, $itemString);
570 return $item->HasDefinitions();
575 # Function: HasDefinition
577 # Returns whether there is a definition for the passed <FileName>.
579 sub HasDefinition #(ExtensionID extension, string itemString, FileName file) => bool
581 my ($self, $extension, $itemString, $file) = @_;
583 my $item = $self->GetItem($extension, $itemString);
588 return $item->HasDefinition($file);
593 ###############################################################################
594 # Group: Watched File Functions
598 # Function: WatchFileForChanges
600 # Begins watching a file for changes. Only one file at a time can be watched.
602 # This should be called before a file is parsed so the file info goes both into the main database and the watched file info.
603 # Afterwards you call <AnalyzeWatchedFileChanges()> so item deletions and definition changes can be detected.
607 # filename - The <FileName> to watch.
609 sub WatchFileForChanges #(FileName filename)
611 my ($self, $filename) = @_;
613 $watchedFileName = $filename;
614 $watchedFile = NaturalDocs::SourceDB::File->New();
615 $watchedFileDefinitions = NaturalDocs::SourceDB::WatchedFileDefinitions->New();
620 # Function: WatchingFileForChanges
622 # Returns whether we're currently watching a file for changes or not.
624 sub WatchingFileForChanges # => bool
627 return defined $watchedFileName;
632 # Function: AnalyzeWatchedFileChanges
634 # Analyzes the watched file for changes. Will delete and change definitions as necessary.
636 sub AnalyzeWatchedFileChanges
640 if (!$self->WatchingFileForChanges())
641 { die "Tried to analyze watched file for changes in SourceDB when no file was being watched."; };
642 if (!$files{$watchedFileName})
646 # Process extensions last registered to first.
648 for (my $extension = scalar @extensions - 1; $extension >= 0; $extension--)
650 my @items = $files{$watchedFileName}->ListItems($extension);
652 foreach my $item (@items)
654 if ($watchedFile->HasItem($extension, $item))
656 if ($extensionUsesDefinitionObjects[$extension])
658 my $originalDefinition = $items[$extension]->GetDefinition($watchedFileName);
659 my $watchedDefinition = $watchedFileDefinitions->GetDefinition($extension, $item);
661 if (!$originalDefinition->Compare($watchedDefinition))
662 { $self->ChangeDefinition($extension, $item, $watchedFileName, $watchedDefinition); };
665 else # !$watchedFile->HasItem($item)
667 $self->DeleteDefinition($extension, $item, $watchedFileName);
673 $watchedFile = undef;
674 $watchedFileName = undef;
675 $watchedFileDefinitions = undef;