6 * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
7 * Copyright (C) 2009, 2010, 2011, MinGW Project
10 * Implementation of package archive processing methods, for reading
11 * and extracting content from tar archives; provides implementations
12 * for each of the pkgTarArchiveProcessor and pkgTarArchiveInstaller
16 * This is free software. Permission is granted to copy, modify and
17 * redistribute this software, under the provisions of the GNU General
18 * Public License, Version 3, (or, at your option, any later version),
19 * as published by the Free Software Foundation; see the file COPYING
20 * for licensing details.
22 * Note, in particular, that this software is provided "as is", in the
23 * hope that it may prove useful, but WITHOUT WARRANTY OF ANY KIND; not
24 * even an implied WARRANTY OF MERCHANTABILITY, nor of FITNESS FOR ANY
25 * PARTICULAR PURPOSE. Under no circumstances will the author, or the
26 * MinGW Project, accept liability for any damages, however caused,
27 * arising from the use of this software.
33 #include <sys/types.h>
49 * Class Implementation: pkgTarArchiveProcessor
53 pkgTarArchiveProcessor::pkgTarArchiveProcessor( pkgXmlNode *pkg )
55 /* Constructor to associate a package tar archive with its
56 * nominated sysroot and respective installation directory path,
57 * and prepare it for processing, using an appropriate streaming
58 * decompression filter; (choice of filter is based on archive
59 * file name extension; file names are restricted to the
60 * POSIX Portable Character Set).
62 * First, we anticipate an invalid initialisation state...
71 /* The 'pkg' XML database entry must be non-NULL, must
72 * represent a package release, and must specify a canonical
73 * tarname to identify the package...
75 if( ((origin = pkg) != NULL) && pkg->IsElementOfType( release_key )
76 && ((tarname = pkg->GetPropVal( tarname_key, NULL )) != NULL) )
78 /* When these pre-conditions are satisfied, we may proceed
79 * to identify and locate the sysroot record with which this
80 * package is to be associated...
82 pkgSpecs lookup( pkgfile = tarname );
83 if( (sysroot = pkg->GetSysRoot( lookup.GetSubSystemName() )) != NULL )
85 /* Having located the requisite sysroot record, we may
86 * retrieve its specified installation path prefix...
89 if( (prefix = sysroot->GetPropVal( pathname_key, NULL )) != NULL )
91 /* ...and incorporate it into a formatting template
92 * for use in deriving the full path names for files
93 * which are installed from this package.
95 const char *template_format = "%F%%/M/%%F";
96 char template_text[mkpath( NULL, template_format, prefix, NULL )];
97 mkpath( template_text, template_format, prefix, NULL );
98 sysroot_len = mkpath( NULL, template_text, "", NULL ) - 1;
99 sysroot_path = strdup( template_text );
102 /* Some older packages don't use the canonical tarname
103 * for the archive file name; identify the real file name
104 * associated with such packages...
106 pkgfile = pkg->ArchiveName();
108 /* Finally, initialise the data stream which we will use
109 * for reading the package content.
111 const char *archive_path_template = pkgArchivePath();
112 char archive_path_name[mkpath( NULL, archive_path_template, pkgfile, NULL )];
113 mkpath( archive_path_name, archive_path_template, pkgfile, NULL );
114 stream = pkgOpenArchiveStream( archive_path_name );
118 pkgTarArchiveProcessor::~pkgTarArchiveProcessor()
120 /* Destructor must release the heap memory allocated in
121 * the constructor, (by strdup and pkgManifest), clean up
122 * the decompression filter state, and close the archive
125 free( (void *)(sysroot_path) );
130 int pkgTarArchiveProcessor::ProcessLinkedEntity( const char *pathname )
132 /* FIXME: Win32 links need special handling; for hard links, we
133 * may be able to create them directly, with >= Win2K and NTFS;
134 * for symlinks on *all* Win32 variants, and for hard links on
135 * FAT32 or Win9x, we need to make physical copies of the source
136 * file, at the link target location.
138 * For now, we simply ignore links.
141 "FIXME:ProcessLinkedEntity<stub>:Ignoring link: %s --> %s\n",
142 pathname, header.field.linkname
148 uint64_t compute_octval( const char *p, size_t len )
149 # define octval( FIELD ) compute_octval( FIELD, sizeof( FIELD ) )
151 /* Helper to convert the ASCII representation of octal values,
152 * (as recorded within tar archive header fields), to their actual
153 * numeric values, ignoring leading or trailing garbage.
155 uint64_t value = 0LL;
157 while( (len > 0) && ((*p < '0') || (*p > '7')) )
159 /* Step over leading garbage.
163 while( (len > 0) && (*p >= '0') && (*p < '8') )
165 /* Accumulate octal digits; (each represents exactly three
166 * bits in the accumulated value), until we either exhaust
167 * the width of the field, or we encounter trailing junk.
169 value = (value << 3) + *p++ - '0'; --len;
174 int pkgTarArchiveProcessor::GetArchiveEntry()
176 /* Read header for next available entry in the tar archive;
177 * check for end-of-archive mark, (all zero header); verify
178 * checksum for active entry.
180 char *buf = header.aggregate;
181 size_t count = stream->Read( buf, sizeof( header ) );
183 if( count < sizeof( header ) )
185 /* Failed to read a complete header; return error code.
192 * Outer loop checks for an all zero header...
196 /* Any non-zero byte transfers control to an inner loop,
197 * to rescan the entire header, accumulating its checksum...
200 for( buf = header.aggregate, count = sizeof( header ); count--; ++buf )
202 if( (buf < header.field.chksum) || (buf >= header.field.typeflag) )
204 * ...counting the actual binary value of each byte,
205 * in all but the checksum field itself...
209 /* ...while treating each byte within the checksum field as
210 * having an effective value equivalent to ASCII <space>.
214 /* After computing the checksum for a non-zero header,
215 * verify it against the value recorded in the checksum field;
216 * return +1 for a successful match, or -2 for failure.
218 return (sum == octval( header.field.chksum )) ? 1 : -2;
221 /* If we get to here, then the inner loop was never entered;
222 * the outer loop has completed, confirming an all zero header;
223 * return zero, to indicate end of archive.
228 int pkgTarArchiveProcessor::Process()
230 /* Generic method for reading tar archives, and extracting their
231 * content; loops over each archive entry in turn...
233 while( GetArchiveEntry() > 0 )
235 char *prefix = *header.field.prefix ? header.field.prefix : NULL;
236 char *name = header.field.name;
238 /* Handle the GNU long name header format.
239 * If the pathname overflows the name field, GNU tar creates a special
240 * entry type, where the data contains the full pathname for the
243 char *longname = NULL;
244 if( *header.field.typeflag == TAR_ENTITY_TYPE_GNU_LONGNAME )
246 /* Extract the full pathname from the data of this entry.
248 longname = EntityDataAsString();
250 dmh_notify( DMH_ERROR, "Unable to read a long name entry\n" );
252 /* Read the entry for which this long name is intended.
254 if( GetArchiveEntry() <= 0 )
255 dmh_notify( DMH_ERROR, "Expected a new entry after a long name entry\n" );
257 /* Use the previously determined long name as the pathname for this entry.
263 /* Found an archive entry; map it to an equivalent file system
264 * path name, within the designated sysroot hierarchy.
266 char pathname[mkpath( NULL, sysroot_path, name, prefix )];
267 mkpath( pathname, sysroot_path, name, prefix );
271 /* Direct further processing to the appropriate handler; (this
272 * is specific to the archive entry classification)...
274 switch( *header.field.typeflag )
278 case TAR_ENTITY_TYPE_DIRECTORY:
280 * We may need to take some action in respect of directories;
281 * e.g. we may need to create a directory, or even a sequence
282 * of directories, to establish a location within the sysroot
285 { /* Note: Microsoft's implementation of stat() appears to choke
286 * on directory path names with trailing slashes; thus, before
287 * we invoke the directory processing routine, (which may need
288 * to call stat(), to check if the specified directory already
289 * exists), we remove any such trailing slashes.
291 char *p = pathname + sizeof( pathname ) - 1;
292 while( (p > pathname) && ((*--p == '/') || (*p == '\\')) )
296 /* We are now ready to process the directory path name entry...
298 status = ProcessDirectory( pathname );
301 case TAR_ENTITY_TYPE_LINK:
302 case TAR_ENTITY_TYPE_SYMLINK:
304 * Links ultimately represent file system entities in
305 * our sysroot hierarchy, but we need special processing
306 * to handle them correctly...
309 status = ProcessLinkedEntity( pathname );
312 case TAR_ENTITY_TYPE_FILE:
313 case TAR_ENTITY_TYPE_ALTFILE:
315 * These represent regular files; the file content is
316 * embedded within the archive stream, so we need to be
317 * prepared to read or copy it, as appropriate...
320 ProcessDataStream( pathname );
324 /* FIXME: we make no provision for handling any other
325 * type of archive entry; we should provide some more
326 * robust error handling, but for now we simply emit
327 * a diagnostic, and return an error condition code...
330 dmh_notify( DMH_ERROR,
331 "unexpected archive entry classification: type %d\n",
332 (int)(*header.field.typeflag)
337 /* If we didn't bail out before getting to here, then the archive
338 * was processed successfully; return the success code.
343 int pkgTarArchiveProcessor::ProcessEntityData( int fd )
345 /* Generic method for reading past the data associated with
346 * a specific header within a tar archive; if given a negative
347 * value for `fd', it will simply skip over the data, otherwise
348 * `fd' is assumed to represent a descriptor for an opened file
349 * stream, to which the data will be copied (extracted).
353 /* Initialise a counter for the length of the data content, and
354 * specify the default size for the transfer buffer in which to
355 * process it; make the initial size of the transfer buffer 16
356 * times the header size.
358 uint64_t bytes_to_copy = octval( header.field.size );
359 size_t block_size = sizeof( header ) << 4;
361 /* While we still have unread data, and no processing error...
363 while( (bytes_to_copy > 0) && (status == 0) )
365 /* Adjust the requested size for the transfer buffer, shrinking
366 * it by 50% at each step, until it is smaller than the remaining
367 * data length, but never smaller than the header record length.
369 while( (bytes_to_copy < block_size) && (block_size > sizeof( header )) )
372 /* Allocate a transfer buffer of the requested size, and populate
373 * it, by reading data from the archive; (since the transfer buffer
374 * is never smaller than the header length, this will also capture
375 * any additional padding bytes, which may be required to keep the
376 * data length equal to an exact multiple of the header length).
378 char buffer[block_size];
379 if( stream->Read( buffer, block_size ) < (int)(block_size) )
381 * Failure to fully populate the transfer buffer, (i.e. a short
382 * read), indicates a corrupt archive; bail out immediately.
386 /* When the number of actual data bytes expected is fewer than the
387 * total number of bytes in the transfer buffer...
389 if( bytes_to_copy < block_size )
391 * ...then we have reached the end of the data for the current
392 * archived entity; adjust the block size to reflect the number
393 * of actual data bytes present in the transfer buffer...
395 block_size = bytes_to_copy;
397 /* With the number of actual data bytes present now accurately
398 * reflected by the block size, we save that data to the stream
399 * specified for archive extraction, (if any).
401 if( (fd >= 0) && (write( fd, buffer, block_size ) != (int)(block_size)) )
403 * An extraction error occurred; set the status code to
408 /* Adjust the count of remaining unprocessed data bytes, and begin
409 * a new processing cycle, to capture any which may be present.
411 bytes_to_copy -= block_size;
414 /* Finally, when all data for the current archive entry has been
415 * processed, we return to the caller with an appropriate completion
421 char *pkgTarArchiveProcessor::EntityDataAsString()
423 /* Read the data associated with a specific header within a tar archive
424 * and return it as a string. The return value is stored in memory which
425 * is allocated by malloc; it should be freed when no longer required.
427 * It is assumed that the return data can be accommodated within available
428 * heap memory. Since the length isn't returned, we assume that the string
429 * is NUL-terminated, and that it contains no embedded NULs.
431 * In the event of any error, NULL is returned.
434 uint64_t bytes_to_copy = octval( header.field.size );
436 /* Round the buffer size to the smallest multiple of the record size.
438 bytes_to_copy += sizeof( header ) - 1;
439 bytes_to_copy -= bytes_to_copy % sizeof( header );
441 /* Allocate the data buffer.
443 data = (char*)(malloc( bytes_to_copy ));
447 /* Read the data into the buffer.
449 size_t count = stream->Read( data, bytes_to_copy );
450 if( count < bytes_to_copy )
452 /* Failure to fully populate the transfer buffer, (i.e. a short
453 * read), indicates a corrupt archive.
463 * Class Implementation: pkgTarArchiveInstaller
468 static int commit_saved_entity( const char *pathname, time_t mtime )
470 /* Helper to set the access and modification times for a file,
471 * after extraction from an archive, to match the specified "mtime";
472 * (typically "mtime" is as recorded within the archive).
474 struct utimbuf timestamp;
476 timestamp.actime = timestamp.modtime = mtime;
477 return utime( pathname, ×tamp );
480 pkgTarArchiveInstaller::
481 pkgTarArchiveInstaller( pkgXmlNode *pkg ):pkgTarArchiveProcessor( pkg )
483 /* Constructor: having successfully set up the pkgTarArchiveProcessor
484 * base class, we attach a pkgManifest to track the installation.
486 if( (tarname != NULL) && (sysroot != NULL) && stream->IsReady() )
487 installed = new pkgManifest( package_key, tarname );
490 int pkgTarArchiveInstaller::Process()
492 /* Specialisation of the base class Process() method.
495 /* First, process the archive as for the base class...
497 if( (status = pkgTarArchiveProcessor::Process()) == 0 )
499 /* ...then, on successful completion...
501 * Update the package installation manifest, to record
502 * the installation in the current sysroot...
504 installed->BindSysRoot( sysroot, package_key );
505 pkgRegister( sysroot, origin, tarname, pkgfile );
510 int pkgTarArchiveInstaller::ProcessDirectory( const char *pathname )
512 /* Create the directory infrastructure required to support
513 * a specific package installation.
515 #if DEBUGLEVEL & DEBUG_SUPPRESS_INSTALLATION
518 * FIXME:maybe adapt for 'dry-run' or 'verbose' use.
522 "FIXME:ProcessDirectory<stub>:not executing: mkdir -p %s\n",
525 # if DEBUGLEVEL & DEBUG_UPDATE_INVENTORY
527 * Although no installation directory has actually been created,
528 * update the inventory to simulate the effect of doing so.
530 installed->AddEntry( dirname_key, pathname + sysroot_len );
535 if( (status = mkdir_recursive( pathname, 0755 )) == 0 )
537 * Either the specified directory already exists,
538 * or we just successfully created it; attach a reference
539 * in the installation manifest for the current package.
541 installed->AddEntry( dirname_key, pathname + sysroot_len );
544 /* A required subdirectory could not be created;
545 * diagnose this failure.
547 dmh_notify( DMH_ERROR, "cannot create directory `%s'\n", pathname );
553 int pkgTarArchiveInstaller::ProcessDataStream( const char *pathname )
555 /* Extract file data from the archive, and copy it to the
556 * associated target file stream, if any.
558 #if DEBUGLEVEL & DEBUG_SUPPRESS_INSTALLATION
561 * FIXME:maybe adapt for 'dry-run' or 'verbose' use.
564 "FIXME:ProcessDataStream<stub>:not extracting: %s\n",
567 # if DEBUGLEVEL & DEBUG_UPDATE_INVENTORY
569 * Although no file has actually been installed, update
570 * the inventory to simulate the effect of doing so.
572 installed->AddEntry( filename_key, pathname + sysroot_len );
574 return ProcessEntityData( -1 );
577 int fd = set_output_stream( pathname, octval( header.field.mode ) );
578 int status = ProcessEntityData( fd );
581 /* File stream was written; close it...
586 /* ...and on successful completion, commit it and
587 * record it in the installation database.
589 commit_saved_entity( pathname, octval( header.field.mtime ) );
590 installed->AddEntry( filename_key, pathname + sysroot_len );
594 { /* The target file was not successfully and completely
595 * written; discard it, and diagnose failure.
598 dmh_notify( DMH_ERROR, "%s: extraction failed\n", pathname );
605 /* $RCSfile$: end of file */