6 * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
7 * Copyright (C) 2009, 2010, MinGW Project
10 * Implementation of the streaming data filters, which will be used
11 * for reading package archives in any supported compression format;
12 * currently supported formats are:--
21 * This is free software. Permission is granted to copy, modify and
22 * redistribute this software, under the provisions of the GNU General
23 * Public License, Version 3, (or, at your option, any later version),
24 * as published by the Free Software Foundation; see the file COPYING
25 * for licensing details.
27 * Note, in particular, that this software is provided "as is", in the
28 * hope that it may prove useful, but WITHOUT WARRANTY OF ANY KIND; not
29 * even an implied WARRANTY OF MERCHANTABILITY, nor of FITNESS FOR ANY
30 * PARTICULAR PURPOSE. Under no circumstances will the author, or the
31 * MinGW Project, accept liability for any damages, however caused,
32 * arising from the use of this software.
40 * MS-Windows nuisances...
41 * Files are expected to be either explicitly text or binary;
42 * (UNIX makes no such specific distinction). We want to force
43 * treatment of all files as binary; define a "no-op" substitute
44 * for the appropriate MS-Windows attribute, for when we compile
45 * on UNIX, so we may henceforth just use it unconditionally.
48 # define O_BINARY _O_BINARY
54 /* We need to enable PKGSTRM_H_SPECIAL awareness, when we compile this...
56 #define PKGSTRM_H_SPECIAL 1
61 * Class Implementation: pkgArchiveStream
63 * This class uses a default constructor and default virtual destructor.
64 * We never instantiate objects of this class directly; all derived classes
65 * provide their own specialised constructors and destructors, together with
66 * a mandatory specialised "Read" method.
68 * We do, however, provide one generic "GetRawData" method, which derived
69 * classes may adopt, or may override, as necessary...
72 int pkgArchiveStream::GetRawData( int fd, uint8_t *buf, size_t max )
74 /* Generic helper function for reading a compressed data stream into
75 * its decompressing filter's input buffer. The default implementation
76 * assumes a file stream, and simply invokes a read() request; however,
77 * we segregate this function, to facilitate an override to handle
78 * other input streaming capabilities.
80 return read( fd, buf, max );
85 * Class Implementation: pkgRawArchiveStream
87 * This is the simplest archive stream class, suitable for archives
88 * which have been stored WITHOUT compression...
91 pkgRawArchiveStream::pkgRawArchiveStream( const char *filename )
93 /* The constructor has little to to, but to open the archive file
94 * and associate a file descriptor with the resultant data stream.
96 fd = open( filename, O_RDONLY | O_BINARY );
99 pkgRawArchiveStream::~pkgRawArchiveStream()
101 /* The destructor needs only to close the data stream.
106 int pkgRawArchiveStream::Read( char *buf, size_t max )
108 /* While the stream reader simply transfers the requested number
109 * of bytes from the stream, to the caller's buffer.
111 return read( fd, buf, max );
116 * Class Implementation: pkgGzipArchiveStream
118 * This class creates an input streaming interface, suitable for
119 * reading archives which have been stored with gzip compression.
120 * The implementation is based on the use of libz.a, which allows
121 * for a similar implementation to that of pkgRawArchiveStream.
124 pkgGzipArchiveStream::pkgGzipArchiveStream( const char *filename )
126 /* Once more, the constructor has little to do but open the stream;
127 * in this case, the method is analogous to C's fopen().
129 stream = gzopen( filename, "rb" );
132 pkgGzipArchiveStream::~pkgGzipArchiveStream()
134 /* Another destructor, with little to do but close the stream; the
135 * gzclose() call suffices for the purpose.
140 int pkgGzipArchiveStream::Read( char *buf, size_t max )
142 /* The reader is again served by a single function call, to transfer
143 * the requested volume of decompressed data from the raw input file
144 * to the caller's buffer.
146 return gzread( stream, buf, max );
151 * Class Implementation: pkgBzipArchiveStream
153 * This class creates an input streaming interface, suitable for
154 * reading archives which have been stored with bzip2 compression.
155 * The implementation is based on the use of libbz2.a, which again
156 * allows for a fairly simple implementation, which is also quite
157 * analogous to that of pkgRawArchiveStream.
160 pkgBzipArchiveStream::pkgBzipArchiveStream( const char *filename )
162 /* The constructor carries a marginal additional overhead, in
163 * that it must first open a regular file, before associating
164 * a bzip2 control structure with it; subsequent stream access
165 * is directed exclusively through that control structure.
167 FILE *streamfile = fopen( filename, "rb" );
168 stream = BZ2_bzReadOpen( &bzerror, streamfile, 0, 0, 0, 0 );
171 pkgBzipArchiveStream::~pkgBzipArchiveStream()
173 /* For the destructor, it is again just a matter of closing
174 * the bzip2 stream; (this also takes care of closing the
175 * associated file stream).
177 BZ2_bzReadClose( &bzerror, stream );
180 int pkgBzipArchiveStream::Read( char *buf, size_t max )
182 /* Once again, reading is a simple matter of transferring
183 * the requisite number of bytes to the caller's buffer.
185 return BZ2_bzRead( &bzerror, stream, buf, max );
190 * Class Implementation: pkgLzmaArchiveStream
192 * This class creates an input streaming interface, suitable for
193 * reading archives which have been stored with lzma compression;
194 * based on the use of liblzma.a, this implements an adaptation of
195 * Lasse Collin's "xzdec" code, as configured for use as an lzma
202 /* Naively cap the memory available to lzma and xz decoders.
204 * FIXME: libarchive appears to use this; however, Lasse Collin
205 * provides a more sophisticated method for xz, based on actual
206 * physical memory footprint; we should adopt it.
208 return 1ULL << 23 + 1ULL << 21;
212 void lzma_stream_initialise( lzma_stream *stream )
214 /* This simple helper provides a static template, which is
215 * used to define initial state for lzma and xz decoders.
217 static const lzma_stream stream_template = LZMA_STREAM_INIT;
218 *stream = stream_template;
220 * ...mark the input buffer as initially empty.
222 stream->avail_in = 0;
225 pkgLzmaArchiveStream::pkgLzmaArchiveStream( const char *filename )
227 /* The constructor must first open a file stream...
229 if( (fd = open( filename, O_RDONLY | O_BINARY )) >= 0 )
231 /* ...then set up the lzma decoder, in appropriately
232 * initialised state...
234 lzma_stream_initialise( &stream );
235 status = lzma_alone_decoder( &stream, memlimit() );
239 pkgLzmaArchiveStream::pkgLzmaArchiveStream( int fileno ):fd( fileno )
241 /* ...then set up the lzma decoder, in appropriately
242 * initialised state...
246 lzma_stream_initialise( &stream );
247 status = lzma_alone_decoder( &stream, memlimit() );
251 pkgLzmaArchiveStream::~pkgLzmaArchiveStream()
253 /* The destructor frees memory resources allocated to the decoder,
254 * and closes the input stream file descriptor.
256 * FIXME: The lzma_alone_decoder may indicate end-of-stream, before
257 * the physical input data stream is exhausted. For now, we silently
258 * ignore any such residual data; (it is likely to be garbage anyway).
259 * Should we handle it any more explicitly?
268 int pkgLzmaArchiveStream::Read( char *buf, size_t max )
270 /* Read an lzma compressed data stream; store up to "max" bytes of
271 * decompressed data into "buf".
275 * We cannot read from a stream with an invalid descriptor;
276 * in this circumstance, just say "nothing was read"...
280 /* Otherwise the stream is ready to read...
281 * Start by directing the decoder to use "buf", initially marking it
284 stream.next_out = (uint8_t *)(buf);
285 stream.avail_out = max;
287 while( (stream.avail_out > 0) && (status == LZMA_OK) )
289 /* "buf" hasn't been filled yet, and the decoder continues to say
290 * that more data may be available.
292 if( stream.avail_in == 0 )
294 /* We exhausted the current content of the raw input buffer;
297 stream.next_in = streambuf;
298 if( (stream.avail_in = GetRawData( fd, streambuf, BUFSIZ )) < 0 )
300 /* FIXME: an I/O error occurred here: need to handle it!!!
305 /* Run the decoder, to decompress as much as possible of the data
306 * currently in the raw input buffer, filling available space in
307 * "buf"; go round again, in case we exhausted the raw input data
308 * before we ran out of available space in "buf".
310 status = lzma_code( &stream, LZMA_RUN );
313 /* When we get to here, we either filled "buf" completely, or we
314 * completely exhausted the raw input stream; in either case, we
315 * return the actual number of bytes stored in "buf", (i.e. its
316 * total size, less any residual free space).
318 return max - stream.avail_out;
323 * Class Implementation: pkgXzArchiveStream
325 * This class creates an input streaming interface, suitable for
326 * reading archives which have been stored with xz compression;
327 * again based on the use of liblzma.a, this implements a further
328 * adaptation of Lasse Collin's "xzdec" code, as configured for
329 * use as an xz decompressor.
332 pkgXzArchiveStream::pkgXzArchiveStream( const char *filename )
334 /* The constructor must first open a file stream...
336 if( (fd = open( filename, O_RDONLY | O_BINARY )) >= 0 )
338 /* ...then set up the lzma decoder, in appropriately
339 * initialised state...
341 lzma_stream_initialise( &stream );
342 status = lzma_stream_decoder( &stream, memlimit(), LZMA_CONCATENATED );
344 /* Finally, recognising that with LZMA_CONCATENATED data,
345 * we will eventually need to switch the decoder from its
346 * initial LZMA_RUN state to LZMA_FINISH, we must provide
347 * a variable to specify the active state, (which we may
348 * initialise for the LZMA_RUN state).
354 pkgXzArchiveStream::~pkgXzArchiveStream()
356 /* This destructor frees memory resources allocated to the decoder,
357 * and closes the input stream file descriptor; unlike the preceding
358 * case of the lzma_alone_decoder, the lzma_stream_decoder guarantees
359 * that there is no trailing garbage remaining from the input stream.
368 int pkgXzArchiveStream::Read( char *buf, size_t max )
370 /* Read an xz compressed data stream; store up to "max" bytes of
371 * decompressed data into "buf".
375 * We cannot read from a stream with an invalid descriptor;
376 * in this circumstance, just say "nothing was read"...
380 /* Otherwise the stream is ready to read...
381 * Start by directing the decoder to use "buf", initially marking it
384 stream.next_out = (uint8_t *)(buf);
385 stream.avail_out = max;
387 while( (stream.avail_out > 0) && (status == LZMA_OK) )
389 /* "buf" hasn't been filled yet, and the decoder continues to say
390 * that more data may be available.
392 if( stream.avail_in == 0 )
394 /* We exhausted the current content of the raw input buffer;
397 stream.next_in = streambuf;
398 if( (stream.avail_in = GetRawData( fd, streambuf, BUFSIZ )) < 0 )
400 /* FIXME: an I/O error occurred here: need to handle it!!!
404 else if( stream.avail_in < BUFSIZ )
406 /* A short read indicates end-of-input...
407 * Unlike the case of the lzma_alone_decoder, (as used for
408 * decompressing lzma streams), the lzma_stream_decoder, (when
409 * initialised for LZMA_CONCATENATED data, as we use here), may
410 * run lzma_code in either LZMA_RUN or LZMA_FINISH mode; the
411 * normal mode is LZMA_RUN, but we switch to LZMA_FINISH
412 * when we have exhausted the input stream.
414 opmode = LZMA_FINISH;
418 /* Run the decoder, to decompress as much as possible of the data
419 * currently in the raw input buffer, filling available space in
420 * "buf"; as noted above, "opmode" will be LZMA_RUN, until we have
421 * exhausted the input stream, when it becomes LZMA_FINISH.
423 status = lzma_code( &stream, opmode );
425 /* We need to go round again, in case we exhausted the raw input
426 * data before we ran out of available space in "buf", except...
428 if( (status == LZMA_OK) && (opmode == LZMA_FINISH) )
430 * ...when we've already achieved the LZMA_FINISH state,
431 * this becomes unnecessary, so we break the cycle.
436 /* When we get to here, we either filled "buf" completely, or we
437 * completely exhausted the raw input stream; in either case, we
438 * return the actual number of bytes stored in "buf", (i.e. its
439 * total size, less any residual free space).
441 return max - stream.avail_out;
446 * Auxiliary function: pkgOpenArchiveStream()
448 * NOTE: Keep this AFTER the class specialisations, so that their derived
449 * class declarations are visible for object instantiation here!
455 extern "C" pkgArchiveStream* pkgOpenArchiveStream( const char* filename )
457 /* Naive decompression filter selection, based on file name extension.
459 * FIXME: adopt more proactive selection method, (similar to that used
460 * by libarchive, perhaps), based on magic patterns within the file.
462 * NOTE: MS-Windows may use UNICODE file names, but distributed package
463 * archives almost certainly do not. For our purposes, use of the POSIX
464 * Portable Character Set should suffice; we offer no concessions for
465 * any usage beyond this.
467 char *ext = strrchr( filename, '.' );
470 if( strcasecmp( ext, ".gz" ) == 0 )
472 * We expect this input stream to be "gzip" compressed,
473 * so we return the appropriate decompressor.
475 return new pkgGzipArchiveStream( filename );
477 else if( strcasecmp( ext, ".bz2" ) == 0 )
479 * We expect this input stream to be "bzip2" compressed,
480 * so again, we return the appropriate decompressor.
482 return new pkgBzipArchiveStream( filename );
484 else if( strcasecmp( ext, ".lzma" ) == 0 )
486 * We expect this input stream to be "lzma" compressed,
487 * so again, we return the appropriate decompressor.
489 return new pkgLzmaArchiveStream( filename );
491 else if( strcasecmp( ext, ".xz" ) == 0 )
493 * We expect this input stream to be "xz" compressed,
494 * so again, we return the appropriate decompressor.
496 return new pkgXzArchiveStream( filename );
499 /* If we get to here, then we didn't recognise any of the standard
500 * compression indicating file name extensions; fall through, to
501 * process the stream as raw (uncompressed) data.
503 return new pkgRawArchiveStream( filename );
506 /* $RCSfile$: end of file */