6 * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
7 * Copyright (C) 2009, 2010, 2011, 2012, MinGW Project
10 * Implementation of the package download machinery for mingw-get.
13 * This is free software. Permission is granted to copy, modify and
14 * redistribute this software, under the provisions of the GNU General
15 * Public License, Version 3, (or, at your option, any later version),
16 * as published by the Free Software Foundation; see the file COPYING
17 * for licensing details.
19 * Note, in particular, that this software is provided "as is", in the
20 * hope that it may prove useful, but WITHOUT WARRANTY OF ANY KIND; not
21 * even an implied WARRANTY OF MERCHANTABILITY, nor of FITNESS FOR ANY
22 * PARTICULAR PURPOSE. Under no circumstances will the author, or the
23 * MinGW Project, accept liability for any damages, however caused,
24 * arising from the use of this software.
27 #define WIN32_LEAN_AND_MEAN
29 #define _WIN32_WINNT 0x0500 /* for GetConsoleWindow() kludge */
32 * FIXME: This kludge allows us to use the standard wininet dialogue
33 * to acquire proxy authentication credentials from the user; this is
34 * expedient for now, (if somewhat anti-social for a CLI application).
35 * We will ultimately need to provide a more robust implementation,
36 * (within the scope of the diagnostic message handler), in order to
37 * obtain a suitable window handle for use when called from the GUI
38 * implementation of mingw-get, (when it becomes available).
40 #define dmh_dialogue_context() GetConsoleWindow()
56 class pkgDownloadMeter
58 /* Abstract base class, from which facilities for monitoring the
59 * progress of file downloads may be derived.
62 /* The working method to refresh the download progress display;
63 * each derived class MUST furnish an implementation for this.
65 virtual int Update( unsigned long length ) = 0;
68 /* Storage for the expected size of the active download...
70 unsigned long content_length;
72 /* ...and a method to format it for human readable display.
74 int SizeFormat( char*, unsigned long );
77 class pkgDownloadMeterTTY : public pkgDownloadMeter
79 /* Implementation of a download meter class, displaying download
80 * statistics within a CLI application context.
83 pkgDownloadMeterTTY( const char*, unsigned long );
84 virtual ~pkgDownloadMeterTTY();
86 virtual int Update( unsigned long );
89 /* This buffer is used to store each compiled status report,
90 * in preparation for display on the console.
92 const char *source_url;
93 char status_report[80];
96 pkgDownloadMeterTTY::pkgDownloadMeterTTY( const char *url, unsigned long length )
99 content_length = length;
102 pkgDownloadMeterTTY::~pkgDownloadMeterTTY()
104 if( source_url == NULL )
108 static inline __attribute__((__always_inline__))
109 unsigned long pow10mul( register unsigned long x, unsigned power )
111 /* Inline helper to perform a fast integer multiplication of "x"
112 * by a specified power of ten.
116 /* Each cycle multiplies by ten, first shifting to get "x * 2"...
120 * ...then further shifting that intermediate result, and adding,
121 * to achieve the effect of "x * 2 + x * 2 * 4".
128 static inline __attribute__((__always_inline__))
129 unsigned long percentage( unsigned long x, unsigned long q )
131 /* Inline helper to compute "x" as an integer percentage of "q".
133 return pow10mul( x, 2 ) / q;
136 int pkgDownloadMeterTTY::Update( unsigned long count )
138 /* Implementation of method to update the download progress report,
139 * displaying the current byte count and anticipated final byte count,
140 * each formatted in a conveniently human readable style, followed by
141 * approximate percentage completion, both as a 48-segment bar graph,
142 * and as a numeric tally.
144 char *p = status_report;
145 int barlen = (content_length > count)
146 ? (((count << 1) + count) << 4) / content_length
147 : (content_length > 0) ? 48 : 0;
149 /* We may safely use sprintf() to assemble the status report, because
150 * we control the field lengths to always fit within the buffer.
152 p += SizeFormat( p, count );
153 p += sprintf( p, " / " );
154 p += (content_length > 0)
155 ? SizeFormat( p, content_length )
156 : sprintf( p, "????.?? ??" );
157 p += sprintf( p, "%*c", status_report + 25 - p, '|' ); p[barlen] = '\0';
158 p += sprintf( p, "%-48s", (char *)(memset( p, '=', barlen )) );
159 p += ( (content_length > 0) && (content_length >= count) )
160 ? sprintf( p, "|%4lu", percentage( count, content_length ) )
161 : sprintf( p, "| ???" );
163 if( source_url != NULL )
165 dmh_printf( "%s\n", source_url );
168 return dmh_printf( "\r%s%%", status_report );
171 int pkgDownloadMeter::SizeFormat( char *buf, unsigned long filesize )
173 /* Helper method to format raw byte counts as B, kB, MB, GB, TB, as
174 * appropriate; (this NEVER requires more than ten characters).
177 const unsigned long sizelimit = 1 << 10; /* 1k */
179 if( filesize < sizelimit )
181 * Raw byte count is less than 1 kB; we may simply emit it.
183 retval = sprintf( buf, "%lu B", filesize );
187 /* The raw byte count is too large to be readily assimilated; we
188 * scale it down to an appropriate value, and append the appropriate
191 unsigned long frac = filesize;
192 const unsigned long fracmask = sizelimit - 1;
193 const char *scale_indicator = "kMGT";
195 /* A ten bit right shift scales by a factor of 1k; continue shifting
196 * until the ultimate value is less than 1k...
198 while( (filesize >>= 10) >= sizelimit )
200 /* ...noting the residual at each shift, and adjusting the scaling
201 * indicator selection to suit.
206 /* Finally, emit the scaled result to the nearest one hundredth part
207 * of the ultimate scaling unit.
210 ( buf, "%lu.%02lu %cB",
211 filesize, percentage( frac & fracmask, sizelimit ), *scale_indicator
217 class pkgInternetAgent
219 /* A minimal, locally implemented class, instantiated ONCE as a
220 * global object, to ensure that wininet's global initialisation is
221 * completed at the proper time, without us doing it explicitly.
224 HINTERNET SessionHandle;
227 inline pkgInternetAgent():SessionHandle( NULL )
231 * This is called during DLL initialisation; thus it seems to be
232 * the ideal place to perform one time internet connection setup.
233 * However, Microsoft caution against doing much here, (especially
234 * creation of threads, either directly or indirectly); thus we
235 * defer the connection setup until we ultimately need it.
238 inline ~pkgInternetAgent()
242 if( SessionHandle != NULL )
243 Close( SessionHandle );
245 HINTERNET OpenURL( const char* );
247 /* Remaining methods are simple inline wrappers for the
248 * wininet functions we plan to use...
250 inline unsigned long QueryStatus( HINTERNET id )
252 unsigned long ok, idx = 0, len = sizeof( ok );
253 if( HttpQueryInfo( id, HTTP_QUERY_FLAG_NUMBER | HTTP_QUERY_STATUS_CODE,
258 inline unsigned long QueryContentLength( HINTERNET id )
260 unsigned long content_len, idx = 0, len = sizeof( content_len );
261 if( HttpQueryInfo( id, HTTP_QUERY_FLAG_NUMBER | HTTP_QUERY_CONTENT_LENGTH,
262 &content_len, &len, &idx )
263 ) return content_len;
266 inline int Read( HINTERNET dl, char *buf, size_t max, unsigned long *count )
268 return InternetReadFile( dl, buf, max, count );
270 inline int Close( HINTERNET id )
272 return InternetCloseHandle( id );
276 /* This is the one and only instantiation of an object of this class.
278 static pkgInternetAgent pkgDownloadAgent;
280 class pkgInternetStreamingAgent
282 /* Another locally implemented class; each individual file download
283 * gets its own instance of this, either as-is for basic data transfer,
284 * or as a specialised derivative of this base class.
287 const char *filename;
288 const char *dest_template;
292 pkgDownloadMeter *dl_meter;
296 virtual int TransferData( int );
299 pkgInternetStreamingAgent( const char*, const char* );
300 virtual ~pkgInternetStreamingAgent();
302 virtual int Get( const char* );
303 inline const char *DestFile(){ return dest_file; }
306 pkgInternetStreamingAgent::pkgInternetStreamingAgent
307 ( const char *local_name, const char *dest_specification )
309 /* Constructor for the pkgInternetStreamingAgent class.
311 filename = local_name;
312 dest_template = dest_specification;
313 dest_file = (char *)(malloc( mkpath( NULL, dest_template, filename, NULL ) ));
314 if( dest_file != NULL )
315 mkpath( dest_file, dest_template, filename, NULL );
318 pkgInternetStreamingAgent::~pkgInternetStreamingAgent()
320 /* Destructor needs to free the heap memory allocated by the
321 * constructor, for storage of "dest_file" name.
323 free( (void *)(dest_file) );
326 HINTERNET pkgInternetAgent::OpenURL( const char *URL )
328 /* Open an internet data stream.
330 HINTERNET ResourceHandle;
332 /* This requires an internet
333 * connection to have been established...
335 if( (SessionHandle == NULL)
336 && (InternetAttemptConnect( 0 ) == ERROR_SUCCESS) )
338 * ...so, on first call, we perform the connection setup
339 * which we deferred from the class constructor; (MSDN
340 * cautions that this MUST NOT be done in the constructor
341 * for any global class object such as ours).
343 SessionHandle = InternetOpen
344 ( "MinGW Installer", INTERNET_OPEN_TYPE_PRECONFIG,
348 /* Aggressively attempt to acquire a resource handle, which we may use
349 * to access the specified URL; (schedule a maximum of five attempts).
352 do { ResourceHandle = InternetOpenUrl
354 /* Here, we attempt to assign a URL specific resource handle,
355 * within the scope of the SessionHandle obtained above, to
356 * manage the connection for the requested URL.
358 * Note: Scott Michel suggests INTERNET_FLAG_EXISTING_CONNECT
359 * here; MSDN tells us it is useful only for FTP connections.
360 * Since we are primarily interested in HTTP connections, it
361 * may not help us. However, it does no harm, and MSDN isn't
362 * always the reliable source of information we might like.
363 * Persistent HTTP connections aren't entirely unknown, (and
364 * indeed, MSDN itself tells us we need to use one, when we
365 * negotiate proxy authentication); thus, we may just as well
366 * specify it anyway, on the off-chance that it may introduce
367 * an undocumented benefit beyond wishful thinking.
369 SessionHandle, URL, NULL, 0, INTERNET_FLAG_EXISTING_CONNECT, 0
371 if( ResourceHandle == NULL )
373 /* We failed to acquire a handle for the URL resource; we may retry
374 * unless we have exhausted the specified retry limit...
378 * ...in which case, we diagnose failure to open the URL.
380 dmh_notify( DMH_ERROR, "%s:cannot open URL\n", URL );
382 else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
383 dmh_printf( "%s\nConnecting ... failed(status=%u); retrying...\n",
384 URL, GetLastError() )
388 { /* We got a handle for the URL resource, but we cannot yet be sure
389 * that it is ready for use; we may still need to address a need for
390 * proxy or server authentication, or other temporary fault.
393 unsigned long ResourceStatus, ResourceErrno;
394 do { /* We must capture any error code which may have been returned,
395 * BEFORE we move on to evaluate the resource status, (since the
396 * procedure for checking status may change the error code).
398 ResourceErrno = GetLastError();
399 ResourceStatus = QueryStatus( ResourceHandle );
400 if( ResourceStatus == HTTP_STATUS_PROXY_AUTH_REQ )
402 /* We've identified a requirement for proxy authentication;
403 * here we simply hand the task off to the Microsoft handler,
404 * to solicit the appropriate response from the user.
406 * FIXME: this may be a reasonable approach when running in
407 * a GUI context, but is rather inelegant in the CLI context.
408 * Furthermore, this particular implementation provides only
409 * for proxy authentication, ignoring the possibility that
410 * server authentication may be required. We may wish to
411 * revisit this later.
413 unsigned long user_response;
414 do { user_response = InternetErrorDlg
415 ( dmh_dialogue_context(), ResourceHandle, ResourceErrno,
416 FLAGS_ERROR_UI_FILTER_FOR_ERRORS |
417 FLAGS_ERROR_UI_FLAGS_CHANGE_OPTIONS |
418 FLAGS_ERROR_UI_FLAGS_GENERATE_DATA,
421 /* Having obtained authentication credentials from
422 * the user, we may retry the open URL request...
424 if( (user_response == ERROR_INTERNET_FORCE_RETRY)
425 && HttpSendRequest( ResourceHandle, NULL, 0, 0, 0 ) )
427 /* ...and, if successful...
429 ResourceErrno = GetLastError();
430 ResourceStatus = QueryStatus( ResourceHandle );
431 if( ResourceStatus == HTTP_STATUS_OK )
433 * ...ensure that the response is anything but 'retry',
434 * so that we will break out of the retry loop...
436 user_response ^= -1L;
438 /* ...otherwise, we keep retrying when appropriate.
440 } while( user_response == ERROR_INTERNET_FORCE_RETRY );
442 else if( ResourceStatus != HTTP_STATUS_OK )
444 /* Other failure modes may not be so readily recoverable;
445 * with little hope of success, retry anyway.
447 DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
448 dmh_printf( "%s: abnormal request status = %u; retrying...\n",
451 if( HttpSendRequest( ResourceHandle, NULL, 0, 0, 0 ) )
452 ResourceStatus = QueryStatus( ResourceHandle );
454 } while( (ResourceStatus != HTTP_STATUS_OK) && (retry-- > 0) );
456 /* Confirm that the URL was (eventually) opened successfully...
458 if( ResourceStatus == HTTP_STATUS_OK )
460 * ...in which case, we have no need to schedule any further
466 { /* The resource handle we've acquired isn't useable; discard it,
467 * so we can reclaim any resources associated with it.
469 Close( ResourceHandle );
471 /* When we have exhausted our retry limit...
475 /* Give up; nullify our resource handle to indicate failure.
477 ResourceHandle = NULL;
479 /* Issue a diagnostic advising the user to refer the problem
480 * to the mingw-get maintainer for possible follow-up.
482 dmh_control( DMH_BEGIN_DIGEST );
483 dmh_notify( DMH_WARNING,
484 "%s: opened with unexpected status: code = %u\n",
487 dmh_notify( DMH_WARNING,
488 "please report this to the mingw-get maintainer\n"
490 dmh_control( DMH_END_DIGEST );
494 /* If we haven't yet acquired a valid resource handle, and we haven't
495 * yet exhausted our retry limit, go back and try again.
497 } while( retries > 0 );
499 /* Ultimately, we return the resource handle for the opened URL,
500 * or NULL if the open request failed.
502 return ResourceHandle;
505 int pkgInternetStreamingAgent::TransferData( int fd )
507 /* In the case of this base class implementation,
508 * we simply read the file's data from the Internet source,
509 * and write a verbatim copy to the destination file.
511 char buf[8192]; unsigned long count, tally = 0;
512 do { dl_status = pkgDownloadAgent.Read( dl_host, buf, sizeof( buf ), &count );
513 dl_meter->Update( tally += count );
514 write( fd, buf, count );
515 } while( dl_status && (count > 0) );
518 DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ) && (dl_status == 0),
519 dmh_printf( "\nInternetReadFile:download error:%d\n", GetLastError() )
524 static const char *get_host_info
525 ( pkgXmlNode *ref, const char *property, const char *fallback = NULL )
527 /* Helper function to retrieve host information from the XML catalogue.
529 * Call with property = "url", to retrieve the URL template to pass as
530 * "fmt" argument to mkpath(), or with property = "mirror", to retrieve
531 * the substitution text for the "modifier" argument.
533 const char *uri = NULL;
536 /* Starting from the "ref" package entry in the catalogue...
538 pkgXmlNode *host = ref->FindFirstAssociate( download_host_key );
539 while( host != NULL )
541 /* Examine its associate tags; if we find one of type
542 * "download-host", with the requisite property, then we
543 * immediately return that property value...
545 if( (uri = host->GetPropVal( property, NULL )) != NULL )
548 /* Otherwise, we look for any other candidate tags
549 * associated with the same catalogue entry...
551 host = host->FindNextAssociate( download_host_key );
553 /* Failing an immediate match, extend the search to the
554 * ancestors of the initial reference entry...
556 ref = ref->GetParent();
558 /* ...and ultimately, if no match is found, we return the
559 * specified "fallback" property value.
565 int set_transit_path( const char *path, const char *file, char *buf = NULL )
567 /* Helper to define the transitional path name for downloaded files,
568 * used to save the file data while the download is in progress.
570 static const char *transit_dir = "/.in-transit";
571 return mkpath( buf, path, file, transit_dir );
574 int pkgInternetStreamingAgent::Get( const char *from_url )
576 /* Download a file from the specified internet URL.
578 * Before download commences, we accept that this may fail...
582 /* Set up a "transit-file" to receive the downloaded content.
584 char transit_file[set_transit_path( dest_template, filename )];
585 int fd; set_transit_path( dest_template, filename, transit_file );
587 if( (fd = set_output_stream( transit_file, 0644 )) >= 0 )
589 /* The "transit-file" is ready to receive incoming data...
590 * Configure and invoke the download handler to copy the data
591 * from the appropriate host URL, to this "transit-file".
593 if( (dl_host = pkgDownloadAgent.OpenURL( from_url )) != NULL )
595 if( pkgDownloadAgent.QueryStatus( dl_host ) == HTTP_STATUS_OK )
597 /* With the download transaction fully specified, we may
598 * request processing of the file transfer...
600 pkgDownloadMeterTTY download_meter
602 from_url, pkgDownloadAgent.QueryContentLength( dl_host )
604 dl_meter = &download_meter;
605 dl_status = TransferData( fd );
607 else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
608 dmh_printf( "OpenURL:error:%d\n", GetLastError() )
611 /* We are done with the URL handle; close it.
613 pkgDownloadAgent.Close( dl_host );
616 /* Always close the "transit-file", whether the download
617 * was successful, or not...
622 * When successful, we move the "transit-file" to its
623 * final downloaded location...
625 rename( transit_file, dest_file );
627 /* ...otherwise, we discard the incomplete "transit-file",
628 * leaving the caller to diagnose the failure.
630 unlink( transit_file );
633 /* Report success or failure to the caller...
638 void pkgActionItem::PrintURI( const char *package_name )
640 /* Private method to display the URI from which a package archive
641 * may be downloaded; called with "package_name" assigned by either
642 * ArchiveName() or SourceArchiveName() as appropriate, and...
644 if( ! match_if_explicit( package_name, value_none ) )
646 /* ...applicable only to real (i.e. not meta) packages.
648 * We begin by retrieving the URI template associated with
649 * the specified package...
651 const char *url_template = get_host_info( Selection(), uri_key );
652 if( url_template != NULL )
654 /* ...then filling in the package name and preferred mirror
655 * assignment, as if preparing to initiate a download...
657 const char *mirror = get_host_info( Selection(), mirror_key );
658 char package_url[mkpath( NULL, url_template, package_name, mirror )];
659 mkpath( package_url, url_template, package_name, mirror );
661 /* ...then, rather than actually initiate the download,
662 * we simply write out the generated URI to stdout.
664 printf( "%s\n", package_url );
669 void pkgActionItem::DownloadSingleArchive
670 ( const char *package_name, const char *archive_cache_path )
672 pkgInternetStreamingAgent download( package_name, archive_cache_path );
674 /* Check if the required archive is already available locally...
676 if( ((flags & ACTION_DOWNLOAD) == ACTION_DOWNLOAD)
677 && ((access( download.DestFile(), R_OK ) != 0) && (errno == ENOENT)) )
679 /* ...if not, ask the download agent to fetch it...
681 const char *url_template = get_host_info( Selection(), uri_key );
682 if( url_template != NULL )
684 /* ...from the URL constructed from the template specified in
685 * the package repository catalogue (configuration database)...
687 const char *mirror = get_host_info( Selection(), mirror_key );
688 char package_url[mkpath( NULL, url_template, package_name, mirror )];
689 mkpath( package_url, url_template, package_name, mirror );
690 if( download.Get( package_url ) > 0 )
692 * Download was successful; clear the pending flag.
694 flags &= ~(ACTION_DOWNLOAD);
696 /* Diagnose failure; leave pending flag set.
698 dmh_notify( DMH_ERROR,
699 "Get package: %s: download failed\n", package_url
703 /* Cannot download; the repository catalogue didn't specify a
704 * template, from which to construct a download URL...
706 dmh_notify( DMH_ERROR,
707 "Get package: %s: no URL specified for download\n", package_name
711 /* There was no need to download any file to satisfy this request.
713 flags &= ~(ACTION_DOWNLOAD);
716 void pkgActionItem::DownloadArchiveFiles( pkgActionItem *current )
718 /* Update the local package cache, to ensure that all packages needed
719 * to complete the current set of scheduled actions are present; if any
720 * are missing, invoke an Internet download agent to fetch them. This
721 * requires us to walk the action list...
723 while( current != NULL )
725 /* ...while we haven't run off the end...
727 if( (current->flags & ACTION_INSTALL) == ACTION_INSTALL )
729 /* For all packages specified in the current action list,
730 * for which an "install" action is scheduled, and for which
731 * no associated archive file is present in the local archive
732 * cache, place an Internet download agent on standby to fetch
733 * the required archive from a suitable internet mirror host.
735 const char *package_name = current->Selection()->ArchiveName();
737 /* An explicit package name of "none" is a special case;
738 * it identifies a "virtual" meta-package...
740 if( match_if_explicit( package_name, value_none ) )
742 * ...which requires nothing to be downloaded...
744 current->flags &= ~(ACTION_DOWNLOAD);
747 /* ...but we expect any other package to provide real content,
748 * for which we may need to download the package archive...
750 current->DownloadSingleArchive( package_name, pkgArchivePath() );
752 /* Repeat download action, for any additional packages specified
753 * in the current "actions" list.
755 current = current->next;
759 #define DATA_CACHE_PATH "%R" "var/cache/mingw-get/data"
760 #define WORKING_DATA_PATH "%R" "var/lib/mingw-get/data"
762 /* Internet servers host package catalogues in lzma compressed format;
763 * we will decompress them "on the fly", as we download them. To achieve
764 * this, we will use a variant of the pkgInternetStreamingAgent, using a
765 * specialised TransferData method; additionally, this will incorporate
766 * a special derivative of a pkgLzmaArchiveStream, with its GetRawData
767 * method adapted to stream data from an internet URI, instead of
768 * reading from a local file.
770 * To derive the pkgInternetLzmaStreamingAgent, we need to include the
771 * specialised declarations of a pkgArchiveStream, in order to make the
772 * declaration of pkgLzmaArchiveStream available as our base class.
774 #define PKGSTRM_H_SPECIAL 1
777 class pkgInternetLzmaStreamingAgent :
778 public pkgInternetStreamingAgent, public pkgLzmaArchiveStream
780 /* Specialisation of the pkgInternetStreamingAgent base class,
781 * providing decompressed copies of LZMA encoded files downloaded
782 * from the Internet; (the LZMA decompression capability is derived
783 * from the pkgLzmaArchiveStream base class).
786 /* We need a specialised constructor...
788 pkgInternetLzmaStreamingAgent( const char*, const char* );
791 /* Specialisation requires overrides for each of this pair of
792 * methods, (the first from the pkgLzmaArchiveStream base class;
793 * the second from pkgInternetStreamingAgent).
795 virtual int GetRawData( int, uint8_t*, size_t );
796 virtual int TransferData( int );
799 /* This specialisation of the pkgInternetStreamingAgent class needs its
800 * own constructor, simply to invoke the constructors for the base classes,
801 * (since neither is instantiated by a default constructor).
803 pkgInternetLzmaStreamingAgent::pkgInternetLzmaStreamingAgent
804 ( const char *local_name, const char *dest_specification ):
805 pkgInternetStreamingAgent( local_name, dest_specification ),
807 * Note that, when we come to initialise the lzma streaming component
808 * of this derived class, we will be streaming directly from the internet,
809 * rather than from a file stream, so we don't require a file descriptor
810 * for the input stream; however, the class semantics still expect one.
811 * To avoid accidental association with an existing file stream, we
812 * use a negative value, (which is never a valid file descriptor);
813 * however, we must not choose -1, since the class implementation
814 * will decline to process the stream; hence, we choose -2.
816 pkgLzmaArchiveStream( -2 ){}
818 int pkgInternetLzmaStreamingAgent::GetRawData( int fd, uint8_t *buf, size_t max )
820 /* Fetch raw (compressed) data from the Internet host, and load it into
821 * the decompression filter's input buffer, whence the TransferData routine
822 * may retrieve it, via the filter, as an uncompressed stream.
825 dl_status = pkgDownloadAgent.Read( dl_host, (char *)(buf), max, &count );
829 int pkgInternetLzmaStreamingAgent::TransferData( int fd )
831 /* In this case, we read the file's data from the Internet source,
832 * stream it through the lzma decompression filter, and write a copy
833 * of the resultant decompressed data to the destination file.
835 char buf[8192]; unsigned long count;
836 do { count = pkgLzmaArchiveStream::Read( buf, sizeof( buf ) );
837 write( fd, buf, count );
838 } while( dl_status && (count > 0) );
841 DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ) && (dl_status == 0),
842 dmh_printf( "\nInternetReadFile:download error:%d\n", GetLastError() )
847 static const char *serial_number( const char *catalogue )
849 /* Local helper function to retrieve issue numbers from any repository
850 * package catalogue; returns the result as a duplicate of the internal
851 * string, allocated on the heap (courtesy of the strdup() function).
854 pkgXmlDocument src( catalogue );
857 && ((issue = src.GetRoot()->GetPropVal( issue_key, NULL )) != NULL) )
859 * Found an issue number; return a copy...
861 return strdup( issue );
863 /* If we get to here, we couldn't get a valid issue number;
864 * whatever the reason, return NULL to indicate failure.
869 void pkgXmlDocument::SyncRepository( const char *name, pkgXmlNode *repository )
871 /* Fetch a named package catalogue from a specified Internet repository.
873 * Package catalogues are XML files; the master copy on the Internet host
874 * must be stored in lzma compressed format, and named to comply with the
875 * convention "%F.xml.lzma", in which "%F" represents the value of the
876 * "name" argument passed to this pkgXmlDocument class method.
878 const char *url_template;
879 if( (url_template = repository->GetPropVal( uri_key, NULL )) != NULL )
881 /* Initialise a streaming agent, to manage the catalogue download;
882 * (note that we must include the "%/M" placeholder in the template
883 * for the local name, to accommodate the name of the intermediate
884 * "in-transit" directory used by the streaming agent).
886 pkgInternetLzmaStreamingAgent download( name, DATA_CACHE_PATH "%/M/%F.xml" );
888 /* Construct the full URI for the master catalogue, and stream it to
889 * a locally cached, decompressed copy of the XML file.
891 const char *mirror = repository->GetPropVal( mirror_key, NULL );
892 char catalogue_url[mkpath( NULL, url_template, name, mirror )];
893 mkpath( catalogue_url, url_template, name, mirror );
894 if( download.Get( catalogue_url ) <= 0 )
895 dmh_notify( DMH_ERROR,
896 "Sync Repository: %s: download failed\n", catalogue_url
900 /* We will only replace our current working copy of this catalogue,
901 * (if one already exists), with the copy we just downloaded, if this
902 * downloaded copy bears an issue number indicating that it is more
903 * recent than the working copy.
905 const char *repository_version, *working_version;
906 if( (repository_version = serial_number( download.DestFile() )) != NULL )
908 /* Identify the location for the working copy, (if it exists).
910 const char *working_copy_path_name = WORKING_DATA_PATH "/%F.xml";
911 char working_copy[mkpath( NULL, working_copy_path_name, name, NULL )];
912 mkpath( working_copy, working_copy_path_name, name, NULL );
914 /* Compare issue serial numbers...
916 if( ((working_version = serial_number( working_copy )) == NULL)
917 || ((strcmp( repository_version, working_version )) > 0) )
919 /* In these circumstances, we couldn't identify an issue number
920 * for the working copy of the catalogue; (maybe there is no such
921 * catalogue, or maybe it doesn't specify a valid issue number);
922 * in either case, we promote the downloaded copy in its place.
924 * FIXME: we assume that the working file and the downloaded copy
925 * are stored on the same physical file system device, so we may
926 * replace the former by simply deleting it, and renaming the
927 * latter with its original path name; we make no provision for
928 * replacing the working version by physical data copying.
930 unlink( working_copy );
931 rename( download.DestFile(), working_copy );
934 /* The issue numbers, returned by the serial_number() function, were
935 * allocated on the heap; free them to avoid leaking memory!
937 free( (void *)(repository_version) );
939 * The working copy issue number may be represented by a NULL pointer;
940 * while it may be safe to call free on this, it just *seems* wrong, so
941 * we check it first, to be certain.
943 if( working_version != NULL )
944 free( (void *)(working_version) );
947 /* If the downloaded copy of the catalogue is still in the download cache,
948 * we have chosen to keep a previous working copy, so we have no further
949 * use for the downloaded copy; discard it, noting that we don't need to
950 * confirm its existence because this will fail silently, if it is no
953 unlink( download.DestFile() );
957 /* $RCSfile$: end of file */