6 * Written by Keith Marshall <keith@users.osdn.me>
7 * Copyright (C) 2009-2013, 2017, 2020, MinGW.org Project
10 * Implementation of the package download machinery for mingw-get.
13 * This is free software. Permission is granted to copy, modify and
14 * redistribute this software, under the provisions of the GNU General
15 * Public License, Version 3, (or, at your option, any later version),
16 * as published by the Free Software Foundation; see the file COPYING
17 * for licensing details.
19 * Note, in particular, that this software is provided "as is", in the
20 * hope that it may prove useful, but WITHOUT WARRANTY OF ANY KIND; not
21 * even an implied WARRANTY OF MERCHANTABILITY, nor of FITNESS FOR ANY
22 * PARTICULAR PURPOSE. Under no circumstances will the author, or the
23 * MinGW Project, accept liability for any damages, however caused,
24 * arising from the use of this software.
27 #define USES_SAFE_STRCMP 1
28 #define WIN32_LEAN_AND_MEAN
30 #define _WIN32_WINNT 0x0500 /* for GetConsoleWindow() kludge */
33 * FIXME: This kludge allows us to use the standard wininet dialogue
34 * to acquire proxy authentication credentials from the user; this is
35 * expedient for now, (if somewhat anti-social for a CLI application).
36 * We will ultimately need to provide a more robust implementation,
37 * (within the scope of the diagnostic message handler), in order to
38 * obtain a suitable window handle for use when called from the GUI
39 * implementation of mingw-get, (when it becomes available).
41 #define dmh_dialogue_context() GetConsoleWindow()
60 /* This static member variable of the pkgDownloadMeter class
61 * provides a mechanism for communicating with a pre-existing
62 * download monitoring dialogue, such as is employed in the GUI.
63 * We MUST define it, and we also initialise it to NULL. While
64 * it remains thus, (as it always does in the CLI), no attempt
65 * will be made to access any pre-existing dialogue, and local
66 * monitoring objects will be employed; the GUI sets this to
67 * refer to its dialogue, when this is activated.
69 pkgDownloadMeter *pkgDownloadMeter::primary = NULL;
71 inline void pkgDownloadMeter::SpinWait( int mode, const char *uri )
73 /* Wrapper to invoke the overridable derived class specific
74 * SpinWait() method, with protection against any attempt to
75 * access a virtual method of a primary class object which
76 * does not exist; (thus, there is no vtable reference).
78 if( primary != NULL ) primary->SpinWaitAction( mode, uri );
81 #if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
83 /* Implementation of a download meter class, displaying download
84 * statistics within a CLI application context.
86 class pkgDownloadMeterTTY: public pkgDownloadMeter
89 pkgDownloadMeterTTY( const char*, unsigned long );
90 virtual ~pkgDownloadMeterTTY();
92 virtual void Update( unsigned long );
95 /* This buffer is used to store each compiled status report,
96 * in preparation for display on the console.
98 const char *source_url;
99 char status_report[80];
104 pkgDownloadMeterTTY::pkgDownloadMeterTTY( const char *url, unsigned long length ):
105 source_url( url ){ content_length = length; }
107 /* ...and destructor.
109 pkgDownloadMeterTTY::~pkgDownloadMeterTTY()
111 if( source_url == NULL )
115 #elif IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
117 /* The setup tool will always use GUI style download metering;
118 * however the download agent remains CLI capable, so we need a
119 * minimal "do nothing" implementation of a TTY style metering
120 * class, to satisfy the linker.
122 class pkgDownloadMeterTTY: public pkgDownloadMeter
124 /* Implementation of a dummy download meter class, sufficient
125 * to promote an illusion of TTY metering capability.
128 pkgDownloadMeterTTY( const char*, unsigned long ){}
129 virtual void Update( unsigned long ){}
134 static inline __attribute__((__always_inline__))
135 unsigned long pow10mul( register unsigned long x, unsigned power )
137 /* Inline helper to perform a fast integer multiplication of "x"
138 * by a specified power of ten.
142 /* Each cycle multiplies by ten, first shifting to get "x * 2"...
146 * ...then further shifting that intermediate result, and adding,
147 * to achieve the effect of "x * 2 + x * 2 * 4".
154 static inline __attribute__((__always_inline__))
155 unsigned long percentage( unsigned long x, unsigned long q )
157 /* Inline helper to compute "x" as an integer percentage of "q".
159 return pow10mul( x, 2 ) / q;
162 #if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
164 void pkgDownloadMeterTTY::Update( unsigned long count )
166 /* Implementation of method to update the download progress report,
167 * displaying the current byte count and anticipated final byte count,
168 * each formatted in a conveniently human readable style, followed by
169 * approximate percentage completion, both as a 48-segment bar graph,
170 * and as a numeric tally.
172 char *p = status_report;
173 int barlen = (content_length > count)
174 ? (((count << 1) + count) << 4) / content_length
175 : (content_length > 0) ? 48 : 0;
177 /* We may safely use sprintf() to assemble the status report, because
178 * we control the field lengths to always fit within the buffer.
180 p += SizeFormat( p, count );
181 p += sprintf( p, " / " );
182 p += (content_length > 0)
183 ? SizeFormat( p, content_length )
184 : sprintf( p, "????.?? ??" );
185 p += sprintf( p, "%*c", status_report + 25 - p, '|' ); p[barlen] = '\0';
186 p += sprintf( p, "%-48s", (char *)(memset( p, '=', barlen )) );
187 p += ( (content_length > 0) && (content_length >= count) )
188 ? sprintf( p, "|%4lu", percentage( count, content_length ) )
189 : sprintf( p, "| ???" );
191 if( source_url != NULL )
193 dmh_printf( "%s\n", source_url );
196 (void)(dmh_printf( "\r%s%%", status_report ));
201 int pkgDownloadMeter::SizeFormat( char *buf, unsigned long filesize )
203 /* Helper method to format raw byte counts as B, kB, MB, GB, TB, as
204 * appropriate; (this NEVER requires more than ten characters).
207 const unsigned long sizelimit = 1 << 10; /* 1k */
209 if( filesize < sizelimit )
211 * Raw byte count is less than 1 kB; we may simply emit it.
213 retval = sprintf( buf, "%lu B", filesize );
217 /* The raw byte count is too large to be readily assimilated; we
218 * scale it down to an appropriate value, and append the appropriate
221 unsigned long frac = filesize;
222 const unsigned long fracmask = sizelimit - 1;
223 const char *scale_indicator = "kMGT";
225 /* A ten bit right shift scales by a factor of 1k; continue shifting
226 * until the ultimate value is less than 1k...
228 while( (filesize >>= 10) >= sizelimit )
230 /* ...noting the residual at each shift, and adjusting the scaling
231 * indicator selection to suit.
236 /* Finally, emit the scaled result to the nearest one hundredth part
237 * of the ultimate scaling unit.
240 ( buf, "%lu.%02lu %cB",
241 filesize, percentage( frac & fracmask, sizelimit ), *scale_indicator
247 /* To facilitate the implementation of the pkgInternetAgent class, which
250 #if IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
252 /* Within the setup tool, the pkgInternetAgent::SetRetryOptions() method
253 * requires a pointer to a pkgSetupAction...
255 typedef pkgSetupAction *INTERNET_RETRY_REQUESTER;
258 /* ...whereas, in the general case, it requires a pointer to a pkgXmlNode.
260 typedef pkgXmlNode *INTERNET_RETRY_REQUESTER;
264 class pkgInternetAgent
266 /* A minimal, locally implemented class, instantiated ONCE as a
267 * global object, to ensure that wininet's global initialisation is
268 * completed at the proper time, without us doing it explicitly.
271 HINTERNET SessionHandle;
272 int connection_delay, delay_factor, retries, retry_interval;
275 inline pkgInternetAgent():SessionHandle( NULL )
279 * This is called during DLL initialisation; thus it seems to be
280 * the ideal place to perform one time internet connection setup.
281 * However, Microsoft caution against doing much here, (especially
282 * creation of threads, either directly or indirectly); thus we
283 * defer the connection setup until we ultimately need it.
286 inline ~pkgInternetAgent()
290 if( SessionHandle != NULL )
291 Close( SessionHandle );
293 void SetRetryOptions( INTERNET_RETRY_REQUESTER, const char* );
294 HINTERNET OpenURL( const char* );
296 /* Remaining methods are simple inline wrappers for the
297 * wininet functions we plan to use...
299 inline unsigned long QueryStatus( HINTERNET id )
301 unsigned long ok, idx = 0, len = sizeof( ok );
302 if( HttpQueryInfo( id, HTTP_QUERY_FLAG_NUMBER | HTTP_QUERY_STATUS_CODE,
307 inline unsigned long QueryContentLength( HINTERNET id )
309 unsigned long content_len, idx = 0, len = sizeof( content_len );
310 if( HttpQueryInfo( id, HTTP_QUERY_FLAG_NUMBER | HTTP_QUERY_CONTENT_LENGTH,
311 &content_len, &len, &idx )
312 ) return content_len;
315 inline int Read( HINTERNET dl, char *buf, size_t max, unsigned long *count )
317 return InternetReadFile( dl, buf, max, count );
319 inline int Close( HINTERNET id )
321 return InternetCloseHandle( id );
325 /* This is the one and only instantiation of an object of this class.
327 static pkgInternetAgent pkgDownloadAgent;
329 class pkgInternetStreamingAgent
331 /* Another locally implemented class; each individual file download
332 * gets its own instance of this, either as-is for basic data transfer,
333 * or as a specialised derivative of this base class.
336 const char *filename;
337 const char *dest_template;
341 pkgDownloadMeter *dl_meter;
345 virtual int TransferData( int );
348 pkgInternetStreamingAgent( const char*, const char* );
349 virtual ~pkgInternetStreamingAgent();
351 virtual int Get( const char* );
352 inline const char *DestFile(){ return dest_file; }
355 pkgInternetStreamingAgent::pkgInternetStreamingAgent
356 ( const char *local_name, const char *dest_specification )
358 /* Constructor for the pkgInternetStreamingAgent class.
360 filename = local_name;
361 dest_template = dest_specification;
362 dest_file = (char *)(malloc( mkpath( NULL, dest_template, filename, NULL ) ));
363 if( dest_file != NULL )
364 mkpath( dest_file, dest_template, filename, NULL );
367 pkgInternetStreamingAgent::~pkgInternetStreamingAgent()
369 /* Destructor needs to free the heap memory allocated by the
370 * constructor, for storage of "dest_file" name.
372 free( (void *)(dest_file) );
375 void pkgInternetAgent::SetRetryOptions
376 ( INTERNET_RETRY_REQUESTER referrer, const char *url_template )
378 /* Initialisation method, invoked immediately prior to the start
379 * of each archive download request, to establish the options for
380 * retrying any failed host connection request.
382 * The first of any sequence of connection attempts may always be
383 * initiated immediately.
385 connection_delay = 0;
387 /* If any further attempts are necessary, we will delay them
388 * at progressively increasing intervals, to give us the best
389 * possible chance to circumvent throttling of excess frequency
390 * connection requests, by the download server.
392 * FIXME: for each change of host domain, we should establish
393 * settings by consulting the configuration profile; for the
394 * time being, we simply assign fixed defaults.
396 delay_factor = INTERNET_DELAY_FACTOR;
397 retry_interval = INTERNET_RETRY_INTERVAL;
398 retries = INTERNET_RETRY_ATTEMPTS;
401 HINTERNET pkgInternetAgent::OpenURL( const char *URL )
403 /* Open an internet data stream.
405 HINTERNET ResourceHandle;
407 /* This requires an internet
408 * connection to have been established...
410 if( (SessionHandle == NULL)
411 && (InternetAttemptConnect( 0 ) == ERROR_SUCCESS) )
413 * ...so, on first call, we perform the connection setup
414 * which we deferred from the class constructor; (MSDN
415 * cautions that this MUST NOT be done in the constructor
416 * for any global class object such as ours).
418 SessionHandle = InternetOpen
419 ( "MinGW Installer", INTERNET_OPEN_TYPE_PRECONFIG,
423 /* Aggressively attempt to acquire a resource handle, which we may use
424 * to access the specified URL; (schedule a maximum of five attempts).
426 do { pkgDownloadMeter::SpinWait( 1, URL );
428 /* Distribute retries at geometrically incrementing intervals.
430 if( connection_delay > 0 )
431 /* This is not the first time we've tried to open this URL;
432 * compute the appropriate retry interval, and wait between
433 * successive attempts to establish the connection.
435 Sleep( connection_delay = delay_factor * connection_delay );
437 /* This is the first attempt to open this URL; don't wait,
438 * but set up the baseline, in milliseconds, for interval
439 * computation, in the event that further attempts may be
442 connection_delay = retry_interval;
444 ResourceHandle = InternetOpenUrl
446 /* Here, we attempt to assign a URL specific resource handle,
447 * within the scope of the SessionHandle obtained above, to
448 * manage the connection for the requested URL.
450 * Note: Scott Michel suggests INTERNET_FLAG_EXISTING_CONNECT
451 * here; MSDN tells us it is useful only for FTP connections.
452 * Since we are primarily interested in HTTP connections, it
453 * may not help us. However, it does no harm, and MSDN isn't
454 * always the reliable source of information we might like.
455 * Persistent HTTP connections aren't entirely unknown, (and
456 * indeed, MSDN itself tells us we need to use one, when we
457 * negotiate proxy authentication); thus, we may just as well
458 * specify it anyway, on the off-chance that it may introduce
459 * an undocumented benefit beyond wishful thinking.
461 SessionHandle, URL, NULL, 0,
462 INTERNET_FLAG_EXISTING_CONNECT
463 | INTERNET_FLAG_IGNORE_CERT_CN_INVALID
464 | INTERNET_FLAG_IGNORE_CERT_DATE_INVALID
465 | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS
466 | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP
467 | INTERNET_FLAG_KEEP_CONNECTION
468 | INTERNET_FLAG_PRAGMA_NOCACHE, 0
470 if( ResourceHandle == NULL )
472 /* We failed to acquire a handle for the URL resource; we may retry
473 * unless we have exhausted the specified retry limit...
477 /* ...in which case, we diagnose failure to open the URL.
479 unsigned int status = GetLastError();
480 DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
481 dmh_printf( "%s\nConnection failed(status=%u); abandoned.\n",
485 DMH_ERROR, "%s:cannot open URL; status = %u\n", URL, status
488 else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
489 dmh_printf( "%s\nConnecting ... failed(status=%u); retrying in %ds...\n",
490 URL, GetLastError(), delay_factor * connection_delay / 1000 )
494 { /* We got a handle for the URL resource, but we cannot yet be sure
495 * that it is ready for use; we may still need to address a need for
496 * proxy or server authentication, or other temporary fault.
499 unsigned long ResourceStatus, ResourceErrno;
500 do { /* We must capture any error code which may have been returned,
501 * BEFORE we move on to evaluate the resource status, (since the
502 * procedure for checking status may change the error code).
504 ResourceErrno = GetLastError();
505 ResourceStatus = QueryStatus( ResourceHandle );
506 if( ResourceStatus == HTTP_STATUS_PROXY_AUTH_REQ )
508 /* We've identified a requirement for proxy authentication;
509 * here we simply hand the task off to the Microsoft handler,
510 * to solicit the appropriate response from the user.
512 * FIXME: this may be a reasonable approach when running in
513 * a GUI context, but is rather inelegant in the CLI context.
514 * Furthermore, this particular implementation provides only
515 * for proxy authentication, ignoring the possibility that
516 * server authentication may be required. We may wish to
517 * revisit this later.
519 unsigned long user_response;
520 do { user_response = InternetErrorDlg
521 ( dmh_dialogue_context(), ResourceHandle, ResourceErrno,
522 FLAGS_ERROR_UI_FILTER_FOR_ERRORS |
523 FLAGS_ERROR_UI_FLAGS_CHANGE_OPTIONS |
524 FLAGS_ERROR_UI_FLAGS_GENERATE_DATA,
527 /* Having obtained authentication credentials from
528 * the user, we may retry the open URL request...
530 if( (user_response == ERROR_INTERNET_FORCE_RETRY)
531 && HttpSendRequest( ResourceHandle, NULL, 0, 0, 0 ) )
533 /* ...and, if successful...
535 ResourceErrno = GetLastError();
536 ResourceStatus = QueryStatus( ResourceHandle );
537 if( ResourceStatus == HTTP_STATUS_OK )
539 * ...ensure that the response is anything but 'retry',
540 * so that we will break out of the retry loop...
542 user_response ^= -1L;
544 /* ...otherwise, we keep retrying when appropriate.
546 } while( user_response == ERROR_INTERNET_FORCE_RETRY );
548 else if( ResourceStatus != HTTP_STATUS_OK )
550 /* Other failure modes may not be so readily recoverable;
551 * with little hope of success, retry anyway.
553 DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
554 dmh_printf( "%s: abnormal request status = %u; retrying...\n",
557 if( HttpSendRequest( ResourceHandle, NULL, 0, 0, 0 ) )
558 ResourceStatus = QueryStatus( ResourceHandle );
560 } while( (ResourceStatus != HTTP_STATUS_OK) && (retry-- > 0) );
562 /* Confirm that the URL was (eventually) opened successfully...
564 if( ResourceStatus == HTTP_STATUS_OK )
566 * ...in which case, we have no need to schedule any further
572 { /* The resource handle we've acquired isn't useable; discard it,
573 * so we can reclaim any resources associated with it.
575 Close( ResourceHandle );
577 /* When we have exhausted our retry limit...
581 /* Give up; nullify our resource handle to indicate failure.
583 ResourceHandle = NULL;
585 /* Issue a diagnostic advising the user to refer the problem
586 * to the mingw-get maintainer for possible follow-up; (note
587 * that we want to group the following messages into a single
588 * message "digest", but if the caller is already doing so,
589 * then we simply incorporate these, and delegate flushing
590 * of the completed "digest" to the caller).
592 uint16_t dmh_cached = dmh_control( DMH_GET_CONTROL_STATE );
593 dmh_cached &= dmh_control( DMH_BEGIN_DIGEST );
594 dmh_notify( DMH_WARNING,
595 "%s: opened with unexpected status: code = %u\n",
598 dmh_notify( DMH_WARNING,
599 "please report this to the mingw-get maintainer\n"
601 if( dmh_cached == 0 ) dmh_control( DMH_END_DIGEST );
605 /* If we haven't yet acquired a valid resource handle, and we haven't
606 * yet exhausted our retry limit, go back and try again.
608 } while( retries > 0 );
609 pkgDownloadMeter::SpinWait( 0 );
611 /* Ultimately, we return the resource handle for the opened URL,
612 * or NULL if the open request failed.
614 return ResourceHandle;
617 int pkgInternetStreamingAgent::TransferData( int fd )
619 /* In the case of this base class implementation,
620 * we simply read the file's data from the Internet source,
621 * and write a verbatim copy to the destination file.
623 char buf[8192]; unsigned long count, tally = 0;
624 do { dl_status = pkgDownloadAgent.Read( dl_host, buf, sizeof( buf ), &count );
625 dl_meter->Update( tally += count );
626 write( fd, buf, count );
627 } while( dl_status && (count > 0) );
630 DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ) && (dl_status == 0),
631 dmh_printf( "\nInternetReadFile:download error:%d\n", GetLastError() )
636 #if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
638 static const char *get_host_info
639 ( pkgXmlNode *ref, const char *property, const char *fallback = NULL )
641 /* Helper function to retrieve host information from the XML catalogue.
643 * Call with property = "url", to retrieve the URL template to pass as
644 * "fmt" argument to mkpath(), or with property = "mirror", to retrieve
645 * the substitution text for the "modifier" argument.
647 const char *uri = NULL;
650 /* Starting from the "ref" package entry in the catalogue...
652 pkgXmlNode *host = ref->FindFirstAssociate( download_host_key );
653 while( host != NULL )
655 /* Examine its associate tags; if we find one of type
656 * "download-host", with the requisite property, then we
657 * immediately return that property value...
659 if( (uri = host->GetPropVal( property, NULL )) != NULL )
662 /* Otherwise, we look for any other candidate tags
663 * associated with the same catalogue entry...
665 host = host->FindNextAssociate( download_host_key );
667 /* Failing an immediate match, extend the search to the
668 * ancestors of the initial reference entry...
670 ref = ref->GetParent();
672 /* ...and ultimately, if no match is found, we return the
673 * specified "fallback" property value.
678 #elif IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
680 static const char *get_host_info
681 ( void *ref, const char *property, const char *fallback = NULL )
683 /* Customised helper to load the download host URI template
684 * directly from the setup tool's resource data section.
686 static const char *uri_template = NULL;
687 if( uri_template == NULL )
688 uri_template = strdup( WTK::StringResource( NULL, ID_DOWNLOAD_HOST_URI ) );
689 return (strcmp( property, uri_key ) == 0) ? uri_template : fallback;
695 int set_transit_path( const char *path, const char *file, char *buf = NULL )
697 /* Helper to define the transitional path name for downloaded files,
698 * used to save the file data while the download is in progress.
700 static const char *transit_dir = "/.in-transit";
701 return mkpath( buf, path, file, transit_dir );
704 int pkgInternetStreamingAgent::Get( const char *from_url )
706 /* Download a file from the specified internet URL.
708 * Before download commences, we accept that this may fail...
712 /* Set up a "transit-file" to receive the downloaded content.
714 char transit_file[set_transit_path( dest_template, filename )];
715 int fd; set_transit_path( dest_template, filename, transit_file );
717 //dmh_printf( "Get: initialise transfer file %s ... ", transit_file );
718 chmod( transit_file, S_IWRITE ); unlink( transit_file );
719 if( (fd = set_output_stream( transit_file, 0644 )) >= 0 )
721 //dmh_printf( "success\nGet: open URL %s ... ", from_url );
722 /* The "transit-file" is ready to receive incoming data...
723 * Configure and invoke the download handler to copy the data
724 * from the appropriate host URL, to this "transit-file".
726 if( (dl_host = pkgDownloadAgent.OpenURL( from_url )) != NULL )
728 //dmh_printf( "success\n" );
729 if( pkgDownloadAgent.QueryStatus( dl_host ) == HTTP_STATUS_OK )
731 /* With the download transaction fully specified, we may
732 * request processing of the file transfer...
734 if( (dl_meter = pkgDownloadMeter::UseGUI()) != NULL )
736 /* ...with progress monitoring delegated to the GUI's
737 * dialogue box, when running under its auspices...
740 filename, pkgDownloadAgent.QueryContentLength( dl_host )
742 dl_status = TransferData( fd );
745 { /* ...otherwise creating our own TTY progress monitor,
746 * when running under the auspices of the CLI.
748 pkgDownloadMeterTTY download_meter(
749 from_url, pkgDownloadAgent.QueryContentLength( dl_host )
751 dl_meter = &download_meter;
753 /* Note that the following call MUST be kept within the
754 * scope in which the progress monitor was created; thus,
755 * it CANNOT be factored out of this "else" block scope,
756 * even though it also appears at the end of the scope
757 * of the preceding "if" block.
759 dl_status = TransferData( fd );
762 else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
763 dmh_printf( "OpenURL:error:%d\n", GetLastError() )
766 /* We are done with the URL handle; close it.
768 pkgDownloadAgent.Close( dl_host );
770 //else dmh_printf( "failed\n" );
772 /* Always close the "transit-file", whether the download
773 * was successful, or not...
778 * When successful, we move the "transit-file" to its
779 * final downloaded location...
781 rename( transit_file, dest_file );
783 /* ...otherwise, we discard the incomplete "transit-file",
784 * leaving the caller to diagnose the failure.
786 unlink( transit_file );
788 //else dmh_printf( "failed\n" );
790 /* Report success or failure to the caller...
795 #if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
797 void pkgActionItem::PrintURI
798 ( const char *package_name, int (*output)( const char * ) )
800 /* Public method to display the URI from which a package archive
801 * may be downloaded; called with "package_name" assigned by either
802 * ArchiveName() or SourceArchiveName() as appropriate, and...
804 if( ! match_if_explicit( package_name, value_none ) )
806 /* ...applicable only to real (i.e. not meta) packages.
808 * We begin by retrieving the URI template associated with
809 * the specified package...
811 const char *url_template = get_host_info( Selection(), uri_key );
812 if( url_template != NULL )
814 /* ...then filling in the package name and preferred mirror
815 * assignment, as if preparing to initiate a download...
817 const char *mirror = get_host_info( Selection(), mirror_key );
818 char package_url[mkpath( NULL, url_template, package_name, mirror )];
819 mkpath( package_url, url_template, package_name, mirror );
821 /* ...then, rather than actually initiate the download,
822 * we simply write out the generated URI to stdout.
824 output( package_url );
829 #endif /* PACKAGE_BASE_COMPONENT */
831 void pkgActionItem::DownloadSingleArchive
832 ( const char *package_name, const char *archive_cache_path )
834 pkgInternetStreamingAgent download( package_name, archive_cache_path );
836 /* Check if the required archive is already available locally...
838 if( ((flags & ACTION_DOWNLOAD) == ACTION_DOWNLOAD)
839 && ((access( download.DestFile(), R_OK ) != 0) && (errno == ENOENT)) )
841 /* ...if not, ask the download agent to fetch it,
842 * anticipating that this may fail...
844 flags |= ACTION_DOWNLOAD_FAILED;
845 const char *url_template = get_host_info( Selection(), uri_key );
846 if( url_template != NULL )
848 /* ...from the URL constructed from the template specified in
849 * the package repository catalogue (configuration database).
851 const char *mirror = get_host_info( Selection(), mirror_key );
852 char package_url[mkpath( NULL, url_template, package_name, mirror )];
853 mkpath( package_url, url_template, package_name, mirror );
855 /* Enable retrying of failed connection attempts, according to the
856 * preferences, if any, which have been configured for the repository
857 * associated with the current URL template, or with default settings
858 * otherwise, then initiate the package download process.
860 pkgDownloadAgent.SetRetryOptions( Selection(), url_template );
861 if( download.Get( package_url ) > 0 )
863 * Download was successful; clear the pending and failure flags.
865 flags &= ~(ACTION_DOWNLOAD | ACTION_DOWNLOAD_FAILED);
867 /* Diagnose failure; leave pending flag set.
869 dmh_notify( DMH_ERROR,
870 "Get package: %s: download failed\n", package_url
874 /* Cannot download; the repository catalogue didn't specify a
875 * template, from which to construct a download URL...
877 dmh_notify( DMH_ERROR,
878 "Get package: %s: no URL specified for download\n", package_name
882 /* There was no need to download any file to satisfy this request.
884 flags &= ~(ACTION_DOWNLOAD);
887 void pkgActionItem::DownloadArchiveFiles( pkgActionItem *current )
889 /* Update the local package cache, to ensure that all packages needed
890 * to complete the current set of scheduled actions are present; if any
891 * are missing, invoke an Internet download agent to fetch them. This
892 * requires us to walk the action list...
894 while( current != NULL )
896 /* ...while we haven't run off the end, and collecting any diagnositic
897 * messages relating to any one package into a common digest...
899 dmh_control( DMH_BEGIN_DIGEST );
900 if( (current->flags & ACTION_INSTALL) == ACTION_INSTALL )
902 /* For all packages specified in the current action list,
903 * for which an "install" action is scheduled, and for which
904 * no associated archive file is present in the local archive
905 * cache, place an Internet download agent on standby to fetch
906 * the required archive from a suitable internet mirror host.
908 const char *package_name = current->Selection()->ArchiveName();
910 /* An explicit package name of "none" is a special case;
911 * it identifies a "virtual" meta-package...
913 if( match_if_explicit( package_name, value_none ) )
915 * ...which requires nothing to be downloaded...
917 current->flags &= ~(ACTION_DOWNLOAD);
920 /* ...but we expect any other package to provide real content,
921 * for which we may need to download the package archive...
923 current->DownloadSingleArchive( package_name, pkgArchivePath() );
926 /* The current action has no associated "install" requirement,
927 * so neither is there any need to request a "download".
929 current->flags &= ~(ACTION_DOWNLOAD);
931 /* Flush out any diagnostics relating to the current package, then
932 * repeat the download action, for any additional packages specified
933 * in the current "actions" list.
935 dmh_control( DMH_END_DIGEST );
936 current = current->next;
940 #if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
942 #define DATA_CACHE_PATH "%R" "var/cache/mingw-get/data"
943 #define WORKING_DATA_PATH "%R" "var/lib/mingw-get/data"
945 /* Internet servers host package catalogues in lzma compressed format;
946 * we will decompress them "on the fly", as we download them. To achieve
947 * this, we will use a variant of the pkgInternetStreamingAgent, using a
948 * specialised TransferData method; additionally, this will incorporate
949 * a special derivative of a pkgLzmaArchiveStream, with its GetRawData
950 * method adapted to stream data from an internet URI, instead of
951 * reading from a local file.
953 * To derive the pkgInternetLzmaStreamingAgent, we need to include the
954 * specialised declarations of a pkgArchiveStream, in order to make the
955 * declaration of pkgLzmaArchiveStream available as our base class.
957 #define PKGSTRM_H_SPECIAL 1
960 class pkgInternetLzmaStreamingAgent:
961 public pkgInternetStreamingAgent, public pkgLzmaArchiveStream
963 /* Specialisation of the pkgInternetStreamingAgent base class,
964 * providing decompressed copies of LZMA encoded files downloaded
965 * from the Internet; (the LZMA decompression capability is derived
966 * from the pkgLzmaArchiveStream base class).
969 /* We need a specialised constructor...
971 pkgInternetLzmaStreamingAgent( const char*, const char* );
974 /* Specialisation requires overrides for each of this pair of
975 * methods, (the first from the pkgLzmaArchiveStream base class;
976 * the second from pkgInternetStreamingAgent).
978 virtual int GetRawData( int, uint8_t*, size_t );
979 virtual int TransferData( int );
982 /* This specialisation of the pkgInternetStreamingAgent class needs its
983 * own constructor, simply to invoke the constructors for the base classes,
984 * (since neither is instantiated by a default constructor).
986 pkgInternetLzmaStreamingAgent::pkgInternetLzmaStreamingAgent
987 ( const char *local_name, const char *dest_specification ):
988 pkgInternetStreamingAgent( local_name, dest_specification ),
990 * Note that, when we come to initialise the lzma streaming component
991 * of this derived class, we will be streaming directly from the internet,
992 * rather than from a file stream, so we don't require a file descriptor
993 * for the input stream; however, the class semantics still expect one.
994 * To avoid accidental association with an existing file stream, we
995 * use a negative value, (which is never a valid file descriptor);
996 * however, we must not choose -1, since the class implementation
997 * will decline to process the stream; hence, we choose -2.
999 pkgLzmaArchiveStream( -2 ){}
1001 int pkgInternetLzmaStreamingAgent::GetRawData( int fd, uint8_t *buf, size_t max )
1003 /* Fetch raw (compressed) data from the Internet host, and load it into
1004 * the decompression filter's input buffer, whence the TransferData routine
1005 * may retrieve it, via the filter, as an uncompressed stream.
1007 unsigned long count;
1008 dl_status = pkgDownloadAgent.Read( dl_host, (char *)(buf), max, &count );
1009 return (int)(count);
1012 int pkgInternetLzmaStreamingAgent::TransferData( int fd )
1014 /* In this case, we read the file's data from the Internet source,
1015 * stream it through the lzma decompression filter, and write a copy
1016 * of the resultant decompressed data to the destination file.
1018 char buf[8192]; unsigned long count;
1019 do { count = pkgLzmaArchiveStream::Read( buf, sizeof( buf ) );
1020 write( fd, buf, count );
1021 } while( dl_status && (count > 0) );
1024 DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ) && (dl_status == 0),
1025 dmh_printf( "\nInternetReadFile:download error:%d\n", GetLastError() )
1030 EXTERN_C const char *serial_number( const char *catalogue )
1032 /* Local helper function to retrieve issue numbers from any repository
1033 * package catalogue; returns the result as a duplicate of the internal
1034 * string, allocated on the heap (courtesy of the strdup() function).
1037 pkgXmlDocument src( catalogue );
1040 && ((issue = src.GetRoot()->GetPropVal( issue_key, NULL )) != NULL) )
1042 * Found an issue number; return a copy...
1044 return strdup( issue );
1046 /* If we get to here, we couldn't get a valid issue number;
1047 * whatever the reason, return NULL to indicate failure.
1052 void pkgXmlDocument::SyncRepository( const char *name, pkgXmlNode *repository )
1054 /* Fetch a named package catalogue from a specified Internet repository.
1056 * Package catalogues are XML files; the master copy on the Internet host
1057 * must be stored in lzma compressed format, and named to comply with the
1058 * convention "%F.xml.lzma", in which "%F" represents the value of the
1059 * "name" argument passed to this pkgXmlDocument class method.
1061 const char *url_template;
1062 if( (url_template = repository->GetPropVal( uri_key, NULL )) != NULL )
1064 /* Initialise a streaming agent, to manage the catalogue download;
1065 * (note that we must include the "%/M" placeholder in the template
1066 * for the local name, to accommodate the name of the intermediate
1067 * "in-transit" directory used by the streaming agent).
1069 pkgInternetLzmaStreamingAgent download( name, DATA_CACHE_PATH "%/M/%F.xml" );
1071 /* Construct the full URI for the master catalogue, and stream it to
1072 * a locally cached, decompressed copy of the XML file.
1074 const char *mirror = repository->GetPropVal( mirror_key, NULL );
1075 char catalogue_url[mkpath( NULL, url_template, name, mirror )];
1076 mkpath( catalogue_url, url_template, name, mirror );
1078 /* Enable retries according to the preferences, if any, as
1079 * configured for the repository, or adopt default settings
1080 * otherwise, then initiate the download process for the
1083 pkgDownloadAgent.SetRetryOptions( repository, url_template );
1084 if( download.Get( catalogue_url ) <= 0 )
1085 dmh_notify( DMH_ERROR,
1086 "Sync Repository: %s: download failed\n", catalogue_url
1090 /* We will only replace our current working copy of this catalogue,
1091 * (if one already exists), with the copy we just downloaded, if this
1092 * downloaded copy bears an issue number indicating that it is more
1093 * recent than the working copy.
1095 const char *repository_version, *working_version;
1096 if( (repository_version = serial_number( download.DestFile() )) != NULL )
1098 /* Identify the location for the working copy, (if it exists).
1100 const char *working_copy_path_name = WORKING_DATA_PATH "/%F.xml";
1101 char working_copy[mkpath( NULL, working_copy_path_name, name, NULL )];
1102 mkpath( working_copy, working_copy_path_name, name, NULL );
1104 /* Compare issue serial numbers...
1106 if( ((working_version = serial_number( working_copy )) == NULL)
1107 || ((strcmp( repository_version, working_version )) > 0) )
1109 /* In these circumstances, we couldn't identify an issue number
1110 * for the working copy of the catalogue; (maybe there is no such
1111 * catalogue, or maybe it doesn't specify a valid issue number);
1112 * in either case, we promote the downloaded copy in its place.
1114 * FIXME: we assume that the working file and the downloaded copy
1115 * are stored on the same physical file system device, so we may
1116 * replace the former by simply deleting it, and renaming the
1117 * latter with its original path name; we make no provision for
1118 * replacing the working version by physical data copying.
1120 unlink( working_copy );
1121 rename( download.DestFile(), working_copy );
1124 /* The issue numbers, returned by the serial_number() function, were
1125 * allocated on the heap; free them to avoid leaking memory!
1127 free( (void *)(repository_version) );
1129 * The working copy issue number may be represented by a NULL pointer;
1130 * while it may be safe to call free on this, it just *seems* wrong, so
1131 * we check it first, to be certain.
1133 if( working_version != NULL )
1134 free( (void *)(working_version) );
1137 /* If the downloaded copy of the catalogue is still in the download cache,
1138 * we have chosen to keep a previous working copy, so we have no further
1139 * use for the downloaded copy; discard it, noting that we don't need to
1140 * confirm its existence because this will fail silently, if it is no
1143 unlink( download.DestFile() );
1147 #endif /* PACKAGE_BASE_COMPONENT */
1149 /* $RCSfile$: end of file */