6 * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
7 * Copyright (C) 2009, 2010, MinGW Project
10 * Implementation of the package download machinery for mingw-get.
13 * This is free software. Permission is granted to copy, modify and
14 * redistribute this software, under the provisions of the GNU General
15 * Public License, Version 3, (or, at your option, any later version),
16 * as published by the Free Software Foundation; see the file COPYING
17 * for licensing details.
19 * Note, in particular, that this software is provided "as is", in the
20 * hope that it may prove useful, but WITHOUT WARRANTY OF ANY KIND; not
21 * even an implied WARRANTY OF MERCHANTABILITY, nor of FITNESS FOR ANY
22 * PARTICULAR PURPOSE. Under no circumstances will the author, or the
23 * MinGW Project, accept liability for any damages, however caused,
24 * arising from the use of this software.
27 #define WIN32_LEAN_AND_MEAN
29 #define _WIN32_WINNT 0x0500 /* for GetConsoleWindow() kludge */
32 * FIXME: This kludge allows us to use the standard wininet dialogue
33 * to acquire proxy authentication credentials from the user; this is
34 * expedient for now, (if somewhat anti-social for a CLI application).
35 * We will ultimately need to provide a more robust implementation,
36 * (within the scope of the diagnostic message handler), in order to
37 * obtain a suitable window handle for use when called from the GUI
38 * implementation of mingw-get, (when it becomes available).
40 #define dmh_dialogue_context() GetConsoleWindow()
55 class pkgInternetAgent
57 /* A minimal, locally implemented class, instantiated ONCE as a
58 * global object, to ensure that wininet's global initialisation is
59 * completed at the proper time, without us doing it explicitly.
62 HINTERNET SessionHandle;
65 inline pkgInternetAgent():SessionHandle( NULL )
69 * This is called during DLL initialisation; thus it seems to be
70 * the ideal place to perform one time internet connection setup.
71 * However, Microsoft caution against doing much here, (especially
72 * creation of threads, either directly or indirectly); thus we
73 * defer the connection setup until we ultimately need it.
76 inline ~pkgInternetAgent()
80 if( SessionHandle != NULL )
81 Close( SessionHandle );
84 /* Remaining methods are simple inline wrappers for the
85 * wininet functions we plan to use...
87 inline HINTERNET OpenURL( const char *URL )
89 /* Open an internet data stream. This requires an internet
90 * connection to have been established...
92 if( (SessionHandle == NULL)
93 && (InternetAttemptConnect( 0 ) == ERROR_SUCCESS) )
95 * ...so, on first call, we perform the connection setup
96 * which we deferred from the class constructor; (MSDN
97 * cautions that this MUST NOT be done in the constructor
98 * for any global class object such as ours).
100 SessionHandle = InternetOpen
101 ( "MinGW Installer", INTERNET_OPEN_TYPE_PRECONFIG,
104 HINTERNET ResourceHandle = InternetOpenUrl
106 /* Here, we attempt to assign a URL specific resource handle,
107 * within the scope of the SessionHandle obtained above, to
108 * manage the connection for the requested URL.
110 * Note: Scott Michel suggests INTERNET_FLAG_EXISTING_CONNECT
111 * here; MSDN tells us it is useful only for FTP connections.
112 * Since we are primarily interested in HTTP connections, it
113 * may not help us. However, it does no harm, and MSDN isn't
114 * always the reliable source of information we might like.
115 * Persistent HTTP connections aren't entirely unknown, (and
116 * indeed, MSDN itself tells us we need to use one, when we
117 * negotiate proxy authentication); thus, we may just as well
118 * specify it anyway, on the off-chance that it may introduce
119 * an undocumented benefit beyond wishful thinking.
121 SessionHandle, URL, NULL, 0, INTERNET_FLAG_EXISTING_CONNECT, 0
123 if( ResourceHandle != NULL )
125 /* We got a handle for the URL resource, but we cannot yet be
126 * sure that it is ready for use; we may still need to handle
127 * proxy or server authentication. Thus, we must capture any
128 * error code which may have been returned, BEFORE we move on
129 * to evaluate the resource status, (since the procedure for
130 * checking status may change the error code).
132 unsigned long ResourceStatus = GetLastError();
133 if( QueryStatus( ResourceHandle ) == HTTP_STATUS_PROXY_AUTH_REQ )
135 /* We've identified a requirement for proxy authentication;
136 * here we simply hand the task off to the Microsoft handler,
137 * to solicit the appropriate response from the user.
139 * FIXME: this may be a reasonable approach when running in
140 * a GUI context, but is rather inelegant in the CLI context.
141 * Furthermore, this particular implementation provides only
142 * for proxy authentication, ignoring the possibility that
143 * server authentication may be required. We may wish to
144 * revisit this later.
146 unsigned long user_response;
147 do { user_response = InternetErrorDlg
148 ( dmh_dialogue_context(), ResourceHandle, ResourceStatus,
149 FLAGS_ERROR_UI_FILTER_FOR_ERRORS |
150 FLAGS_ERROR_UI_FLAGS_CHANGE_OPTIONS |
151 FLAGS_ERROR_UI_FLAGS_GENERATE_DATA,
154 /* Having obtained authentication credentials from
155 * the user, we may retry the open URL request...
157 if( (user_response == ERROR_INTERNET_FORCE_RETRY)
158 && HttpSendRequest( ResourceHandle, NULL, 0, 0, 0 ) )
160 /* ...and, if successful...
162 ResourceStatus = GetLastError();
163 if( QueryStatus( ResourceHandle ) == HTTP_STATUS_OK )
165 * ...ensure that the response is anything but 'retry',
166 * so that we will break out of the retry loop...
168 user_response ^= -1L;
170 /* ...otherwise, we keep retrying when appropriate.
172 } while( user_response == ERROR_INTERNET_FORCE_RETRY );
175 /* Ultimately, we return the resource handle for the opened URL,
176 * or NULL if the open request failed.
178 return ResourceHandle;
180 inline unsigned long QueryStatus( HINTERNET id )
182 unsigned long ok, idx = 0, len = sizeof( ok );
183 if( HttpQueryInfo( id, HTTP_QUERY_FLAG_NUMBER | HTTP_QUERY_STATUS_CODE,
188 inline int Read( HINTERNET dl, char *buf, size_t max, unsigned long *count )
190 return InternetReadFile( dl, buf, max, count );
192 inline int Close( HINTERNET id )
194 return InternetCloseHandle( id );
198 /* This is the one and only instantiation of an object of this class.
200 static pkgInternetAgent pkgDownloadAgent;
202 class pkgInternetStreamingAgent
204 /* Another locally implemented class; each individual file download
205 * gets its own instance of this, either as-is for basic data transfer,
206 * or as a specialised derivative of this base class.
209 const char *filename;
210 const char *dest_template;
217 virtual int TransferData( int );
220 pkgInternetStreamingAgent( const char*, const char* );
221 virtual ~pkgInternetStreamingAgent();
223 virtual int Get( const char* );
224 inline const char *DestFile(){ return dest_file; }
227 pkgInternetStreamingAgent::pkgInternetStreamingAgent
228 ( const char *local_name, const char *dest_specification )
230 /* Constructor for the pkgInternetStreamingAgent class.
232 filename = local_name;
233 dest_template = dest_specification;
234 dest_file = (char *)(malloc( mkpath( NULL, dest_template, filename, NULL ) ));
235 if( dest_file != NULL )
236 mkpath( dest_file, dest_template, filename, NULL );
239 pkgInternetStreamingAgent::~pkgInternetStreamingAgent()
241 /* Destructor needs to free the heap memory allocated by the
242 * constructor, for storage of "dest_file" name.
244 free( (void *)(dest_file) );
247 int pkgInternetStreamingAgent::TransferData( int fd )
249 /* In the case of this base class implementation,
250 * we simply read the file's data from the Internet source,
251 * and write a verbatim copy to the destination file.
253 char buf[8192]; DWORD count, tally = 0;
254 do { dl_status = pkgDownloadAgent.Read( dl_host, buf, sizeof( buf ), &count );
255 dmh_printf( "\rdownloading: %s: %I32d b", filename, tally += count );
256 write( fd, buf, count );
257 } while( dl_status && (count > 0) );
258 dmh_printf( "\rdownloading: %s: %I32d b\n", filename, tally );
262 static const char *get_host_info
263 ( pkgXmlNode *ref, const char *property, const char *fallback = NULL )
265 /* Helper function to retrieve host information from the XML catalogue.
267 * Call with property = "url", to retrieve the URL template to pass as
268 * "fmt" argument to mkpath(), or with property = "mirror", to retrieve
269 * the substitution text for the "modifier" argument.
271 const char *uri = NULL;
274 /* Starting from the "ref" package entry in the catalogue...
276 pkgXmlNode *host = ref->FindFirstAssociate( download_host_key );
277 while( host != NULL )
279 /* Examine its associate tags; if we find one of type
280 * "download-host", with the requisite property, then we
281 * immediately return that property value...
283 if( (uri = host->GetPropVal( property, NULL )) != NULL )
286 /* Otherwise, we look for any other candidate tags
287 * associated with the same catalogue entry...
289 host = host->FindNextAssociate( download_host_key );
291 /* Failing an immediate match, extend the search to the
292 * ancestors of the initial reference entry...
294 ref = ref->GetParent();
296 /* ...and ultimately, if no match is found, we return the
297 * specified "fallback" property value.
303 int set_transit_path( const char *path, const char *file, char *buf = NULL )
305 /* Helper to define the transitional path name for downloaded files,
306 * used to save the file data while the download is in progress.
308 static const char *transit_dir = "/.in-transit";
309 return mkpath( buf, path, file, transit_dir );
312 int pkgInternetStreamingAgent::Get( const char *from_url )
314 /* Download a file from the specified internet URL.
316 * Before download commences, we accept that this may fail...
320 /* Set up a "transit-file" to receive the downloaded content.
322 char transit_file[set_transit_path( dest_template, filename )];
323 int fd; set_transit_path( dest_template, filename, transit_file );
325 if( (fd = set_output_stream( transit_file, 0644 )) >= 0 )
327 /* The "transit-file" is ready to receive incoming data...
328 * Configure and invoke the download handler to copy the data
329 * from the appropriate host URL, to this "transit-file".
331 if( (dl_host = pkgDownloadAgent.OpenURL( from_url )) != NULL )
333 if( pkgDownloadAgent.QueryStatus( dl_host ) == HTTP_STATUS_OK )
335 /* With the download transaction fully specified, we may
336 * request processing of the file transfer...
338 dl_status = TransferData( fd );
341 /* We are done with the URL handle; close it.
343 pkgDownloadAgent.Close( dl_host );
346 /* Always close the "transit-file", whether the download
347 * was successful, or not...
352 * When successful, we move the "transit-file" to its
353 * final downloaded location...
355 rename( transit_file, dest_file );
357 /* ...otherwise, we discard the incomplete "transit-file",
358 * leaving the caller to diagnose the failure.
360 unlink( transit_file );
363 /* Report success or failure to the caller...
368 void pkgActionItem::DownloadArchiveFiles( pkgActionItem *current )
370 /* Update the local package cache, to ensure that all packages needed
371 * to complete the current set of scheduled actions are present; if any
372 * are missing, invoke an Internet download agent to fetch them. This
373 * requires us to walk the action list...
375 while( current != NULL )
377 /* ...while we haven't run off the end...
379 if( (current->flags & ACTION_INSTALL) == ACTION_INSTALL )
381 /* For all packages specified in the current action list,
382 * for which an "install" action is scheduled, and for which
383 * no associated archive file is present in the local archive
384 * cache, place an Internet download agent on standby to fetch
385 * the required archive from a suitable internet mirror host.
387 const char *package_name = current->Selection()->ArchiveName();
389 /* An explicit package name of "none" is a special case, indicating
390 * a "virtual" meta-package; it requires nothing to be downloaded...
392 if( ! match_if_explicit( package_name, value_none ) )
394 /* ...but we expect any other package to provide real content,
395 * for which we may need to download the package archive...
397 pkgInternetStreamingAgent download( package_name, pkgArchivePath() );
399 /* Check if the required archive is already available locally...
401 if( (access( download.DestFile(), R_OK ) != 0) && (errno == ENOENT) )
403 /* ...if not, ask the download agent to fetch it...
405 const char *url_template = get_host_info( current->Selection(), uri_key );
406 if( url_template != NULL )
408 /* ...from the URL constructed from the template specified in
409 * the package repository catalogue (configuration database)...
411 const char *mirror = get_host_info( current->Selection(), mirror_key );
412 char package_url[mkpath( NULL, url_template, package_name, mirror )];
413 mkpath( package_url, url_template, package_name, mirror );
414 if( ! (download.Get( package_url ) > 0) )
415 dmh_notify( DMH_ERROR,
416 "Get package: %s: download failed\n", package_url
420 /* Cannot download; the repository catalogue didn't specify a
421 * template, from which to construct a download URL...
423 dmh_notify( DMH_ERROR,
424 "Get package: %s: no URL specified for download\n", package_name
429 /* Repeat download action, for any additional packages specified
430 * in the current "actions" list.
432 current = current->next;
436 #define DATA_CACHE_PATH "%R" "var/cache/mingw-get/data"
437 #define WORKING_DATA_PATH "%R" "var/lib/mingw-get/data"
439 /* Internet servers host package catalogues in lzma compressed format;
440 * we will decompress them "on the fly", as we download them. To achieve
441 * this, we will use a variant of the pkgInternetStreamingAgent, using a
442 * specialised TransferData method; additionally, this will incorporate
443 * a special derivative of a pkgLzmaArchiveStream, with its GetRawData
444 * method adapted to stream data from an internet URI, instead of
445 * reading from a local file.
447 * To derive the pkgInternetLzmaStreamingAgent, we need to include the
448 * specialised declarations of a pkgArchiveStream, in order to make the
449 * declaration of pkgLzmaArchiveStream available as our base class.
451 #define PKGSTRM_H_SPECIAL 1
454 class pkgInternetLzmaStreamingAgent :
455 public pkgInternetStreamingAgent, public pkgLzmaArchiveStream
457 /* Specialisation of the pkgInternetStreamingAgent base class,
458 * providing decompressed copies of LZMA encoded files downloaded
459 * from the Internet; (the LZMA decompression capability is derived
460 * from the pkgLzmaArchiveStream base class).
463 /* We need a specialised constructor...
465 pkgInternetLzmaStreamingAgent( const char*, const char* );
468 /* Specialisation requires overrides for each of this pair of
469 * methods, (the first from the pkgLzmaArchiveStream base class;
470 * the second from pkgInternetStreamingAgent).
472 virtual int GetRawData( int, uint8_t*, size_t );
473 virtual int TransferData( int );
476 /* This specialisation of the pkgInternetStreamingAgent class needs its
477 * own constructor, simply to invoke the constructors for the base classes,
478 * (since neither is instantiated by a default constructor).
480 pkgInternetLzmaStreamingAgent::pkgInternetLzmaStreamingAgent
481 ( const char *local_name, const char *dest_specification ):
482 pkgInternetStreamingAgent( local_name, dest_specification ),
484 * Note that, when we come to initialise the lzma streaming component
485 * of this derived class, we will be streaming directly from the internet,
486 * rather than from a file stream, so we don't require a file descriptor
487 * for the input stream; however, the class semantics still expect one.
488 * To avoid accidental association with an existing file stream, we
489 * use a negative value, (which is never a valid file descriptor);
490 * however, we must not choose -1, since the class implementation
491 * will decline to process the stream; hence, we choose -2.
493 pkgLzmaArchiveStream( -2 ){}
495 int pkgInternetLzmaStreamingAgent::GetRawData( int fd, uint8_t *buf, size_t max )
497 /* Fetch raw (compressed) data from the Internet host, and load it into
498 * the decompression filter's input buffer, whence the TransferData routine
499 * may retrieve it, via the filter, as an uncompressed stream.
502 dl_status = pkgDownloadAgent.Read( dl_host, (char *)(buf), max, &count );
506 int pkgInternetLzmaStreamingAgent::TransferData( int fd )
508 /* In this case, we read the file's data from the Internet source,
509 * stream it through the lzma decompression filter, and write a copy
510 * of the resultant decompressed data to the destination file.
512 char buf[8192]; DWORD count;
513 do { count = pkgLzmaArchiveStream::Read( buf, sizeof( buf ) );
514 write( fd, buf, count );
515 } while( dl_status && (count > 0) );
519 static const char *serial_number( const char *catalogue )
521 /* Local helper function to retrieve issue numbers from any repository
522 * package catalogue; returns the result as a duplicate of the internal
523 * string, allocated on the heap (courtesy of the strdup() function).
526 pkgXmlDocument src( catalogue );
529 && ((issue = src.GetRoot()->GetPropVal( issue_key, NULL )) != NULL) )
531 * Found an issue number; return a copy...
533 return strdup( issue );
535 /* If we get to here, we couldn't get a valid issue number;
536 * whatever the reason, return NULL to indicate failure.
541 void pkgXmlDocument::SyncRepository( const char *name, pkgXmlNode *repository )
543 /* Fetch a named package catalogue from a specified Internet repository.
545 * Package catalogues are XML files; the master copy on the Internet host
546 * must be stored in lzma compressed format, and named to comply with the
547 * convention "%F.xml.lzma", in which "%F" represents the value of the
548 * "name" argument passed to this pkgXmlDocument class method.
550 const char *url_template;
551 if( (url_template = repository->GetPropVal( uri_key, NULL )) != NULL )
553 /* Initialise a streaming agent, to manage the catalogue download;
554 * (note that we must include the "%/M" placeholder in the template
555 * for the local name, to accommodate the name of the intermediate
556 * "in-transit" directory used by the streaming agent).
558 pkgInternetLzmaStreamingAgent download( name, DATA_CACHE_PATH "%/M/%F.xml" );
560 /* Construct the full URI for the master catalogue, and stream it to
561 * a locally cached, decompressed copy of the XML file.
563 const char *mirror = repository->GetPropVal( mirror_key, NULL );
564 char catalogue_url[mkpath( NULL, url_template, name, mirror )];
565 mkpath( catalogue_url, url_template, name, mirror );
566 if( download.Get( catalogue_url ) <= 0 )
567 dmh_notify( DMH_ERROR,
568 "Sync Repository: %s: download failed\n", catalogue_url
572 /* We will only replace our current working copy of this catalogue,
573 * (if one already exists), with the copy we just downloaded, if this
574 * downloaded copy bears an issue number indicating that it is more
575 * recent than the working copy.
577 const char *repository_version, *working_version;
578 if( (repository_version = serial_number( download.DestFile() )) != NULL )
580 /* Identify the location for the working copy, (if it exists).
582 const char *working_copy_path_name = WORKING_DATA_PATH "/%F.xml";
583 char working_copy[mkpath( NULL, working_copy_path_name, name, NULL )];
584 mkpath( working_copy, working_copy_path_name, name, NULL );
586 /* Compare issue serial numbers...
588 if( ((working_version = serial_number( working_copy )) == NULL)
589 || ((strcmp( repository_version, working_version )) > 0) )
591 /* In these circumstances, we couldn't identify an issue number
592 * for the working copy of the catalogue; (maybe there is no such
593 * catalogue, or maybe it doesn't specify a valid issue number);
594 * in either case, we promote the downloaded copy in its place.
596 * FIXME: we assume that the working file and the downloaded copy
597 * are stored on the same physical file system device, so we may
598 * replace the former by simply deleting it, and renaming the
599 * latter with its original path name; we make no provision for
600 * replacing the working version by physical data copying.
602 unlink( working_copy );
603 rename( download.DestFile(), working_copy );
606 /* The issue numbers, returned by the serial_number() function, were
607 * allocated on the heap; free them to avoid leaking memory!
609 free( (void *)(repository_version) );
611 * The working copy issue number may be represented by a NULL pointer;
612 * while it may be safe to call free on this, it just *seems* wrong, so
613 * we check it first, to be certain.
615 if( working_version != NULL )
616 free( (void *)(working_version) );
619 /* If the downloaded copy of the catalogue is still in the download cache,
620 * we have chosen to keep a previous working copy, so we have no further
621 * use for the downloaded copy; discard it, noting that we don't need to
622 * confirm its existence because this will fail silently, if it is no
625 unlink( download.DestFile() );
629 /* $RCSfile$: end of file */