*
* $Id$
*
- * Written by Keith Marshall <keithmarshall@users.sourceforge.net>
- * Copyright (C) 2009, 2010, 2011, 2012, MinGW Project
+ * Written by Keith Marshall <keith@users.osdn.me>
+ * Copyright (C) 2009-2013, 2017, 2020, MinGW.org Project
*
*
* Implementation of the package download machinery for mingw-get.
* arising from the use of this software.
*
*/
+#define USES_SAFE_STRCMP 1
#define WIN32_LEAN_AND_MEAN
#define _WIN32_WINNT 0x0500 /* for GetConsoleWindow() kludge */
*/
#define dmh_dialogue_context() GetConsoleWindow()
+#include <sys/stat.h>
+
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include "debug.h"
#include "pkgbase.h"
+#include "pkginet.h"
#include "pkgkeys.h"
#include "pkgtask.h"
-class pkgDownloadMeter
+/* This static member variable of the pkgDownloadMeter class
+ * provides a mechanism for communicating with a pre-existing
+ * download monitoring dialogue, such as is employed in the GUI.
+ * We MUST define it, and we also initialise it to NULL. While
+ * it remains thus, (as it always does in the CLI), no attempt
+ * will be made to access any pre-existing dialogue, and local
+ * monitoring objects will be employed; the GUI sets this to
+ * refer to its dialogue, when this is activated.
+ */
+pkgDownloadMeter *pkgDownloadMeter::primary = NULL;
+
+inline void pkgDownloadMeter::SpinWait( int mode, const char *uri )
{
- /* Abstract base class, from which facilities for monitoring the
- * progress of file downloads may be derived.
+ /* Wrapper to invoke the overridable derived class specific
+ * SpinWait() method, with protection against any attempt to
+ * access a virtual method of a primary class object which
+ * does not exist; (thus, there is no vtable reference).
*/
- public:
- /* The working method to refresh the download progress display;
- * each derived class MUST furnish an implementation for this.
- */
- virtual int Update( unsigned long length ) = 0;
-
- protected:
- /* Storage for the expected size of the active download...
- */
- unsigned long content_length;
+ if( primary != NULL ) primary->SpinWaitAction( mode, uri );
+}
- /* ...and a method to format it for human readable display.
- */
- int SizeFormat( char*, unsigned long );
-};
+#if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
-class pkgDownloadMeterTTY : public pkgDownloadMeter
+/* Implementation of a download meter class, displaying download
+ * statistics within a CLI application context.
+ */
+class pkgDownloadMeterTTY: public pkgDownloadMeter
{
- /* Implementation of a download meter class, displaying download
- * statistics within a CLI application context.
- */
public:
pkgDownloadMeterTTY( const char*, unsigned long );
virtual ~pkgDownloadMeterTTY();
- virtual int Update( unsigned long );
+ virtual void Update( unsigned long );
private:
/* This buffer is used to store each compiled status report,
char status_report[80];
};
-pkgDownloadMeterTTY::pkgDownloadMeterTTY( const char *url, unsigned long length )
-{
- source_url = url;
- content_length = length;
-}
+/* Constructor...
+ */
+pkgDownloadMeterTTY::pkgDownloadMeterTTY( const char *url, unsigned long length ):
+ source_url( url ){ content_length = length; }
+/* ...and destructor.
+ */
pkgDownloadMeterTTY::~pkgDownloadMeterTTY()
{
if( source_url == NULL )
dmh_printf( "\n" );
}
+#elif IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
+
+/* The setup tool will always use GUI style download metering;
+ * however the download agent remains CLI capable, so we need a
+ * minimal "do nothing" implementation of a TTY style metering
+ * class, to satisfy the linker.
+ */
+class pkgDownloadMeterTTY: public pkgDownloadMeter
+{
+ /* Implementation of a dummy download meter class, sufficient
+ * to promote an illusion of TTY metering capability.
+ */
+ public:
+ pkgDownloadMeterTTY( const char*, unsigned long ){}
+ virtual void Update( unsigned long ){}
+};
+
+#endif
+
static inline __attribute__((__always_inline__))
unsigned long pow10mul( register unsigned long x, unsigned power )
{
return pow10mul( x, 2 ) / q;
}
-int pkgDownloadMeterTTY::Update( unsigned long count )
+#if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
+
+void pkgDownloadMeterTTY::Update( unsigned long count )
{
/* Implementation of method to update the download progress report,
* displaying the current byte count and anticipated final byte count,
dmh_printf( "%s\n", source_url );
source_url = NULL;
}
- return dmh_printf( "\r%s%%", status_report );
+ (void)(dmh_printf( "\r%s%%", status_report ));
}
+#endif
+
int pkgDownloadMeter::SizeFormat( char *buf, unsigned long filesize )
{
/* Helper method to format raw byte counts as B, kB, MB, GB, TB, as
return retval;
}
+/* To facilitate the implementation of the pkgInternetAgent class, which
+ * is declared below:
+ */
+#if IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
+
+/* Within the setup tool, the pkgInternetAgent::SetRetryOptions() method
+ * requires a pointer to a pkgSetupAction...
+ */
+ typedef pkgSetupAction *INTERNET_RETRY_REQUESTER;
+
+#else
+/* ...whereas, in the general case, it requires a pointer to a pkgXmlNode.
+ */
+ typedef pkgXmlNode *INTERNET_RETRY_REQUESTER;
+
+#endif
+
class pkgInternetAgent
{
/* A minimal, locally implemented class, instantiated ONCE as a
*/
private:
HINTERNET SessionHandle;
+ int connection_delay, delay_factor, retries, retry_interval;
public:
inline pkgInternetAgent():SessionHandle( NULL )
if( SessionHandle != NULL )
Close( SessionHandle );
}
+ void SetRetryOptions( INTERNET_RETRY_REQUESTER, const char* );
HINTERNET OpenURL( const char* );
/* Remaining methods are simple inline wrappers for the
free( (void *)(dest_file) );
}
+void pkgInternetAgent::SetRetryOptions
+( INTERNET_RETRY_REQUESTER referrer, const char *url_template )
+{
+ /* Initialisation method, invoked immediately prior to the start
+ * of each archive download request, to establish the options for
+ * retrying any failed host connection request.
+ *
+ * The first of any sequence of connection attempts may always be
+ * initiated immediately.
+ */
+ connection_delay = 0;
+
+ /* If any further attempts are necessary, we will delay them
+ * at progressively increasing intervals, to give us the best
+ * possible chance to circumvent throttling of excess frequency
+ * connection requests, by the download server.
+ *
+ * FIXME: for each change of host domain, we should establish
+ * settings by consulting the configuration profile; for the
+ * time being, we simply assign fixed defaults.
+ */
+ delay_factor = INTERNET_DELAY_FACTOR;
+ retry_interval = INTERNET_RETRY_INTERVAL;
+ retries = INTERNET_RETRY_ATTEMPTS;
+}
+
HINTERNET pkgInternetAgent::OpenURL( const char *URL )
{
/* Open an internet data stream.
/* Aggressively attempt to acquire a resource handle, which we may use
* to access the specified URL; (schedule a maximum of five attempts).
*/
- int retries = 5;
- do { ResourceHandle = InternetOpenUrl
+ do { pkgDownloadMeter::SpinWait( 1, URL );
+
+ /* Distribute retries at geometrically incrementing intervals.
+ */
+ if( connection_delay > 0 )
+ /* This is not the first time we've tried to open this URL;
+ * compute the appropriate retry interval, and wait between
+ * successive attempts to establish the connection.
+ */
+ Sleep( connection_delay = delay_factor * connection_delay );
+ else
+ /* This is the first attempt to open this URL; don't wait,
+ * but set up the baseline, in milliseconds, for interval
+ * computation, in the event that further attempts may be
+ * required.
+ */
+ connection_delay = retry_interval;
+
+ ResourceHandle = InternetOpenUrl
(
/* Here, we attempt to assign a URL specific resource handle,
* within the scope of the SessionHandle obtained above, to
* specify it anyway, on the off-chance that it may introduce
* an undocumented benefit beyond wishful thinking.
*/
- SessionHandle, URL, NULL, 0, INTERNET_FLAG_EXISTING_CONNECT, 0
+ SessionHandle, URL, NULL, 0,
+ INTERNET_FLAG_EXISTING_CONNECT
+ | INTERNET_FLAG_IGNORE_CERT_CN_INVALID
+ | INTERNET_FLAG_IGNORE_CERT_DATE_INVALID
+ | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS
+ | INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTP
+ | INTERNET_FLAG_KEEP_CONNECTION
+ | INTERNET_FLAG_PRAGMA_NOCACHE, 0
);
if( ResourceHandle == NULL )
{
* unless we have exhausted the specified retry limit...
*/
if( --retries < 1 )
- /*
- * ...in which case, we diagnose failure to open the URL.
+ {
+ /* ...in which case, we diagnose failure to open the URL.
*/
- dmh_notify( DMH_ERROR, "%s:cannot open URL\n", URL );
-
+ unsigned int status = GetLastError();
+ DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
+ dmh_printf( "%s\nConnection failed(status=%u); abandoned.\n",
+ URL, status )
+ );
+ dmh_notify(
+ DMH_ERROR, "%s:cannot open URL; status = %u\n", URL, status
+ );
+ }
else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
- dmh_printf( "%s\nConnecting ... failed(status=%u); retrying...\n",
- URL, GetLastError() )
+ dmh_printf( "%s\nConnecting ... failed(status=%u); retrying in %ds...\n",
+ URL, GetLastError(), delay_factor * connection_delay / 1000 )
);
}
else
ResourceHandle = NULL;
/* Issue a diagnostic advising the user to refer the problem
- * to the mingw-get maintainer for possible follow-up.
+ * to the mingw-get maintainer for possible follow-up; (note
+ * that we want to group the following messages into a single
+ * message "digest", but if the caller is already doing so,
+ * then we simply incorporate these, and delegate flushing
+ * of the completed "digest" to the caller).
*/
- dmh_control( DMH_BEGIN_DIGEST );
+ uint16_t dmh_cached = dmh_control( DMH_GET_CONTROL_STATE );
+ dmh_cached &= dmh_control( DMH_BEGIN_DIGEST );
dmh_notify( DMH_WARNING,
"%s: opened with unexpected status: code = %u\n",
URL, ResourceStatus
dmh_notify( DMH_WARNING,
"please report this to the mingw-get maintainer\n"
);
- dmh_control( DMH_END_DIGEST );
+ if( dmh_cached == 0 ) dmh_control( DMH_END_DIGEST );
}
}
}
* yet exhausted our retry limit, go back and try again.
*/
} while( retries > 0 );
+ pkgDownloadMeter::SpinWait( 0 );
/* Ultimately, we return the resource handle for the opened URL,
* or NULL if the open request failed.
return dl_status;
}
+#if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
+
static const char *get_host_info
( pkgXmlNode *ref, const char *property, const char *fallback = NULL )
{
return fallback;
}
+#elif IMPLEMENTATION_LEVEL == SETUP_TOOL_COMPONENT
+
+static const char *get_host_info
+( void *ref, const char *property, const char *fallback = NULL )
+{
+ /* Customised helper to load the download host URI template
+ * directly from the setup tool's resource data section.
+ */
+ static const char *uri_template = NULL;
+ if( uri_template == NULL )
+ uri_template = strdup( WTK::StringResource( NULL, ID_DOWNLOAD_HOST_URI ) );
+ return (strcmp( property, uri_key ) == 0) ? uri_template : fallback;
+}
+
+#endif
+
static inline
int set_transit_path( const char *path, const char *file, char *buf = NULL )
{
char transit_file[set_transit_path( dest_template, filename )];
int fd; set_transit_path( dest_template, filename, transit_file );
+//dmh_printf( "Get: initialise transfer file %s ... ", transit_file );
+ chmod( transit_file, S_IWRITE ); unlink( transit_file );
if( (fd = set_output_stream( transit_file, 0644 )) >= 0 )
{
+//dmh_printf( "success\nGet: open URL %s ... ", from_url );
/* The "transit-file" is ready to receive incoming data...
* Configure and invoke the download handler to copy the data
* from the appropriate host URL, to this "transit-file".
*/
if( (dl_host = pkgDownloadAgent.OpenURL( from_url )) != NULL )
{
+//dmh_printf( "success\n" );
if( pkgDownloadAgent.QueryStatus( dl_host ) == HTTP_STATUS_OK )
{
/* With the download transaction fully specified, we may
* request processing of the file transfer...
*/
- pkgDownloadMeterTTY download_meter
- (
- from_url, pkgDownloadAgent.QueryContentLength( dl_host )
- );
- dl_meter = &download_meter;
- dl_status = TransferData( fd );
+ if( (dl_meter = pkgDownloadMeter::UseGUI()) != NULL )
+ {
+ /* ...with progress monitoring delegated to the GUI's
+ * dialogue box, when running under its auspices...
+ */
+ dl_meter->ResetGUI(
+ filename, pkgDownloadAgent.QueryContentLength( dl_host )
+ );
+ dl_status = TransferData( fd );
+ }
+ else
+ { /* ...otherwise creating our own TTY progress monitor,
+ * when running under the auspices of the CLI.
+ */
+ pkgDownloadMeterTTY download_meter(
+ from_url, pkgDownloadAgent.QueryContentLength( dl_host )
+ );
+ dl_meter = &download_meter;
+
+ /* Note that the following call MUST be kept within the
+ * scope in which the progress monitor was created; thus,
+ * it CANNOT be factored out of this "else" block scope,
+ * even though it also appears at the end of the scope
+ * of the preceding "if" block.
+ */
+ dl_status = TransferData( fd );
+ }
}
else DEBUG_INVOKE_IF( DEBUG_REQUEST( DEBUG_TRACE_INTERNET_REQUESTS ),
dmh_printf( "OpenURL:error:%d\n", GetLastError() )
*/
pkgDownloadAgent.Close( dl_host );
}
+//else dmh_printf( "failed\n" );
/* Always close the "transit-file", whether the download
* was successful, or not...
*/
unlink( transit_file );
}
+//else dmh_printf( "failed\n" );
/* Report success or failure to the caller...
*/
return dl_status;
}
-void pkgActionItem::PrintURI( const char *package_name )
+#if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
+
+void pkgActionItem::PrintURI
+( const char *package_name, int (*output)( const char * ) )
{
- /* Private method to display the URI from which a package archive
+ /* Public method to display the URI from which a package archive
* may be downloaded; called with "package_name" assigned by either
* ArchiveName() or SourceArchiveName() as appropriate, and...
*/
/* ...then, rather than actually initiate the download,
* we simply write out the generated URI to stdout.
*/
- printf( "%s\n", package_url );
+ output( package_url );
}
}
}
+#endif /* PACKAGE_BASE_COMPONENT */
+
void pkgActionItem::DownloadSingleArchive
( const char *package_name, const char *archive_cache_path )
{
if( ((flags & ACTION_DOWNLOAD) == ACTION_DOWNLOAD)
&& ((access( download.DestFile(), R_OK ) != 0) && (errno == ENOENT)) )
{
- /* ...if not, ask the download agent to fetch it...
+ /* ...if not, ask the download agent to fetch it,
+ * anticipating that this may fail...
*/
+ flags |= ACTION_DOWNLOAD_FAILED;
const char *url_template = get_host_info( Selection(), uri_key );
if( url_template != NULL )
{
/* ...from the URL constructed from the template specified in
- * the package repository catalogue (configuration database)...
+ * the package repository catalogue (configuration database).
*/
const char *mirror = get_host_info( Selection(), mirror_key );
char package_url[mkpath( NULL, url_template, package_name, mirror )];
mkpath( package_url, url_template, package_name, mirror );
+
+ /* Enable retrying of failed connection attempts, according to the
+ * preferences, if any, which have been configured for the repository
+ * associated with the current URL template, or with default settings
+ * otherwise, then initiate the package download process.
+ */
+ pkgDownloadAgent.SetRetryOptions( Selection(), url_template );
if( download.Get( package_url ) > 0 )
/*
- * Download was successful; clear the pending flag.
+ * Download was successful; clear the pending and failure flags.
*/
- flags &= ~(ACTION_DOWNLOAD);
+ flags &= ~(ACTION_DOWNLOAD | ACTION_DOWNLOAD_FAILED);
else
/* Diagnose failure; leave pending flag set.
*/
*/
while( current != NULL )
{
- /* ...while we haven't run off the end...
+ /* ...while we haven't run off the end, and collecting any diagnositic
+ * messages relating to any one package into a common digest...
*/
+ dmh_control( DMH_BEGIN_DIGEST );
if( (current->flags & ACTION_INSTALL) == ACTION_INSTALL )
{
/* For all packages specified in the current action list,
*/
current->DownloadSingleArchive( package_name, pkgArchivePath() );
}
- /* Repeat download action, for any additional packages specified
+ else
+ /* The current action has no associated "install" requirement,
+ * so neither is there any need to request a "download".
+ */
+ current->flags &= ~(ACTION_DOWNLOAD);
+
+ /* Flush out any diagnostics relating to the current package, then
+ * repeat the download action, for any additional packages specified
* in the current "actions" list.
*/
+ dmh_control( DMH_END_DIGEST );
current = current->next;
}
}
-#define DATA_CACHE_PATH "%R" "var/cache/mingw-get/data"
+#if IMPLEMENTATION_LEVEL == PACKAGE_BASE_COMPONENT
+
+#define DATA_CACHE_PATH "%R" "var/cache/mingw-get/data"
#define WORKING_DATA_PATH "%R" "var/lib/mingw-get/data"
/* Internet servers host package catalogues in lzma compressed format;
#define PKGSTRM_H_SPECIAL 1
#include "pkgstrm.h"
-class pkgInternetLzmaStreamingAgent :
+class pkgInternetLzmaStreamingAgent:
public pkgInternetStreamingAgent, public pkgLzmaArchiveStream
{
/* Specialisation of the pkgInternetStreamingAgent base class,
return dl_status;
}
-static const char *serial_number( const char *catalogue )
+EXTERN_C const char *serial_number( const char *catalogue )
{
/* Local helper function to retrieve issue numbers from any repository
* package catalogue; returns the result as a duplicate of the internal
const char *mirror = repository->GetPropVal( mirror_key, NULL );
char catalogue_url[mkpath( NULL, url_template, name, mirror )];
mkpath( catalogue_url, url_template, name, mirror );
+
+ /* Enable retries according to the preferences, if any, as
+ * configured for the repository, or adopt default settings
+ * otherwise, then initiate the download process for the
+ * catalogue file.
+ */
+ pkgDownloadAgent.SetRetryOptions( repository, url_template );
if( download.Get( catalogue_url ) <= 0 )
dmh_notify( DMH_ERROR,
"Sync Repository: %s: download failed\n", catalogue_url
}
}
+#endif /* PACKAGE_BASE_COMPONENT */
+
/* $RCSfile$: end of file */
OSDN Git Service
