3 # Generates to stdout a ranked Antergos mirror list that can be directly used as
4 # file /etc/pacman.d/antergos-mirrorlist.
6 # For optimizing performance
7 # - while searching the mirrors with this tool
9 # - while updating your system using the ranked mirrors
10 # you may exclude mirror countries (option --exclude-countries)
11 # or exclude groups of countries (--exclude-countrygroups), or individual mirrors (--exclude-mirrors).
13 # You can specify which protocols (https, http) are used (option --protocols).
15 # For more details, see below.
18 # - Copy this script into a file (e.g. /usr/bin/reflector-antergos or /usr/local/bin/reflector-antergos)
19 # and make it executable (e.g. chmod +x /usr/bin/reflector-antergos).
21 # NOTE1: for option --exclude-countries, in addition to country names you can
22 # give "reserved" word 'Automated'. The Automated excludes all
23 # automated mirrors (usually one or two).
24 # Single mirrors (URLs) can be excluded with option --exclude-mirrors.
26 # NOTE2: it is possible to manually edit and re-order mirrors
27 # in /etc/pacman.d/antergos-mirrorlist.
30 # reflector-antergos --exclude-countrygroup=Asia2,Europe > ~/antergos-mirrorlist
31 # sudo mv -f /etc/pacman.d/antergos-mirrorlist /etc/pacman.d/antergos-mirrorlist.bak # optional: backup old file
32 # sudo mv -f ~/antergos-mirrorlist /etc/pacman.d/antergos-mirrorlist
35 # - Only http and https protocols are currently supported.
38 # - This script assumes $INPUTFILE (antergos-mirrorlist, see function GetLatestInputFile below)
39 # is written using a certain format.
40 # If the format changes, this script must be modified accordingly
41 # (which may or may not be trivial).
44 OUTPUTFILE=/etc/pacman.d/antergos-mirrorlist
45 INPUTFILE="" # the latest available antergos-mirrorlist, downloaded!
47 EXCLUDEDCOUNTRYGROUPS=""
50 PROTOCOLS="http" # default protocol
56 TMPDIR="$HOME/.rankingstuff.tmp.$$"
57 #TMPDIR="/tmp/.rankingstuff.tmp.$$"
58 RANKEDMIRRORS="$TMPDIR/rankedmirrors"
59 INSTALLER="$TMPDIR/installer.bash"
61 REFLECTOR_ANTERGOS_CONF=/etc/reflector-antergos.conf # may contain e.g. REFERENCEURL
62 if [ -f $REFLECTOR_ANTERGOS_CONF ] ; then
63 source $REFLECTOR_ANTERGOS_CONF
65 if [ "$REFERENCEURL" = "" ] ; then
66 REFERENCEURL=http://mirrors.antergos.com/$REPO/$ARCH/main.files.tar.gz
69 REFERENCEFILE="$TMPDIR/referencefile"
70 REFERENCESTAMP=0 # timestamp
73 SAVE_TO_FILE=false # false=output to standard output, true=to /etc/pacman.d/antergos-mirrorlist
75 HIDE_ORIGINAL_MIRRORS=yes # Controls how the original mirror list is appended after ranked mirrors.
76 # yes=hides original mirrors, no=don't hide original mirrors.
78 VERSION_STUB="$Id: reflector-antergos,v 1.34 2018/06/20 14:38:01 manuel Exp $"
79 VERSION=0.$(echo "$VERSION_STUB" | awk '{print $3}')
84 TIME_CMD=/usr/bin/time
85 WGET_CMD=/usr/bin/wget
87 WGET_TIMEOUT=30 # seconds to wait for the mirror to respond
89 echo2() { echo "$@" >&2 ; }
90 verbose2() { test $VERBOSE -ne 0 && echo2 "$@" ; }
91 verbose2more() { test $VERBOSE -eq 2 && echo2 "$@" ; }
92 debug2() { test $DEBUG -eq 1 && echo2 "$@" ; }
97 printf "$fmt" "$@" >&2
103 if [ $? -ne 0 ] ; then
104 echo2 "ERROR: command '$*' failed."
115 reflector-antergos generates (to stdout) a ranked Antergos mirror list
116 that can be directly used as file /etc/pacman.d/antergos-mirrorlist.
120 Usage: $(basename $0) [options]
122 --exclude-countries=X,Y,Z,...
123 Exclude all mirrors from given countries.
124 Note1: If a country name includes spaces, e.g. Czech Republic,
125 convert spaces into underscores, e.g. Czech_Republic.
126 Note2: The word 'Automated' excludes all mirrors from section
127 'Automated Mirror Selection'.
129 --exclude-countrygroups=X,Y,Z,...
130 Exclude countries using group names: America, Asia, Asia2, Europe, Europe2
131 where Asia2 and Europe2 include Russia.
133 --exclude-mirrors=X,Y,Z,...
134 Exclude mirror urls that contain any of strings X, Y or Z as substring.
136 --include-countries=X,Y,Z,...
137 The opposite of --exclude-countries.
138 Note that this option cannot be used together with options to
142 Saves the ranked mirror list to file /etc/pacman.d/antergos-mirrorlist
143 and makes a backup of the previous file.
144 Note that usage of this option requires root permissions.
145 If this option is not used, the ranked mirror list is printed to standard output.
147 --hide-original-mirrors=X
148 As the original mirror list is appended to the ranking result,
149 this option controls whether original mirrors are hidden or not.
150 X=yes hides original mirrors (default).
151 X=no does not hide them.
152 Note that the original mirror list may contain some out-of-order mirrors.
155 Repo protocols in preference order (currently only https and http
159 Max time in seconds to wait for a mirror response. Default is $WGET_TIMEOUT.
162 X defaults to 'antergos'
165 X defaults to $(uname -m)
168 Don't show progress info.
171 Show more details about operation
178 $(basename $0) --exclude-countries=China,Russia,USA,Japan,Automated \\
179 --exclude-mirrors=sourceforge,mirrors.tuxns,antergos-mirror-a.alpix \\
180 --protocols=https,http
190 # return as one word!
193 1) echo "generic_error" ;;
194 2) echo "parse_error" ;;
195 3) echo "file_i/o_error" ;;
196 4) echo "network_failure" ;;
197 5) echo "ssl_verification_failure" ;;
198 6) echo "username/password_authentication_failure" ;;
199 7) echo "protocol_error" ;;
200 8) echo "server_issued_error" ;;
201 *) echo "(unknown_error)" ;;
209 local mirror=$(echo "$mirrorarg" | sed -e 's|\$repo|'$REPO'|' -e 's|\$arch|'$ARCH'|')
210 local mirrorurl="$mirror/$(basename $REFERENCEURL)"
211 local mirrorfile="$TMPDIR/mirrorfile"
212 local mirrordata="$TMPDIR/mirrordata"
214 local downloadtime=-1
220 for xx in $EXCLUDEDMIRRORS
222 if [[ "$mirror" =~ "$xx" ]] ; then
227 verbose2 " $mirrorarg"
231 $TIME_CMD -f %e $WGET_CMD --timeout=$WGET_TIMEOUT --tries=1 -q -O "$mirrorfile" "$mirrorurl" 2>"$mirrordata" 1>/dev/null
234 if [ $result -eq 0 ] ; then
235 downloadtime=$(cat "$mirrordata" | tr '.' ',')
236 timestamp=$(stat -c %Y "$mirrorfile")
238 #verbose2 "vertailu: $timestamp $REFERENCESTAMP"
239 if [[ "$timestamp" < "$REFERENCESTAMP" ]] ; then
242 elif [ "$(cmp "$mirrorfile" "$REFERENCEFILE")" != "" ] ; then
243 status="contents_differ"
247 status=$(wget_error $result) # file download problem
250 if [ "$status" != "ok" ] ; then
253 echo2 "status = $status"
255 outdated) echo2 "timestamp = $timestamp refstamp = $REFERENCESTAMP" ;;
261 printf "%sServer = %s ############# downloadtime: %5s status: %s %s\n" \
262 "$hide" "$mirrorarg" "$downloadtime" "$status" "$country" >> "$RANKEDMIRRORS"
274 for protocol in $PROTOCOLS
276 m=$(echo "$mirror" | sed 's|^[a-z]*://|'$protocol'://|')
277 RankMirrorProtocol "$m" "$country"
287 if [ "$country" = "Automated" ] ; then
288 mirrors=$(grep -A 5 "^# [ ]*Automated Mirror" "$INPUTFILE" | grep "^Server = " | awk '{print $3}')
290 local country2=$(echo $country | sed 's|_.*$||') # remove first underscore (and tail) that may have been put there
291 country=$(echo $country | sed 's|_| |g') # restore the original country name!
292 mirrors=$(sed -n '/# '$country2'/,/^$/ p' $INPUTFILE | sed '1d;$d' | awk '{print $3}')
296 for mirror in $mirrors
298 RankMirror "$mirror" "$country"
306 mv "$RANKEDMIRRORS" "$RANKEDMIRRORS".tmp
308 for protocol in $PROTOCOLS
310 grep "Server = $protocol://" "$RANKEDMIRRORS".tmp | sort -g -k 6 >> "$RANKEDMIRRORS"
312 rm -f "$RANKEDMIRRORS".tmp
315 RemoveCommentsOnServerLines()
317 mv "$RANKEDMIRRORS" "$RANKEDMIRRORS".tmp
318 cat "$RANKEDMIRRORS".tmp | sed 's|/\$arch[ ]*#|/\$arch\n#|' > "$RANKEDMIRRORS"
319 rm -f "$RANKEDMIRRORS".tmp
324 local program=$(basename $0)
326 mv "$RANKEDMIRRORS" "$RANKEDMIRRORS".tmp
327 echo "## Generated by $program (version $VERSION) at $(date)." >> "$RANKEDMIRRORS"
328 echo "##" >> "$RANKEDMIRRORS"
329 echo "## Options used: $COMMAND_LINE" >> "$RANKEDMIRRORS"
330 echo "##" >> "$RANKEDMIRRORS"
331 cat "$RANKEDMIRRORS".tmp >> "$RANKEDMIRRORS"
332 if [ -f "$INPUTFILE" ] ; then
333 echo "" >> "$RANKEDMIRRORS"
334 echo "" >> "$RANKEDMIRRORS"
335 echo "#############################################################" >> "$RANKEDMIRRORS"
336 echo "############## Original antergos-mirrorlist: ################" >> "$RANKEDMIRRORS"
337 echo "#############################################################" >> "$RANKEDMIRRORS"
338 echo "" >> "$RANKEDMIRRORS"
339 case "$HIDE_ORIGINAL_MIRRORS" in
341 cat "$INPUTFILE" | sed -e 's|^#Server = |##Server = |' -e 's|^Server = |#Server = |' >> "$RANKEDMIRRORS" # original mirrors commented out
344 cat "$INPUTFILE" >> "$RANKEDMIRRORS"
348 rm -f "$RANKEDMIRRORS".tmp
353 case "$SAVE_TO_FILE" in
355 sudo mv -f "$OUTPUTFILE" "$OUTPUTFILE".bak
356 sudo cp -f "$RANKEDMIRRORS" "$OUTPUTFILE"
364 RankAllMirrors() # this mainly controls the operation
366 # If a country name for allcountries includes spaces, they are turned to underscores.
367 # Note that this implementation handles max 3 words in a country name.
368 # If more words can exist on a country name, change the awk command on the next line.
369 local allcountries=$(grep "^# [A-Z]" "$INPUTFILE" | awk '{print $2"_"$3"_"$4}' | sed 's|[_]*$||')
370 local automatedmirrors=$(grep -A 5 "^# [ ]*Automated Mirror" "$INPUTFILE" | grep "^Server = " | awk '{print $3}')
373 rm -f "$RANKEDMIRRORS"
375 if [ -n "$INCLUDEDCOUNTRIES" ] ; then
376 # only the included countries are ranked
377 for country in $INCLUDEDCOUNTRIES
379 RankCountryMirrors "$country"
382 if [[ ! "$EXCLUDEDCOUNTRIES" =~ "Automated" ]] ; then
383 allcountries+=" Automated"
385 printf2 "Automated Mirrors:\n <excluded>\n"
388 for country in $allcountries
390 if [[ ! "$EXCLUDEDCOUNTRIES" =~ "$country" ]] ; then
391 RankCountryMirrors "$country"
393 printf2 "$(echo $country | sed 's|_| |g'):\n <excluded>\n"
398 if [ $FOUND_MIRRORS -eq 0 ] ; then
400 echo2 -n "Fail: no good mirrors currently found! "
401 if [ "$EXCLUDEDCOUNTRIES" = "" ] ; then
402 echo2 "Please try again later."
404 echo2 "Please try again later, or reduce your list of excluded countries."
408 echo2 "$FOUND_MIRRORS mirrors found."
413 RemoveCommentsOnServerLines
419 # Download the latest available antergos-mirrorlist as a reference input file.
421 echo2 -n "Downloading latest mirror info ... "
422 _runcheck pushd "$TMPDIR" >/dev/null
424 local filelist=fileinfo-$$.html
425 _runcheck wget -q -O "$filelist" https://repo.antergos.info/antergos/x86_64/
427 local inputfile=$(cat "$filelist" | sed 's|</a>|</a>\n|g' | grep "antergos-mirrorlist" | grep -v "xz\.sig" | sed 's|^.*>\(antergos\-mirrorlist\-[0-9\-]*\-any\.pkg\.tar\.xz\)<.*$|\1|')
430 _runcheck wget -q https://repo.antergos.info/antergos/x86_64/"$inputfile"
431 _runcheck tar xf "$inputfile"
433 if [ -f etc/pacman.d/antergos-mirrorlist ] ; then
434 mv etc/pacman.d/antergos-mirrorlist .
436 rm -f "$inputfile" .BUILDINFO .MTREE .PKGINFO
438 echo2 "Error: file antergos-mirrorlist not found in downloaded package '$PWD/$inputfile'!"
442 _runcheck popd >/dev/null
446 INPUTFILE="$TMPDIR/antergos-mirrorlist"
448 if [ "$(cat "$INPUTFILE")" = "" ] ; then
449 echo2 "Error: your '$INPUTFILE' is empty??"
456 # make sure we have a reference file to read!
459 $WGET_CMD -q -O "$REFERENCEFILE" $REFERENCEURL
460 REFERENCESTAMP=$(stat -c %Y "$REFERENCEFILE")
465 rm -rf ${TMPDIR%.*}.*
468 AddToExcludedCountries()
470 local america="Canada USA"
471 local asia="China Japan"
472 local europe="Bulgaria Czech Denmark France England Germany Greece Netherlands Portugal Spain Sweden"
475 for xx in $EXCLUDEDCOUNTRYGROUPS
478 America) EXCLUDEDCOUNTRIES+=" $america" ;;
479 Asia) EXCLUDEDCOUNTRIES+=" $asia" ;;
480 Asia2) EXCLUDEDCOUNTRIES+=" $asia Russia" ;;
481 Europe) EXCLUDEDCOUNTRIES+=" $europe" ;;
482 Europe2) EXCLUDEDCOUNTRIES+=" $europe Russia" ;;
484 echo2 "Warning: word '$xx' is not recognized as a country group."
499 CleanAll ; Usage ; doexit=1
511 PROTOCOLS="$(echo ${arg:12} | tr ',' ' ')"
513 --exclude-countries=*)
514 EXCLUDEDCOUNTRIES+=" $(echo ${arg:20} | tr ',' ' ')"
516 --exclude-countrygroups=*)
517 EXCLUDEDCOUNTRYGROUPS="$(echo ${arg:24} | tr ',' ' ')"
518 AddToExcludedCountries
520 --include-countries=*)
521 INCLUDEDCOUNTRIES+=" $(echo ${arg:20} | tr ',' ' ')"
527 EXCLUDEDMIRRORS="$(echo ${arg:18} | tr ',' ' ')"
529 --hide-original-mirrors=*)
530 HIDE_ORIGINAL_MIRRORS="${arg:24}"
533 WGET_TIMEOUT="${arg:10}"
536 REPO=$(echo ${arg:7})
539 ARCH=$(echo ${arg:7})
543 if [ -n "$INCLUDEDCOUNTRIES" ] && [ -n "$EXCLUDEDCOUNTRIES" ] ; then
544 echo2 "Sorry, options to both include and exclude countries cannot be used together."
545 echo2 "Note however that it is allowed to include countries but exclude mirrors."
549 debug2 "Command line: $0 $@"
551 verbose2more "EXCLUDEDCOUNTRYGROUPS = $EXCLUDEDCOUNTRYGROUPS"
552 verbose2more "EXCLUDEDCOUNTRIES = $EXCLUDEDCOUNTRIES"
553 verbose2more "EXCLUDEDMIRRORS = $EXCLUDEDMIRRORS"
554 verbose2more "INCLUDEDCOUNTRIES = $INCLUDEDCOUNTRIES"
555 verbose2more "PROTOCOLS = $PROTOCOLS"
556 verbose2more "WGET_TIMEOUT = $WGET_TIMEOUT"
558 test $doexit -eq 1 && exit 0
563 # check required programs
564 if [ ! -x $TIME_CMD ] ; then
565 echo2 "Sorry, you need to install package 'time' first!"
566 echo2 "pacman -S --needed time"
569 if [ ! -x $WGET_CMD ] ; then
570 echo2 "Sorry, you need to install package 'wget' first!"
571 echo2 "pacman -S --needed wget"