fs/afs/rotate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* Handle fileserver selection and rotation.
   3  *
   4  * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #include <linux/kernel.h>
   9 #include <linux/slab.h>
  10 #include <linux/fs.h>
  11 #include <linux/sched.h>
  12 #include <linux/delay.h>
  13 #include <linux/sched/signal.h>
  14 #include "internal.h"
  15 #include "afs_fs.h"
  16
  17 /*
  18  * Begin iteration through a server list, starting with the vnode's last used
  19  * server if possible, or the last recorded good server if not.
  20  */
  21 static bool afs_start_fs_iteration(struct afs_operation *op,
  22                                    struct afs_vnode *vnode)
  23 {
  24         struct afs_cb_interest *cbi;
  25         int i;
  26
  27         read_lock(&op->volume->servers_lock);
  28         op->server_list = afs_get_serverlist(op->volume->servers);
  29         read_unlock(&op->volume->servers_lock);
  30
  31         op->untried = (1UL << op->server_list->nr_servers) - 1;
  32         op->index = READ_ONCE(op->server_list->preferred);
  33
  34         cbi = rcu_dereference_protected(vnode->cb_interest,
  35                                         lockdep_is_held(&vnode->io_lock));
  36         if (cbi) {
  37                 /* See if the vnode's preferred record is still available */
  38                 for (i = 0; i < op->server_list->nr_servers; i++) {
  39                         if (op->server_list->servers[i].cb_interest == cbi) {
  40                                 op->index = i;
  41                                 goto found_interest;
  42                         }
  43                 }
  44
  45                 /* If we have a lock outstanding on a server that's no longer
  46                  * serving this vnode, then we can't switch to another server
  47                  * and have to return an error.
  48                  */
  49                 if (op->flags & AFS_OPERATION_CUR_ONLY) {
  50                         op->error = -ESTALE;
  51                         return false;
  52                 }
  53
  54                 /* Note that the callback promise is effectively broken */
  55                 write_seqlock(&vnode->cb_lock);
  56                 ASSERTCMP(cbi, ==, rcu_access_pointer(vnode->cb_interest));
  57                 rcu_assign_pointer(vnode->cb_interest, NULL);
  58                 if (test_and_clear_bit(AFS_VNODE_CB_PROMISED, &vnode->flags))
  59                         vnode->cb_break++;
  60                 write_sequnlock(&vnode->cb_lock);
  61
  62                 afs_put_cb_interest(op->net, cbi);
  63                 cbi = NULL;
  64         }
  65
  66 found_interest:
  67         return true;
  68 }
  69
  70 /*
  71  * Post volume busy note.
  72  */
  73 static void afs_busy(struct afs_volume *volume, u32 abort_code)
  74 {
  75         const char *m;
  76
  77         switch (abort_code) {
  78         case VOFFLINE:          m = "offline";          break;
  79         case VRESTARTING:       m = "restarting";       break;
  80         case VSALVAGING:        m = "being salvaged";   break;
  81         default:                m = "busy";             break;
  82         }
  83
  84         pr_notice("kAFS: Volume %llu '%s' is %s\n", volume->vid, volume->name, m);
  85 }
  86
  87 /*
  88  * Sleep and retry the operation to the same fileserver.
  89  */
  90 static bool afs_sleep_and_retry(struct afs_operation *op)
  91 {
  92         if (!(op->flags & AFS_OPERATION_UNINTR)) {
  93                 msleep_interruptible(1000);
  94                 if (signal_pending(current)) {
  95                         op->error = -ERESTARTSYS;
  96                         return false;
  97                 }
  98         } else {
  99                 msleep(1000);
 100         }
 101
 102         return true;
 103 }
 104
 105 /*
 106  * Select the fileserver to use.  May be called multiple times to rotate
 107  * through the fileservers.
 108  */
 109 bool afs_select_fileserver(struct afs_operation *op)
 110 {
 111         struct afs_addr_list *alist;
 112         struct afs_server *server;
 113         struct afs_vnode *vnode = op->file[0].vnode;
 114         struct afs_error e;
 115         u32 rtt;
 116         int error = op->ac.error, i;
 117
 118         _enter("%lx[%d],%lx[%d],%d,%d",
 119                op->untried, op->index,
 120                op->ac.tried, op->ac.index,
 121                error, op->ac.abort_code);
 122
 123         if (op->flags & AFS_OPERATION_STOP) {
 124                 _leave(" = f [stopped]");
 125                 return false;
 126         }
 127
 128         op->nr_iterations++;
 129
 130         /* Evaluate the result of the previous operation, if there was one. */
 131         switch (error) {
 132         case SHRT_MAX:
 133                 goto start;
 134
 135         case 0:
 136         default:
 137                 /* Success or local failure.  Stop. */
 138                 op->error = error;
 139                 op->flags |= AFS_OPERATION_STOP;
 140                 _leave(" = f [okay/local %d]", error);
 141                 return false;
 142
 143         case -ECONNABORTED:
 144                 /* The far side rejected the operation on some grounds.  This
 145                  * might involve the server being busy or the volume having been moved.
 146                  */
 147                 switch (op->ac.abort_code) {
 148                 case VNOVOL:
 149                         /* This fileserver doesn't know about the volume.
 150                          * - May indicate that the VL is wrong - retry once and compare
 151                          *   the results.
 152                          * - May indicate that the fileserver couldn't attach to the vol.
 153                          */
 154                         if (op->flags & AFS_OPERATION_VNOVOL) {
 155                                 op->error = -EREMOTEIO;
 156                                 goto next_server;
 157                         }
 158
 159                         write_lock(&op->volume->servers_lock);
 160                         op->server_list->vnovol_mask |= 1 << op->index;
 161                         write_unlock(&op->volume->servers_lock);
 162
 163                         set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags);
 164                         error = afs_check_volume_status(op->volume, op);
 165                         if (error < 0)
 166                                 goto failed_set_error;
 167
 168                         if (test_bit(AFS_VOLUME_DELETED, &op->volume->flags)) {
 169                                 op->error = -ENOMEDIUM;
 170                                 goto failed;
 171                         }
 172
 173                         /* If the server list didn't change, then assume that
 174                          * it's the fileserver having trouble.
 175                          */
 176                         if (op->volume->servers == op->server_list) {
 177                                 op->error = -EREMOTEIO;
 178                                 goto next_server;
 179                         }
 180
 181                         /* Try again */
 182                         op->flags |= AFS_OPERATION_VNOVOL;
 183                         _leave(" = t [vnovol]");
 184                         return true;
 185
 186                 case VSALVAGE: /* TODO: Should this return an error or iterate? */
 187                 case VVOLEXISTS:
 188                 case VNOSERVICE:
 189                 case VONLINE:
 190                 case VDISKFULL:
 191                 case VOVERQUOTA:
 192                         op->error = afs_abort_to_error(op->ac.abort_code);
 193                         goto next_server;
 194
 195                 case VOFFLINE:
 196                         if (!test_and_set_bit(AFS_VOLUME_OFFLINE, &op->volume->flags)) {
 197                                 afs_busy(op->volume, op->ac.abort_code);
 198                                 clear_bit(AFS_VOLUME_BUSY, &op->volume->flags);
 199                         }
 200                         if (op->flags & AFS_OPERATION_NO_VSLEEP) {
 201                                 op->error = -EADV;
 202                                 goto failed;
 203                         }
 204                         if (op->flags & AFS_OPERATION_CUR_ONLY) {
 205                                 op->error = -ESTALE;
 206                                 goto failed;
 207                         }
 208                         goto busy;
 209
 210                 case VSALVAGING:
 211                 case VRESTARTING:
 212                 case VBUSY:
 213                         /* Retry after going round all the servers unless we
 214                          * have a file lock we need to maintain.
 215                          */
 216                         if (op->flags & AFS_OPERATION_NO_VSLEEP) {
 217                                 op->error = -EBUSY;
 218                                 goto failed;
 219                         }
 220                         if (!test_and_set_bit(AFS_VOLUME_BUSY, &op->volume->flags)) {
 221                                 afs_busy(op->volume, op->ac.abort_code);
 222                                 clear_bit(AFS_VOLUME_OFFLINE, &op->volume->flags);
 223                         }
 224                 busy:
 225                         if (op->flags & AFS_OPERATION_CUR_ONLY) {
 226                                 if (!afs_sleep_and_retry(op))
 227                                         goto failed;
 228
 229                                  /* Retry with same server & address */
 230                                 _leave(" = t [vbusy]");
 231                                 return true;
 232                         }
 233
 234                         op->flags |= AFS_OPERATION_VBUSY;
 235                         goto next_server;
 236
 237                 case VMOVED:
 238                         /* The volume migrated to another server.  We consider
 239                          * consider all locks and callbacks broken and request
 240                          * an update from the VLDB.
 241                          *
 242                          * We also limit the number of VMOVED hops we will
 243                          * honour, just in case someone sets up a loop.
 244                          */
 245                         if (op->flags & AFS_OPERATION_VMOVED) {
 246                                 op->error = -EREMOTEIO;
 247                                 goto failed;
 248                         }
 249                         op->flags |= AFS_OPERATION_VMOVED;
 250
 251                         set_bit(AFS_VOLUME_WAIT, &op->volume->flags);
 252                         set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags);
 253                         error = afs_check_volume_status(op->volume, op);
 254                         if (error < 0)
 255                                 goto failed_set_error;
 256
 257                         /* If the server list didn't change, then the VLDB is
 258                          * out of sync with the fileservers.  This is hopefully
 259                          * a temporary condition, however, so we don't want to
 260                          * permanently block access to the file.
 261                          *
 262                          * TODO: Try other fileservers if we can.
 263                          *
 264                          * TODO: Retry a few times with sleeps.
 265                          */
 266                         if (op->volume->servers == op->server_list) {
 267                                 op->error = -ENOMEDIUM;
 268                                 goto failed;
 269                         }
 270
 271                         goto restart_from_beginning;
 272
 273                 default:
 274                         clear_bit(AFS_VOLUME_OFFLINE, &op->volume->flags);
 275                         clear_bit(AFS_VOLUME_BUSY, &op->volume->flags);
 276                         op->error = afs_abort_to_error(op->ac.abort_code);
 277                         goto failed;
 278                 }
 279
 280         case -ETIMEDOUT:
 281         case -ETIME:
 282                 if (op->error != -EDESTADDRREQ)
 283                         goto iterate_address;
 284                 /* Fall through */
 285         case -ERFKILL:
 286         case -EADDRNOTAVAIL:
 287         case -ENETUNREACH:
 288         case -EHOSTUNREACH:
 289         case -EHOSTDOWN:
 290         case -ECONNREFUSED:
 291                 _debug("no conn");
 292                 op->error = error;
 293                 goto iterate_address;
 294
 295         case -ECONNRESET:
 296                 _debug("call reset");
 297                 op->error = error;
 298                 goto failed;
 299         }
 300
 301 restart_from_beginning:
 302         _debug("restart");
 303         afs_end_cursor(&op->ac);
 304         afs_put_cb_interest(op->net, op->cbi);
 305         op->cbi = NULL;
 306         afs_put_serverlist(op->net, op->server_list);
 307         op->server_list = NULL;
 308 start:
 309         _debug("start");
 310         /* See if we need to do an update of the volume record.  Note that the
 311          * volume may have moved or even have been deleted.
 312          */
 313         error = afs_check_volume_status(op->volume, op);
 314         if (error < 0)
 315                 goto failed_set_error;
 316
 317         if (!afs_start_fs_iteration(op, vnode))
 318                 goto failed;
 319
 320         _debug("__ VOL %llx __", op->volume->vid);
 321
 322 pick_server:
 323         _debug("pick [%lx]", op->untried);
 324
 325         error = afs_wait_for_fs_probes(op->server_list, op->untried);
 326         if (error < 0)
 327                 goto failed_set_error;
 328
 329         /* Pick the untried server with the lowest RTT.  If we have outstanding
 330          * callbacks, we stick with the server we're already using if we can.
 331          */
 332         if (op->cbi) {
 333                 _debug("cbi %u", op->index);
 334                 if (test_bit(op->index, &op->untried))
 335                         goto selected_server;
 336                 afs_put_cb_interest(op->net, op->cbi);
 337                 op->cbi = NULL;
 338                 _debug("nocbi");
 339         }
 340
 341         op->index = -1;
 342         rtt = U32_MAX;
 343         for (i = 0; i < op->server_list->nr_servers; i++) {
 344                 struct afs_server *s = op->server_list->servers[i].server;
 345
 346                 if (!test_bit(i, &op->untried) || !s->probe.responded)
 347                         continue;
 348                 if (s->probe.rtt < rtt) {
 349                         op->index = i;
 350                         rtt = s->probe.rtt;
 351                 }
 352         }
 353
 354         if (op->index == -1)
 355                 goto no_more_servers;
 356
 357 selected_server:
 358         _debug("use %d", op->index);
 359         __clear_bit(op->index, &op->untried);
 360
 361         /* We're starting on a different fileserver from the list.  We need to
 362          * check it, create a callback intercept, find its address list and
 363          * probe its capabilities before we use it.
 364          */
 365         ASSERTCMP(op->ac.alist, ==, NULL);
 366         server = op->server_list->servers[op->index].server;
 367
 368         if (!afs_check_server_record(op, server))
 369                 goto failed;
 370
 371         _debug("USING SERVER: %pU", &server->uuid);
 372
 373         /* Make sure we've got a callback interest record for this server.  We
 374          * have to link it in before we send the request as we can be sent a
 375          * break request before we've finished decoding the reply and
 376          * installing the vnode.
 377          */
 378         error = afs_register_server_cb_interest(vnode, op->server_list,
 379                                                 op->index);
 380         if (error < 0)
 381                 goto failed_set_error;
 382
 383         op->cbi = afs_get_cb_interest(
 384                 rcu_dereference_protected(vnode->cb_interest,
 385                                           lockdep_is_held(&vnode->io_lock)));
 386
 387         read_lock(&server->fs_lock);
 388         alist = rcu_dereference_protected(server->addresses,
 389                                           lockdep_is_held(&server->fs_lock));
 390         afs_get_addrlist(alist);
 391         read_unlock(&server->fs_lock);
 392
 393         memset(&op->ac, 0, sizeof(op->ac));
 394
 395         if (!op->ac.alist)
 396                 op->ac.alist = alist;
 397         else
 398                 afs_put_addrlist(alist);
 399
 400         op->ac.index = -1;
 401
 402 iterate_address:
 403         ASSERT(op->ac.alist);
 404         /* Iterate over the current server's address list to try and find an
 405          * address on which it will respond to us.
 406          */
 407         if (!afs_iterate_addresses(&op->ac))
 408                 goto next_server;
 409
 410         _debug("address [%u] %u/%u", op->index, op->ac.index, op->ac.alist->nr_addrs);
 411
 412         _leave(" = t");
 413         return true;
 414
 415 next_server:
 416         _debug("next");
 417         afs_end_cursor(&op->ac);
 418         goto pick_server;
 419
 420 no_more_servers:
 421         /* That's all the servers poked to no good effect.  Try again if some
 422          * of them were busy.
 423          */
 424         if (op->flags & AFS_OPERATION_VBUSY)
 425                 goto restart_from_beginning;
 426
 427         e.error = -EDESTADDRREQ;
 428         e.responded = false;
 429         for (i = 0; i < op->server_list->nr_servers; i++) {
 430                 struct afs_server *s = op->server_list->servers[i].server;
 431
 432                 afs_prioritise_error(&e, READ_ONCE(s->probe.error),
 433                                      s->probe.abort_code);
 434         }
 435
 436         error = e.error;
 437
 438 failed_set_error:
 439         op->error = error;
 440 failed:
 441         op->flags |= AFS_OPERATION_STOP;
 442         afs_end_cursor(&op->ac);
 443         _leave(" = f [failed %d]", op->error);
 444         return false;
 445 }
 446
 447 /*
 448  * Select the same fileserver we used for a vnode before and only that
 449  * fileserver.  We use this when we have a lock on that file, which is backed
 450  * only by the fileserver we obtained it from.
 451  */
 452 bool afs_select_current_fileserver(struct afs_operation *op)
 453 {
 454         struct afs_cb_interest *cbi;
 455         struct afs_addr_list *alist;
 456         int error = op->ac.error;
 457
 458         _enter("");
 459
 460         switch (error) {
 461         case SHRT_MAX:
 462                 cbi = op->cbi;
 463                 if (!cbi) {
 464                         op->error = -ESTALE;
 465                         op->flags |= AFS_OPERATION_STOP;
 466                         return false;
 467                 }
 468
 469                 read_lock(&cbi->server->fs_lock);
 470                 alist = rcu_dereference_protected(cbi->server->addresses,
 471                                                   lockdep_is_held(&cbi->server->fs_lock));
 472                 afs_get_addrlist(alist);
 473                 read_unlock(&cbi->server->fs_lock);
 474                 if (!alist) {
 475                         op->error = -ESTALE;
 476                         op->flags |= AFS_OPERATION_STOP;
 477                         return false;
 478                 }
 479
 480                 memset(&op->ac, 0, sizeof(op->ac));
 481                 op->ac.alist = alist;
 482                 op->ac.index = -1;
 483                 goto iterate_address;
 484
 485         case 0:
 486         default:
 487                 /* Success or local failure.  Stop. */
 488                 op->error = error;
 489                 op->flags |= AFS_OPERATION_STOP;
 490                 _leave(" = f [okay/local %d]", error);
 491                 return false;
 492
 493         case -ECONNABORTED:
 494                 op->error = afs_abort_to_error(op->ac.abort_code);
 495                 op->flags |= AFS_OPERATION_STOP;
 496                 _leave(" = f [abort]");
 497                 return false;
 498
 499         case -ERFKILL:
 500         case -EADDRNOTAVAIL:
 501         case -ENETUNREACH:
 502         case -EHOSTUNREACH:
 503         case -EHOSTDOWN:
 504         case -ECONNREFUSED:
 505         case -ETIMEDOUT:
 506         case -ETIME:
 507                 _debug("no conn");
 508                 op->error = error;
 509                 goto iterate_address;
 510         }
 511
 512 iterate_address:
 513         /* Iterate over the current server's address list to try and find an
 514          * address on which it will respond to us.
 515          */
 516         if (afs_iterate_addresses(&op->ac)) {
 517                 _leave(" = t");
 518                 return true;
 519         }
 520
 521         afs_end_cursor(&op->ac);
 522         return false;
 523 }
 524
 525 /*
 526  * Dump cursor state in the case of the error being EDESTADDRREQ.
 527  */
 528 void afs_dump_edestaddrreq(const struct afs_operation *op)
 529 {
 530         static int count;
 531         int i;
 532
 533         if (!IS_ENABLED(CONFIG_AFS_DEBUG_CURSOR) || count > 3)
 534                 return;
 535         count++;
 536
 537         rcu_read_lock();
 538
 539         pr_notice("EDESTADDR occurred\n");
 540         pr_notice("FC: cbb=%x cbb2=%x fl=%x err=%hd\n",
 541                   op->file[0].cb_break_before,
 542                   op->file[1].cb_break_before, op->flags, op->error);
 543         pr_notice("FC: ut=%lx ix=%d ni=%u\n",
 544                   op->untried, op->index, op->nr_iterations);
 545
 546         if (op->server_list) {
 547                 const struct afs_server_list *sl = op->server_list;
 548                 pr_notice("FC: SL nr=%u pr=%u vnov=%hx\n",
 549                           sl->nr_servers, sl->preferred, sl->vnovol_mask);
 550                 for (i = 0; i < sl->nr_servers; i++) {
 551                         const struct afs_server *s = sl->servers[i].server;
 552                         pr_notice("FC: server fl=%lx av=%u %pU\n",
 553                                   s->flags, s->addr_version, &s->uuid);
 554                         if (s->addresses) {
 555                                 const struct afs_addr_list *a =
 556                                         rcu_dereference(s->addresses);
 557                                 pr_notice("FC:  - av=%u nr=%u/%u/%u pr=%u\n",
 558                                           a->version,
 559                                           a->nr_ipv4, a->nr_addrs, a->max_addrs,
 560                                           a->preferred);
 561                                 pr_notice("FC:  - R=%lx F=%lx\n",
 562                                           a->responded, a->failed);
 563                                 if (a == op->ac.alist)
 564                                         pr_notice("FC:  - current\n");
 565                         }
 566                 }
 567         }
 568
 569         pr_notice("AC: t=%lx ax=%u ac=%d er=%d r=%u ni=%u\n",
 570                   op->ac.tried, op->ac.index, op->ac.abort_code, op->ac.error,
 571                   op->ac.responded, op->ac.nr_iterations);
 572         rcu_read_unlock();
 573 }