ApplyCache.C

Go to the documentation of this file.

// Copyright (C) 2001, Compaq Computer Corporation
// 
// This file is part of Vesta.
// 
// Vesta is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// Vesta is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with Vesta; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

// File: ApplyCache.C

#include <Basics.H>
#include <FP.H>
#include <Model.H>
#include <FV.H>
#include <CacheIndex.H>
#include <CacheC.H>
#include "ApplyCache.H"
#include "Expr.H"
#include "Val.H"
#include "Pickle.H"
#include "Err.H"
#include "PrimRunTool.H"
#include "Debug.H"
#include "ThreadData.H"
#include <Table.H>

using std::ostream;
using std::cerr;
using std::endl;
using Basics::OBufStream;

// KCS, 2001-12-6: Changed both evaluatorRunToolVersion and
// evaluatorFunctionVersion, as integer values are now fingerprinted
// as 32-bit values.

// KCS, 2002-8-20: Changed evaluatorFunctionVersion due to a fix to
// dependency analysis in several primitive functions for boundary
// cases with empty lists and bindings.

// KCS, 2002-11-6: Changed evaluatorFunctionVersion due to a bug fix
// to dependency collection code which iadvertently munged dependency
// paths.  This bug *seemed* to only add additional bogus secondary
// dependencies, but it's safer to update evaluatorFunctionVersion.

// KCS, 2003-2-11: Changed both evaluatorRunToolVersion and
// evaluatorFunctionVersion, as the pickle version has changed for
// the network byte order conversion.

// KCS, 2004-04-05: Changed evaluatorFunctionVersion due to
// dependency analysis bug fixes in CollectModel.

// STV&KCS, 2004-12-25: Changed evaluatorRunToolVersion after adding
// "dumped_core" to _run_tool result.

// Runtool call version:
const char *evaluatorRunToolVersion = "Fri Feb 11 13:30:46 2005";
const FP::Tag evaluatorRunToolVersionTag(evaluatorRunToolVersion);

// Function call version:
const char *evaluatorFunctionVersion = "Fri Feb 11 13:30:46 2005";
const FP::Tag evaluatorFunctionVersionTag(evaluatorFunctionVersion);

// The number of addentry calls to the cache:
static int addEntryCount = 0;

// The mutex protects all the orphans in all the workers and the
// addEntryCount.
static Basics::mutex frontierMu;

// The mutex protects the variables lastRenewed and
// renewLease_failure.
static Basics::mutex leaseMu;
static time_t lastRenewed;
static bool renewLease_failure = false;

// A table to keep from running identical tools in parallel.
// This checks for a PK and waits on a condition if that PK
// is already in progress.
static Basics::mutex runToolTableMu;
class rtt_cond {
        Basics::cond cond;
        int count; // number of in progress _run_tool()
                   // calls with this PK
        bool dec_and_del() {
                assert(count > 0);
                if(--count == 0) {
                        delete this;
                        return false;
                } else {
                        return true;
                }
        };
    public:
        rtt_cond() { count = 1; };
        void wait() {
                count++;
                cond.wait(runToolTableMu);
                (void) dec_and_del();
        };
        void wake() {
                if(dec_and_del()) {
                        cond.broadcast();
                }
        };
        
};
static Table<FP::Tag, rtt_cond *>::Default runToolTable;


void Tags(const CompactFV::List& names, const Context& c, 
          /*OUT*/ FP::List& tags) {
  // Used for cache lookup:
  int len = names.num;
  if (len > 0) {
    tags.len = len;
    tags.fp = NEW_PTRFREE_ARRAY(FP::Tag, len);
    int size = names.tbl.NumArcs();
    Val *vals = NEW_ARRAY(Val, size);
    int i;
    for (i = 0; i < size; i++) { vals[i] = NULL; }
    for (i = 0; i < len; i++)   {
      short int idx = names.idx[i];
      PathKind pkind = (PathKind)names.types[i];
      tags.fp[i] = LookupPath(idx, pkind, names.tbl, c, vals)->FingerPrint();
    }
  } else {
    tags.len = 0;
    tags.fp = NULL;
  }
  return;
}

bool NamesTagsPickle(Val val, bool cutoff, const Context& inPKArgs,
                     /*OUT*/ char*& types, /*OUT*/ FV2::List& names,
                     /*OUT*/ FP::List& tags, /*OUT*/ VestaVal::T& vval) {
  /* Create all the stuff needed for an AddEntry call. */

  // We now create the secondary key of the entry. Since some dependencies
  // on the function arguments have already been encoded in the primary key,
  // they are redundant, and therefore removed.
  DPaths *sk = NEW_CONSTR(DPaths, (val->SizeOfDPS()));
  sk->Merge(val->dps);
  ValueDpnd(val, sk);
  DepPathTbl::TIter iter(sk);
  DepPathTbl::KVPairPtr ptr;
  Context work = inPKArgs;
  Text id;
  while (!work.Null()) {
    id = work.Pop()->name;
    iter.Reset();
    while (iter.Next(ptr)) {
      if (ptr->key.content->path->getlo() == id)
        sk->Delete(ptr->key, ptr, false);
    }
  }
  sk->Resize();
  // Set names and tags:
  int len = sk->Size();
  names.len = len;
  tags.len = len;
  if (len > 0) {
    types = NEW_PTRFREE_ARRAY(char, len);    
    names.name = NEW_ARRAY(FV2::TPtr, len);
    tags.fp = NEW_PTRFREE_ARRAY(FP::Tag, len);
    iter.Reset();
    int index = 0;
    while (iter.Next(ptr)) {
      types[index] = (char)ptr->key.content->pKind;
      names.name[index] = ptr->key.content->path;
      if (ptr->val->vKind != FpVK && ptr->key.content->pKind == ExprPK) {
        if (ptr->val->vKind == ClosureVK)
          tags.fp[index++] = ((ClosureVC*)ptr->val)->FingerPrintExpr();
        else {
          assert(ptr->val->vKind == ModelVK);
          tags.fp[index++] = ((ModelVC*)ptr->val)->FingerPrintFile();
        }
      }
      else
        tags.fp[index++] = ptr->val->FingerPrint();
    }
  }
  // Pickle the value:
  if (cutoff) ModelCutOff(val);
  return Pickle(val, vval);
}

static void CheckLeases() {
  leaseMu.lock();    
  bool expired = (time(NULL) - lastRenewed) > leaseDuration;
  bool failed = renewLease_failure;
  leaseMu.unlock();
  if (expired) {
    throw(Evaluator::failure("Leases timed out. Start over.", true));
  } else if(failed) {
    throw(Evaluator::failure("Lease renewal failed.", true));
  }
}

void CollectKids(CacheEntry::IndicesApp* orphanCIs, int kidsIndex,
                 CacheEntry::Indices& kids) {
  CheckLeases();
  kids.len = orphanCIs->len - kidsIndex;
  kids.index = orphanCIs->index + kidsIndex;
}
    
void UpdateOrphans(CacheEntry::IndicesApp* orphanCIs, int kidsIndex,
                   CacheEntry::Index ci) {
  frontierMu.lock();
  orphanCIs->len = kidsIndex;
  orphanCIs->Append(ci);
  frontierMu.unlock();  
}

void Checkpoint() {
  CheckLeases();
  frontierMu.lock();        
  addEntryCount++;
  if ((addEntryCount & 0x3f) == 0) {
    CacheEntry::IndicesApp orphanCIs;
    ThreadData::mu.lock();
    ThreadData* cur = ThreadData::head;
    while (cur) {
      assert(cur->orphanCIs != 0);
      for (int i = 0; i < cur->orphanCIs->len; i++) {
        orphanCIs.Append(cur->orphanCIs->index[i]);
      }
      cur = cur->next;
    }
    ThreadData::mu.unlock();
    theCache->Checkpoint(topModelRoot->fptag, topModelSid, orphanCIs, false);
  }
  frontierMu.unlock();
}

void FreeNamesTags(CompactFV::List& names, FP::List& tags) {
  // Free names and tags after cache lookup:
  names.tbl = PrefixTbl();
  tags.len = 0;
  tags.fp = NULL;
}

// General function application:
Context BindApplicationArgs(ClosureVC *clos, ArgList actuals, const Context& c) {
  Context result, closCon = clos->con;
  Exprs forms = clos->func->args->elems;
  Exprs acts = actuals->elems;
  int fSize = forms.size();
  int aSize = acts.size();
  Expr fe;
  AssignEC *formal;
  Text name;
  int i;

  if (aSize > fSize + 1) {
    outputMu.lock();    
    actuals->EError("Too many actual parameters:");
    outputMu.unlock();        
    return conEmpty;
  }
  for (i = 0; i < aSize; i++) { // Actuals to formals
    if (i == fSize)
      name = nameDot;
    else {
      fe = forms.get(i);
      if (fe->kind == NameEK)
        name = ((Name)fe)->id;
      else if (fe->kind == AssignEK)
        name = ((AssignEC*)fe)->lhs->id;
      else {
        outputMu.lock();        
        actuals->EError("Bad parameter list. It is neither actual nor default: `");
        ErrorExpr(fe);
        ErrorDetail("'.");
        outputMu.unlock();      
        return conEmpty;
      }
    }
    PushToContext(name, acts.get(i), c, result);
  }
  for (i = aSize; i < fSize; i++) { // Default rest of formals
    fe = forms.get(i);
    if (fe->kind == AssignEK) {
      formal = (AssignEC*)fe;
      PushToContext(formal->lhs->id, formal->rhs, closCon, result);
    }
    else {
      outputMu.lock();          
      actuals->EError("Bad parameter list. Must have default: `");
      ErrorExpr(fe);
      ErrorDetail("'.");
      outputMu.unlock();          
      return conEmpty;
    }
  }
  if (aSize <= fSize) {
    Name eDot = NEW_CONSTR(NameEC, (nameDot, actuals->loc));
    PushToContext(nameDot, eDot, c, result);
  }
  return result;
}

Val ApplicationFromCache(ClosureVC* clos, const Context& argsCon, SrcLoc *loc) {
  FP::Tag pk(evaluatorFunctionVersionTag);
  CompactFV::List fvNames;
  FV2::List entryNames;
  FP::List tags;
  CacheEntry::Index cIndex;
  VestaVal::T cVal;
  CacheIntf::LookupRes cResult = CacheIntf::Miss;
  Model::T model = clos->func->body->loc->shortId;
  Val result;

  IncCounter(&appCallCounter);
  Context evalCon = argsCon.Append(clos->con);

  ostream *traceRes = ThreadDataGet()->traceRes;
  // Evaluate in the nocaching cases:
  if (cacheOption != 3 || clos->func->noCache) {
    // Normal function caching is disabled.
    if (traceRes) *traceRes << ": disabled\n";
    result = clos->func->body->Eval(evalCon);
    if (cacheOption < 2) return result;
    return FuncDpnd(result, clos, argsCon);
  }

  // Compute the primary key:
  // The pk consists of the fingerprints of the closure body, the argumnets
  // that either is specified by pk pragma or evaluates to simple values.
  pk.Extend(clos->FingerPrintExpr());
  Context work = argsCon;
  (void)work.Pop();   // Do not put the dot into pk.
  int last = clos->func->args->elems.size();
  Bit64 inPKs = clos->func->args->inPKs;
  Context inPKArgs;
  while (!work.Null()) {
    Assoc elem = work.Pop();
    if (inPKs & ((Bit64)1 << --last)) {
      // Add this argument into pk.
      pk.Extend(elem->FingerPrint());
      inPKArgs.Push(elem);
    }
    else {
      switch (elem->val->vKind) {
      case BooleanVK: case IntegerVK: case PrimitiveVK:
      case TextVK: case ErrorVK:
        pk.Extend(elem->FingerPrint());
        inPKArgs.Push(elem);
        break;
      case ClosureVK:
        {
          FP::Tag tag(elem->name);
          tag.Extend(((ClosureVC*)elem->val)->FingerPrintExpr());
          pk.Extend(tag);
          break;
        }
      case ModelVK:
        {
          FP::Tag tag(elem->name);
          tag.Extend(((ModelVC*)elem->val)->FingerPrintFile());
          pk.Extend(tag);
          break;
        }
      case FpVK:
        assert(false);
      default:
        break;
      }
    }
  }
  // Evaluate with caching:
  ThreadData *thdata = ThreadDataGet();
  CacheEntry::IndicesApp* orphanCIs = thdata->orphanCIs;  
  try {
    while (true) {
      bool noEntry;
      FV::Epoch epoch = theCache->FreeVariables(pk, /*OUT*/ fvNames, /*OUT*/ noEntry);
      if (noEntry)
        cResult = CacheIntf::Miss;
      else {
        Tags(fvNames, evalCon, /*OUT*/ tags);
        cResult = theCache->Lookup(pk, epoch, tags, /*OUT*/ cIndex, /*OUT*/ cVal);
      }
      FreeNamesTags(fvNames, tags);
      if (cResult == CacheIntf::Hit) {
        if (Unpickle(cVal, evalCon, result)) {
          if (traceRes) {
            *traceRes << ": hit (ci=" << cIndex << ")\n";
          }
          IncCounter(&appHitCounter);
          UpdateOrphans(orphanCIs, orphanCIs->len, cIndex);
          /****
          while (!inPKArgs.Null()) {
            Assoc elem = inPKArgs.Pop();
            result->AddToDPS(NEW_CONSTR(DepPath,(elem->name, NormPK)), 
                             elem->val);
          }
          ****/
          return FuncDpnd(result, clos, argsCon);
        }
        outputMu.lock();
        OBufStream l_msg;
        l_msg << "ApplicationFromCache: Cache hit but bad cache data." << endl << endl
              << "\tpk = " << pk << endl
              << "\tci = " << cIndex << endl << endl
              << "Evaluate as cache miss." << endl;
        Error(Text(l_msg.str()), loc);
        outputMu.unlock();      
        cResult = CacheIntf::Miss;
      }
      if (cResult == CacheIntf::Miss) {
        if (traceRes) *traceRes << ": miss\n";
        int kidsIndex = orphanCIs->len;
        result = clos->func->body->Eval(evalCon);
        if (!noAddEntry) {
          // Create a new cache entry:
          char *types;
          if (NamesTagsPickle(result, false, inPKArgs, /*OUT*/ types, /*OUT*/ entryNames,
                              /*OUT*/ tags, /*OUT*/ cVal)) {
            CacheEntry::Indices kids;
            CollectKids(orphanCIs, kidsIndex, kids);
            Text pksource(clos->func->loc->file);
            pksource += "/" + clos->func->name;
            pksource += "(), line " + IntToText(clos->func->loc->line);
            pksource += ", col " + IntToText(clos->func->loc->character);
            // Add the entry, capturing the result of the call.
            CacheIntf::AddEntryRes ae_res =
              theCache->AddEntry(pk, types, entryNames, tags, cVal, model,
                                 kids, pksource, /*OUT*/ cIndex);
            // If the AddEntry fails, die with an error.
            if(ae_res != CacheIntf::EntryAdded)
              {
                outputMu.lock();        
                Error(Text("AddEntry failed: ")+
                      CacheIntf::AddEntryResName(ae_res)+"\n"
                      "This may be an evaluator bug.  "
                      "Please report it.\n",
                      loc);
                outputMu.unlock();
                if (traceRes) *traceRes << endl;
                throw(Evaluator::failure(Text("AddEntry failed (may be an "
                                              "evaluator bug)"),
                                         false));
              }
            UpdateOrphans(orphanCIs, kidsIndex, cIndex);
            Checkpoint();
            if (traceRes) {
              *traceRes << "  " << thdata->funcCallDepth << ". "
                << clos->func->args->loc->file << ": "
                << clos->func->name << "(): add (ci=" << cIndex << ")\n";
            }
          }
        }
        return FuncDpnd(result, clos, argsCon);
      } 
      if (cResult != CacheIntf::FVMismatch) {
        if (traceRes) *traceRes << ": mismatch\n";
        OBufStream err_msg;
        err_msg << "ApplicationFromCache got bad result: "
                << CacheIntf::LookupResName(cResult) << endl
                << "\tpk = " << pk << endl
                << "\tFV epoch = " << epoch << endl
                << "\ttags.len = " << tags.len << endl
                << "This may be an evaluator bug.  Please report it." << endl;
        outputMu.lock();
        Error(Text(err_msg.str()), loc);
        outputMu.unlock();      
        result = NEW(ErrorVC);
        return result;
      }
    }
  } catch (PrefixTbl::Overflow) {
    outputMu.lock();        
    Error(Text("Internal limits exceeded: PrefixTbl overflow\n"),
          loc);
    cerr << "Caching function \"" << clos->func->name
         << "\" defined at:" << endl << endl
         << "  " << clos->func->loc->file
         << ", line " << clos->func->loc->line
         << ", char " << clos->func->loc->character
         << endl << endl
         << Text("This usually means that there are too many free variables "
                 "in the secondary key of the function being cached.  This "
                 "can often be alleviated by marking more arguments with "
                 "the /**pk**/ pragma.  (Fixing this will also reduce the CPU "
                 "load of your evaluator and cache server, and improve "
                 "build performance.)").WordWrap()
         << endl << endl;
    // This is a fatal error.  Exit and dump core.
    abort();
    outputMu.unlock();
  } catch (SRPC::failure f) {
    outputMu.lock();        
    Error(Text("ApplicationFromCache: SRPC failure (")
             + IntToText(f.r) + "): " + f.msg + ".\n",
          loc);
    outputMu.unlock();        
    if (traceRes) *traceRes << endl;
    throw(Evaluator::failure(Text("Cache server is possibly down"), false));
  }
  //return result; // not reached
}

Val ApplyFunction(ClosureVC* clos, ApplyEC *ae, const Context& c) {
  ArgList args = ae->args;
  Context argsCon;

  argsCon = BindApplicationArgs(clos, args, c);

  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;
  thdata->funcCallDepth++;
  if (traceRes) {
    *traceRes << "  " << thdata->funcCallDepth << ". "
      << clos->func->args->loc->file << ": "
      << clos->func->name << "()";
  }
  if (argsCon == conEmpty) {
    // when argsCon is correct, it can not be empty since `dot' must be
    // defined in argsCon.
    outputMu.lock();    
    ErrorDetail("\n  Formal list is ");
    ErrorExpr(clos->func->args);
    ErrorDetail("\n  Actual list is ");
    ErrorExpr(args);
    ErrorDetail("\n");
    outputMu.unlock();        
    if (traceRes) *traceRes << ": badargs\n";
    return NEW(ErrorVC);
  }

  // Evaluate:
  Val result = ApplicationFromCache(clos, argsCon, ae->loc);
  thdata->funcCallDepth--;
  return result;
}
  
// Model application:
Val NormalModelFromCache(ModelVC *fun, const Context& evalCon, SrcLoc *loc) {
  Model::T model = fun->content->sid;
  FP::Tag pk(evaluatorFunctionVersionTag);
  CompactFV::List fvNames;
  FV2::List entryNames;
  FP::List tags;
  CacheEntry::Index cIndex;
  VestaVal::T cVal;
  CacheIntf::LookupRes cResult = CacheIntf::Miss;
  Val result;

  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;
  
  // Evaluate as nocache if the model imports a local model:
  if (((ModelEC*)fun->content->model)->ImportLocalModel()) {
    result = ((ModelEC*)fun->content->model)->block->Eval(evalCon);
    ModelCutOff(result);
    return result;
  }
  
  IncCounter(&nModelCallCounter);
  // Record this call if traced:
  if (traceRes) {
    *traceRes << "  " << thdata->funcCallDepth << ". " << fun->content->name;
  }

  // Compute the primary key:
  pk.Extend(fun->FingerPrintFile());

  // Evaluate:
  CacheEntry::IndicesApp* orphanCIs = ThreadDataGet()->orphanCIs;  
  try {
    while (true) {
      bool noEntry;
      FV::Epoch epoch = theCache->FreeVariables(pk, /*OUT*/ fvNames, /*OUT*/ noEntry);
      if (noEntry)
        cResult = CacheIntf::Miss;
      else {
        Tags(fvNames, evalCon, /*OUT*/ tags);
        cResult = theCache->Lookup(pk, epoch, tags, /*OUT*/ cIndex, /*OUT*/ cVal);
      }
      FreeNamesTags(fvNames, tags);
      if (cResult == CacheIntf::Hit) {
        if (Unpickle(cVal, evalCon, result)) {
          if (traceRes) {
            *traceRes << ": hit (ci=" << cIndex << ")\n";
          }
          IncCounter(&nModelHitCounter);
          UpdateOrphans(orphanCIs, orphanCIs->len, cIndex);
          return result;
        }
        outputMu.lock();
        OBufStream l_msg;
        l_msg << "NormalModelFromCache: Cache hit but bad cache data." << endl << endl
              << "\tpk = " << pk << endl
              << "\tci = " << cIndex << endl << endl
              << "Evaluate as cache miss." << endl; 
        Error(Text(l_msg.str()), loc);  
        outputMu.unlock();      
        throw(Evaluator::failure(Text("Bad cache data"), false));
      }
      if (cResult == CacheIntf::Miss) {
        if (traceRes) *traceRes << ": miss\n";
        int kidsIndex = orphanCIs->len;
        result = ((ModelEC*)fun->content->model)->block->Eval(evalCon);
        if (!noAddEntry) {
          // Create the normal-case model entry:
          char *types;
          if (NamesTagsPickle(result, true, conEmpty, /*OUT*/ types, /*OUT*/ entryNames,
                              /*OUT*/ tags, /*OUT*/ cVal)) {
            CacheEntry::Indices kids;
            CollectKids(orphanCIs, kidsIndex, kids);
            Text pksource(fun->content->name);
            pksource += "() (normal)";
            // Add the entry, capturing the result of the call.
            CacheIntf::AddEntryRes ae_res =
              theCache->AddEntry(pk, types, entryNames, tags, cVal, model,
                                 kids, pksource, /*OUT*/ cIndex);
            // If the AddEntry fails, die with an error.
            if(ae_res != CacheIntf::EntryAdded)
              {
                outputMu.lock();        
                Error(Text("AddEntry failed: ")+
                      CacheIntf::AddEntryResName(ae_res)+"\n"
                      "This may be an evaluator bug.  "
                      "Please report it.\n",
                      loc);
                outputMu.unlock();
                if (traceRes) *traceRes << endl;
                throw(Evaluator::failure(Text("AddEntry failed (may be an "
                                              "evaluator bug)"),
                                         false));
              }
            UpdateOrphans(orphanCIs, kidsIndex, cIndex);
            Checkpoint();
            if (traceRes) {
              *traceRes << "  " << thdata->funcCallDepth << ". " << fun->content->name
                        << ": add (ci=" << cIndex << ")\n";
            }
          }
        }
        return result;
      }
      if (cResult != CacheIntf::FVMismatch) {
        if (traceRes) *traceRes << ": mismatch\n";
        OBufStream err_msg;
        err_msg << "NormalModelFromCache got bad result: "
                << CacheIntf::LookupResName(cResult) << endl
                << "\tpk = " << pk << endl
                << "\tFV epoch = " << epoch << endl
                << "\ttags.len = " << tags.len << endl
                << "This may be an evaluator bug.  Please report it." << endl;
        outputMu.lock();        
        Error(Text(err_msg.str()), loc);
        outputMu.unlock();      
        result = NEW(ErrorVC);
        return result;
      }
    }
  } catch (PrefixTbl::Overflow) {
    outputMu.lock();        
    Error(Text("Internal limits exceeded: PrefixTbl overflow\n"),
          loc);
    cerr << "Caching model (normal):" << endl << endl
         << "  " << fun->content->name
         << endl << endl
         << Text("This usually means that there are too many free variables "
                 "in the secondary key of the function being cached.  For "
                 "models, this may mean too much work is being done in a "
                 "single model file.  Try splitting up the work of this "
                 "model file into several separate model files each doing "
                 "a smaller amount of work.  (Fixing this will also reduce "
                 "the CPU load of your evaluator and cache server, and "
                 "improve build performance.)").WordWrap()
         << endl << endl;
    // This is a fatal error.  Exit and dump core.
    abort();
    outputMu.unlock();
  } catch (SRPC::failure f) {
    outputMu.lock();        
    fun->VError(Text("NormalModelFromCache: SRPC failure (")
                + IntToText(f.r) + "): " + f.msg + ".\n");
    outputMu.unlock();        
    if (traceRes) *traceRes << endl;
    throw(Evaluator::failure(Text("Cache server is possibly down"), false));
  }
}

Val ModelFromCache(ModelVC* fun, const Context& argsCon, SrcLoc *loc) {
  Context evalCon;
  Model::T model = fun->content->sid;
  FP::Tag pk(evaluatorFunctionVersionTag);
  CompactFV::List fvNames;
  FV2::List entryNames;
  FP::List tags;
  CacheEntry::Index cIndex;
  VestaVal::T cVal;
  CacheIntf::LookupRes cResult = CacheIntf::Miss;
  Val result;

  IncCounter(&sModelCallCounter);
  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;
  
  // Evaluate in the nocache cases:
  if (cacheOption < 2) {
    if (traceRes) *traceRes << ": disabled\n";
    Val fun1 = ((ModelVC*)fun)->Force();
    if (fun1->vKind == ModelVK) {
      ModelEC *modelExpr = (ModelEC*)((ModelVC*)fun1)->content->model;
      evalCon = argsCon.Append(((ModelVC*)fun1)->content->c);
      return modelExpr->block->Eval(evalCon);
    }
    outputMu.lock();        
    Error("Trying to apply a model that failed to parse: `", loc);
    ErrorVal(fun1);
    ErrorDetail("'.\n");
    outputMu.unlock();        
    result = NEW(ErrorVC);
    return result;
  }
  
  // Compute the primary key:
  pk.Extend(fun->FingerPrint());

  // Evaluate with caching:
  CacheEntry::IndicesApp* orphanCIs = ThreadDataGet()->orphanCIs;  
  try {
    while (true) {
      bool noEntry;
      FV::Epoch epoch = theCache->FreeVariables(pk, /*OUT*/ fvNames, /*OUT*/ noEntry);
      if (noEntry)
        cResult = CacheIntf::Miss;
      else {
        Tags(fvNames, argsCon, /*OUT*/ tags);
        cResult = theCache->Lookup(pk, epoch, tags, /*OUT*/ cIndex, /*OUT*/ cVal);
      }
      FreeNamesTags(fvNames, tags);
      if (cResult == CacheIntf::Hit) {
        if (Unpickle(cVal, argsCon, result)) {
          if (traceRes) {
            *traceRes << ": hit (ci=" << cIndex << ")\n";
          }
          IncCounter(&sModelHitCounter);
          UpdateOrphans(orphanCIs, orphanCIs->len, cIndex);
          return ModelDpnd(result, fun, argsCon);
        }
        outputMu.lock();        
        OBufStream l_msg;
        l_msg << "ModelFromCache: Cache hit but bad cache data." << endl << endl
              << "\tpk = " << pk << endl
              << "\tci = " << cIndex << endl << endl
              << "Evaluate as cache miss." << endl;
        Error(Text(l_msg.str()), loc);
        outputMu.unlock();      
        throw(Evaluator::failure(Text("Bad cache data"), false));
      }
      if (cResult == CacheIntf::Miss) {
        if (traceRes) *traceRes << ": miss\n";
        int kidsIndex = orphanCIs->len;
        Val fun1 = ((ModelVC*)fun)->Force();
        if (fun1->vKind == ModelVK) {
          evalCon = argsCon.Append(((ModelVC*)fun1)->content->c);
          result = NormalModelFromCache(fun, evalCon, loc);
          if (noAddEntry) {
            ModelCutOff(result);
          }
          else {
            // Create the special-case model entry:
            char *types;
            if (NamesTagsPickle(result, false, conEmpty, /*OUT*/ types, /*OUT*/ entryNames,
                                /*OUT*/ tags, /*OUT*/ cVal)) {
              CacheEntry::Indices kids;
              CollectKids(orphanCIs, kidsIndex, kids);
              Text pksource(fun->content->name);
              pksource += "() (special)";             
              // Add the entry, capturing the result of the call.
              CacheIntf::AddEntryRes ae_res =
                theCache->AddEntry(pk, types, entryNames, tags, cVal, model,
                                   kids, pksource, /*OUT*/ cIndex);
              // If the AddEntry fails, die with an error.
              if(ae_res != CacheIntf::EntryAdded)
                {
                  outputMu.lock();      
                  Error(Text("AddEntry failed: ")+
                        CacheIntf::AddEntryResName(ae_res)+"\n"
                        "This may be an evaluator bug.  "
                        "Please report it.\n",
                        loc);
                  outputMu.unlock();
                  if (traceRes) *traceRes << endl;
                  throw(Evaluator::failure(Text("AddEntry failed (may be an "
                                                "evaluator bug)"),
                                           false));
                }
              UpdateOrphans(orphanCIs, kidsIndex, cIndex);
              Checkpoint();
              if (traceRes) {
                *traceRes << "  " << thdata->funcCallDepth << ". " << fun->content->name
                          << ": add (ci=" << cIndex << ")\n";
              }
            }
          }
          return ModelDpnd(result, fun, argsCon);
        }
        outputMu.lock();        
        Error("Trying to apply a model that failed to parse: `", loc);
        ErrorVal(fun1);
        ErrorDetail("'.\n");
        outputMu.unlock();      
        result = NEW(ErrorVC);
        return result->Merge(fun);
      }
      if (cResult != CacheIntf::FVMismatch) {
        if (traceRes) *traceRes << ": mismatch\n";
        OBufStream err_msg;
        err_msg << "ModelFromCache got bad result: "
                << CacheIntf::LookupResName(cResult) << endl
                << "\tpk = " << pk << endl
                << "\tFV epoch = " << epoch << endl
                << "\ttags.len = " << tags.len << endl
                << "This may be an evaluator bug.  Please report it." << endl;
        outputMu.lock();        
        Error(Text(err_msg.str()), loc);
        outputMu.unlock();      
        result = NEW(ErrorVC);
        return result;
      }
    }
  } catch (PrefixTbl::Overflow) {
    outputMu.lock();        
    Error(Text("Internal limits exceeded: PrefixTbl overflow\n"),
          loc);
    cerr << "Caching model (special):" << endl << endl
         << "  " << fun->content->name
         << endl << endl
         << Text("This usually means that there are too many free variables "
                 "in the secondary key of the function being cached.  For "
                 "models, this may mean too much work is being done in a "
                 "single model file.  Try splitting up the work of this "
                 "model file into several separate model files each doing "
                 "a smaller amount of work.  (Fixing this will also reduce "
                 "the CPU load of your evaluator and cache server, and "
                 "improve build performance.)").WordWrap()
         << endl << endl;
    // This is a fatal error.  Exit and dump core.
    abort();
    outputMu.unlock();
  } catch (SRPC::failure f) {
    outputMu.lock();        
    Error(Text("ModelFromCache: SRPC failure (") + IntToText(f.r) + "): " + f.msg + ".\n",
          loc);
    outputMu.unlock();        
    if (traceRes) *traceRes << endl;
    throw(Evaluator::failure(Text("Cache server is possibly down"), false));
  }
  //return result; // not reached
}

Val ApplyModel(ModelVC* fun, ApplyEC *ae, const Context& c) {
  /* There are two cache entries created.  The first one created in
     NormalModelFromCache is the normal-case model entry, the second one
     is the special-case model entry.  In cache lookup, we always first
     look up the special-case model entry, and then the normal-case
     model entry.   */
  ArgList args = ae->args;
  Exprs acts = args->elems;
  Context argsCon;

  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;

  // Bind model argument:
  int aSize = acts.size();
  if (aSize == 0) {
    Name eDot = NEW_CONSTR(NameEC, (nameDot, args->loc));
    PushToContext(nameDot, eDot, c, argsCon);
  }
  else if (aSize == 1)
    PushToContext(nameDot, acts.getlo(), c, argsCon);
  else {
    outputMu.lock();        
    args->EError("Too many actual parameters:");
    outputMu.unlock();        
    thdata->funcCallDepth++;
    if (traceRes) {
      *traceRes << "  " << thdata->funcCallDepth << ". " << fun->content->name;
      *traceRes << ": badargs\n";
    }
    thdata->funcCallDepth--;
    return NEW(ErrorVC);
  }

  thdata->funcCallDepth++;
  // Record this call if traced:
  if (traceRes) {
    *traceRes << "  " << thdata->funcCallDepth << ". " << fun->content->name;
  }

  // Evaluate:
  Val result = ModelFromCache(fun, argsCon, ae->loc);
  thdata->funcCallDepth--;
  return result;
}  

// RunTool application:
bool CheckTreatment(TextVC *tr) {
  Text txt(tr->NDS());
  return (txt == "ignore" || 
          txt == "report" || 
          txt == "report_nocache" ||
          txt == "value" ||
          txt == "report_value");         
}

bool AllTexts(BindingVC *b) {
  Context work = b->elems;
  while (!work.Null()) {
    Assoc a = work.Pop();
    if (a->val->vKind != TextVK) return false;
  }
  return true; 
}

bool BindRunToolArgs(ArgList args, const Context& c,
                     RunToolArgs& runToolArgs) {
  // Handle type-checking and defaulting of arguments to _run_tool
  int nArgs = args->Length();
  Exprs elems = args->elems;
  Val val;

  if (nArgs < 2) {
    outputMu.lock();        
    args->EError("Too few actual parameters:");
    outputMu.unlock();        
    return false;
  }
  if (nArgs > 11) {
    outputMu.lock();        
    args->EError("Too many actual parameters:");
    outputMu.unlock();        
    return false;
  }
  
  // platform:
  val = elems.get(0)->Eval(c);
  if (val->vKind != TextVK) {
    outputMu.lock();        
    args->EError("The `platform' argument to _run_tool must be a text:");
    outputMu.unlock();        
    return false;
  }
  runToolArgs.platform = (TextVC*)val;

  // command:
  val = elems.get(1)->Eval(c);
  bool ok = false;
  if (val->vKind == ListVK) {
    Vals vs = ((ListVC*)val)->elems;
    ok = true;
    while (!vs.Null())
      if (vs.Pop()->vKind != TextVK) { ok = false; break; }
  }
  if (!ok) {
    outputMu.lock();        
    args->EError("The `command' argument to _run_tool must be a list of texts:");
    outputMu.unlock();        
    return false;
  }
  runToolArgs.command_line = (ListVC*)val;

  // stdin:
  if (nArgs <= 2) {
    Text stdin_default(emptyText);
    val = NEW_CONSTR(TextVC, (stdin_default));
  }
  else {
    val = elems.get(2)->Eval(c);
    if (val->vKind != TextVK) {
      outputMu.lock();          
      args->EError("The `stdin' argument to _run_tool must be a text:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.stdin_data = (TextVC*)val;

  // stdout_treatment:
  if (nArgs <= 3) {
    Text stdout_treatment_default("report");
    val = NEW_CONSTR(TextVC, (stdout_treatment_default));
  }
  else {
    val = elems.get(3)->Eval(c);
    if (val->vKind != TextVK || !CheckTreatment((TextVC*)val)) {
      outputMu.lock();          
      args->EError("The `stdout_treatment' argument to _run_tool is not a suitable text:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.stdout_treatment = (TextVC*)val;

  // stderr_treatment:
  if (nArgs <= 4) {
    Text stderr_treatment_default("report");
    val = NEW_CONSTR(TextVC, (stderr_treatment_default));
  }
  else {
    val = elems.get(4)->Eval(c);
    if (val->vKind != TextVK || !CheckTreatment((TextVC *)val)) {
      outputMu.lock();          
      args->EError("The `stderr_treatment' argument to _run_tool is not a suitable text:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.stderr_treatment = (TextVC*)val;

  // status_treatment:
  if (nArgs <= 5) {
    Text status_treatment_default("report_nocache");
    val = NEW_CONSTR(TextVC, (status_treatment_default));
  }
  else {
    val = elems.get(5)->Eval(c);
    if (val->vKind != TextVK || !CheckTreatment((TextVC *)val)) {
      outputMu.lock();    
      args->EError("The `status_treatment' argument to _run_tool is not a suitable text:");
      outputMu.unlock();    
      return false;
    }
  }
  runToolArgs.status_treatment = (TextVC*)val;

  // signal_treatment:
  if (nArgs <= 6) {
    Text signal_treatment_default("report_nocache");
    val = NEW_CONSTR(TextVC, (signal_treatment_default));
  }
  else {
    val = elems.get(6)->Eval(c);
    if (val->vKind != TextVK || !CheckTreatment((TextVC *)val)) {
      outputMu.lock();          
      args->EError("The `signal_treatment' argument to _run_tool is not a suitable text:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.signal_treatment = (TextVC*)val;

  // fp_content:
  if (nArgs <= 7) {
    val = fpContent;
  }
  else {
    val = elems.get(7)->Eval(c);
    if (val->vKind == BooleanVK) {
      val = NEW_CONSTR(IntegerVC, (((BooleanVC*)val)->b ? -1 : 0));
    }
    else if (val->vKind != IntegerVK) {
      outputMu.lock();          
      args->EError("The `fp_content' argument to _run_tool must be an integer or boolean:");
      outputMu.unlock();          
      return false;
    }
    else if (((IntegerVC*)val)->num == -2) {
      val = fpContent;
    }
  }
  runToolArgs.fp_content = (IntegerVC*)val;

  // wd:
  if (nArgs <= 8) {
    Text wd_name(".WD");
    val = NEW_CONSTR(TextVC, (wd_name));
  }
  else {
    val = elems.get(8)->Eval(c);
    if (val->vKind != TextVK) {
      outputMu.lock();          
      args->EError("The `wd' argument to _run_tool must be a text:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.wd_name = (TextVC*)val;

  // existing_writable:
  if (nArgs <= 9) {
    val = NEW_CONSTR(BooleanVC, (false));
  }
  else {
    val = elems.get(9)->Eval(c);
    if (val->vKind != BooleanVK) {
      outputMu.lock();          
      args->EError("The `existing_writable' argument to _run_tool must be a boolean:");
      outputMu.unlock();          
      return false;
    }      
  }
  runToolArgs.existing_writable = (BooleanVC*)val;

  // .:
  if (nArgs <= 10) {
    val = LookupInContext(nameDot, c);
    if (val->vKind != BindingVK) {
      outputMu.lock();          
      args->EError("The `.' argument to _run_tool must be a binding:");
      outputMu.unlock();          
      return false;
    }
  }
  else {
    val = elems.get(10)->Eval(c);
    if (val->vKind != BindingVK) {
      outputMu.lock();          
      args->EError("The `.' argument to _run_tool must be a binding:");
      outputMu.unlock();          
      return false;
    }
  }
  runToolArgs.dot = (BindingVC *)val;
  Val envVars = runToolArgs.dot->Lookup("envVars");
  if (envVars->vKind != BindingVK || !AllTexts((BindingVC*)envVars)) {
    outputMu.lock();        
    args->EError("./envVars isn't a binding of texts:");
    outputMu.unlock();        
    return false;
  }
  Val root = runToolArgs.dot->Lookup("root"); 
  if (root->vKind != BindingVK) {
    outputMu.lock();        
    args->EError("./root isn't a binding:");
    outputMu.unlock();        
    return false;
  }

  // RunTool arguments are ok.
  runToolArgs.loc = args->loc;
  return true;
}

void RunToolPK(const RunToolArgs& runToolArgs, FP::Tag &pk) {
  // Primary key for a runtool call:
  // We combine fingerprints of relevant arguments to _run_tool, 
  // except for ./root, whose dependencies are fine-grained. 
  pk.Extend("_run_tool");
  pk.Extend(runToolArgs.platform->FingerPrint());
  pk.Extend(runToolArgs.command_line->FingerPrint());
  pk.Extend(runToolArgs.stdin_data->FingerPrint());
  pk.Extend(runToolArgs.stdout_treatment->FingerPrint());
  pk.Extend(runToolArgs.stderr_treatment->FingerPrint());
  pk.Extend(runToolArgs.status_treatment->FingerPrint());
  pk.Extend(runToolArgs.signal_treatment->FingerPrint());
  pk.Extend(runToolArgs.fp_content->FingerPrint());
  pk.Extend(runToolArgs.wd_name->FingerPrint());
  if (runToolArgs.existing_writable->b)
    pk.Extend(runToolArgs.existing_writable->FingerPrint());
  Val envVars = runToolArgs.dot->Lookup("envVars");
  pk.Extend(envVars->FingerPrint());
}

Val MergeArgsDpnd(Val result, const RunToolArgs& args) {
  // Add the dependency of the arguments of this runtool call.
  result->Merge(args.platform);

  ListVC *command = args.command_line;
  result->Merge(command);
  if (!command->path) {
    if (command->lenDps != NULL &&
        command->lenDps->Size() != 0) {
      DPaths psTemp;
      ValueDpnd(command, &psTemp);
      result->MergeDPS(&psTemp);
    }
    else {
      Vals vs = command->elems;
      while (!vs.Null())
        result->Merge(vs.Pop());
    }
  }
  result->Merge(args.stdin_data);
  result->Merge(args.stdout_treatment);
  result->Merge(args.stderr_treatment);
  result->Merge(args.wd_name);
  return result;
}

static Basics::thread renewLeaseThread;

static void* RenewLeases(void *arg) throw () {
  time_t tsStart;
  // time_t sleepDuration = (time_t)(leaseDuration * 0.8);
  time_t sleepDuration = 10;

  // Remember how many time we've hit the "server busy" case.
  unsigned int server_busy_count = 0;
  
  while (true) {
    int left = sleepDuration;
    do {
      left = sleep(left);
    } while (left > 0);

    // If lease already expired, quit.
    leaseMu.lock();
    if (time(&tsStart) - lastRenewed > leaseDuration) {
      leaseMu.unlock();
      // If we've had trouble getting the cache server to accept our
      // calls and the leases expired, print a message about that.
      if(server_busy_count > 0)
        {
          outputMu.lock();
          Error(Text("RenewLeases thread got \"connection refused: server busy\" ") +
                IntToText(server_busy_count) + " times, and now leases have expired!\n");
          outputMu.unlock();
        }
      return NULL;
    }
    leaseMu.unlock();
    
    // Renew leases:
    CacheEntry::IndicesApp orphanCIs;    
    frontierMu.lock();
    ThreadData::mu.lock();
    ThreadData* cur = ThreadData::head;
    while (cur) {
      assert(cur->orphanCIs != 0);
      for (int i = 0; i < cur->orphanCIs->len; i++) {
        orphanCIs.Append(cur->orphanCIs->index[i]);
      }
      cur = cur->next;
    }
    ThreadData::mu.unlock();
    frontierMu.unlock();

    bool renewed = false;
    try
      {
        // Renew the leases, remembering whether we succeeded.
        renewed = theCache->RenewLeases(orphanCIs);

        // No "server busy" SRPC failure.
        server_busy_count = 0;

        // If we didn't renew successfully, print an error message,
        // remember that we've failed, and exit the thread.
        if(!renewed)
          {
            outputMu.lock();
            Error(Text("RenewLeases returned failure!\n") +
                  "This may be a bug.  Please reposrt it.\n");
            outputMu.unlock();

            leaseMu.lock();
            renewLease_failure = true;
            leaseMu.unlock();

            return NULL;
          }
      }
    // If we get an SRPC failure...
    catch (SRPC::failure f)
      {
        // If this looks like hitting the cache server's connection
        // limits, ignore it (but remember it).  (Presumably we'll get
        // in on a subsequent call.)
        if((f.r == 1) && (f.msg.FindText("connection refused: server busy") != -1))
          {
            server_busy_count++;
          }
        // If it doesn't look like the LimService message...
        else
          {
            // ...assume the cache server went down.  Print an error
            // message with the SRPC failure, note that we've failed,
            // and exit the thread.
            outputMu.lock();
            Error(Text("RenewLeases thread: SRPC failure (")
                  + IntToText(f.r) + "): " + f.msg + ".\n");
            outputMu.unlock();

            leaseMu.lock();
            renewLease_failure = true;
            leaseMu.unlock();

            return NULL;
          }
      }
    
    // If lease expires during renewing, quit.
    leaseMu.lock();
    if (time(NULL) - lastRenewed > leaseDuration) {
      leaseMu.unlock();
      // If we've had trouble getting the cache server to accept our
      // calls and the leases expired, print a message about that.
      if(server_busy_count > 0)
        {
          outputMu.lock();
          Error(Text("RenewLeases thread got \"connection refused: server busy\" ") +
                IntToText(server_busy_count) + " times, and now leases have expired!\n");
          outputMu.unlock();
        }
      return NULL;
    }
    
    if(renewed)
      {
        // Leases are renewed successfully:
        lastRenewed = tsStart;
      }
    leaseMu.unlock();    
  }
}

void StartRenewLeaseThread() {
  if (cacheOption != 0) {
    time(&lastRenewed);  
    renewLeaseThread.fork_and_detach(RenewLeases, (void*)NULL);
  }
}

Val RunToolFromCache(const RunToolArgs& runToolArgs, const Context& c, SrcLoc *loc) {
  FP::Tag pk(evaluatorRunToolVersionTag);
  CompactFV::List fvNames;
  FV2::List entryNames;
  FP::List tags;
  CacheEntry::Index cIndex;
  VestaVal::T cVal;
  CacheIntf::LookupRes cResult = CacheIntf::Miss;
  Model::T model = NullShortId;
  Val result;
  VestaSource* rootForTool;

  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;

  // Evaluate in the nocaching cases:
  if (cacheOption == 0) {
    if (traceRes) *traceRes << ": disabled\n";
    result = RunTool(runToolArgs, rootForTool);
    DeleteRootForTool(rootForTool);
    return result;
  }

  // Compute the primary key:
  RunToolPK(runToolArgs, pk);

  // Evaluate with caching:
  CacheEntry::IndicesApp* orphanCIs = ThreadDataGet()->orphanCIs;    
  try {
    while (true) {
      bool noEntry;
      FV::Epoch epoch = theCache->FreeVariables(pk, /*OUT*/ fvNames, /*OUT*/ noEntry);
      if (noEntry)
        cResult = CacheIntf::Miss;
      else {
        Tags(fvNames, c, /*OUT*/ tags);
        cResult = theCache->Lookup(pk, epoch, tags, /*OUT*/ cIndex, /*OUT*/ cVal);
      }
      FreeNamesTags(fvNames, tags);
      if (cResult == CacheIntf::Hit) {
        if (Unpickle(cVal, c, result)) {
          if (traceRes) *traceRes << ": hit(ci=" << cIndex << ")\n";
          IncCounter(&toolHitCounter);
          if (cacheOption == 1) {
            result->dps = NULL;
            return result;
          }
          UpdateOrphans(orphanCIs, orphanCIs->len, cIndex);
          return MergeArgsDpnd(result, runToolArgs);
        }
        outputMu.lock();        
        OBufStream l_msg;
        l_msg << "RunToolFromCache: Cache hit but bad cache data." << endl << endl
              << "\tpk = " << pk << endl
              << "\tci = " << cIndex << endl << endl
              << "Evaluate as cache miss." << endl;
        Error(Text(l_msg.str()), loc);
        outputMu.unlock();      
        cResult = CacheIntf::Miss;
      }
      if (cResult == CacheIntf::Miss) {
        if (traceRes) *traceRes << ": miss\n";
        int kidsIndex = orphanCIs->len;

        // Check to make sure there isn't an outstanding tool run with the same pk
        rtt_cond *rttCond;
        runToolTableMu.lock();
        bool found = runToolTable.Get(pk, /*OUT*/ rttCond);
        if(!found) {
                rttCond = NEW(rtt_cond);
                runToolTable.Put(pk, rttCond);
                runToolTableMu.unlock();
        } else {
                // There's already a tool with this PK running...
                // We'll wait for it to finish
                outputMu.lock();
                cerr << ThreadLabel() << "Waiting on possbily identical tool run" << endl;
                outputMu.unlock();
                rttCond->wait();
                // Woken up, tool must have finished.  Go around the
                // loop and try another cache lookup.
                runToolTableMu.unlock();
                continue;
        }
        // run the tool
        try {
                result = RunTool(runToolArgs, rootForTool);
                char *types;
                if (!noAddEntry &&
                    NamesTagsPickle(result, false, conEmpty, /*OUT*/ types, /*OUT*/ entryNames,
                                    /*OUT*/ tags, /*OUT*/ cVal)) {
                  CacheEntry::Indices kids;
                  CollectKids(orphanCIs, kidsIndex, kids);
                  // kids should be empty when evaluating eagerly.

                  Text pksource("_run_tool, command line: ");
                  pksource += ToolCommandLineAsText(runToolArgs);

                  // Add the entry, capturing the result of the call.
                  CacheIntf::AddEntryRes ae_res =
                    theCache->AddEntry(pk, types, entryNames, tags, cVal, model,
                                       kids, pksource, /*OUT*/ cIndex);
                  // If the AddEntry fails, die with an error.
                  if(ae_res != CacheIntf::EntryAdded)
                    {
                      outputMu.lock();          
                      Error(Text("AddEntry failed: ")+
                            CacheIntf::AddEntryResName(ae_res)+"\n"
                            "This may be an evaluator bug.  "
                            "Please report it.\n",
                            loc);
                      outputMu.unlock();
                      if (traceRes) *traceRes << endl;
                      throw(Evaluator::failure(Text("AddEntry failed (may be an "
                                                    "evaluator bug)"),
                                               false));
                    }
                  // cIndex is added to the orphans.
                  UpdateOrphans(orphanCIs, kidsIndex, cIndex);
                  Checkpoint();
                  if (traceRes) {
                    *traceRes << "  " << thdata->funcCallDepth << ". " << loc->file << ": "
                              << "_run_tool(): add (ci=" << cIndex << ")\n";
                  }
                }
                // now wake up the other threads waiting for the same PK
                runToolTableMu.lock();
                runToolTable.Delete(pk, /*OUT*/ rttCond);
                rttCond->wake();
                runToolTableMu.unlock();
        } catch(...) {
                runToolTableMu.lock();
                runToolTable.Delete(pk, /*OUT*/ rttCond);
                rttCond->wake();
                runToolTableMu.unlock();
                throw;
        }

        // delete the volatile dir for root after adding cache entry.
        // The new cache entry is now protecting the deriveds from weeding.
        DeleteRootForTool(rootForTool);   
        if (cacheOption == 1) {
          result->dps = NULL;
          return result;
        }
        return MergeArgsDpnd(result, runToolArgs);
      }
      if (cResult != CacheIntf::FVMismatch) {
        if (traceRes) *traceRes << ": mismatch\n";
        OBufStream err_msg;
        err_msg << "RunToolFromCache got bad result: "
                << CacheIntf::LookupResName(cResult) << endl
                << "\tpk = " << pk << endl
                << "\tFV epoch = " << epoch << endl
                << "\ttags.len = " << tags.len << endl
                << "This may be an evaluator bug.  Please report it." << endl;
        outputMu.lock();        
        Error(Text(err_msg.str()), loc);
        outputMu.unlock();      
        result = NEW(ErrorVC);
        return result;
      }
    }
  } catch (PrefixTbl::Overflow) {
    outputMu.lock();        
    Error(Text("Internal limits exceeded: PrefixTbl overflow\n"),
          loc);
    cerr << "Caching _run_tool with command line:" << endl << endl
         << "  " << ToolCommandLineAsText(runToolArgs) << endl << endl
         << Text("This usually means that there are too many free variables "
                 "in the secondary key of the function being cached.  For "
                 "_run_tool calls, this may mean you're trying to do too "
                 "much work in a single tool invocation.  Try splitting the "
                 "work of this tool invocation into several separate tool "
                 "invocations each doing a smaller amount of work and using "
                 "a smaller number of source files.  (Fixing this will also "
                 "reduce the CPU load of your evaluator and cache server, and "
                 "improve build performance.)").WordWrap()
         << endl << endl;
    // This is a fatal error.  Exit and dump core.
    abort();
    outputMu.unlock();
  } catch (SRPC::failure f) {
    outputMu.lock();        
    Error(Text("RunToolFromCache: SRPC failure (")
             + IntToText(f.r) + "): " + f.msg + ".\n",
          loc);
    outputMu.unlock();        
    if (traceRes) *traceRes << endl;
    throw(Evaluator::failure(Text("Cache server is possibly down"), false));
  }
  // return result;  // not reached
}

Val ApplyRunTool(ArgList args, const Context& c) {
  RunToolArgs runToolArgs;

  ThreadData *thdata = ThreadDataGet();
  ostream *traceRes = thdata->traceRes;
  
  IncCounter(&toolCallCounter);
  if (!BindRunToolArgs(args, c, runToolArgs)) {
    thdata->funcCallDepth++;
    if (traceRes) {
      *traceRes << "  " << thdata->funcCallDepth << ". " << args->loc->file << ": ";
      *traceRes << "_run_tool()";
      *traceRes << ": badargs\n";
    }
    thdata->funcCallDepth--;
    return NEW(ErrorVC);
  }
  thdata->funcCallDepth++;
  if (traceRes) {
    *traceRes << "  " << thdata->funcCallDepth << ". " << args->loc->file << ": ";
    *traceRes << "_run_tool()";
  }

  // Evaluate:
  Val result = RunToolFromCache(runToolArgs, c, args->loc);
  thdata->funcCallDepth--;
  return result;
}

---