VASTi.C

Go to the documentation of this file.

// Copyright (C) 2001, Compaq Computer Corporation
// 
// This file is part of Vesta.
// 
// Vesta is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// Vesta is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with Vesta; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

/* File: VASTi.C                                               */

#include <dirent.h>
#include <time.h>
extern "C" {
    // add declaration not included by broken header files
    struct tm *_Plocaltime_r(const time_t *timer, struct tm *result);
}
#include <Basics.H>
#include <VestaConfig.H>
#include <SourceOrDerived.H>
#include <ParCacheC.H>
#include <Timer.H>
#include <ReposUI.H>
#include "VASTi.H"
#include "Lex.H"
#include "Parser.H"
#include "Expr.H"
#include "Val.H"
#include "PrimRunTool.H"
#include "ToolDirectoryServer.H"
#include "Location.H"
#include "Err.H"
#include "Files.H"
#include "ApplyCache.H"
#include "ThreadData.H"

using std::ios;
using std::ostream;
using std::fstream;
using std::cout;
using std::cerr;
using std::endl;
using std::flush;

// On some platforms (at least Linux) errno.h doesn't define ESUCCESS
#if !defined(ESUCCESS)
#define ESUCCESS 0
#endif

static Text modelPath;
static mode_t newUMask = 022, oldUMask;

// In parallel build, we only want to print out one error stack.
// The variable callStackPrinted controls that. It is protected
// by callStackMu.
static bool callStackPrinted = false;

void PrintCacheStat(ostream *vout) {
  *vout << "\nCaching Stats:\n";
  *vout << "    Cache: [" << "Server=" << *ParCacheC::Locate() << ", "
        << "MetaData=" << VestaConfig::get_Text("CacheServer", "MetaDataRoot")
        << "]\n";
  *vout << "    Function: [" << "Calls=" << appCallCounter << ", "
        << "Hits=" << appHitCounter << "]\n";
  *vout << "    Model: ["
        << "Calls=[" << "Special=" << sModelCallCounter << ", "
        << "Normal=" << nModelCallCounter << "], "
        << "Hits=[" << "Special=" << sModelHitCounter << ", "
        << "Normal=" << nModelHitCounter << "]" << "]\n";
  *vout << "    RunTool: [" << "Calls=" << toolCallCounter << ", "
        << "Hits=" << toolHitCounter << "]\n";
}

void PrintFuncTrace(ostream *vout) {
  if (recordTrace) {
    *vout << "\nFunction call graph:\n"
          << ThreadDataGet()->traceRes->str();
  }
}

void PrintErrorStack(ostream *vout) {
  if (!recordCallStack) return;
  callStackMu.lock();
  if (callStackPrinted) {
    callStackMu.unlock();
    return;
  }
  callStackPrinted = true;

  ThreadData *thdata = ThreadDataGet();
  *vout << "\nError stack trace:\n";
  int counter = 0;
  int st = 0;
  for (;;) {
    int sz = thdata->callStack->size();
    for (int i = st; i < sz; i++) {
      Expr expr = thdata->callStack->get(i);
      *vout << counter++ << ". " << expr->loc->file << ": line "
            << expr->loc->line << ", char " << expr->loc->character
            << endl;
    }
    if (thdata->parent == NULL) break;
    st = thdata->parent->callStack->size() - thdata->parentCallStackSize;
    thdata = thdata->parent;
  }
  callStackMu.unlock();  
}

AssocVC* Lookup(Val val) {
  /* Get the component of the value specified by shipFromPath. */
  Val result = val;
  const char *path = shipFromPath.cchars();
  int len = shipFromPath.Length();

  Text arc;
  int i = 0;
  while (i < len) {
    int j = i;
    while (j < len && !IsDelimiter(path[j])) j++;
    arc = shipFromPath.Sub(i, j);
    if (result->vKind == BindingVK) {
      AssocVC *as = FindInContext(arc, ((BindingVC*)result)->elems);
      if (as == nullAssoc) {
        cerr << "Warning: the path `" << shipFromPath
             << "' does not exist in the result value.\n";
        return as;
      }
      result = as->val;
      i = j + 1;
    }
    else {
      Error(Text("Failed to find the value to ship in ") +
            shipFromPath + ".\n");
      return NULL;
    }
  }
  return NEW_CONSTR(AssocVC, (arc, result));
}

int Copy(const char *fromPath, const char *toPath) {
  /* Copy the file specified by the fromPath to the file specified
     by the toPath.  It returns the errno if fails. */
  const int BuffSz = 8192;
  char buf[BuffSz];
  int fromFd = open(fromPath, O_RDONLY);
  if (fromFd == -1) return errno;

  // Determine the mode we should use for the shipped copy.  Start
  // with rw priviledges.  (Note that these will be modified by
  // umask.)
  mode_t to_mode = (S_IRUSR | S_IWUSR |
                    S_IRGRP | S_IWGRP |
                    S_IROTH | S_IWOTH);
  // If the file we're copying has execute permission, then give
  // execute permission on the shipped copy.
  struct stat from_stat;
  if(fstat(fromFd, &from_stat) == 0)
    {
      if(from_stat.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))
        {
          to_mode |= S_IXUSR | S_IXGRP | S_IXOTH;
        }
    }
  else
    {
      // This shouldn't happen.  We opened it, so we should be able to
      // stat it.
      int err = errno;
      cerr << "Warning: couldn't stat file to be shipped (`" << fromPath
           << "'): " << strerror(err) << endl;

    }

  remove(toPath);
  int toFd = creat(toPath, to_mode);
  if (toFd == -1) {
    close(fromFd);
    return errno;
  }

  int size;
  int eno = ESUCCESS;
  while (size = read(fromFd, buf, BuffSz)) {
    if (write(toFd, buf, size) == -1) {
      eno = errno;
      break;
    }
  }
  close(fromFd);
  close(toFd);
  return eno;
}

int CleanDir(Text path) {
  int err = 0;
  int entries = 0;

  // Check if .log exists
  struct stat statbuf;
  bool havelog = (stat(".log", &statbuf) == 0);

  DIR *dirPt = opendir(".");
  if (dirPt == NULL) {
    Error(Text("Failed to open the directory ") + path + ".\n");
    return -1;
  }

  for (dirent *dp = readdir(dirPt); dp != NULL; dp = readdir(dirPt)) {
    if (strcmp(dp->d_name, ".") && strcmp(dp->d_name, "..") && 
        strcmp(dp->d_name, ".log")) {
      entries++;
      if (!forceClean && !havelog) continue;
      err = lstat(dp->d_name, &statbuf);
      if (S_ISDIR(statbuf.st_mode)) {
        // must be a directory. Remove it recursively.
        err = chdir(dp->d_name);
        if (err) return err;
        err = CleanDir(path + dp->d_name + "/");
        if (err) return err;
        err = chdir("..");
        if (err) return err;

        if(!hushedShipping)
          cout << "Cleaning " << path << dp->d_name << endl;
        err = rmdir(dp->d_name);
        if (err) return err;

      } else {
        // must be a nondirectory. So, remove it.
        if(!hushedShipping)
          cout << "Cleaning " << path << dp->d_name << endl;
        err = remove(dp->d_name);
        if (err) return err;
      }
    }
  }
  if (havelog) {
    err = remove(".log");
    if (err) return err;
  }
  (void)closedir(dirPt);

  if (!forceClean && !havelog && entries > 0) {
    Error(Text("Tried to clean a nonempty directory with no .log file: ")
          + shipToPath + ".\n");
    return -1;
  }

  return 0;
}

enum ShipKind { CopyShip, SymLinkShip };
const char *KindName[] = { "copy", "link" };



void LogShip(ShortId sid, const char *path,
  fstream &logFile, struct tm *shipTM, bool is_dir = false) throw ()
{
    char dateStr[20];
    sprintf(dateStr, "%02d:%02d:%02d %02d-%02d-%04d",
      shipTM->tm_hour, shipTM->tm_min, shipTM->tm_sec,
      (shipTM->tm_mon + 1), shipTM->tm_mday, (1900 + shipTM->tm_year));

    logFile << modelPath << ": " << path;
    if(is_dir) {
      logFile << " -> directory";
    }
    else {
      if (sid) {
        char sidHex[20];
        int err = sprintf(sidHex, "0x%08x", sid); assert(err >= 0);
        logFile << " -> " << sidHex;
      } else {
        logFile << " -> literal";
      }
    }
    
    logFile << ' ' << dateStr << endl;
    logFile.flush();
}

int ShipShortId(ShortId sid, const Text &name, ShipKind kind,
                fstream &logFile, struct tm *shipTM)
throw () {
  int err = ESUCCESS;
  if (sid != NullShortId) {
    if (!hushedShipping)
      cout << "Shipping " << name << "..." << endl;
    char *path = SourceOrDerived::shortIdToName(sid, /*tailonly=*/ false);
    switch (kind) {
    case CopyShip:
      err = Copy(path, name.cchars());
      break;
    case SymLinkShip:
      remove(name.cchars());
      if (symlink(path, name.cchars()) == -1) {
        err = errno;
      }
      break;
    default:
      assert(false);
    }
    if (err != ESUCCESS) {
      Error(Text("Failed to ") + KindName[(int)kind] + " the file " +
            name + ": " + strerror(err) + ".\n");
    }
    else {
      LogShip(sid, name.cchars(), logFile, shipTM);
    }
  }
  return err;
}

int ShipText(const Text &txt, const Text &name,
             fstream &logFile, struct tm *shipTM)
throw () {
    if (!hushedShipping)
      cout << "Shipping " << name << "..." << endl;
    remove(name.cchars());
    fstream outFile;
    outFile.open(name.cchars(), ios::out);
    if (outFile.fail()) {
      Error(Text("Failed to create the file ") + name +
            ": " + strerror(errno) + ".\n");
      return errno;
    }
    outFile << txt << flush;
    if (outFile.fail()) {
      Error(Text("Failed to write to the file ") + name +
            ": " + strerror(errno) + ".\n");
      return errno;
    }
    outFile.close();
    LogShip(0, name.cchars(), logFile, shipTM);
    return 0;
}

int ShipValue(Val value, ShipKind kind, struct tm *shipTM) throw () {
  fstream logFile;
  logFile.open(dotLogFiles?".log":"/dev/null",
               ios::out | ios::app);
  if (logFile.fail()) {
    Error(Text("Failed to open .log file: ") + strerror(errno) + ".\n");
    return -1;
  }
   
  /* Assume that the work dir is the place to ship. */
  if (value->vKind == BindingVK) {
    BindingVC *bv = (BindingVC*)value;
    Context work = bv->elems;
    while (!work.Null()) {
      Assoc elem = work.Pop();
      Val val = elem->val;
      ShortId sid;
      int err = 0;      
      switch (val->vKind) {
        case TextVK:
          if (((TextVC*)val)->HasSid()) {
            sid = ((TextVC*)val)->Sid();
            err = ShipShortId(sid, elem->name, kind, logFile, shipTM);
          }
          else { 
            err = ShipText(((TextVC*)val)->NDS(), elem->name, logFile, shipTM);
          }
          if (err) return err;
          break;
        case ModelVK:
          sid = ((ModelVC*)val)->Sid();
          err = ShipShortId(sid, elem->name, kind, logFile, shipTM);
          if (err) return err;
          break;
        case BindingVK:
          {
            if(!FS::IsDirectory(elem->name.cchars())) {
              // directory does not exist; create it
              err = mkdir(elem->name.cchars(), 0xfff);
              if (err) {
                Error(Text("Failed to create the directory ") +
                     elem->name + ": " + strerror(errno) + ".\n");
              }
            }
            LogShip(0, elem->name.cchars(), logFile, shipTM, true);           
          }
          if (err) return err;
          err = chdir(elem->name.cchars());
          if (err) return err;
          err = ShipValue(val, kind, shipTM);
          if (err) return err;    
          err = chdir("..");
          if (err) return err;
          break;
        default:
          // ignore other cases
          break;
      }
    }
  }
  logFile.close();
  return 0;
}

bool VestaShip(AssocVC *namedVal) {
  /* Change to the directory at shipToPath, empty the directory, if requested,
     and then ship the result.  */
  int err = 0;
  const char *path = shipToPath.cchars();
  Val shipValue;
  Text log_dir = shipToPath;

  err = chdir(path);
  if (!err) {
    if (shipClean || forceClean) {
      // Empty the directory
      if(hushedShipping) {
        cout << "Cleaning..." << endl;
        cout.flush();
      }
      Text dirPath = shipToPath;
      if(dirPath[dirPath.Length()-1] != '/')
        dirPath += "/";
      err = CleanDir(dirPath);
      if (err) {
        Error(Text("Failed to delete the contents of the directory ") +
              shipToPath + ".\n");
        return false;
      }
    }
    if (namedVal->val->vKind == BindingVK) {
      shipValue = namedVal->val;
    }
    else {
      shipValue = NEW_CONSTR(BindingVC, (Context(namedVal)));
    }
  }
  else if ((errno == ENOTDIR || errno == ENOENT) &&
           (namedVal->val->vKind == TextVK ||
            namedVal->val->vKind == ModelVK)) {
    // Shipping to a file
    Text filename;
    try {
      FS::SplitPath(shipToPath, filename, log_dir);
    }
    catch(FS::Failure f) {
        Error(Text("FS failure: ") + f.get_op() + ": " + f.get_errno() + ".\n");
        return false;
    }
    err = chdir(log_dir.cchars());
    if(!err) {
      shipValue = 
        NEW_CONSTR(BindingVC, 
                   (Context(NEW_CONSTR(AssocVC, (filename, namedVal->val)))));
    }
  }

  if(err) {
    Error(Text("Failed to cd to the directory ") + log_dir + ": "
               + strerror(errno) + ".\n");
    return false;
  }

  // ship the requested value if no errors have occurred
  if (hushedShipping) {
    cout << "Shipping...\n";
    cout.flush();
  }
  time_t nowT;
  if (time(&nowT) == (time_t)(-1)) {
    Error(Text("Getting time of day from time(): ") + strerror(errno) + ".\n");
    return false;
  }
  else {
    struct tm buffTM, *nowTM;
    nowTM = localtime_r(&nowT, &buffTM);
    ShipKind kind = (shipBySymLink ? SymLinkShip : CopyShip);
      
    /* We still need to add a file lock, since there can be multiple
       writers to the same log file. */
    err = ShipValue(shipValue, kind, nowTM);
  }

  if (hushedShipping) {
    cout << "Done!\n";
    cout.flush();
  }
  return (err == 0);
}

bool Interpret(const Text& model) {
  Text prefix;
  VestaSource *vSource;
  fstream *iFile;
  VestaSource::errorCode err;
  Expr expr;
  Val result;
  bool success = true;

  uid_t euid = geteuid();
  if (seteuid(getuid()) < 0) {
    throw(Evaluator::failure(Text("Failed to switch to the real user-id.\n"),
                             false));
  }

  try {
    modelPath = ReposUI::canonicalize(model);
  }
  catch(ReposUI::failure f) {
    Error(Text("ReposUI failure: ") + f.msg + ".\n");
    return false;
  }

  if (seteuid(euid) < 0) {
    throw(Evaluator::failure(
             Text("Failed to switch back to the effective user-id.\n"),
             false));      
  }
  if (!OpenSource(NULL, modelPath, noLoc, /*OUT*/ iFile, /*OUT*/ topModelRoot,
                  /*OUT*/ topModelSid, /*OUT*/ vSource)) {
    // failed to open the top-level model.
    success = false;
  }
  else {
    // Parsing the top-level model:
    if (iFile == NULL) {
      iFile = NEW(SourceOrDerived);
      ((SourceOrDerived*)iFile)->open(topModelSid);
      if (iFile->bad()) {
        throw(Evaluator::failure(Text("Can't open the model ") + modelPath +
                                 "; giving up!\n",
                                 false));
      }
    }
    try {
      expr = Parse(iFile, modelPath, topModelSid, topModelRoot);
    } catch (const char* report) {
      // ErrorDetail(report);    // Handle parsing error exception.
      success = false;
    }
    iFile->close();
  }
  if (!success) {
    Error("Failed to parse `" + model + "'.\n\n");
    return false;
  }
  if (parseOnly) {
    cout << "Parsing completed.\n";
    return true;
  }

  // Evaluate:
  ModelVC *fun = NULL;
  try {
    ModelEC *modelExpr = (ModelEC*)expr;
    Context cc = ProcessModelHead(modelExpr);
    fun = NEW_CONSTR(ModelVC, (modelPath, topModelSid, topModelRoot, 
                               modelExpr, cc, vSource));
    ArgList args = NEW_CONSTR(ArgListEC, (0, modelExpr->loc));
    ApplyEC *ae = NEW_CONSTR(ApplyEC, (NEW_CONSTR(NameEC, (model, noLoc)), 
                                       args, modelExpr->loc));
    if (recordCallStack) {
      ThreadDataGet()->callStack->addlo(modelExpr);
    }
    result = ApplyModel(fun, ae, conInitial);
  } catch (SRPC::failure f) {
    Error(Text("SRPC failure (") + IntToText(f.r) + "): " + f.msg + ".\n");
    success = false;
  } catch (Evaluator::failure f) {
    //Error(Text("Vesta evaluation failure; ") + f.msg + ".\n");
    success = false;
    if (f.r) {
      // This feature lets us retry the evaluation on certain errors.
      // It is not currently used, and might not work properly.
      success = StartEval(model);
      exit(success ? 0 : 1);
    }
  } catch (const char* report) {
    // ErrorDetail(report);    // Handle parsing error exception.
    success = false;
  } catch (ReposUI::failure f) {
    Error(Text("ReposUI failure: ") + f.msg + ".\n");
    success = false;
  } catch (VestaConfig::failure f) {
    Error(Text("VestaConfig failure: ") + f.msg + ".\n");
    success = false;
  }

  // Print evaluation result:
  if (success && printResult) {
    // When there is no exception, always print the evaluation result,
    // which may still contains error.
    cout << "\nReturn value of `" << model << "':\n";
    result->PrintD(&cout);
    cout << "\n";
  }

  // Make sure we know if there was an error
  if (ErrorCount() > 0) success = false;

  // Print error stack, if the evaluation failed and the stack
  // has not already been printed.
  if (!success) PrintErrorStack(&cerr);
    
  // Print function call trace:
  PrintFuncTrace(&cout);

  // Print cache stats & Checkpoint cache:
  if (cacheOption != 0) {
    if (printCacheStats) PrintCacheStat(&cout);
    // orphanCIs contains all the cache entries that have no parent 
    // in the cache.
    CacheEntry::IndicesApp* orphanCIs = ThreadDataGet()->orphanCIs;    
    if (fun != NULL && orphanCIs->len > 0)
      try {
        theCache->Checkpoint(topModelRoot->fptag, topModelSid, *orphanCIs, true);
      } catch (SRPC::failure f) {
        Error(Text("SRPC failure when checkpointing cache (")
              + IntToText(f.r) + "): " + f.msg + ".\n");        
        success = false;
      }
  }

  // Ship the value if the evaluation succeeded.
  // First follow the shipFromPath to get the component to be shipped
  // from the result, and ship it to the place specified by the shipToPath.
  if (success && !shipToPath.Empty()) {
    AssocVC *shipValue = Lookup(result);
    if (shipValue != NULL) {
      if (seteuid(getuid()) < 0) {
        Error("Failed to switch to the real user-id.\n");
        success = false;
      }
      else {
        umask(oldUMask);
        success = VestaShip(shipValue);
      }
    }
  }

  // Print this process's status:
  if (psStat) {
    cout << "\nEvaluator Process Stats:\n   ";
    cout.flush();
    Text command("ps -o pid,user,vsz,rss,time,comm -A | grep ");
    command += IntToText(getpid());
    system(command.chars());
  }

  // Flush all the output streams:
  cout.flush();
  return success;
}

bool StartEval(const Text& filename) {
  // Initialization:
  ThreadDataInit();
  ErrInit();
  LexInit();
  ValInit();
  PrimInit();
  PrimRunToolInit();
  StartRenewLeaseThread();

  // Make sure umask used when directly writing deriveds into the
  // repository is not too restrictive.  Use the same value that
  // the repository uses internally.
  try {
    newUMask = 022; // documented default
    newUMask = VestaConfig::get_int("Repository", "umask");
  } catch (VestaConfig::failure) {
    // ignore errors, use default
  }
  oldUMask = umask(newUMask);

  // Evaluation:
  return Interpret(filename);
}

---